KittyPatch/kitty
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor the compilation code

Browse Source 
Makes it a bit more efficient. All compiling is done in parallel,
followed by all linking.
This commit is contained in:
Kovid Goyal
2019-07-05 18:13:21 +05:30
parent 29ad6fba22
commit 5e7fbfefe9
1 changed files with 73 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
setup.py
View File

@@ -15,6 +15,7 @@ import subprocess
import sys import sys
import sysconfig import sysconfig
import time import time
from functools import partial
from collections import namedtuple from collections import namedtuple
from contextlib import suppress from contextlib import suppress
from pathlib import Path from pathlib import Path
@@ -367,12 +368,12 @@ def dependecies_for(src, obj, all_headers):
yield path yield path
def parallel_run(todo, desc='Compiling {} ...'): def parallel_run(items):
try: try:
num_workers = max(2, os.cpu_count()) num_workers = max(2, os.cpu_count())
except Exception: except Exception:
num_workers = 2 num_workers = 2
items = list(reversed(tuple(todo.items()))) items = list(reversed(items))
workers = {} workers = {}
failed = None failed = None
@@ -381,37 +382,66 @@ def parallel_run(todo, desc='Compiling {} ...'):
if not workers: if not workers:
return return
pid, s = os.wait() pid, s = os.wait()
name, cmd, w = workers.pop(pid, (None, None, None)) compile_cmd, w = workers.pop(pid, (None, None))
if name is not None and ((s & 0xff) != 0 or ((s >> 8) & 0xff) != 0) and failed is None: if compile_cmd is not None and ((s & 0xff) != 0 or ((s >> 8) & 0xff) != 0) and failed is None:
failed = name, cmd failed = compile_cmd
elif compile_cmd.on_success is not None:
compile_cmd.on_success()
while items and failed is None: while items and failed is None:
while len(workers) < num_workers and items: while len(workers) < num_workers and items:
name, cmd = items.pop() compile_cmd = items.pop()
if verbose: if verbose:
print(' '.join(cmd)) print(' '.join(compile_cmd.cmd))
else: else:
print(desc.format(emphasis(name))) print(compile_cmd.desc)
w = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) w = subprocess.Popen(compile_cmd.cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
workers[w.pid] = name, cmd, w workers[w.pid] = compile_cmd, w
wait() wait()
while len(workers): while len(workers):
wait() wait()
if failed: if failed:
run_tool(failed[1]) run_tool(failed.cmd)
CompileKey = namedtuple('CompileKey', 'src dest') CompileKey = namedtuple('CompileKey', 'src dest')
Command = namedtuple('Command', 'desc cmd is_newer_func on_success key keyfile')
class CompilationDatabase: class CompilationDatabase:
def cmd_changed(self, key, cmd): def __init__(self, incremental):
self.all_keys.add(key) self.incremental = incremental
return self.db.get(key) != cmd self.compile_commands = []
self.link_commands = []
def update_cmd(self, key, cmd): def add_command(self, desc, cmd, is_newer_func, key=None, on_success=None, keyfile=None):
self.db[key] = cmd queue = self.link_commands if key is None else self.compile_commands
queue.append(Command(desc, cmd, is_newer_func, on_success, key, keyfile))
def build_all(self):
items = []
def sort_key(compile_cmd):
if compile_cmd.keyfile:
return os.path.getsize(compile_cmd.keyfile)
return 0
for compile_cmd in self.compile_commands:
if not self.incremental or self.cmd_changed(compile_cmd) or compile_cmd.is_newer_func():
items.append(compile_cmd)
items.sort(key=sort_key, reverse=True)
parallel_run(items)
items = []
for compile_cmd in self.link_commands:
if not self.incremental or compile_cmd.is_newer_func():
items.append(compile_cmd)
parallel_run(items)
def cmd_changed(self, compile_cmd):
key, cmd = compile_cmd.key, compile_cmd.cmd
return self.db.get(key) != cmd
def __enter__(self): def __enter__(self):
self.all_keys = set() self.all_keys = set()
@@ -432,21 +462,19 @@ class CompilationDatabase:
for key in set(cdb) - self.all_keys: for key in set(cdb) - self.all_keys:
del cdb[key] del cdb[key]
compilation_database = [ compilation_database = [
{'file': k.src, 'arguments': v, 'directory': base, 'output': k.dest} for k, v in cdb.items() {'file': c.key.src, 'arguments': c.cmd, 'directory': base, 'output': c.key.dest} for c in self.compile_commands
] ]
with open(self.dbpath, 'w') as f: with open(self.dbpath, 'w') as f:
json.dump(compilation_database, f, indent=2, sort_keys=True) json.dump(compilation_database, f, indent=2, sort_keys=True)
def compile_c_extension(kenv, module, incremental, compilation_database, sources, headers): def compile_c_extension(kenv, module, compilation_database, sources, headers):
prefix = os.path.basename(module) prefix = os.path.basename(module)
objects = [ objects = [
os.path.join(build_dir, prefix + '-' + os.path.basename(src) + '.o') os.path.join(build_dir, prefix + '-' + os.path.basename(src) + '.o')
for src in sources for src in sources
] ]
todo = {}
for original_src, dest in zip(sources, objects): for original_src, dest in zip(sources, objects):
src = original_src src = original_src
cppflags = kenv.cppflags[:] cppflags = kenv.cppflags[:]
@@ -460,32 +488,23 @@ def compile_c_extension(kenv, module, incremental, compilation_database, sources
cmd = [kenv.cc, '-MMD'] + cppflags + kenv.cflags cmd = [kenv.cc, '-MMD'] + cppflags + kenv.cflags
cmd += ['-c', src] + ['-o', dest] cmd += ['-c', src] + ['-o', dest]
key = CompileKey(original_src, os.path.basename(dest)) key = CompileKey(original_src, os.path.basename(dest))
cmd_changed = compilation_database.cmd_changed(key, cmd) desc = 'Compiling {} ...'.format(emphasis(src))
must_compile = not incremental or cmd_changed compilation_database.add_command(desc, cmd, partial(newer, dest, *dependecies_for(os.path.join(base, src), dest, headers)), key=key, keyfile=src)
src = os.path.join(base, src)
if must_compile or newer(
dest, *dependecies_for(src, dest, headers)
):
compilation_database.update_cmd(key, cmd)
todo[original_src] = cmd
if todo:
parallel_run(todo)
dest = os.path.join(build_dir, module + '.so') dest = os.path.join(build_dir, module + '.so')
real_dest = os.path.join(base, module + '.so') real_dest = os.path.join(base, module + '.so')
os.makedirs(os.path.dirname(dest), exist_ok=True) os.makedirs(os.path.dirname(dest), exist_ok=True)
if not incremental or newer(real_dest, *objects): desc = 'Linking {} ...'.format(emphasis(module))
# Old versions of clang don't like -pthread being passed to the linker # Old versions of clang don't like -pthread being passed to the linker
# Don't treat linker warnings as errors (linker generates spurious # Don't treat linker warnings as errors (linker generates spurious
# warnings on some old systems) # warnings on some old systems)
unsafe = {'-pthread', '-Werror', '-pedantic-errors'} unsafe = {'-pthread', '-Werror', '-pedantic-errors'}
linker_cflags = list(filter(lambda x: x not in unsafe, kenv.cflags)) linker_cflags = list(filter(lambda x: x not in unsafe, kenv.cflags))
try: cmd = [kenv.cc] + linker_cflags + kenv.ldflags + objects + kenv.ldpaths + ['-o', dest]
run_tool([kenv.cc] + linker_cflags + kenv.ldflags + objects + kenv.ldpaths + ['-o', dest], desc='Linking {} ...'.format(emphasis(module)))
except Exception: def on_success():
with suppress(EnvironmentError): os.rename(dest, real_dest)
os.remove(dest)
else: compilation_database.add_command(desc, cmd, partial(newer, real_dest, *objects), on_success=on_success)
os.rename(dest, real_dest)
def find_c_files(): def find_c_files():
@@ -505,7 +524,7 @@ def find_c_files():
return tuple(ans), tuple(headers) return tuple(ans), tuple(headers)
def compile_glfw(incremental, compilation_database): def compile_glfw(compilation_database):
modules = 'cocoa' if is_macos else 'x11 wayland' modules = 'cocoa' if is_macos else 'x11 wayland'
for module in modules.split(): for module in modules.split():
try: try:
@@ -525,7 +544,7 @@ def compile_glfw(incremental, compilation_database):
print(err, file=sys.stderr) print(err, file=sys.stderr)
print(error('Disabling building of wayland backend'), file=sys.stderr) print(error('Disabling building of wayland backend'), file=sys.stderr)
continue continue
compile_c_extension(genv, 'kitty/glfw-' + module, incremental, compilation_database, sources, all_headers) compile_c_extension(genv, 'kitty/glfw-' + module, compilation_database, sources, all_headers)
def kittens_env(): def kittens_env():
@@ -538,7 +557,7 @@ def kittens_env():
return kenv return kenv
def compile_kittens(incremental, compilation_database): def compile_kittens(compilation_database):
kenv = kittens_env() kenv = kittens_env()
def list_files(q): def list_files(q):
@@ -559,17 +578,17 @@ def compile_kittens(incremental, compilation_database):
filter_sources=lambda x: 'windows_compat.c' not in x), filter_sources=lambda x: 'windows_compat.c' not in x),
): ):
compile_c_extension( compile_c_extension(
kenv, dest, incremental, compilation_database, sources, all_headers + ['kitty/data-types.h']) kenv, dest, compilation_database, sources, all_headers + ['kitty/data-types.h'])
def build(args, native_optimizations=True): def build(args, native_optimizations=True):
global env global env
env = init_env(args.debug, args.sanitize, native_optimizations, args.profile, args.extra_logging) env = init_env(args.debug, args.sanitize, native_optimizations, args.profile, args.extra_logging)
compile_c_extension( compile_c_extension(
kitty_env(), 'kitty/fast_data_types', args.incremental, args.compilation_database, *find_c_files() kitty_env(), 'kitty/fast_data_types', args.compilation_database, *find_c_files()
) )
compile_glfw(args.incremental, args.compilation_database) compile_glfw(args.compilation_database)
compile_kittens(args.incremental, args.compilation_database) compile_kittens(args.compilation_database)
def safe_makedirs(path): def safe_makedirs(path):
@@ -618,10 +637,9 @@ def build_launcher(args, launcher_dir='.', bundle_type='source'):
cmd = [env.cc] + cppflags + cflags + [ cmd = [env.cc] + cppflags + cflags + [
src, '-o', dest] + ldflags + libs + pylib src, '-o', dest] + ldflags + libs + pylib
key = CompileKey('launcher.c', 'kitty') key = CompileKey('launcher.c', 'kitty')
must_compile = not args.incremental or args.compilation_database.cmd_changed(key, cmd) desc = 'Building {}...'.format(emphasis('launcher'))
if must_compile or newer(dest, src): args.compilation_database.add_command(desc, cmd, partial(newer, dest, src), key=key, keyfile=src)
run_tool(cmd, 'Building {}...'.format(emphasis('launcher'))) args.compilation_database.build_all()
args.compilation_database.update_cmd(key, cmd)
# Packaging {{{ # Packaging {{{
@@ -959,7 +977,7 @@ def main():
clean() clean()
return return
with CompilationDatabase() as cdb: with CompilationDatabase(args.incremental) as cdb:
args.compilation_database = cdb args.compilation_database = cdb
if args.action == 'build': if args.action == 'build':
build(args) build(args)