c2a924a5ea
Much more comprehensive test coverage at the cost of slightly increasing the frozen build size.
483 lines
17 KiB
Python
483 lines
17 KiB
Python
#!/usr/bin/env python
|
|
# vim:fileencoding=utf-8
|
|
# License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
|
|
|
|
import glob
|
|
import json
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zipfile
|
|
|
|
from bypy.constants import PREFIX, PYTHON, SW, python_major_minor_version
|
|
from bypy.freeze import (
|
|
extract_extension_modules, freeze_python, path_to_freeze_dir
|
|
)
|
|
from bypy.macos_sign import (
|
|
codesign, create_entitlements_file, make_certificate_useable, notarize_app,
|
|
verify_signature
|
|
)
|
|
from bypy.utils import (
|
|
current_dir, mkdtemp, py_compile, run_shell, timeit, walk
|
|
)
|
|
|
|
iv = globals()['init_env']
|
|
kitty_constants = iv['kitty_constants']
|
|
self_dir = os.path.dirname(os.path.abspath(__file__))
|
|
join = os.path.join
|
|
basename = os.path.basename
|
|
dirname = os.path.dirname
|
|
abspath = os.path.abspath
|
|
APPNAME = kitty_constants['appname']
|
|
VERSION = kitty_constants['version']
|
|
py_ver = '.'.join(map(str, python_major_minor_version()))
|
|
|
|
|
|
def flush(func):
|
|
def ff(*args, **kwargs):
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
ret = func(*args, **kwargs)
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
return ret
|
|
|
|
return ff
|
|
|
|
|
|
def flipwritable(fn, mode=None):
|
|
"""
|
|
Flip the writability of a file and return the old mode. Returns None
|
|
if the file is already writable.
|
|
"""
|
|
if os.access(fn, os.W_OK):
|
|
return None
|
|
old_mode = os.stat(fn).st_mode
|
|
os.chmod(fn, stat.S_IWRITE | old_mode)
|
|
return old_mode
|
|
|
|
|
|
STRIPCMD = ('/usr/bin/strip', '-x', '-S', '-')
|
|
|
|
|
|
def strip_files(files, argv_max=(256 * 1024)):
|
|
"""
|
|
Strip a list of files
|
|
"""
|
|
tostrip = [(fn, flipwritable(fn)) for fn in files if os.path.exists(fn)]
|
|
while tostrip:
|
|
cmd = list(STRIPCMD)
|
|
flips = []
|
|
pathlen = sum(len(s) + 1 for s in cmd)
|
|
while pathlen < argv_max:
|
|
if not tostrip:
|
|
break
|
|
added, flip = tostrip.pop()
|
|
pathlen += len(added) + 1
|
|
cmd.append(added)
|
|
flips.append((added, flip))
|
|
else:
|
|
cmd.pop()
|
|
tostrip.append(flips.pop())
|
|
os.spawnv(os.P_WAIT, cmd[0], cmd)
|
|
for args in flips:
|
|
flipwritable(*args)
|
|
|
|
|
|
def files_in(folder):
|
|
for record in os.walk(folder):
|
|
for f in record[-1]:
|
|
yield os.path.join(record[0], f)
|
|
|
|
|
|
def expand_dirs(items, exclude=lambda x: x.endswith('.so')):
|
|
items = set(items)
|
|
dirs = set(x for x in items if os.path.isdir(x))
|
|
items.difference_update(dirs)
|
|
for x in dirs:
|
|
items.update({y for y in files_in(x) if not exclude(y)})
|
|
return items
|
|
|
|
|
|
def do_sign(app_dir):
|
|
with current_dir(os.path.join(app_dir, 'Contents')):
|
|
# Sign all .so files
|
|
so_files = {x for x in files_in('.') if x.endswith('.so')}
|
|
codesign(so_files)
|
|
# Sign everything else in Frameworks
|
|
with current_dir('Frameworks'):
|
|
fw = set(glob.glob('*.framework'))
|
|
codesign(fw)
|
|
items = set(os.listdir('.')) - fw
|
|
codesign(expand_dirs(items))
|
|
|
|
# Now sign the main app
|
|
codesign(app_dir)
|
|
verify_signature(app_dir)
|
|
|
|
|
|
def sign_app(app_dir, notarize):
|
|
# Copied from iTerm2: https://github.com/gnachman/iTerm2/blob/master/iTerm2.entitlements
|
|
create_entitlements_file({
|
|
'com.apple.security.automation.apple-events': True,
|
|
'com.apple.security.cs.allow-jit': True,
|
|
'com.apple.security.device.audio-input': True,
|
|
'com.apple.security.device.camera': True,
|
|
'com.apple.security.personal-information.addressbook': True,
|
|
'com.apple.security.personal-information.calendars': True,
|
|
'com.apple.security.personal-information.location': True,
|
|
'com.apple.security.personal-information.photos-library': True,
|
|
})
|
|
with make_certificate_useable():
|
|
do_sign(app_dir)
|
|
if notarize:
|
|
notarize_app(app_dir)
|
|
|
|
|
|
class Freeze(object):
|
|
|
|
FID = '@executable_path/../Frameworks'
|
|
|
|
def __init__(self, build_dir, dont_strip=False, sign_installers=False, notarize=False, skip_tests=False):
|
|
self.build_dir = build_dir
|
|
self.skip_tests = skip_tests
|
|
self.sign_installers = sign_installers
|
|
self.notarize = notarize
|
|
self.dont_strip = dont_strip
|
|
self.contents_dir = join(self.build_dir, 'Contents')
|
|
self.resources_dir = join(self.contents_dir, 'Resources')
|
|
self.frameworks_dir = join(self.contents_dir, 'Frameworks')
|
|
self.to_strip = []
|
|
self.warnings = []
|
|
self.py_ver = py_ver
|
|
self.python_stdlib = join(self.resources_dir, 'Python', 'lib', 'python' + self.py_ver)
|
|
self.site_packages = self.python_stdlib # hack to avoid needing to add site-packages to path
|
|
self.obj_dir = mkdtemp('launchers-')
|
|
|
|
self.run()
|
|
|
|
def run_shell(self):
|
|
with current_dir(self.contents_dir):
|
|
run_shell()
|
|
|
|
def run(self):
|
|
ret = 0
|
|
self.add_python_framework()
|
|
self.add_site_packages()
|
|
self.add_stdlib()
|
|
self.add_misc_libraries()
|
|
self.freeze_python()
|
|
if not self.dont_strip:
|
|
self.strip_files()
|
|
if not self.skip_tests:
|
|
self.run_tests()
|
|
# self.run_shell()
|
|
|
|
ret = self.makedmg(self.build_dir, APPNAME + '-' + VERSION)
|
|
|
|
return ret
|
|
|
|
@flush
|
|
def strip_files(self):
|
|
print('\nStripping files...')
|
|
strip_files(self.to_strip)
|
|
|
|
@flush
|
|
def run_tests(self):
|
|
iv['run_tests'](os.path.join(self.contents_dir, 'MacOS', 'kitty'))
|
|
|
|
@flush
|
|
def set_id(self, path_to_lib, new_id):
|
|
old_mode = flipwritable(path_to_lib)
|
|
subprocess.check_call(
|
|
['install_name_tool', '-id', new_id, path_to_lib])
|
|
if old_mode is not None:
|
|
flipwritable(path_to_lib, old_mode)
|
|
|
|
@flush
|
|
def get_dependencies(self, path_to_lib):
|
|
install_name = subprocess.check_output(
|
|
['otool', '-D', path_to_lib]).decode('utf-8').splitlines()[-1].strip()
|
|
raw = subprocess.check_output(['otool', '-L', path_to_lib]).decode('utf-8')
|
|
for line in raw.splitlines():
|
|
if 'compatibility' not in line or line.strip().endswith(':'):
|
|
continue
|
|
idx = line.find('(')
|
|
path = line[:idx].strip()
|
|
yield path, path == install_name
|
|
|
|
@flush
|
|
def get_local_dependencies(self, path_to_lib):
|
|
for x, is_id in self.get_dependencies(path_to_lib):
|
|
for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/'):
|
|
if x.startswith(y):
|
|
if y == PREFIX + '/python/Python.framework/':
|
|
y = PREFIX + '/python/'
|
|
yield x, x[len(y):], is_id
|
|
break
|
|
|
|
@flush
|
|
def change_dep(self, old_dep, new_dep, is_id, path_to_lib):
|
|
cmd = ['-id', new_dep] if is_id else ['-change', old_dep, new_dep]
|
|
subprocess.check_call(['install_name_tool'] + cmd + [path_to_lib])
|
|
|
|
@flush
|
|
def fix_dependencies_in_lib(self, path_to_lib):
|
|
self.to_strip.append(path_to_lib)
|
|
old_mode = flipwritable(path_to_lib)
|
|
for dep, bname, is_id in self.get_local_dependencies(path_to_lib):
|
|
ndep = self.FID + '/' + bname
|
|
self.change_dep(dep, ndep, is_id, path_to_lib)
|
|
ldeps = list(self.get_local_dependencies(path_to_lib))
|
|
if ldeps:
|
|
print('\nFailed to fix dependencies in', path_to_lib)
|
|
print('Remaining local dependencies:', ldeps)
|
|
raise SystemExit(1)
|
|
if old_mode is not None:
|
|
flipwritable(path_to_lib, old_mode)
|
|
|
|
@flush
|
|
def add_python_framework(self):
|
|
print('\nAdding Python framework')
|
|
src = join(PREFIX + '/python', 'Python.framework')
|
|
x = join(self.frameworks_dir, 'Python.framework')
|
|
curr = os.path.realpath(join(src, 'Versions', 'Current'))
|
|
currd = join(x, 'Versions', basename(curr))
|
|
rd = join(currd, 'Resources')
|
|
os.makedirs(rd)
|
|
shutil.copy2(join(curr, 'Resources', 'Info.plist'), rd)
|
|
shutil.copy2(join(curr, 'Python'), currd)
|
|
self.set_id(
|
|
join(currd, 'Python'),
|
|
self.FID + '/Python.framework/Versions/%s/Python' % basename(curr))
|
|
# The following is needed for codesign
|
|
with current_dir(x):
|
|
os.symlink(basename(curr), 'Versions/Current')
|
|
for y in ('Python', 'Resources'):
|
|
os.symlink('Versions/Current/%s' % y, y)
|
|
|
|
@flush
|
|
def install_dylib(self, path, set_id=True):
|
|
shutil.copy2(path, self.frameworks_dir)
|
|
if set_id:
|
|
self.set_id(
|
|
join(self.frameworks_dir, basename(path)),
|
|
self.FID + '/' + basename(path))
|
|
self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path)))
|
|
|
|
@flush
|
|
def add_misc_libraries(self):
|
|
for x in (
|
|
'sqlite3.0',
|
|
'z.1',
|
|
'harfbuzz.0',
|
|
'png16.16',
|
|
'lcms2.2',
|
|
'crypto.1.1',
|
|
'ssl.1.1',
|
|
'ffi.7',
|
|
):
|
|
print('\nAdding', x)
|
|
x = 'lib%s.dylib' % x
|
|
src = join(PREFIX, 'lib', x)
|
|
shutil.copy2(src, self.frameworks_dir)
|
|
dest = join(self.frameworks_dir, x)
|
|
self.set_id(dest, self.FID + '/' + x)
|
|
self.fix_dependencies_in_lib(dest)
|
|
|
|
@flush
|
|
def add_package_dir(self, x, dest=None):
|
|
def ignore(root, files):
|
|
ans = []
|
|
for y in files:
|
|
ext = os.path.splitext(y)[1]
|
|
if ext not in ('', '.py', '.so') or \
|
|
(not ext and not os.path.isdir(join(root, y))):
|
|
ans.append(y)
|
|
|
|
return ans
|
|
|
|
if dest is None:
|
|
dest = self.site_packages
|
|
dest = join(dest, basename(x))
|
|
shutil.copytree(x, dest, symlinks=True, ignore=ignore)
|
|
for f in walk(dest):
|
|
if f.endswith('.so'):
|
|
self.fix_dependencies_in_lib(f)
|
|
|
|
@flush
|
|
def add_stdlib(self):
|
|
print('\nAdding python stdlib')
|
|
src = PREFIX + '/python/Python.framework/Versions/Current/lib/python' + self.py_ver
|
|
dest = self.python_stdlib
|
|
if not os.path.exists(dest):
|
|
os.makedirs(dest)
|
|
for x in os.listdir(src):
|
|
if x in ('site-packages', 'config', 'test', 'lib2to3', 'lib-tk',
|
|
'lib-old', 'idlelib', 'plat-mac', 'plat-darwin',
|
|
'site.py', 'distutils', 'turtledemo', 'tkinter'):
|
|
continue
|
|
x = join(src, x)
|
|
if os.path.isdir(x):
|
|
self.add_package_dir(x, dest)
|
|
elif os.path.splitext(x)[1] in ('.so', '.py'):
|
|
shutil.copy2(x, dest)
|
|
dest2 = join(dest, basename(x))
|
|
if dest2.endswith('.so'):
|
|
self.fix_dependencies_in_lib(dest2)
|
|
|
|
@flush
|
|
def freeze_python(self):
|
|
print('\nFreezing python')
|
|
kitty_dir = join(self.resources_dir, 'kitty')
|
|
bases = ('kitty', 'kittens', 'kitty_tests')
|
|
for x in bases:
|
|
dest = os.path.join(self.python_stdlib, x)
|
|
os.rename(os.path.join(kitty_dir, x), dest)
|
|
if x == 'kitty':
|
|
shutil.rmtree(os.path.join(dest, 'launcher'))
|
|
os.rename(os.path.join(kitty_dir, '__main__.py'), os.path.join(self.python_stdlib, 'kitty_main.py'))
|
|
shutil.rmtree(os.path.join(kitty_dir, '__pycache__'))
|
|
pdir = os.path.join(dirname(self.python_stdlib), 'kitty-extensions')
|
|
os.mkdir(pdir)
|
|
print('Extracting extension modules from', self.python_stdlib, 'to', pdir)
|
|
ext_map = extract_extension_modules(self.python_stdlib, pdir)
|
|
shutil.copy(os.path.join(os.path.dirname(self_dir), 'site.py'), os.path.join(self.python_stdlib, 'site.py'))
|
|
for x in bases:
|
|
iv['sanitize_source_folder'](os.path.join(self.python_stdlib, x))
|
|
self.compile_py_modules()
|
|
freeze_python(self.python_stdlib, pdir, self.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM')
|
|
iv['build_frozen_launcher']([path_to_freeze_dir(), self.obj_dir])
|
|
os.rename(join(dirname(self.contents_dir), 'bin', 'kitty'), join(self.contents_dir, 'MacOS', 'kitty'))
|
|
shutil.rmtree(join(dirname(self.contents_dir), 'bin'))
|
|
self.fix_dependencies_in_lib(join(self.contents_dir, 'MacOS', 'kitty'))
|
|
for f in walk(pdir):
|
|
if f.endswith('.so') or f.endswith('.dylib'):
|
|
self.fix_dependencies_in_lib(f)
|
|
|
|
@flush
|
|
def add_site_packages(self):
|
|
print('\nAdding site-packages')
|
|
os.makedirs(self.site_packages)
|
|
sys_path = json.loads(subprocess.check_output([
|
|
PYTHON, '-c', 'import sys, json; json.dump(sys.path, sys.stdout)']))
|
|
paths = reversed(tuple(map(abspath, [x for x in sys_path if x.startswith('/') and not x.startswith('/Library/')])))
|
|
upaths = []
|
|
for x in paths:
|
|
if x not in upaths and (x.endswith('.egg') or x.endswith('/site-packages')):
|
|
upaths.append(x)
|
|
for x in upaths:
|
|
print('\t', x)
|
|
tdir = None
|
|
try:
|
|
if not os.path.isdir(x):
|
|
zf = zipfile.ZipFile(x)
|
|
tdir = tempfile.mkdtemp()
|
|
zf.extractall(tdir)
|
|
x = tdir
|
|
self.add_modules_from_dir(x)
|
|
self.add_packages_from_dir(x)
|
|
finally:
|
|
if tdir is not None:
|
|
shutil.rmtree(tdir)
|
|
self.remove_bytecode(self.site_packages)
|
|
|
|
@flush
|
|
def add_modules_from_dir(self, src):
|
|
for x in glob.glob(join(src, '*.py')) + glob.glob(join(src, '*.so')):
|
|
shutil.copy2(x, self.site_packages)
|
|
if x.endswith('.so'):
|
|
self.fix_dependencies_in_lib(x)
|
|
|
|
@flush
|
|
def add_packages_from_dir(self, src):
|
|
for x in os.listdir(src):
|
|
x = join(src, x)
|
|
if os.path.isdir(x) and os.path.exists(join(x, '__init__.py')):
|
|
if self.filter_package(basename(x)):
|
|
continue
|
|
self.add_package_dir(x)
|
|
|
|
@flush
|
|
def filter_package(self, name):
|
|
return name in ('Cython', 'modulegraph', 'macholib', 'py2app',
|
|
'bdist_mpkg', 'altgraph')
|
|
|
|
@flush
|
|
def remove_bytecode(self, dest):
|
|
for x in os.walk(dest):
|
|
root = x[0]
|
|
for f in x[-1]:
|
|
if os.path.splitext(f) == '.pyc':
|
|
os.remove(join(root, f))
|
|
|
|
@flush
|
|
def compile_py_modules(self):
|
|
self.remove_bytecode(join(self.resources_dir, 'Python'))
|
|
py_compile(join(self.resources_dir, 'Python'))
|
|
|
|
@flush
|
|
def makedmg(self, d, volname, internet_enable=True, format='ULFO'):
|
|
''' Copy a directory d into a dmg named volname '''
|
|
print('\nMaking dmg...')
|
|
sys.stdout.flush()
|
|
destdir = os.path.join(SW, 'dist')
|
|
try:
|
|
shutil.rmtree(destdir)
|
|
except FileNotFoundError:
|
|
pass
|
|
os.mkdir(destdir)
|
|
dmg = os.path.join(destdir, volname + '.dmg')
|
|
if os.path.exists(dmg):
|
|
os.unlink(dmg)
|
|
tdir = tempfile.mkdtemp()
|
|
appdir = os.path.join(tdir, os.path.basename(d))
|
|
shutil.copytree(d, appdir, symlinks=True)
|
|
if self.sign_installers:
|
|
with timeit() as times:
|
|
sign_app(appdir, self.notarize)
|
|
print('Signing completed in %d minutes %d seconds' % tuple(times))
|
|
os.symlink('/Applications', os.path.join(tdir, 'Applications'))
|
|
size_in_mb = int(
|
|
subprocess.check_output(['du', '-s', '-k', tdir]).decode('utf-8')
|
|
.split()[0]) / 1024.
|
|
cmd = [
|
|
'/usr/bin/hdiutil', 'create', '-srcfolder', tdir, '-volname',
|
|
volname, '-format', format
|
|
]
|
|
if 190 < size_in_mb < 250:
|
|
# We need -size 255m because of a bug in hdiutil. When the size of
|
|
# srcfolder is close to 200MB hdiutil fails with
|
|
# diskimages-helper: resize request is above maximum size allowed.
|
|
cmd += ['-size', '255m']
|
|
print('\nCreating dmg...')
|
|
with timeit() as times:
|
|
subprocess.check_call(cmd + [dmg])
|
|
if internet_enable:
|
|
subprocess.check_call(
|
|
['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg])
|
|
print('dmg created in %d minutes and %d seconds' % tuple(times))
|
|
shutil.rmtree(tdir)
|
|
size = os.stat(dmg).st_size / (1024 * 1024.)
|
|
print('\nInstaller size: %.2fMB\n' % size)
|
|
return dmg
|
|
|
|
|
|
def main():
|
|
args = globals()['args']
|
|
ext_dir = globals()['ext_dir']
|
|
Freeze(
|
|
os.path.join(ext_dir, kitty_constants['appname'] + '.app'),
|
|
dont_strip=args.dont_strip,
|
|
sign_installers=args.sign_installers,
|
|
notarize=args.notarize,
|
|
skip_tests=args.skip_tests
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|