338 lines
17 KiB
Python
338 lines
17 KiB
Python
"""Reproducible Windows release build. Run with the independent .build-venv Python."""
|
|
import argparse
|
|
import ast
|
|
import hashlib
|
|
import importlib.metadata
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
WORK = ROOT / '.windows-build'
|
|
STAGE = WORK / 'stage'
|
|
OWN = ('app', 'framework', 'monitor_runtime')
|
|
|
|
def run(args, **kwargs):
|
|
print('RUN', subprocess.list2cmdline([str(x) for x in args]), flush=True)
|
|
subprocess.run([str(x) for x in args], check=True, **kwargs)
|
|
|
|
def clean(directory):
|
|
directory = directory.resolve()
|
|
if not directory.is_relative_to(WORK.resolve()) or directory == WORK.resolve():
|
|
raise ValueError('Refusing to remove a path outside build staging')
|
|
if directory.exists():
|
|
shutil.rmtree(directory)
|
|
directory.mkdir(parents=True)
|
|
|
|
def compiler_environment(vcvars):
|
|
if not vcvars:
|
|
candidates = list(Path('C:/Program Files (x86)/Microsoft Visual Studio').glob('*/BuildTools/VC/Auxiliary/Build/vcvars64.bat'))
|
|
candidates += list(Path('C:/Program Files/Microsoft Visual Studio').glob('*/*/VC/Auxiliary/Build/vcvars64.bat'))
|
|
if not candidates:
|
|
raise RuntimeError('Install Visual Studio C++ Build Tools and Windows SDK, or pass --vcvars')
|
|
vcvars = str(sorted(candidates)[-1])
|
|
# Generated batch contains only validated local paths, never arbitrary shell input.
|
|
if any(c in vcvars for c in '"\r\n%'):
|
|
raise ValueError('Unsupported vcvars path')
|
|
script = WORK / 'compiler-env.cmd'
|
|
script.write_text('@echo off\r\ncall "' + vcvars + '" >nul\r\nif errorlevel 1 exit /b 1\r\nset\r\n', encoding='utf-8')
|
|
raw = subprocess.check_output(['cmd.exe', '/d', '/c', str(script)])
|
|
env = {k.upper(): v for k, v in os.environ.items()}
|
|
for line in raw.decode('mbcs', errors='replace').splitlines():
|
|
if '=' in line:
|
|
key, value = line.split('=', 1)
|
|
env[key.upper()] = value
|
|
compiler_bin = Path(env['VCTOOLSINSTALLDIR']) / 'bin' / 'Hostx64' / 'x64'
|
|
sdk_bin = Path(env['WINDOWSSDKVERBINPATH']) / 'x64'
|
|
env['PATH'] = str(compiler_bin) + ';' + str(sdk_bin) + ';' + env['PATH']
|
|
if not (compiler_bin / 'cl.exe').exists():
|
|
raise RuntimeError('C++ compiler is missing: ' + str(compiler_bin))
|
|
env['DISTUTILS_USE_SDK'] = '1'
|
|
env['MSSdk'] = '1'
|
|
return env
|
|
|
|
def verify_environment():
|
|
if sys.version_info[:3] != (3, 12, 13):
|
|
raise RuntimeError('This release is pinned to Python 3.12.13 x64')
|
|
for lock in (ROOT/'requirements-windows-cpu.lock.txt',ROOT/'deploy/windows/build-tools.lock.txt'):
|
|
for line in lock.read_text(encoding='utf-8-sig').splitlines():
|
|
line=line.strip()
|
|
if not line or line.startswith('#'):
|
|
continue
|
|
name,expected=line.split('==',1)
|
|
actual=importlib.metadata.version(name)
|
|
if actual != expected:
|
|
raise RuntimeError(f'Locked dependency mismatch: {name}: {actual} != {expected}')
|
|
if '+cpu' not in importlib.metadata.version('torch'):
|
|
raise RuntimeError('Only the locked CPU torch build is supported')
|
|
|
|
def stage(public_key):
|
|
from cryptography.hazmat.primitives import serialization
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
|
if not isinstance(serialization.load_pem_public_key(public_key.read_bytes()), Ed25519PublicKey):
|
|
raise ValueError('An Ed25519 PUBLIC key is required')
|
|
clean(STAGE)
|
|
source = WORK / 'sources'
|
|
clean(source)
|
|
modules = []
|
|
glue = []
|
|
external = set()
|
|
for package in OWN:
|
|
for path in sorted((ROOT / package).rglob('*.py')):
|
|
if '__pycache__' in path.parts or path.name == 'tests.py':
|
|
continue
|
|
relative = path.relative_to(ROOT)
|
|
if not path.stem.isidentifier():
|
|
# Django's numbered migration names need an import-only glue module.
|
|
stub = STAGE / relative
|
|
stub.parent.mkdir(parents=True, exist_ok=True)
|
|
private_name = '_m' + path.stem
|
|
stub.write_text('from .' + private_name + ' import Migration\n', encoding='utf-8')
|
|
glue.append('.'.join(relative.with_suffix('').parts))
|
|
relative = relative.with_name(private_name + '.py')
|
|
body = path.read_text(encoding='utf-8-sig')
|
|
if relative.as_posix() == 'monitor_runtime/release.py':
|
|
body = 'RELEASE_BUILD = True\n'
|
|
tree = ast.parse(body)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
external.update(alias.name for alias in node.names if alias.name.split('.')[0] not in OWN)
|
|
if isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
|
if node.module.split('.')[0] not in OWN:
|
|
external.add(node.module)
|
|
if path.name == '__init__.py':
|
|
meaningful = [n for n in tree.body if not (isinstance(n, ast.Expr) and isinstance(n.value, ast.Constant))]
|
|
output = STAGE / relative
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
glue.append('.'.join(relative.parent.parts))
|
|
output.write_text('from ._package import *\n' if meaningful else '', encoding='utf-8')
|
|
if not meaningful:
|
|
continue
|
|
relative = relative.with_name('_package.py')
|
|
output = source / relative
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(body, encoding='utf-8')
|
|
modules.append(('.'.join(relative.with_suffix('').parts), str(output)))
|
|
manifest = {'modules': modules, 'glue': glue, 'external': sorted(external)}
|
|
(WORK / 'modules.json').write_text(json.dumps(manifest, indent=2), encoding='utf-8')
|
|
for folder in ('templates',):
|
|
shutil.copytree(ROOT / folder, STAGE / folder)
|
|
shutil.copytree(ROOT / 'static', STAGE / 'static', ignore=shutil.ignore_patterns('upload', 'storage', '__pycache__'))
|
|
shutil.copytree(STAGE / 'static', STAGE / 'public-static')
|
|
shutil.copytree(ROOT / 'deploy/windows', STAGE / 'deploy/windows',
|
|
ignore=shutil.ignore_patterns('*.py', '*.ps1', '*.iss', '*.md', '__pycache__'))
|
|
for path in ROOT.glob('language-*.json'):
|
|
shutil.copyfile(path, STAGE / path.name)
|
|
shutil.copyfile(public_key, STAGE / 'license-public.pem')
|
|
(STAGE / 'zlm').mkdir()
|
|
shutil.copyfile(ROOT / 'zlm/bin.x86.windows10/monitor_zlm.exe', STAGE / 'zlm/monitor_zlm.exe')
|
|
import imageio_ffmpeg
|
|
(STAGE / 'tools').mkdir()
|
|
shutil.copyfile(imageio_ffmpeg.get_ffmpeg_exe(), STAGE / 'tools/ffmpeg.exe')
|
|
shutil.copyfile(ROOT / 'monitor_entry.py', STAGE / 'monitor_entry.py')
|
|
versions = {d.metadata['Name']: d.version for d in importlib.metadata.distributions() if d.metadata['Name']}
|
|
(STAGE / 'dependency-versions.json').write_text(json.dumps(versions, indent=2), encoding='utf-8')
|
|
# Include installed dependency license notices without copying package source trees.
|
|
notices = STAGE / 'third-party-notices'
|
|
notices.mkdir()
|
|
for distribution in importlib.metadata.distributions():
|
|
name = distribution.metadata['Name']
|
|
for item in distribution.files or []:
|
|
if '.dist-info' not in str(item):
|
|
continue
|
|
if Path(item).name.lower().startswith(('license', 'copying', 'notice')):
|
|
target = notices / name / Path(item).name
|
|
target.parent.mkdir(exist_ok=True)
|
|
origin = distribution.locate_file(item)
|
|
if origin.is_file():
|
|
shutil.copyfile(origin, target)
|
|
|
|
def compile_modules(vcvars):
|
|
script = WORK / 'compile_extensions.py'
|
|
script.write_text("""import json
|
|
from pathlib import Path
|
|
from setuptools import setup, Extension
|
|
from Cython.Build import cythonize
|
|
work=Path(__file__).resolve().parent
|
|
modules=json.loads((work/'modules.json').read_text())['modules']
|
|
extensions=[Extension(name,[source]) for name,source in modules]
|
|
setup(name='monitor-private', ext_modules=cythonize(extensions,
|
|
build_dir=str(work/'cython'), compiler_directives={
|
|
'language_level':3,'binding':True,'annotation_typing':False,
|
|
'infer_types':False,'embedsignature':False}, nthreads=0),
|
|
script_args=['build_ext','--build-lib',str(work/'stage'),'--build-temp',str(work/'objects'),'-j','2'])
|
|
""", encoding='utf-8')
|
|
run([sys.executable, script], env=compiler_environment(vcvars), cwd=WORK)
|
|
|
|
def bundle():
|
|
manifest = json.loads((WORK / 'modules.json').read_text())
|
|
spec = WORK / 'monitor.spec'
|
|
spec.write_text("""import json
|
|
from pathlib import Path
|
|
from PyInstaller.utils.hooks import collect_all, collect_submodules, copy_metadata
|
|
work=Path(SPECPATH)
|
|
stage=work/'stage'
|
|
manifest=json.loads((work/'modules.json').read_text())
|
|
datas=[]
|
|
binaries=[]
|
|
hidden=[name for name,_ in manifest['modules']]+manifest['glue']+manifest['external']
|
|
hidden += ['pystray._win32','django.db.backends.sqlite3','django.contrib.auth.hashers',
|
|
'django.contrib.sessions.backends.db','django.template.backends.django']
|
|
for package in ['django','ultralytics','torch','torchvision','onnxruntime','openvino','onvif','pystray','whitenoise']:
|
|
d,b,h=collect_all(package)
|
|
datas+=d;binaries+=b;hidden+=h
|
|
for name in ['templates','static','public-static','deploy','zlm','tools',
|
|
'license-public.pem','dependency-versions.json','third-party-notices']:
|
|
child=stage/name
|
|
datas.append((str(child),child.name if child.is_dir() else '.'))
|
|
for child in stage.glob('language-*.json'):
|
|
datas.append((str(child),'.'))
|
|
a=Analysis([str(stage/'monitor_entry.py')],pathex=[str(stage)],
|
|
binaries=binaries,datas=datas,hiddenimports=sorted(set(hidden)),
|
|
excludes=['pytest','IPython','notebook','tkinter','matplotlib.tests'],
|
|
noarchive=False)
|
|
own=('app','framework','monitor_runtime')
|
|
unexpected=[name for name,_,_ in a.pure if name.split('.')[0] in own and name not in manifest['glue']]
|
|
if unexpected:
|
|
raise RuntimeError('Uncompiled private modules: '+repr(unexpected))
|
|
pyz=PYZ(a.pure)
|
|
exe=EXE(pyz,a.scripts,[],exclude_binaries=True,name='Monitor',console=False,
|
|
debug=False,upx=False,icon=str(work/'monitor.ico'))
|
|
coll=COLLECT(exe,a.binaries,a.datas,strip=False,upx=False,name='Monitor')
|
|
""", encoding='utf-8')
|
|
from PIL import Image
|
|
Image.open(ROOT / 'static/images/logo.png').save(WORK / 'monitor.ico', sizes=[(16,16),(32,32),(48,48),(128,128),(256,256)])
|
|
env = os.environ.copy()
|
|
env['PYTHONPATH'] = str(STAGE)
|
|
env['DJANGO_SETTINGS_MODULE'] = 'framework.settings'
|
|
env['MONITOR_DATA_DIR'] = str(WORK / 'build-data')
|
|
env['MONITOR_DESKTOP'] = '1'
|
|
env['MONITOR_SERVICE_MODE'] = 'disabled'
|
|
env['MONITOR_BOOTSTRAP_SERVICES'] = 'false'
|
|
env['CI'] = 'true'
|
|
env['YOLO_OFFLINE'] = 'true'
|
|
env['YOLO_AUTOINSTALL'] = 'false'
|
|
(WORK / 'yolo/Ultralytics').mkdir(parents=True, exist_ok=True)
|
|
env['YOLO_CONFIG_DIR'] = str(WORK / 'yolo')
|
|
env['PYINSTALLER_CONFIG_DIR'] = str(WORK / 'pyinstaller-cache')
|
|
run([sys.executable, '-m', 'PyInstaller', '--noconfirm', '--clean', '--distpath', WORK / 'dist',
|
|
'--workpath', WORK / 'pyinstaller', spec], env=env, cwd=STAGE)
|
|
finalize_payload()
|
|
|
|
def finalize_payload():
|
|
payload = WORK / 'dist/Monitor'
|
|
if not (payload/'Monitor.exe').exists():
|
|
raise RuntimeError('Bundle must succeed before payload finalization')
|
|
internal = payload/'_internal'
|
|
# Extension modules are immutable release code, refreshed after incremental compilation.
|
|
for package in OWN:
|
|
for compiled in (STAGE/package).rglob('*.pyd'):
|
|
target=internal/compiled.relative_to(STAGE)
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
shutil.copyfile(compiled,target)
|
|
# onvif-zeep installs WSDL beside its package, outside collect_all('onvif').
|
|
import onvif
|
|
wsdl=Path(onvif.__file__).resolve().parent.parent/'wsdl'
|
|
if not (wsdl/'devicemgmt.wsdl').is_file():
|
|
raise RuntimeError('ONVIF WSDL resources are missing from the build environment')
|
|
shutil.copytree(wsdl,internal/'wsdl',dirs_exist_ok=True)
|
|
audit(payload)
|
|
|
|
def audit(folder):
|
|
forbidden = {'.runtime-secrets.json','.trusted-models.json','monitor.sqlite3','config.json',
|
|
'license.json','license-clock.json'}
|
|
for path in folder.rglob('*'):
|
|
relative = path.relative_to(folder)
|
|
parts = relative.parts[1:] if relative.parts[0] == '_internal' else relative.parts
|
|
if parts and parts[0] in OWN:
|
|
if path.suffix.lower() in ('.py','.pyc','.pyo','.c','.cpp','.html'):
|
|
raise RuntimeError('Private source/code artifact: '+str(path))
|
|
if path.suffix.lower() == '.log':
|
|
raise RuntimeError('Runtime log in release: '+str(path))
|
|
if path.name in forbidden or 'private' in path.name.lower() and path.suffix == '.pem':
|
|
raise RuntimeError('Runtime data in release: '+str(path))
|
|
digest = lambda p: hashlib.file_digest(p.open('rb'), 'sha256').hexdigest()
|
|
manifest = {str(p.relative_to(folder)):digest(p) for p in folder.rglob('*') if p.is_file()}
|
|
(WORK / 'release-files.sha256.json').write_text(json.dumps(manifest, indent=2), encoding='utf-8')
|
|
print('Release payload audit passed:', len(manifest), 'files')
|
|
|
|
def validate_payload():
|
|
import datetime
|
|
payload=WORK/'dist/Monitor'
|
|
test_data=WORK/('release-self-test-'+datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f'))
|
|
env=os.environ.copy()
|
|
env['MONITOR_DATA_DIR']=str(test_data)
|
|
env.pop('PYTHONPATH',None);env.pop('PYTHONHOME',None)
|
|
windows=env['SYSTEMROOT']
|
|
env['PATH']=windows+'/System32;'+windows+';'+windows+'/System32/WindowsPowerShell/v1.0'
|
|
if not (payload/'_internal/wsdl/devicemgmt.wsdl').is_file():
|
|
raise RuntimeError('Packaged ONVIF resources are missing')
|
|
run([payload/'Monitor.exe','--self-test'],env=env,cwd=WORK,timeout=180)
|
|
report=test_data/'.runtime/self-test.json'
|
|
result=json.loads(report.read_text(encoding='utf-8'))
|
|
if result.get('status') != 'passed':
|
|
raise RuntimeError('Frozen dependency self-test failed')
|
|
shutil.copyfile(report,WORK/'frozen-self-test.json')
|
|
|
|
|
|
def installer(iscc, sign_thumbprint):
|
|
finalize_payload()
|
|
validate_payload()
|
|
payload = WORK / 'dist/Monitor'
|
|
if not iscc:
|
|
candidates = [WORK/'tools/Inno/ISCC.exe',Path('C:/Program Files (x86)/Inno Setup 6/ISCC.exe')]
|
|
iscc = next((p for p in candidates if p.exists()), None)
|
|
if not iscc:
|
|
raise RuntimeError('Inno Setup ISCC.exe is required; pass --iscc')
|
|
if sign_thumbprint:
|
|
run(['signtool.exe','sign','/sha1',sign_thumbprint,'/fd','SHA256',payload/'Monitor.exe'])
|
|
manifest_path=WORK/'release-files.sha256.json'
|
|
manifest=json.loads(manifest_path.read_text(encoding='utf-8'))
|
|
with (payload/'Monitor.exe').open('rb') as handle:
|
|
manifest['Monitor.exe']=hashlib.file_digest(handle,'sha256').hexdigest()
|
|
manifest_path.write_text(json.dumps(manifest,indent=2),encoding='utf-8')
|
|
output = ROOT / 'dist/windows'
|
|
output.mkdir(parents=True,exist_ok=True)
|
|
run([iscc, '/DPayload='+str(payload), '/DOutput='+str(output), ROOT/'deploy/windows/Monitor.iss'])
|
|
exe = output / 'Monitor-Setup-x64.exe'
|
|
if sign_thumbprint:
|
|
run(['signtool.exe','sign','/sha1',sign_thumbprint,'/fd','SHA256',exe])
|
|
with exe.open('rb') as handle:
|
|
checksum = hashlib.file_digest(handle,'sha256').hexdigest()
|
|
(output/'SHA256SUMS.txt').write_text(checksum+' '+exe.name+'\n',encoding='ascii')
|
|
shutil.copyfile(WORK/'release-files.sha256.json',output/'release-files.sha256.json')
|
|
shutil.copyfile(payload/'_internal/dependency-versions.json',output/'dependency-versions.json')
|
|
shutil.copyfile(ROOT/'deploy/windows/DEPLOYMENT.md',output/'DEPLOYMENT.md')
|
|
shutil.copyfile(WORK/'frozen-self-test.json',output/'frozen-self-test.json')
|
|
print('INSTALLER:', exe)
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--public-key',type=Path,required=True)
|
|
parser.add_argument('--phase',choices=['all','stage','compile','bundle','finalize','validate','installer'],default='all')
|
|
parser.add_argument('--vcvars')
|
|
parser.add_argument('--iscc')
|
|
parser.add_argument('--sign-thumbprint')
|
|
args=parser.parse_args()
|
|
if os.name != 'nt' or sys.maxsize <= 2**32:
|
|
parser.error('Windows x64 Python is required')
|
|
if Path(sys.prefix).resolve() != (ROOT/'.build-venv').resolve():
|
|
parser.error('Run with .build-venv/Scripts/python.exe; do not modify the development environment')
|
|
WORK.mkdir(exist_ok=True)
|
|
verify_environment()
|
|
if args.phase not in ('all','stage'):
|
|
if args.public_key.read_bytes() != (STAGE/'license-public.pem').read_bytes():
|
|
parser.error('Public key differs from staged release; run the stage and compile phases first')
|
|
if args.phase in ('all','stage'):stage(args.public_key.resolve())
|
|
if args.phase in ('all','compile'):compile_modules(args.vcvars)
|
|
if args.phase in ('all','bundle'):bundle()
|
|
if args.phase == 'finalize':finalize_payload()
|
|
if args.phase == 'validate':validate_payload()
|
|
if args.phase in ('all','installer'):installer(args.iscc,args.sign_thumbprint)
|
|
|
|
if __name__=='__main__':
|
|
main()
|