406 lines
19 KiB
Python
406 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Release byte-domain checks. No installation, activation, or signing authority."""
|
|
import argparse
|
|
import hashlib
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
from urllib.parse import urlsplit
|
|
from contextlib import contextmanager
|
|
|
|
|
|
RELEASE_ROOT = 'did:key:z6Mkfu5LT8d4DjETtrkATvHh9Dvcbnr7zBCUwfau8Sw7DLWT'
|
|
APPOINTMENT = Path('/etc/archipelago/release-publication/appointment.json')
|
|
TARGET = 'x86_64-unknown-linux-gnu'
|
|
|
|
# Keep publication checks self-contained: the appointed verifier is installed
|
|
# as a single pinned file, independently of candidate source/modules.
|
|
PRIVATE_TEST_APPS = frozenset({'kithmoot', 'v4v', 'node-demo-v4v'})
|
|
|
|
|
|
def excluded(app_id):
|
|
return isinstance(app_id, str) and app_id.casefold() in PRIVATE_TEST_APPS
|
|
|
|
|
|
def public_catalog(value):
|
|
"""Remove private app entries and storefront references from either format."""
|
|
if isinstance(value, dict):
|
|
return {key: public_catalog(item) for key, item in value.items()
|
|
if not excluded(key) and not excluded(item)}
|
|
if isinstance(value, list):
|
|
return [public_catalog(item) for item in value
|
|
if not excluded(item) and not (isinstance(item, dict) and
|
|
any(excluded(item.get(key)) for key in ('id', 'appId', 'app_id')))]
|
|
return value
|
|
|
|
|
|
def check_catalog(value):
|
|
if public_catalog(value) != value:
|
|
raise ValueError('private test app in public catalog')
|
|
|
|
|
|
def check_member(name, raw=None):
|
|
parts = Path(name).parts
|
|
for index, part in enumerate(parts[:-1]):
|
|
if part in ('apps', 'docker') and excluded(parts[index + 1]):
|
|
raise ValueError(f'private test app in public payload: {name}')
|
|
if raw is not None and Path(name).name in ('catalog.json', 'app-catalog.json', 'node-app-catalog.json'):
|
|
check_catalog(json.loads(raw))
|
|
|
|
|
|
def strict_json(raw):
|
|
def pairs(items):
|
|
result = {}
|
|
for key, value in items:
|
|
if key in result:
|
|
raise ValueError(f'duplicate JSON key: {key}')
|
|
result[key] = value
|
|
return result
|
|
return json.loads(raw, object_pairs_hook=pairs)
|
|
|
|
|
|
def protected_bytes(path):
|
|
"""Walk from / using pinned directory FDs; no symlinks or user writers."""
|
|
path = Path(path)
|
|
if not path.is_absolute() or '..' in path.parts:
|
|
raise ValueError('custody requires canonical absolute path')
|
|
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
for index, part in enumerate(path.parts[1:]):
|
|
parent = os.fstat(fd)
|
|
if parent.st_uid != 0 or parent.st_mode & 0o022:
|
|
raise ValueError('mutable custody directory')
|
|
flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
|
|
if index < len(path.parts) - 2:
|
|
flags |= os.O_DIRECTORY
|
|
child = os.open(part, flags, dir_fd=fd)
|
|
os.close(fd)
|
|
fd = child
|
|
info = os.fstat(fd)
|
|
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o222:
|
|
raise ValueError('mutable custody file (requires root-owned read-only regular file)')
|
|
with os.fdopen(os.dup(fd), 'rb') as stream:
|
|
return stream.read()
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def pinned_record(reference):
|
|
raw = protected_bytes(reference['path'])
|
|
if not re.fullmatch('[0-9a-f]{64}', reference['sha256']) or digest(raw) != reference['sha256']:
|
|
raise ValueError('appointed receipt hash mismatch')
|
|
return strict_json(raw)
|
|
|
|
|
|
def production_receipt(receipt):
|
|
if (receipt.get('schema') != 'archipelago-production-build-v1'
|
|
or receipt.get('clean') is not True
|
|
or receipt.get('release_root') != RELEASE_ROOT
|
|
or receipt.get('target') != TARGET
|
|
or receipt.get('build_profile') != 'release'
|
|
or receipt.get('test_root_override') is not False):
|
|
raise ValueError('not a clean production-root build receipt')
|
|
for key, length in (('source_commit', 40), ('source_tree', 40),
|
|
('cargo_lock_sha256', 64), ('backend_sha256', 64)):
|
|
if not re.fullmatch('[0-9a-f]{' + str(length) + '}', receipt.get(key, '')):
|
|
raise ValueError(f'invalid build receipt {key}')
|
|
if not receipt.get('version') or not receipt.get('build_command'):
|
|
raise ValueError('missing build provenance')
|
|
# A frozen build log/input record is appointed by root, never by the candidate.
|
|
evidence = receipt['evidence']
|
|
if digest(protected_bytes(evidence['path'])) != evidence['sha256']:
|
|
raise ValueError('build evidence hash mismatch')
|
|
|
|
|
|
def appointed_verifier():
|
|
appointment = strict_json(protected_bytes(APPOINTMENT))
|
|
if (appointment.get('schema') != 'archipelago-release-appointment-v1'
|
|
or appointment.get('release_root') != RELEASE_ROOT):
|
|
raise ValueError('wrong production root appointment')
|
|
verifier = appointment['verifier']
|
|
receipt = pinned_record(verifier['build_receipt'])
|
|
production_receipt(receipt)
|
|
raw = protected_bytes(verifier['path'])
|
|
if digest(raw) != verifier['sha256'] or digest(raw) != receipt['backend_sha256']:
|
|
raise ValueError('appointed verifier hash mismatch')
|
|
return appointment, raw
|
|
|
|
|
|
def check_build(root, backend, version, appointment):
|
|
commit = check_source(root, version)
|
|
receipt = pinned_record(appointment['releases'][version])
|
|
production_receipt(receipt)
|
|
tree = subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD^{tree}'], text=True).strip()
|
|
expected = dict(source_commit=commit, source_tree=tree, version=version,
|
|
cargo_lock_sha256=digest(regular(root / 'core/Cargo.lock')),
|
|
backend_sha256=digest(regular(backend)))
|
|
if any(receipt.get(key) != value for key, value in expected.items()):
|
|
raise ValueError('candidate differs from appointed clean build receipt')
|
|
check_native(backend)
|
|
return expected
|
|
|
|
|
|
@contextmanager
|
|
def sealed_bytes(name, raw):
|
|
"""Execute/verify exactly the checked bytes, never reopen a mutable pathname."""
|
|
fd = os.memfd_create(name, os.MFD_ALLOW_SEALING)
|
|
try:
|
|
with os.fdopen(os.dup(fd), 'wb') as stream:
|
|
stream.write(raw)
|
|
os.fchmod(fd, 0o500)
|
|
fcntl.fcntl(fd, fcntl.F_ADD_SEALS, fcntl.F_SEAL_WRITE | fcntl.F_SEAL_GROW |
|
|
fcntl.F_SEAL_SHRINK | fcntl.F_SEAL_SEAL)
|
|
yield fd
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def verify_production(root, backend, version, document, *, iso=False):
|
|
appointment, verifier = appointed_verifier()
|
|
build = check_build(root, backend, version, appointment)
|
|
if digest(verifier) == build['backend_sha256']:
|
|
raise ValueError('candidate cannot be its own appointed verifier')
|
|
raw = regular(document)
|
|
value = strict_json(raw)
|
|
if value.get('signed_by') != RELEASE_ROOT or not value.get('signature'):
|
|
raise ValueError('unsigned or wrong-root document; UAT cannot publish')
|
|
if not iso:
|
|
components = [c for c in value['components'] if c['name'] == 'archipelago']
|
|
candidate = regular(backend)
|
|
if (value.get('version') != version or value.get('source_commit') != build['source_commit']
|
|
or digest(candidate) != build['backend_sha256']
|
|
or len(components) != 1 or components[0]['sha256'] != digest(candidate)
|
|
or components[0]['size_bytes'] != len(candidate)):
|
|
raise ValueError('manifest candidate/build identity mismatch')
|
|
with sealed_bytes('release-verifier', verifier) as executable, sealed_bytes('release-document', raw) as doc:
|
|
subprocess.run([f'/proc/self/fd/{executable}', 'ceremony', 'verify', f'/proc/self/fd/{doc}'],
|
|
pass_fds=(executable, doc), check=True, cwd='/',
|
|
env={'PATH': '/usr/bin:/bin', 'HOME': '/nonexistent', 'LANG': 'C'})
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def regular(path):
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError(f'not a regular source file: {path}')
|
|
return path.read_bytes()
|
|
|
|
|
|
def os_component(path, version, base_url, *, content_addressed=False):
|
|
"""Bind the actual reviewed OS transaction, not merely an installer helper."""
|
|
from os_update_plan import validate
|
|
from d09_wire import pairs
|
|
raw = regular(path)
|
|
if not raw or len(raw) > 1024 * 1024:
|
|
raise ValueError('OS release plan size invalid')
|
|
validate(json.loads(raw, object_pairs_hook=pairs))
|
|
if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?', version):
|
|
raise ValueError('OS component release version')
|
|
if not base_url.startswith('https://'):
|
|
raise ValueError('OS release requires HTTPS distribution')
|
|
filename = 'os-update-plan-' + digest(raw) + '.json' if content_addressed else 'os-update-plan.json'
|
|
component = dict(name='os-update-plan.json', current_version='', new_version=version,
|
|
download_url=base_url.rstrip('/') + '/v' + version + '/' + filename,
|
|
sha256=digest(raw), size_bytes=len(raw))
|
|
component_asset_name(component, version)
|
|
return component
|
|
|
|
|
|
def component_asset_name(component, version):
|
|
"""Map signed logical components to finite public filenames, never arbitrary paths."""
|
|
if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?', version):
|
|
raise ValueError('invalid release version')
|
|
if not re.fullmatch('[0-9a-f]{64}', component.get('sha256', '')):
|
|
raise ValueError('invalid component hash')
|
|
name = component['name']
|
|
if name not in ('archipelago', f'archipelago-frontend-{version}.tar.gz', 'os-update-plan.json'):
|
|
raise ValueError('unknown release component')
|
|
allowed = {name}
|
|
if name == 'os-update-plan.json':
|
|
allowed.add('os-update-plan-' + component['sha256'] + '.json')
|
|
url = urlsplit(component['download_url'])
|
|
filename = url.path.rsplit('/', 1)[-1]
|
|
if (url.scheme != 'https' or not url.netloc or url.username is not None
|
|
or url.password is not None or url.query or url.fragment
|
|
or filename not in allowed or not url.path.endswith('/v' + version + '/' + filename)):
|
|
raise ValueError('component URL version/name/hash mismatch')
|
|
return filename
|
|
|
|
|
|
def runtime_sources(root):
|
|
# Discover the complete helper domain, including new typed helpers added by
|
|
# concurrent lanes. Final release entry points separately require clean Git.
|
|
paths = sorted((root / 'scripts').rglob('*'))
|
|
paths += sorted((root / 'image-recipe/configs').rglob('*'))
|
|
result = {}
|
|
for path in paths:
|
|
rel = path.relative_to(root)
|
|
if any(p in ('__pycache__', 'reports', 'node_modules', '.git') for p in rel.parts):
|
|
continue
|
|
if path.name.endswith(('.pyc', '.log', '.bak')) or '.bak-' in path.name or path.name.startswith('._'):
|
|
continue
|
|
if path.is_symlink():
|
|
raise ValueError(f'symlink in runtime source: {rel}')
|
|
if path.is_file():
|
|
result['archipelago-runtime/' + rel.as_posix()] = regular(path)
|
|
if not result:
|
|
raise ValueError('empty runtime source domain')
|
|
return result
|
|
|
|
|
|
def declaration(root, version, commit):
|
|
return {'schema_version': 1, 'version': version, 'source_commit': commit,
|
|
'files': {name: digest(raw) for name, raw in runtime_sources(root).items()}}
|
|
|
|
|
|
def stage(root, dest, version, commit):
|
|
for name, raw in runtime_sources(root).items():
|
|
target = dest / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(raw)
|
|
vpn_ui = root / 'docker/nostr-vpn-ui/index.html'
|
|
if vpn_ui.exists():
|
|
(dest / 'nostr-vpn').mkdir(parents=True, exist_ok=True)
|
|
(dest / 'nostr-vpn/index.html').write_bytes(regular(vpn_ui))
|
|
(dest / 'archipelago-runtime/RELEASE-PAYLOAD.json').write_text(
|
|
json.dumps(declaration(root, version, commit), sort_keys=True, indent=2) + '\n')
|
|
|
|
|
|
def check_archive(root, archive, version, commit):
|
|
expected = runtime_sources(root)
|
|
vpn_ui = root / 'docker/nostr-vpn-ui/index.html'
|
|
if vpn_ui.exists():
|
|
expected['nostr-vpn/index.html'] = regular(vpn_ui)
|
|
for name in ('archipelago-companion.apk', 'archipelago-companion.json'):
|
|
expected['packages/' + name] = regular(root / 'neode-ui/public/packages' / name)
|
|
required = set(expected) | {'index.html', 'aiui/index.html', 'aiui/BUILD-INFO',
|
|
'archipelago-runtime/RELEASE-PAYLOAD.json'}
|
|
found = {}
|
|
seen = set()
|
|
with tarfile.open(archive, 'r|gz') as package:
|
|
for member in package:
|
|
name = member.name.removeprefix('./').rstrip('/') or '.'
|
|
if name.startswith('/') or (name != '.' and any(p in ('', '.', '..') for p in name.split('/'))):
|
|
raise ValueError('noncanonical archive path')
|
|
if name in seen:
|
|
raise ValueError(f'duplicate archive member: {name}')
|
|
seen.add(name)
|
|
check_member(name)
|
|
if not (member.isdir() or member.isreg()):
|
|
raise ValueError(f'archive links/special files forbidden: {name}')
|
|
if member.mode != (0o755 if member.isdir() else 0o644):
|
|
raise ValueError(f'archive mode mismatch: {name}')
|
|
if member.isreg() and Path(name).name in ('catalog.json', 'app-catalog.json', 'node-app-catalog.json'):
|
|
if member.size > 16 * 1024 * 1024:
|
|
raise ValueError('oversize public app catalog')
|
|
check_member(name, package.extractfile(member).read())
|
|
if name in required:
|
|
limit = len(expected[name]) if name in expected else 16 * 1024 * 1024
|
|
if not member.isreg() or member.size > limit:
|
|
raise ValueError(f'archive type/size mismatch: {name}')
|
|
found[name] = package.extractfile(member).read()
|
|
if name in expected and found[name] != expected[name]:
|
|
raise ValueError(f'archive source mismatch: {name}')
|
|
if required - found.keys():
|
|
raise ValueError(f'archive missing payload: {sorted(required - found.keys())}')
|
|
if json.loads(found['archipelago-runtime/RELEASE-PAYLOAD.json']) != declaration(root, version, commit):
|
|
raise ValueError('runtime hash/version/source declaration mismatch')
|
|
info = dict(line.split('=', 1) for line in found['aiui/BUILD-INFO'].decode().splitlines() if '=' in line)
|
|
if info.get('commit') != commit or info.get('base_path') != '/aiui/':
|
|
raise ValueError('AIUI source/base-path mismatch')
|
|
|
|
|
|
def check_native(path, architecture='x86_64'):
|
|
with path.open('rb') as stream:
|
|
header = stream.read(64)
|
|
machine = {'x86_64': 62, 'aarch64': 183}[architecture]
|
|
if (len(header) != 64 or header[:6] != b'\x7fELF\x02\x01'
|
|
or struct.unpack_from('<H', header, 18)[0] != machine
|
|
or struct.unpack_from('<H', header, 16)[0] not in (2, 3)):
|
|
raise ValueError(f'backend is not a native {architecture} ELF executable')
|
|
|
|
|
|
def check_source(root, version):
|
|
def git(*args):
|
|
return subprocess.check_output(['git', '-C', str(root), *args], text=True).strip()
|
|
if git('status', '--porcelain', '--untracked-files=all'):
|
|
raise ValueError('release requires clean reviewed source, including untracked files')
|
|
tag = f'refs/tags/v{version}'
|
|
if git('cat-file', '-t', tag) != 'tag':
|
|
raise ValueError('release requires an annotated tag')
|
|
commit = git('rev-parse', 'HEAD')
|
|
if git('rev-parse', tag + '^{commit}') != commit:
|
|
raise ValueError('annotated tag does not target the build commit')
|
|
import tomllib
|
|
cargo = tomllib.loads((root / 'core/archipelago/Cargo.toml').read_text())
|
|
npm = json.loads((root / 'neode-ui/package.json').read_text())
|
|
if cargo['package']['version'] != version or npm['version'] != version:
|
|
raise ValueError('source version mismatch')
|
|
return commit
|
|
|
|
|
|
def check_companion(root):
|
|
apk = root / 'neode-ui/public/packages/archipelago-companion.apk'
|
|
metadata = json.loads(regular(apk.with_suffix('.json')))
|
|
aapt = os.environ.get('AAPT', 'aapt')
|
|
signer = os.environ.get('APKSIGNER', 'apksigner')
|
|
badging = subprocess.check_output([aapt, 'dump', 'badging', str(apk)], text=True)
|
|
match = re.search(r"^package: name='([^']+)' versionCode='([^']+)' versionName='([^']+)'", badging, re.M)
|
|
if not match or int(match[2]) != metadata['versionCode'] or match[3] != metadata['versionName']:
|
|
raise ValueError('APK actual version differs from node metadata')
|
|
if match[1] != 'com.archipelago.app.debug':
|
|
raise ValueError('APK package identity differs from the deployed companion')
|
|
output = subprocess.check_output([signer, 'verify', '--verbose', '--min-sdk-version', '21', '--print-certs', str(apk)], text=True)
|
|
for scheme in ('v1', 'v2', 'v3'):
|
|
if not re.search(rf'^Verified using {scheme} scheme .*: true$', output, re.M):
|
|
raise ValueError(f'APK missing verified {scheme} signature')
|
|
certs = re.findall(r'^Signer #\d+ certificate SHA-256 digest: ([0-9a-f]+)$', output, re.M)
|
|
if certs != ['d622e07e7f474246e8e2bcee828ca9bf6c6cdf9434c065c58fe89bb4eec2664d']:
|
|
raise ValueError('APK signer continuity mismatch')
|
|
return {'sha256': digest(regular(apk)), **metadata, 'package_name': match[1], 'certificate_sha256': certs[0]}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('stage', 'archive', 'native', 'companion', 'source', 'production', 'production-iso', 'build'))
|
|
parser.add_argument('path', nargs='?', type=Path)
|
|
parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument('--version')
|
|
parser.add_argument('--commit')
|
|
parser.add_argument('--backend', type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.action in ('production', 'production-iso', 'build'):
|
|
if not args.backend or not args.version:
|
|
parser.error('production/build require --backend and --version')
|
|
if args.action == 'build':
|
|
appointment, _ = appointed_verifier()
|
|
check_build(args.root, args.backend, args.version, appointment)
|
|
else:
|
|
verify_production(args.root, args.backend, args.version, args.path,
|
|
iso=args.action == 'production-iso')
|
|
elif args.action in ('stage', 'archive'):
|
|
if not args.path or not args.version or not args.commit:
|
|
parser.error('stage/archive require path, --version and --commit')
|
|
(stage if args.action == 'stage' else check_archive)(args.root, args.path, args.version, args.commit)
|
|
elif args.action == 'source':
|
|
print(check_source(args.root, args.version))
|
|
elif args.action == 'native':
|
|
check_native(args.path)
|
|
else:
|
|
print(json.dumps(check_companion(args.root), sort_keys=True))
|
|
except (OSError, ValueError, KeyError, tarfile.TarError, subprocess.CalledProcessError) as error:
|
|
parser.exit(1, f'FAIL: {error}\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|