#!/usr/bin/env python3 """Real apt/dpkg/filesystem faults on an independent prepared disposable disk.""" import fcntl import errno import json import os from pathlib import Path import signal import subprocess import sys import time import urllib.error assert 'QEMU' in Path('/sys/class/dmi/id/sys_vendor').read_text() assert subprocess.check_output(['findmnt', '-n', '-o', 'SOURCE', '/'], text=True).strip() == '/dev/vda' sys.path.insert(0, '/opt/ota-source/scripts') import os_update_executor as executor from os_update_plan import canonical, digest evidence = Path('/var/log/os-ota-stage') mode = Path('/opt/os-ota-scenario').read_text() binding = json.loads((evidence/'binding.json').read_text()) plan = json.loads((evidence/'plan.json').read_text()) canaries = Path('/var/lib/archipelago/ota-synthetic-canaries') hashes = lambda: {p.name: executor.file_hash(p) for p in canaries.iterdir()} before = hashes() inventory = executor.inventory() result = dict(scenario=mode, production_approved=False, passed=False, recovery_policy='no automatic dpkg replay, no financial-data rollback', events=[]) def refused(label, phase='apply', expected=None): observed = None try: executor.stage(binding, 'APPLY', phase) except Exception as error: observed = error if expected is not None: assert expected in str(error), (label,type(error).__name__,str(error)) result['events'].append(dict(label=label, exception=type(error).__name__, error=str(error))) else: raise AssertionError(label+' unexpectedly succeeded') assert hashes() == before return observed def fresh_appointment(): """A distinct root fixture candidate; never reset an existing consumed status.""" global binding old = executor.Executor(binding['os_update']) preservation = json.loads((old.source/'preservation.json').read_text()) authority = digest(('independent-VM-fault-'+mode).encode()) preservation['authority_digest'] = authority plan['preservation_sha256'] = digest(canonical(preservation)) plan_sha = digest(canonical(plan)) manifest = dict(os_update=dict(version=1, plan_sha256=plan_sha), qualification='ROOT-APPOINTED-DISPOSABLE-FAULT-NOT-PRODUCTION', scenario=mode) source = executor.ROOT/'plans'/plan_sha source.mkdir(mode=0o700) for name, value in (('plan.json',plan),('manifest.json',manifest),('preservation.json',preservation)): (source/name).write_bytes(canonical(value)) # Retain the initial appointed verifier as evidence before appointing the # separate fault candidate on this independent VM only. executor.VERIFIER.rename(executor.VERIFIER.with_name(executor.VERIFIER.name+'.initial-fixture')) verifier = '''#!/usr/bin/python3 import hashlib,pathlib,sys assert 'QEMU' in pathlib.Path('/sys/class/dmi/id/sys_vendor').read_text() assert sys.argv[1:3] == ['ceremony','verify'] and len(sys.argv)==4 assert hashlib.sha256(pathlib.Path(sys.argv[3]).read_bytes()).hexdigest()==%r ''' % digest(canonical(manifest)) executor.VERIFIER.write_text(verifier) executor.VERIFIER.chmod(0o755) binding = dict(os_update=dict(version=1, plan_sha256=plan_sha, manifest_sha256=digest(canonical(manifest)), authority_digest=authority, verifier_sha256=executor.file_hash(executor.VERIFIER))) (evidence/('fault-'+mode+'-binding.json')).write_bytes(canonical(binding)) def fill_private_filesystem(filler): """Reserve evidence space first; ext4 f_bfree is not all fallocatable.""" reserve=filler.with_suffix('.reserve') with reserve.open('xb') as stream: os.posix_fallocate(stream.fileno(),0,8*1024**2) try: with filler.open('xb') as stream: offset=0 for chunk in (16*1024**2,64*1024): while offset < 8*1024**3: try: os.posix_fallocate(stream.fileno(),offset,chunk) except OSError as error: if error.errno != errno.ENOSPC: raise break offset+=chunk result['space_filler_bytes']=offset finally: reserve.unlink() try: if mode == 'locked-apt': holder=subprocess.Popen([sys.executable,'-c', 'import fcntl,time;f=open("/var/lib/dpkg/lock-frontend","a");' 'fcntl.lockf(f,fcntl.LOCK_EX|fcntl.LOCK_NB);print("LOCKED",flush=True);time.sleep(600)'], stdout=subprocess.PIPE,text=True) try: assert holder.stdout.readline().strip()=='LOCKED' result['competing_package_writer_pid']=holder.pid refused('actual-dpkg-frontend-lock',expected='lock') finally: holder.terminate() holder.wait() holder.stdout.close() assert executor.inventory() == inventory if executor.Executor(binding['os_update']).status['phase']=='VERIFIED': status=executor.stage(binding,'APPLY','apply') assert status['phase']=='REBOOT_REQUIRED' assert executor.stage(binding,'APPLY','apply')==status result['recovery']='pre-effect real lock refusal; released lock; actual apply completed once under retained libapt locks' else: refused('lock-released-consumed-authority-not-replayed',expected='never retry dpkg') result['reconciliation'] = executor.stage(binding,'APPLY','reconcile') result['recovery'] = 'lock released; dpkg unchanged; externally reconciled authority still required' elif mode == 'space': filler = Path('/var/tmp/os-ota-private-space-fault') try: fill_private_filesystem(filler) refused('actual-filesystem-space-exhaustion') finally: filler.unlink(missing_ok=True) assert executor.inventory() == inventory if executor.Executor(binding['os_update']).status['phase'] == 'VERIFIED': recovered = executor.stage(binding,'APPLY','apply') assert recovered['phase']=='REBOOT_REQUIRED' result['recovery'] = 'pre-effect space refusal; owned filler removed; original prepared authority applied successfully' else: refused('space-recovered-authority-not-replayed') result['reconciliation'] = executor.stage(binding,'APPLY','reconcile') result['recovery'] = 'only owned filler removed; dpkg unchanged; evidence-only reconciliation; external authority required' elif mode == 'partial-dpkg': preparation = executor.Executor(binding['os_update']).status['phase'] != 'VERIFIED' if preparation: fresh_appointment() download = subprocess.run(['apt-get','download','curl=8.14.1-2+deb13u5'], cwd=evidence, text=True, capture_output=True) print(download.stdout, download.stderr) download.check_returncode() archive = next(evidence.glob('curl_*deb13u5_amd64.deb')) subprocess.run(['dpkg','--unpack',str(archive)],check=True) audit = subprocess.check_output(['dpkg','--audit'],text=True) assert audit.strip() result['partial_dpkg_audit'] = audit refused('actual-unpacked-unconfigured-package',phase='prepare' if preparation else 'apply', expected='dpkg inventory changed') subprocess.run(['dpkg','--configure','curl'],check=True) assert executor.inventory() == inventory assert not subprocess.check_output(['dpkg','--audit'],text=True).strip() executor.Executor(binding['os_update']).preflight() if preparation: refused('dpkg-repaired-preparation-authority-not-replayed','prepare',expected='external reconciliation') result['recovery'] = 'explicit configure of same-version curl; clean dpkg and original inventory; no snapshot restore' elif mode in ('offline','missing-dependency','interrupted-before','bad-hash','space-prepare'): if mode == 'missing-dependency': removed = next(p for p in plan['packages'] if p['name']=='libcurl4t64') plan['packages'].remove(removed) result['missing_dependency'] = removed if mode == 'bad-hash': next(p for p in plan['packages'] if p['name']=='curl')['sha256']='0'*64 fresh_appointment() if mode == 'bad-hash': refused('package-hash-disagrees-with-authenticated-index','prepare',expected='Packages/archive chain') refused('bad-hash-candidate-not-replayed','prepare',expected='external reconciliation') result['recovery']='bad candidate retained; no package effect; corrected new authority required' elif mode == 'space-prepare': filler=Path('/var/tmp/os-ota-private-space-fault') try: fill_private_filesystem(filler) error=refused('real-ENOSPC-during-preparation','prepare') assert isinstance(error,OSError) and error.errno==errno.ENOSPC, str(error) finally: filler.unlink(missing_ok=True) refused('space-restored-preparation-not-replayed','prepare',expected='external reconciliation') result['recovery']='only owned filler removed; disk space restored; partial preparation retained; dpkg unchanged' elif mode == 'offline': # Actual private guest network is removed; no transport monkeypatch. subprocess.run(['ip','link','set','eth0','down'],check=True) error = refused('actual-offline-preparation', 'prepare') assert isinstance(error,urllib.error.URLError), 'must be real transport failure, not another preparation refusal' subprocess.run(['ip','link','set','eth0','up'],check=True) subprocess.run(['ip','route','replace','default','via','10.0.2.2'],check=True) refused('network-restored-preparation-not-replayed', 'prepare',expected='external reconciliation') result['recovery'] = 'guest route restored; partial preparation retained for external reconciliation' elif mode == 'missing-dependency': refused('incomplete-authenticated-dependency-closure', 'prepare',expected='incomplete archive transaction') result['recovery'] = 'incomplete candidate refused without dpkg effects; corrected new authority required' else: path = evidence/('fault-'+mode+'-binding.json') child = subprocess.Popen([sys.executable,'-B','-c', 'import sys,json;sys.path.insert(0,"/opt/ota-source/scripts");' 'import os_update_executor as e;e.stage(json.load(open(sys.argv[1])),"APPLY","prepare")',str(path)], start_new_session=True) try: deadline = time.monotonic()+60 while time.monotonic()