diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index dd29f09e..d2bced68 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -110,3 +110,21 @@ command stderr was removed by fixture cleanup, so no exact cause is claimed. The stale backend compile was interrupted after the readiness edit; a fresh backend build/suite remains required for the final embedded controller. Volume-archive restore and the full supervised app cutover remain open gates. + +## Four-volume restore barrier — 2026-10-07 + +The controller now restores each fresh volume archive into separate owned storage +before target startup. It rejects unsafe paths/links and unsupported special files, +checks restored bytes, links, ownership and modes, and rearchives the restored tree +to compare ACLs and extended attributes explicitly (GNU tar compare alone does not +check xattrs). Durable proof binds all four archive hashes to the operation. Failure +retains ingress and lifecycle holds; retry removes only the operation-owned fixture. +No production volume is mounted or modified by restore verification. + +All24 pure controller tests pass. The real rootless fixture passes four archives +containing hidden files, hardlinks, symlinks, mapped numeric ownership, ACLs and +xattrs; changed restored bytes/xattrs and unreadable archives are rejected. Evidence: +`/tmp/archy-20261007-volume-controller-tests.log` and +`/tmp/archy-20261007-volume-restore.log`. Repeatable fixture: +`tests/regression/test_indeehub_maintenance_volumes.py`. +Full seven-member application cutover and final backend build remain separate gates. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index e7961b59..a87f4222 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -3,7 +3,7 @@ No live execution is part of source qualification. Original writable-layer images must already be durable. Never unlock ARCHY_UPDATE_LOCK_FD or release another hold. """ -import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid +import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid, tarfile NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay') VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data') DATA = pathlib.Path('/var/lib/archipelago') @@ -99,6 +99,46 @@ def validate_nginx_guards(config): matched+=1 require(matched>=3,'Expected complete legacy IndeeHub route guards') return matched +def validate_volume_archive(path): + """Validate the complete inventory before extracting into private storage.""" + entries={};size=0 + with tarfile.open(path, 'r:') as archive: + for member in archive: + name=pathlib.PurePosixPath(member.name) + require(not name.is_absolute() and '..' not in name.parts,'Unsafe volume archive path') + key=str(name) + require(key not in entries and len(entries)<1000000,'Duplicate or oversized volume archive inventory') + require(member.isfile() or member.isdir() or member.issym() or member.islnk(),'Unsupported volume archive entry') + entries[key]=member;size+=member.size + require(entries and entries.get('.') and entries['.'].isdir(),'Volume archive root is missing') + for key,member in entries.items(): + for parent in pathlib.PurePosixPath(key).parents: + ancestor=entries.get(str(parent)) + require(ancestor is None or ancestor.isdir(),'Volume archive writes through a link') + if member.issym() or member.islnk(): + target=pathlib.PurePosixPath(member.linkname) + require(not target.is_absolute(),'Volume archive link escapes restore storage') + parts=list(pathlib.PurePosixPath(key).parent.parts) if member.issym() else [] + for part in target.parts: + if part=='..': + require(parts,'Volume archive link escapes restore storage');parts.pop() + elif part!='.':parts.append(part) + if member.islnk(): + linked=entries.get(str(pathlib.PurePosixPath(*parts))) + require(linked is not None and linked.isfile(),'Volume archive hardlink target is not a regular file') + return size + +def volume_archive_metadata(path): + entries={};links={} + with tarfile.open(path,'r:') as archive: + for member in archive: + name=str(pathlib.PurePosixPath(member.name)) + entries[name]={'mode':member.mode,'uid':member.uid,'gid':member.gid, + 'attributes':{key:value for key,value in member.pax_headers.items() if key.startswith('SCHILY.')}} + if member.islnk():links[name]=str(pathlib.PurePosixPath(member.linkname)) + for name,target in links.items():entries[name]=entries[target] + return entries + class Controller: def __init__(self, data, operation, lock_fd, runner=None): self.data=pathlib.Path(data);self.operation=operation;self.lock_fd=lock_fd;self.runner=runner @@ -271,7 +311,7 @@ class Controller: require(time.monotonic()measured*2+512*1024*1024,'Insufficient volume restore space') + name='volume-restore-'+uuid.uuid4().hex;path=self.root/name + path.mkdir(mode=0o700) + with (path/'owner').open('x') as owner: + owner.write(self.operation);owner.flush();os.fsync(owner.fileno()) + self.record['volume_restore_fixture']=name;self.save() + try: + (path/'payload').mkdir(mode=0o700) + self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner', + '--same-owner','--same-permissions','-C',str(path/'payload'),'-xpf',str(archive)],timeout=1800) + self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner', + '-C',str(path/'payload'),'-df',str(archive)],timeout=1800) + # GNU tar --compare omits extended attributes. Re-archive the + # restored tree and compare numeric ownership, modes, ACLs and + # xattrs explicitly (normalizing hardlink traversal order). + roundtrip=path/'roundtrip.tar' + with roundtrip.open('xb') as output: + self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner', + '-C',str(path/'payload'),'-cpf','-','.'],timeout=1800,output=output) + require(volume_archive_metadata(archive)==volume_archive_metadata(roundtrip),'Restored volume metadata differs from backup') + finally:self.cleanup_volume_fixture() + self.verify_artifacts() + self.record['volume_restore_verified']=terms;self.save() def verify_artifacts(self): expected_artifacts={'database.dump',*(volume+'.tar' for volume in VOLUMES)} require(set(self.record.get('artifacts',{}))==expected_artifacts,'Backup artifact inventory incomplete or unexpected') @@ -354,6 +447,7 @@ class Controller: for name in NAMES:require(self.record.get('stopped',{}).get(name,{}).get('confirmed'),'Original writer stop evidence missing') self.verify_artifacts() require(self.record.get('backup_restore_verified')==self.backup_restore_terms(),'Fresh database backup restore is not verified') + require(self.record.get('volume_restore_verified')==self.volume_restore_terms(),'Fresh volume backup restore is not verified') return {'operation_id':self.operation,'state':'held'} def release(self, outcome): require(outcome in ('committed','restored','aborted'),'Invalid release outcome') diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 65ed2ac4..6529ff36 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -59,6 +59,7 @@ class MaintenanceTests(unittest.TestCase): c.record['original_members']=members() c.record['database_before']={'operation_id':self.operation,'tables':{},'migrations':[]} c.record['backup_restore_verified']=c.backup_restore_terms() + c.record['volume_restore_verified']=c.volume_restore_terms() c.save() return c def test_complete_backup_checksums_allow_verification(self): @@ -71,6 +72,39 @@ class MaintenanceTests(unittest.TestCase): self.assertEqual(c.fence.read_text(),self.operation) c.record.pop('backup_restore_verified') with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify() + def test_volume_restore_proof_must_match_all_archives_and_operation(self): + import copy + c=self.completed_backup();proof=copy.deepcopy(c.record['volume_restore_verified']) + for field in ('operation_id',*module.VOLUMES): + changed=copy.deepcopy(proof) + if field=='operation_id':changed[field]='changed' + else:changed['archives'][field+'.tar']='changed' + c.record['volume_restore_verified']=changed + with self.assertRaisesRegex(RuntimeError,'volume backup restore is not verified'):c.verify() + c.record.pop('volume_restore_verified') + with self.assertRaisesRegex(RuntimeError,'volume backup restore is not verified'):c.verify() + self.assertEqual(c.fence.read_text(),self.operation) + def test_foreign_volume_fixture_is_never_removed(self): + c=self.completed_backup();name='volume-restore-'+'a'*32 + path=c.root/name;path.mkdir();(path/'owner').write_text(str(uuid.uuid4())) + c.record['volume_restore_fixture']=name + with self.assertRaisesRegex(RuntimeError,'ownership changed'):c.cleanup_volume_fixture() + self.assertEqual(self.calls,[]);self.assertTrue(path.exists()) + def test_unsafe_archive_paths_and_links_are_rejected_before_extraction(self): + import tarfile,io + c=self.completed_backup();path=c.root/'unsafe.tar' + cases=[('../escape',tarfile.REGTYPE,''),('/escape',tarfile.REGTYPE,''), + ('link',tarfile.SYMTYPE,'../../escape'),('link',tarfile.LNKTYPE,'../escape'), + ('device',tarfile.CHRTYPE,''),('hard',tarfile.LNKTYPE,'missing')] + for name,kind,target in cases: + with tarfile.open(path,'w') as archive: + root=tarfile.TarInfo('.');root.type=tarfile.DIRTYPE;archive.addfile(root) + entry=tarfile.TarInfo(name);entry.type=kind;entry.linkname=target;archive.addfile(entry) + with self.assertRaises(RuntimeError):module.validate_volume_archive(path) + with tarfile.open(path,'w') as archive: + for name,kind,target in [('.',tarfile.DIRTYPE,''),('dir',tarfile.SYMTYPE,'safe'),('dir/file',tarfile.REGTYPE,'')]: + entry=tarfile.TarInfo(name);entry.type=kind;entry.linkname=target;archive.addfile(entry) + with self.assertRaisesRegex(RuntimeError,'writes through a link'):module.validate_volume_archive(path) def test_foreign_restore_fixture_is_never_removed(self): c=self.completed_backup();name='archy-backup-restore-'+'a'*32 c.record['restore_fixture']={'name':name,'image_id':'a'*64} diff --git a/tests/regression/test_indeehub_maintenance_volumes.py b/tests/regression/test_indeehub_maintenance_volumes.py new file mode 100644 index 00000000..8e63c8ef --- /dev/null +++ b/tests/regression/test_indeehub_maintenance_volumes.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Restore fixture-only volume archives with the production maintenance gate. + +Requires rootless Podman. Creates no containers and never opens live app volumes. +""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import uuid + +MODULE = Path(__file__).resolve().parents[2] / 'scripts/indeehub-maintenance-controller.py' +spec = importlib.util.spec_from_file_location('maintenance', MODULE) +maintenance = importlib.util.module_from_spec(spec) +spec.loader.exec_module(maintenance) + + +def main(): + with tempfile.TemporaryDirectory(prefix='archy-volume-restore-') as temporary: + root = Path(temporary) + source = root/'source' + source.mkdir() + (source/'.hidden').write_bytes(b'retained hidden object\x00') + (source/'nested').mkdir() + original = source/'nested'/'media' + original.write_bytes(bytes(range(256))*4096) + original.chmod(0o640) + os.link(original, source/'hardlink') + (source/'symlink').symlink_to('nested/media') + os.setxattr(original, 'user.archy-fixture', b'retained metadata') + # Exercise numeric ownership which the calling host user cannot reproduce + # without the rootless user namespace used by production backup/restore. + subprocess.run(['podman','unshare','chown','101:102',str(original)],check=True) + subprocess.run(['podman','unshare','setfacl','-m','u:103:r--',str(original)],check=True) + operation = str(uuid.uuid4()) + controller = maintenance.Controller(root/'data',operation,0) + controller.record = {'operation_id':operation,'artifacts':{}} + holds = controller.data/'update-transactions'/'holds' + holds.mkdir(parents=True) + for name in maintenance.NAMES:(holds/name).write_text(operation) + controller.fence.parent.mkdir(parents=True) + controller.fence.write_text(operation) + backup = controller.root/'backup' + backup.mkdir(parents=True) + for name in ('database.dump',*(v+'.tar' for v in maintenance.VOLUMES)): + path = backup/name + if name=='database.dump':path.write_bytes(b'database checked by separate fixture') + else: + subprocess.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner', + '-C',str(source),'-cpf',str(path),'.'],check=True) + controller.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':maintenance.sha(path)} + controller.save() + try: + controller.verify_volume_backups() + assert controller.record['volume_restore_verified']==controller.volume_restore_terms() + assert 'volume_restore_fixture' not in controller.record + controller.verify_volume_backups() # durable proof is reusable + controller.record.pop('volume_restore_verified') + actual_run = controller.run + def corrupt_restored(argv,**kwargs): + if '-df' in argv: + payload = Path(argv[argv.index('-C')+1]) + subprocess.run(['podman','unshare','sh','-c','printf changed > "$1/.hidden"','fixture',str(payload)],check=True) + return actual_run(argv,**kwargs) + controller.run=corrupt_restored + try:controller.verify_volume_backups() + except subprocess.CalledProcessError:pass + else:raise AssertionError('Changed restoration accepted') + assert 'volume_restore_verified' not in controller.record + assert 'volume_restore_fixture' not in controller.record + assert controller.fence.read_text()==operation + controller.run=actual_run + def corrupt_metadata(argv,**kwargs): + result=actual_run(argv,**kwargs) + if '-xpf' in argv: + payload=Path(argv[argv.index('-C')+1]) + subprocess.run(['podman','unshare','python3','-c', + "import os,sys;os.setxattr(sys.argv[1],'user.archy-fixture',b'changed')", + str(payload/'nested'/'media')],check=True) + return result + controller.run=corrupt_metadata + try:controller.verify_volume_backups() + except RuntimeError as error:assert 'metadata differs' in str(error) + else:raise AssertionError('Changed xattr restoration accepted') + assert 'volume_restore_verified' not in controller.record + assert 'volume_restore_fixture' not in controller.record + controller.run=actual_run + path=backup/(maintenance.VOLUMES[0]+'.tar') + path.write_bytes(b'not a tar archive') + controller.record['artifacts'][path.name]={'bytes':path.stat().st_size,'sha256':maintenance.sha(path)} + try:controller.verify_volume_backups() + except maintenance.tarfile.ReadError:pass + else:raise AssertionError('Unreadable archive accepted') + assert 'volume_restore_verified' not in controller.record + assert controller.fence.read_text()==operation + print(json.dumps({'production_volume_restore_barrier':'passed','volumes':4, + 'hidden_files':True,'hardlinks':True,'symlinks':True,'numeric_ownership':True, + 'xattrs':True,'acls':True,'corrupt_xattr_rejected':True,'corrupt_restore_rejected':True,'unreadable_archive_rejected':True, + 'live_volumes_opened':False})) + finally: + subprocess.run(['podman','unshare','rm','-rf','--',str(source)],check=True) + +if __name__=='__main__':main()