Verify fresh IndeeHub volume restores before supervised cutover

docs/managed-update-recovery-implementation.md

@@ -110,3 +110,21 @@ command stderr was removed by fixture cleanup, so no exact cause is claimed.
 The stale backend compile was interrupted after the readiness edit; a fresh
 backend build/suite remains required for the final embedded controller.
 Volume-archive restore and the full supervised app cutover remain open gates.
+
+## Four-volume restore barrier — 2026-10-07
+
+The controller now restores each fresh volume archive into separate owned storage
+before target startup. It rejects unsafe paths/links and unsupported special files,
+checks restored bytes, links, ownership and modes, and rearchives the restored tree
+to compare ACLs and extended attributes explicitly (GNU tar compare alone does not
+check xattrs). Durable proof binds all four archive hashes to the operation. Failure
+retains ingress and lifecycle holds; retry removes only the operation-owned fixture.
+No production volume is mounted or modified by restore verification.
+
+All24 pure controller tests pass. The real rootless fixture passes four archives
+containing hidden files, hardlinks, symlinks, mapped numeric ownership, ACLs and
+xattrs; changed restored bytes/xattrs and unreadable archives are rejected. Evidence:
+`/tmp/archy-20261007-volume-controller-tests.log` and
+`/tmp/archy-20261007-volume-restore.log`. Repeatable fixture:
+`tests/regression/test_indeehub_maintenance_volumes.py`.
+Full seven-member application cutover and final backend build remain separate gates.

scripts/indeehub-maintenance-controller.py

@@ -3,7 +3,7 @@
 No live execution is part of source qualification. Original writable-layer images
 must already be durable. Never unlock ARCHY_UPDATE_LOCK_FD or release another hold.
 """
-import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid
+import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid, tarfile
 NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay')
 VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data')
 DATA = pathlib.Path('/var/lib/archipelago')
@@ -99,6 +99,46 @@ def validate_nginx_guards(config):
             matched+=1
     require(matched>=3,'Expected complete legacy IndeeHub route guards')
     return matched
+def validate_volume_archive(path):
+    """Validate the complete inventory before extracting into private storage."""
+    entries={};size=0
+    with tarfile.open(path, 'r:') as archive:
+        for member in archive:
+            name=pathlib.PurePosixPath(member.name)
+            require(not name.is_absolute() and '..' not in name.parts,'Unsafe volume archive path')
+            key=str(name)
+            require(key not in entries and len(entries)<1000000,'Duplicate or oversized volume archive inventory')
+            require(member.isfile() or member.isdir() or member.issym() or member.islnk(),'Unsupported volume archive entry')
+            entries[key]=member;size+=member.size
+    require(entries and entries.get('.') and entries['.'].isdir(),'Volume archive root is missing')
+    for key,member in entries.items():
+        for parent in pathlib.PurePosixPath(key).parents:
+            ancestor=entries.get(str(parent))
+            require(ancestor is None or ancestor.isdir(),'Volume archive writes through a link')
+        if member.issym() or member.islnk():
+            target=pathlib.PurePosixPath(member.linkname)
+            require(not target.is_absolute(),'Volume archive link escapes restore storage')
+            parts=list(pathlib.PurePosixPath(key).parent.parts) if member.issym() else []
+            for part in target.parts:
+                if part=='..':
+                    require(parts,'Volume archive link escapes restore storage');parts.pop()
+                elif part!='.':parts.append(part)
+            if member.islnk():
+                linked=entries.get(str(pathlib.PurePosixPath(*parts)))
+                require(linked is not None and linked.isfile(),'Volume archive hardlink target is not a regular file')
+    return size
+
+def volume_archive_metadata(path):
+    entries={};links={}
+    with tarfile.open(path,'r:') as archive:
+        for member in archive:
+            name=str(pathlib.PurePosixPath(member.name))
+            entries[name]={'mode':member.mode,'uid':member.uid,'gid':member.gid,
+                'attributes':{key:value for key,value in member.pax_headers.items() if key.startswith('SCHILY.')}}
+            if member.islnk():links[name]=str(pathlib.PurePosixPath(member.linkname))
+    for name,target in links.items():entries[name]=entries[target]
+    return entries
+
 class Controller:
     def __init__(self, data, operation, lock_fd, runner=None):
         self.data=pathlib.Path(data);self.operation=operation;self.lock_fd=lock_fd;self.runner=runner
@@ -271,7 +311,7 @@ class Controller:
                 require(time.monotonic()<deadline,'Transcodes still active; retained job state, no forced completion')
                 time.sleep(1)
             self.graceful_stop('indeedhub-ffmpeg');self.legacy_api_idle();self.graceful_stop('indeedhub-api')
-        self.backup();self.verify_database_backup();self.verify();return {'operation_id':self.operation,'state':'drained'}
+        self.backup();self.verify_database_backup();self.verify_volume_backups();self.verify();return {'operation_id':self.operation,'state':'drained'}
     def backup_restore_terms(self):
         baseline=self.record.get('database_before')
         require(baseline and baseline.get('operation_id')==self.operation,'Backup database baseline missing')
@@ -335,6 +375,59 @@ class Controller:
         finally:
             self.cleanup_restore_fixture()
         self.record['backup_restore_verified']=terms;self.save()
+    def volume_restore_terms(self):
+        return {'operation_id':self.operation,'archives':{
+            name:self.record['artifacts'][name]['sha256'] for name in (v+'.tar' for v in VOLUMES)}}
+    def cleanup_volume_fixture(self):
+        name=self.record.get('volume_restore_fixture')
+        if name is None:return
+        require(bool(re.fullmatch('volume-restore-[0-9a-f]{32}',name)),'Invalid volume restore fixture')
+        path=self.root/name
+        if path.exists() or path.is_symlink():
+            require(path.is_dir() and not path.is_symlink() and path.stat().st_uid==os.getuid(),'Volume restore ownership changed')
+            owner=path/'owner'
+            require(owner.is_file() and not owner.is_symlink() and owner.read_text()==self.operation,'Volume restore ownership changed')
+            self.run(['podman','unshare','rm','-rf','--',str(path/'payload')],timeout=1800)
+            roundtrip=path/'roundtrip.tar'
+            if roundtrip.exists():
+                require(roundtrip.is_file() and not roundtrip.is_symlink(),'Volume restore output changed')
+                roundtrip.unlink()
+            owner.unlink();path.rmdir()
+        del self.record['volume_restore_fixture'];self.save()
+    def verify_volume_backups(self):
+        # Restore each entire volume to fresh owned storage, then have GNU tar
+        # compare its bytes, links, ownership, modes and metadata to the archive.
+        # Live volumes are neither mounted nor written by this verification.
+        self.holds();self.fence_matches();self.verify_artifacts()
+        terms=self.volume_restore_terms();self.cleanup_volume_fixture()
+        if self.record.get('volume_restore_verified'):
+            require(self.record['volume_restore_verified']==terms,'Volume restore proof changed');return
+        for volume in VOLUMES:
+            archive=self.root/'backup'/(volume+'.tar')
+            measured=validate_volume_archive(archive)
+            require(shutil.disk_usage(self.root).free>measured*2+512*1024*1024,'Insufficient volume restore space')
+            name='volume-restore-'+uuid.uuid4().hex;path=self.root/name
+            path.mkdir(mode=0o700)
+            with (path/'owner').open('x') as owner:
+                owner.write(self.operation);owner.flush();os.fsync(owner.fileno())
+            self.record['volume_restore_fixture']=name;self.save()
+            try:
+                (path/'payload').mkdir(mode=0o700)
+                self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner',
+                    '--same-owner','--same-permissions','-C',str(path/'payload'),'-xpf',str(archive)],timeout=1800)
+                self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner',
+                    '-C',str(path/'payload'),'-df',str(archive)],timeout=1800)
+                # GNU tar --compare omits extended attributes. Re-archive the
+                # restored tree and compare numeric ownership, modes, ACLs and
+                # xattrs explicitly (normalizing hardlink traversal order).
+                roundtrip=path/'roundtrip.tar'
+                with roundtrip.open('xb') as output:
+                    self.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner',
+                        '-C',str(path/'payload'),'-cpf','-','.'],timeout=1800,output=output)
+                require(volume_archive_metadata(archive)==volume_archive_metadata(roundtrip),'Restored volume metadata differs from backup')
+            finally:self.cleanup_volume_fixture()
+        self.verify_artifacts()
+        self.record['volume_restore_verified']=terms;self.save()
     def verify_artifacts(self):
         expected_artifacts={'database.dump',*(volume+'.tar' for volume in VOLUMES)}
         require(set(self.record.get('artifacts',{}))==expected_artifacts,'Backup artifact inventory incomplete or unexpected')
@@ -354,6 +447,7 @@ class Controller:
         for name in NAMES:require(self.record.get('stopped',{}).get(name,{}).get('confirmed'),'Original writer stop evidence missing')
         self.verify_artifacts()
         require(self.record.get('backup_restore_verified')==self.backup_restore_terms(),'Fresh database backup restore is not verified')
+        require(self.record.get('volume_restore_verified')==self.volume_restore_terms(),'Fresh volume backup restore is not verified')
         return {'operation_id':self.operation,'state':'held'}
     def release(self, outcome):
         require(outcome in ('committed','restored','aborted'),'Invalid release outcome')

tests/regression/test_indeehub_maintenance_controller.py

@@ -59,6 +59,7 @@ class MaintenanceTests(unittest.TestCase):
   c.record['original_members']=members()
   c.record['database_before']={'operation_id':self.operation,'tables':{},'migrations':[]}
   c.record['backup_restore_verified']=c.backup_restore_terms()
+  c.record['volume_restore_verified']=c.volume_restore_terms()
   c.save()
   return c
  def test_complete_backup_checksums_allow_verification(self):
@@ -71,6 +72,39 @@ class MaintenanceTests(unittest.TestCase):
    self.assertEqual(c.fence.read_text(),self.operation)
   c.record.pop('backup_restore_verified')
   with self.assertRaisesRegex(RuntimeError,'restore is not verified'):c.verify()
+ def test_volume_restore_proof_must_match_all_archives_and_operation(self):
+  import copy
+  c=self.completed_backup();proof=copy.deepcopy(c.record['volume_restore_verified'])
+  for field in ('operation_id',*module.VOLUMES):
+   changed=copy.deepcopy(proof)
+   if field=='operation_id':changed[field]='changed'
+   else:changed['archives'][field+'.tar']='changed'
+   c.record['volume_restore_verified']=changed
+   with self.assertRaisesRegex(RuntimeError,'volume backup restore is not verified'):c.verify()
+  c.record.pop('volume_restore_verified')
+  with self.assertRaisesRegex(RuntimeError,'volume backup restore is not verified'):c.verify()
+  self.assertEqual(c.fence.read_text(),self.operation)
+ def test_foreign_volume_fixture_is_never_removed(self):
+  c=self.completed_backup();name='volume-restore-'+'a'*32
+  path=c.root/name;path.mkdir();(path/'owner').write_text(str(uuid.uuid4()))
+  c.record['volume_restore_fixture']=name
+  with self.assertRaisesRegex(RuntimeError,'ownership changed'):c.cleanup_volume_fixture()
+  self.assertEqual(self.calls,[]);self.assertTrue(path.exists())
+ def test_unsafe_archive_paths_and_links_are_rejected_before_extraction(self):
+  import tarfile,io
+  c=self.completed_backup();path=c.root/'unsafe.tar'
+  cases=[('../escape',tarfile.REGTYPE,''),('/escape',tarfile.REGTYPE,''),
+         ('link',tarfile.SYMTYPE,'../../escape'),('link',tarfile.LNKTYPE,'../escape'),
+         ('device',tarfile.CHRTYPE,''),('hard',tarfile.LNKTYPE,'missing')]
+  for name,kind,target in cases:
+   with tarfile.open(path,'w') as archive:
+    root=tarfile.TarInfo('.');root.type=tarfile.DIRTYPE;archive.addfile(root)
+    entry=tarfile.TarInfo(name);entry.type=kind;entry.linkname=target;archive.addfile(entry)
+   with self.assertRaises(RuntimeError):module.validate_volume_archive(path)
+  with tarfile.open(path,'w') as archive:
+   for name,kind,target in [('.',tarfile.DIRTYPE,''),('dir',tarfile.SYMTYPE,'safe'),('dir/file',tarfile.REGTYPE,'')]:
+    entry=tarfile.TarInfo(name);entry.type=kind;entry.linkname=target;archive.addfile(entry)
+  with self.assertRaisesRegex(RuntimeError,'writes through a link'):module.validate_volume_archive(path)
  def test_foreign_restore_fixture_is_never_removed(self):
   c=self.completed_backup();name='archy-backup-restore-'+'a'*32
   c.record['restore_fixture']={'name':name,'image_id':'a'*64}

tests/regression/test_indeehub_maintenance_volumes.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Restore fixture-only volume archives with the production maintenance gate.
+
+Requires rootless Podman. Creates no containers and never opens live app volumes.
+"""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import uuid
+
+MODULE = Path(__file__).resolve().parents[2] / 'scripts/indeehub-maintenance-controller.py'
+spec = importlib.util.spec_from_file_location('maintenance', MODULE)
+maintenance = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(maintenance)
+
+
+def main():
+    with tempfile.TemporaryDirectory(prefix='archy-volume-restore-') as temporary:
+        root = Path(temporary)
+        source = root/'source'
+        source.mkdir()
+        (source/'.hidden').write_bytes(b'retained hidden object\x00')
+        (source/'nested').mkdir()
+        original = source/'nested'/'media'
+        original.write_bytes(bytes(range(256))*4096)
+        original.chmod(0o640)
+        os.link(original, source/'hardlink')
+        (source/'symlink').symlink_to('nested/media')
+        os.setxattr(original, 'user.archy-fixture', b'retained metadata')
+        # Exercise numeric ownership which the calling host user cannot reproduce
+        # without the rootless user namespace used by production backup/restore.
+        subprocess.run(['podman','unshare','chown','101:102',str(original)],check=True)
+        subprocess.run(['podman','unshare','setfacl','-m','u:103:r--',str(original)],check=True)
+        operation = str(uuid.uuid4())
+        controller = maintenance.Controller(root/'data',operation,0)
+        controller.record = {'operation_id':operation,'artifacts':{}}
+        holds = controller.data/'update-transactions'/'holds'
+        holds.mkdir(parents=True)
+        for name in maintenance.NAMES:(holds/name).write_text(operation)
+        controller.fence.parent.mkdir(parents=True)
+        controller.fence.write_text(operation)
+        backup = controller.root/'backup'
+        backup.mkdir(parents=True)
+        for name in ('database.dump',*(v+'.tar' for v in maintenance.VOLUMES)):
+            path = backup/name
+            if name=='database.dump':path.write_bytes(b'database checked by separate fixture')
+            else:
+                subprocess.run(['podman','unshare','tar','--xattrs','--acls','--numeric-owner',
+                    '-C',str(source),'-cpf',str(path),'.'],check=True)
+            controller.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':maintenance.sha(path)}
+        controller.save()
+        try:
+            controller.verify_volume_backups()
+            assert controller.record['volume_restore_verified']==controller.volume_restore_terms()
+            assert 'volume_restore_fixture' not in controller.record
+            controller.verify_volume_backups() # durable proof is reusable
+            controller.record.pop('volume_restore_verified')
+            actual_run = controller.run
+            def corrupt_restored(argv,**kwargs):
+                if '-df' in argv:
+                    payload = Path(argv[argv.index('-C')+1])
+                    subprocess.run(['podman','unshare','sh','-c','printf changed > "$1/.hidden"','fixture',str(payload)],check=True)
+                return actual_run(argv,**kwargs)
+            controller.run=corrupt_restored
+            try:controller.verify_volume_backups()
+            except subprocess.CalledProcessError:pass
+            else:raise AssertionError('Changed restoration accepted')
+            assert 'volume_restore_verified' not in controller.record
+            assert 'volume_restore_fixture' not in controller.record
+            assert controller.fence.read_text()==operation
+            controller.run=actual_run
+            def corrupt_metadata(argv,**kwargs):
+                result=actual_run(argv,**kwargs)
+                if '-xpf' in argv:
+                    payload=Path(argv[argv.index('-C')+1])
+                    subprocess.run(['podman','unshare','python3','-c',
+                        "import os,sys;os.setxattr(sys.argv[1],'user.archy-fixture',b'changed')",
+                        str(payload/'nested'/'media')],check=True)
+                return result
+            controller.run=corrupt_metadata
+            try:controller.verify_volume_backups()
+            except RuntimeError as error:assert 'metadata differs' in str(error)
+            else:raise AssertionError('Changed xattr restoration accepted')
+            assert 'volume_restore_verified' not in controller.record
+            assert 'volume_restore_fixture' not in controller.record
+            controller.run=actual_run
+            path=backup/(maintenance.VOLUMES[0]+'.tar')
+            path.write_bytes(b'not a tar archive')
+            controller.record['artifacts'][path.name]={'bytes':path.stat().st_size,'sha256':maintenance.sha(path)}
+            try:controller.verify_volume_backups()
+            except maintenance.tarfile.ReadError:pass
+            else:raise AssertionError('Unreadable archive accepted')
+            assert 'volume_restore_verified' not in controller.record
+            assert controller.fence.read_text()==operation
+            print(json.dumps({'production_volume_restore_barrier':'passed','volumes':4,
+                'hidden_files':True,'hardlinks':True,'symlinks':True,'numeric_ownership':True,
+                'xattrs':True,'acls':True,'corrupt_xattr_rejected':True,'corrupt_restore_rejected':True,'unreadable_archive_rejected':True,
+                'live_volumes_opened':False}))
+        finally:
+            subprocess.run(['podman','unshare','rm','-rf','--',str(source)],check=True)
+
+if __name__=='__main__':main()