fix(indeehub): compare logical PostgreSQL restore schema

docs/managed-update-recovery-implementation.md

@@ -457,3 +457,41 @@ retain `preserve_original: null`. Relay lineage now distinguishes that verified
 post-target state from pre-target `preserve_original: false`, and rejects missing
 or nonboolean startup markers and inconsistent pairs. The expanded 53 Python
 cases pass; the 2,025-test receipt predates this final helper-only correction.
+
+### Fresh RPC drain and pre-target recovery evidence (2026-10-08)
+
+The prior child unexpectedly rebooted while the manager was barred; its original
+PostgreSQL identity changed, so the recovery preflight correctly refused before
+starting the manager. Child `indeehub-v2-20261007T232717` was powered off with
+operation `3b3c564b-cce6-4727-8be8-369a95c479e4` explicitly **unrecovered**.
+The cause is not proven. The next disposable child has serial kernel logging,
+QEMU `-no-reboot`, boot-ID gates and fixture-only `panic=0`/`hardlockup_panic=0`;
+lockup detection remains enabled. This is application qualification, not kernel
+watchdog or production reboot acceptance.
+
+Fresh child `indeehub-v2-20261008T012000`, matching bca8bad8 executable
+`626afa7563cc3c47f65d31ec6788af18bad2cc3ad057ee008da03440f32ab6cd`,
+passed manager startup with all seven identities retained and the real missing-plan
+RPC refusal with Updating cleared. Operation
+`bfeefcc8-fe33-4925-9252-98cc0c84ac84` then drained all seven members, including
+relay exit 0, and captured the complete backup. Fresh database verification
+refused before target startup. The native controller restored all seven original
+writable layers and exact pinned recipes, restored API/relay Restart=always,
+and released all holds/fence on the same boot. Independent receipt:
+`pretarget-restored-bfeefcc8.receipt.json`. **Pre-target recovery passed; full
+post-target rollback and successful cutover have not passed.**
+
+A separate networkless dump-restore diagnostic confirmed 70 differences, all
+physical PostgreSQL column-slot numbers (`schema.columns[i][0]`). Historical
+DROP COLUMN leaves gaps that pg_dump correctly compacts. The commitment now
+retains `ORDER BY attnum` and every logical column field, but excludes physical
+slot numbers. Actual candidate SQL on the original and freshly restored database
+then matched with **zero differences**. All four real volume archives separately
+passed extraction, comparison and metadata round-trip verification under an
+independent component operation; the real transaction record was not modified.
+
+**55 pure controller tests pass**, including logical column order/type/removal
+refusal. Bounded private failure diagnostics now preserve the controller's reason
+without exposing stderr in the public RPC response. The previously recorded
+2,025 Rust tests predate these final helper changes; a matching executable and
+final combined receipt remain required.

scripts/indeehub-maintenance-controller.py

@@ -132,12 +132,14 @@ ADDITIVE_MIGRATIONS = {
     'AddMediaRegistrationRetirements1791374401000':1791374401000,
 }
 ADDITIVE_TABLES = {'archipelago_media_registrations','archipelago_publications','archipelago_publication_outbox','archipelago_rental_entitlements','archipelago_registration_intents'}
+# Keep logical column order, but not physical attnum values: pg_dump compacts
+# slots left behind by DROP COLUMN while preserving the surviving column order.
 DB_COMMITMENTS_SQL = r'''
 BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY;
 SELECT format($query$
  SELECT jsonb_build_object('table',%L,'schema',
    jsonb_build_object(
-    'columns',(SELECT coalesce(jsonb_agg(jsonb_build_array(a.attnum,a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,pg_get_expr(d.adbin,d.adrelid)) ORDER BY a.attnum),'[]'::jsonb) FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attrelid=%s AND a.attnum>0 AND NOT a.attisdropped),
+    'columns',(SELECT coalesce(jsonb_agg(jsonb_build_array(a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,pg_get_expr(d.adbin,d.adrelid)) ORDER BY a.attnum),'[]'::jsonb) FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attrelid=%s AND a.attnum>0 AND NOT a.attisdropped),
     'constraints',(SELECT coalesce(jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid,true)) ORDER BY conname),'[]'::jsonb) FROM pg_constraint WHERE conrelid=%s),
     'indexes',(SELECT coalesce(jsonb_agg(pg_get_indexdef(indexrelid) ORDER BY indexrelid::regclass::text),'[]'::jsonb) FROM pg_index WHERE indrelid=%s),
     'triggers',(SELECT coalesce(jsonb_agg(pg_get_triggerdef(oid,true) ORDER BY tgname),'[]'::jsonb) FROM pg_trigger WHERE tgrelid=%s AND NOT tgisinternal),
@@ -261,6 +263,11 @@ class Controller:
         self.record=json.loads(self.path.read_text()) if self.path.exists() else None
         if self.record:require(self.record['operation_id']==operation,'Maintenance journal changed')
     def save(self): atomic(self.path,self.record)
+    def save_failure(self, action, error):
+        # The native adapter deliberately withholds stderr from public errors.
+        # Retain bounded diagnostics privately so an operator can inspect refusal.
+        atomic(self.root/'last-failure.private.json',{'operation_id':self.operation,'action':action,
+            'error_type':type(error).__name__,'message':str(error)[:4096],'at':time.time()})
     def run(self, argv, timeout=30, output=None, input_bytes=None, input_file=None):
         if self.runner:return self.runner(argv,timeout,output)
         self.root.mkdir(mode=0o700,parents=True,exist_ok=True)
@@ -826,7 +833,12 @@ def main():
     if 'recovery' in request:require(type(request['recovery']) is bool,'Invalid recovery flag')
     require(os.getuid()==1000,'Expected node service user');fd=int(os.environ['ARCHY_UPDATE_LOCK_FD']);actual=os.fstat(fd);expected=(DATA/'update-transactions'/'lock').stat();require((actual.st_dev,actual.st_ino)==(expected.st_dev,expected.st_ino),'Inherited lifecycle lock is not the expected file')
     controller=Controller(DATA,request['operation_id'],fd)
-    result=controller.acquire(request['original_members'],request.get('recovery',False)) if sys.argv[1]=='acquire' else controller.verify() if sys.argv[1]=='verify' else controller.release(request['outcome'])
+    try:
+        result=controller.acquire(request['original_members'],request.get('recovery',False)) if sys.argv[1]=='acquire' else controller.verify() if sys.argv[1]=='verify' else controller.release(request['outcome'])
+    except Exception as error:
+        try:controller.save_failure(sys.argv[1],error)
+        except Exception:pass  # Diagnostic failure must not replace the original refusal.
+        raise
     encoded=json.dumps(result);require(len(encoded)<=4096,'Maintenance response exceeds bound');print(encoded)
 if __name__=='__main__':
     try:main()

tests/regression/test_indeehub_maintenance_controller.py

@@ -562,4 +562,17 @@ console.log('process identity cases passed');'''
    if change=='wrong-body':bad['members'][0]['pinned_original_body']='changed'
    if change=='cycle':bad['members'][0]['original']['image']=image
    with self.assertRaises(RuntimeError):module.verify_relay_image_lineage(image,digest,[bad],installed)
+ def test_column_commitments_retain_logical_order_and_every_semantic_field(self):
+  import copy
+  columns=[['first','integer',True,'','',''],['last','text',False,'','','']]
+  before={'operation_id':self.operation,'tables':{'typeorm_migrations':{'schema':{},'rows':0,'rows_sha256':'a'},'items':{'schema':{'columns':columns},'rows':0,'rows_sha256':'b'}},'migrations':[]}
+  module.verify_database_compatibility(before,copy.deepcopy(before))
+  for replacement in (list(reversed(columns)),columns[:-1],[['first','bigint',True,'','',''],columns[1]]):
+   after=copy.deepcopy(before);after['tables']['items']['schema']['columns']=replacement
+   with self.assertRaisesRegex(RuntimeError,'changed original table schema'):module.verify_database_compatibility(before,after)
+ def test_failure_diagnostics_are_private_bounded_and_do_not_change_journal(self):
+  c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared'};c.save();before=c.path.read_bytes()
+  c.save_failure('acquire',RuntimeError('diagnostic'*1000));path=c.root/'last-failure.private.json';record=json.loads(path.read_text())
+  self.assertEqual(record['operation_id'],self.operation);self.assertEqual(record['action'],'acquire');self.assertEqual(record['error_type'],'RuntimeError');self.assertEqual(len(record['message']),4096)
+  self.assertEqual(path.stat().st_mode&0o777,0o600);self.assertEqual(c.path.read_bytes(),before)
 if __name__=='__main__':unittest.main()