From c58d1180e77ed081e06a88f7342bb0c542c711ab Mon Sep 17 00:00:00 2001 From: archipelago Date: Thu, 8 Oct 2026 01:51:21 -0400 Subject: [PATCH] fix(indeehub): compare logical PostgreSQL restore schema --- .../managed-update-recovery-implementation.md | 38 +++++++++++++++++++ scripts/indeehub-maintenance-controller.py | 16 +++++++- .../test_indeehub_maintenance_controller.py | 13 +++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index dd55be16..1ede00ea 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -457,3 +457,41 @@ retain `preserve_original: null`. Relay lineage now distinguishes that verified post-target state from pre-target `preserve_original: false`, and rejects missing or nonboolean startup markers and inconsistent pairs. The expanded 53 Python cases pass; the 2,025-test receipt predates this final helper-only correction. + +### Fresh RPC drain and pre-target recovery evidence (2026-10-08) + +The prior child unexpectedly rebooted while the manager was barred; its original +PostgreSQL identity changed, so the recovery preflight correctly refused before +starting the manager. Child `indeehub-v2-20261007T232717` was powered off with +operation `3b3c564b-cce6-4727-8be8-369a95c479e4` explicitly **unrecovered**. +The cause is not proven. The next disposable child has serial kernel logging, +QEMU `-no-reboot`, boot-ID gates and fixture-only `panic=0`/`hardlockup_panic=0`; +lockup detection remains enabled. This is application qualification, not kernel +watchdog or production reboot acceptance. + +Fresh child `indeehub-v2-20261008T012000`, matching bca8bad8 executable +`626afa7563cc3c47f65d31ec6788af18bad2cc3ad057ee008da03440f32ab6cd`, +passed manager startup with all seven identities retained and the real missing-plan +RPC refusal with Updating cleared. Operation +`bfeefcc8-fe33-4925-9252-98cc0c84ac84` then drained all seven members, including +relay exit 0, and captured the complete backup. Fresh database verification +refused before target startup. The native controller restored all seven original +writable layers and exact pinned recipes, restored API/relay Restart=always, +and released all holds/fence on the same boot. Independent receipt: +`pretarget-restored-bfeefcc8.receipt.json`. **Pre-target recovery passed; full +post-target rollback and successful cutover have not passed.** + +A separate networkless dump-restore diagnostic confirmed 70 differences, all +physical PostgreSQL column-slot numbers (`schema.columns[i][0]`). Historical +DROP COLUMN leaves gaps that pg_dump correctly compacts. The commitment now +retains `ORDER BY attnum` and every logical column field, but excludes physical +slot numbers. Actual candidate SQL on the original and freshly restored database +then matched with **zero differences**. All four real volume archives separately +passed extraction, comparison and metadata round-trip verification under an +independent component operation; the real transaction record was not modified. + +**55 pure controller tests pass**, including logical column order/type/removal +refusal. Bounded private failure diagnostics now preserve the controller's reason +without exposing stderr in the public RPC response. The previously recorded +2,025 Rust tests predate these final helper changes; a matching executable and +final combined receipt remain required. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index eec1f838..f04cd5d5 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -132,12 +132,14 @@ ADDITIVE_MIGRATIONS = { 'AddMediaRegistrationRetirements1791374401000':1791374401000, } ADDITIVE_TABLES = {'archipelago_media_registrations','archipelago_publications','archipelago_publication_outbox','archipelago_rental_entitlements','archipelago_registration_intents'} +# Keep logical column order, but not physical attnum values: pg_dump compacts +# slots left behind by DROP COLUMN while preserving the surviving column order. DB_COMMITMENTS_SQL = r''' BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY; SELECT format($query$ SELECT jsonb_build_object('table',%L,'schema', jsonb_build_object( - 'columns',(SELECT coalesce(jsonb_agg(jsonb_build_array(a.attnum,a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,pg_get_expr(d.adbin,d.adrelid)) ORDER BY a.attnum),'[]'::jsonb) FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attrelid=%s AND a.attnum>0 AND NOT a.attisdropped), + 'columns',(SELECT coalesce(jsonb_agg(jsonb_build_array(a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull,a.attidentity,a.attgenerated,pg_get_expr(d.adbin,d.adrelid)) ORDER BY a.attnum),'[]'::jsonb) FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum WHERE a.attrelid=%s AND a.attnum>0 AND NOT a.attisdropped), 'constraints',(SELECT coalesce(jsonb_agg(jsonb_build_array(conname,pg_get_constraintdef(oid,true)) ORDER BY conname),'[]'::jsonb) FROM pg_constraint WHERE conrelid=%s), 'indexes',(SELECT coalesce(jsonb_agg(pg_get_indexdef(indexrelid) ORDER BY indexrelid::regclass::text),'[]'::jsonb) FROM pg_index WHERE indrelid=%s), 'triggers',(SELECT coalesce(jsonb_agg(pg_get_triggerdef(oid,true) ORDER BY tgname),'[]'::jsonb) FROM pg_trigger WHERE tgrelid=%s AND NOT tgisinternal), @@ -261,6 +263,11 @@ class Controller: self.record=json.loads(self.path.read_text()) if self.path.exists() else None if self.record:require(self.record['operation_id']==operation,'Maintenance journal changed') def save(self): atomic(self.path,self.record) + def save_failure(self, action, error): + # The native adapter deliberately withholds stderr from public errors. + # Retain bounded diagnostics privately so an operator can inspect refusal. + atomic(self.root/'last-failure.private.json',{'operation_id':self.operation,'action':action, + 'error_type':type(error).__name__,'message':str(error)[:4096],'at':time.time()}) def run(self, argv, timeout=30, output=None, input_bytes=None, input_file=None): if self.runner:return self.runner(argv,timeout,output) self.root.mkdir(mode=0o700,parents=True,exist_ok=True) @@ -826,7 +833,12 @@ def main(): if 'recovery' in request:require(type(request['recovery']) is bool,'Invalid recovery flag') require(os.getuid()==1000,'Expected node service user');fd=int(os.environ['ARCHY_UPDATE_LOCK_FD']);actual=os.fstat(fd);expected=(DATA/'update-transactions'/'lock').stat();require((actual.st_dev,actual.st_ino)==(expected.st_dev,expected.st_ino),'Inherited lifecycle lock is not the expected file') controller=Controller(DATA,request['operation_id'],fd) - result=controller.acquire(request['original_members'],request.get('recovery',False)) if sys.argv[1]=='acquire' else controller.verify() if sys.argv[1]=='verify' else controller.release(request['outcome']) + try: + result=controller.acquire(request['original_members'],request.get('recovery',False)) if sys.argv[1]=='acquire' else controller.verify() if sys.argv[1]=='verify' else controller.release(request['outcome']) + except Exception as error: + try:controller.save_failure(sys.argv[1],error) + except Exception:pass # Diagnostic failure must not replace the original refusal. + raise encoded=json.dumps(result);require(len(encoded)<=4096,'Maintenance response exceeds bound');print(encoded) if __name__=='__main__': try:main() diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 9f3f91a0..de990025 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -562,4 +562,17 @@ console.log('process identity cases passed');''' if change=='wrong-body':bad['members'][0]['pinned_original_body']='changed' if change=='cycle':bad['members'][0]['original']['image']=image with self.assertRaises(RuntimeError):module.verify_relay_image_lineage(image,digest,[bad],installed) + def test_column_commitments_retain_logical_order_and_every_semantic_field(self): + import copy + columns=[['first','integer',True,'','',''],['last','text',False,'','','']] + before={'operation_id':self.operation,'tables':{'typeorm_migrations':{'schema':{},'rows':0,'rows_sha256':'a'},'items':{'schema':{'columns':columns},'rows':0,'rows_sha256':'b'}},'migrations':[]} + module.verify_database_compatibility(before,copy.deepcopy(before)) + for replacement in (list(reversed(columns)),columns[:-1],[['first','bigint',True,'','',''],columns[1]]): + after=copy.deepcopy(before);after['tables']['items']['schema']['columns']=replacement + with self.assertRaisesRegex(RuntimeError,'changed original table schema'):module.verify_database_compatibility(before,after) + def test_failure_diagnostics_are_private_bounded_and_do_not_change_journal(self): + c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared'};c.save();before=c.path.read_bytes() + c.save_failure('acquire',RuntimeError('diagnostic'*1000));path=c.root/'last-failure.private.json';record=json.loads(path.read_text()) + self.assertEqual(record['operation_id'],self.operation);self.assertEqual(record['action'],'acquire');self.assertEqual(record['error_type'],'RuntimeError');self.assertEqual(len(record['message']),4096) + self.assertEqual(path.stat().st_mode&0o777,0o600);self.assertEqual(c.path.read_bytes(),before) if __name__=='__main__':unittest.main()