diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index a7209e27..677113af 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -380,3 +380,41 @@ fingerprints for fresh transactions are being qualified separately; legacy records must retain strict comparison. The pre-target writer-preservation fix in 07c7eb0f also awaits combined compilation/tests. No Yaya migration or catalog activation has occurred. + +### Actual seven-service rehearsal, 2026-10-08 + +The matching VM executable for 32317236 passed the full isolated suite: +**2,021 passed, zero failed, five existing ignores**, 532 inputs unchanged. +Its fresh child VM uses the real `Restart=always`, 30-second Podman stop and +45-second systemd stop settings. Manager startup preserved all seven registered +container IDs. The actual authenticated missing-plan update refusal retained +all seven IDs and cleared Updating. The subsequent controller recorded the +frontend exit 0, narrowly qualified idle-worker exit 137, and empty-business +API wrapper exit 1, including its operation-owned runtime restart override. + +The database barrier then correctly refused an invalid table assumption: +IndeeHub's actual history is `public.typeorm_migrations`, not `public.migrations`. +Both compiled configuration files in exact original API image +`364a8d5dd4114349b9c09ac0b16b00aa066195294ae41399d72a85122db7b5a9` +and a read-only database existence query confirmed this. The helper now uses +that configured table, requires it in both compatibility snapshots and gives +no row-change exemption to an unrelated table called `migrations`. +**48 pure controller tests pass.** No history table or database row was edited. + +Recovery also exposed that the native no-external-overrides guard rejects the +controller's own runtime restart fence. API-only exact ownership recognition +is checkpointed in 884ea492, with regression cases; its combined Rust validation +and executable remain pending. It does not admit arbitrary drop-ins. + +A separate candidate-helper rehearsal, with the manager stopped and real +lifecycle flock retained, passed actual database commitments/dump and clean +MinIO/Redis stops. It then refused relay exit 137. The original relay image +`061516573b143b44e331f960036a6a3dc43c9b256ef8ca71afedbeb2cf797a4b` +uses a shell PID 1 around `nostr-rs-relay`. A disposable, networkless child-SIGINT +probe exited 130 without OOM; this is **not accepted as graceful**. Relay shutdown +remains under investigation. The held operation is +`3b3c564b-cce6-4727-8be8-369a95c479e4` in child fixture +`indeehub-v2-20261007T232717`. PostgreSQL remains intact; all original recovery +images and failure evidence are retained. The manager is stopped and startup +barred. The candidate helper was separate from the installed pinned helper. +Neither full target rollback nor successful update has passed. Yaya is unchanged. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index 13a745c5..de528bb4 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -7,6 +7,7 @@ import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay') VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data') DATA = pathlib.Path('/var/lib/archipelago') +MIGRATION_TABLE = 'typeorm_migrations' QUEUE_COUNTS = frozenset(('active','waiting','paused','delayed','failed','completed')) def valid_queue_counts(counts): return isinstance(counts,dict) and set(counts)==QUEUE_COUNTS and all(type(v) is int and v>=0 for v in counts.values()) @@ -69,16 +70,16 @@ SELECT format($query$ $query$,c.relname,c.oid,c.oid,c.oid,c.oid,c.relrowsecurity::text||':'||c.relforcerowsecurity::text,c.relname,c.relname) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname \gexec -SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.migrations m; +SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.typeorm_migrations m; COMMIT; ''' def verify_database_compatibility(before, after): require(before.get('operation_id')==after.get('operation_id'),'Data compatibility operation changed') - old=before['tables'];new=after['tables'];require(set(old)<=set(new),'Data compatibility lost original tables') + old=before['tables'];new=after['tables'];require(MIGRATION_TABLE in old and MIGRATION_TABLE in new,'Data compatibility lacks configured migration history table');require(set(old)<=set(new),'Data compatibility lost original tables') extra=set(new)-set(old);require(extra<=ADDITIVE_TABLES,'Data compatibility contains unreviewed tables') for name in old: require(old[name]['schema']==new[name]['schema'],'Data compatibility changed original table schema') - if name!='migrations': + if name!=MIGRATION_TABLE: require(old[name]['rows']==new[name]['rows'] and old[name]['rows_sha256']==new[name]['rows_sha256'],'Data compatibility changed original rows') for name in extra:require(new[name]['rows']==0,'Data compatibility contains new application data') previous=before['migrations'];current=after['migrations'] @@ -458,7 +459,7 @@ class Controller: require(migrations is None,'Duplicate database migration observation');migrations=row['migration_rows'] else: name=row.pop('table');require(name not in tables and re.fullmatch('[a-zA-Z_][a-zA-Z0-9_]*',name),'Invalid database table observation');tables[name]=row - require(tables and 'migrations' in tables and isinstance(migrations,list),'Database compatibility observation incomplete') + require(tables and MIGRATION_TABLE in tables and isinstance(migrations,list),'Database compatibility observation incomplete') return {'operation_id':self.operation,'tables':tables,'migrations':migrations} def verify_restored_data(self): baseline=self.record.get('database_before') diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index 4b110334..77f48a9e 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -447,19 +447,38 @@ console.log('process identity cases passed');''' def test_rollback_compatibility_binds_operation_preserves_rows_and_allows_only_empty_additions(self): import copy table={'schema':{'columns':['original']},'rows':0,'rows_sha256':'a'*64} - before={'operation_id':self.operation,'tables':{'migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]} + before={'operation_id':self.operation,'tables':{'typeorm_migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]} after=copy.deepcopy(before) self.assertEqual(module.verify_database_compatibility(before,after)['original_tables'],2) names=list(module.ADDITIVE_MIGRATIONS) after['migrations'] += [{'id':i+2,'timestamp':module.ADDITIVE_MIGRATIONS[name],'name':name} for i,name in enumerate(names)] - after['tables']['migrations']['rows']=4;after['tables']['migrations']['rows_sha256']='b'*64 + after['tables']['typeorm_migrations']['rows']=4;after['tables']['typeorm_migrations']['rows_sha256']='b'*64 for name in module.ADDITIVE_TABLES:after['tables'][name]=copy.deepcopy(table) self.assertEqual(len(module.verify_database_compatibility(before,after)['new_empty_tables']),5) for mutate in [lambda d:d.update(operation_id=str(uuid.uuid4())),lambda d:d['tables']['contents'].update(rows_sha256='c'*64),lambda d:d['tables']['contents']['schema'].update(columns=['changed']),lambda d:d['tables']['archipelago_publications'].update(rows=1),lambda d:d['migrations'][0].update(name='Altered'),lambda d:d['migrations'][-1].update(name='Unreviewed'),lambda d:d['tables'].update(unreviewed=copy.deepcopy(table))]: damaged=copy.deepcopy(after);mutate(damaged) with self.assertRaisesRegex(RuntimeError,'Data compatibility'):module.verify_database_compatibility(before,damaged) + def test_migration_history_uses_configured_table_and_never_exempts_unrelated_migrations(self): + import copy + self.assertEqual(module.MIGRATION_TABLE,'typeorm_migrations') + self.assertIn('FROM public.typeorm_migrations m;',module.DB_COMMITMENTS_SQL) + self.assertNotIn('FROM public.migrations m;',module.DB_COMMITMENTS_SQL) + table={'schema':{},'rows':0,'rows_sha256':'a'*64} + before={'operation_id':self.operation,'tables':{'typeorm_migrations':copy.deepcopy(table),'migrations':copy.deepcopy(table)},'migrations':[]} + after=copy.deepcopy(before);after['tables']['migrations']['rows_sha256']='b'*64 + with self.assertRaisesRegex(RuntimeError,'changed original rows'):module.verify_database_compatibility(before,after) + def test_database_compatibility_rejects_missing_configured_history(self): + baseline={'operation_id':self.operation,'tables':{},'migrations':[]} + with self.assertRaisesRegex(RuntimeError,'lacks configured migration history'):module.verify_database_compatibility(baseline,baseline) + def test_database_observation_requires_actual_typeorm_history_table(self): + c=self.controller;table={'table':'migrations','schema':{},'rows':0,'rows_sha256':'a'*64} + def result(argv,timeout,output):return (json.dumps(table)+'\n'+json.dumps({'migration_rows':[]})+'\n').encode() + c.runner=result + with self.assertRaisesRegex(RuntimeError,'observation incomplete'):c.database_commitments() + table['table']='typeorm_migrations' + self.assertIn('typeorm_migrations',c.database_commitments()['tables']) def test_verified_rollback_records_operation_proof_before_releasing_fence(self): - c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'migrations':table},'migrations':[]} + c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'typeorm_migrations':table},'migrations':[]} c.record={'operation_id':self.operation,'phase':'Recovering','database_before':baseline};c.save() c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) module.atomic(c.data/'update-transactions'/'supervised'/(self.operation+'.json'),{'phase':'Restored','target_startup_began':True})