lfg2025/archy
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 12 Wiki Activity

fix(indeehub): bind backup checks to configured migration history

Browse Source
This commit is contained in:
archipelago
2026-10-08 00:17:14 -04:00
parent 884ea49238
commit 838ad745f3
3 changed files with 65 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/managed-update-recovery-implementation.md
+38
View File
@@ -380,3 +380,41 @@ fingerprints for fresh transactions are being qualified separately; legacy
records must retain strict comparison. The pre-target writer-preservation fix
in 07c7eb0f also awaits combined compilation/tests. No Yaya migration or catalog
activation has occurred.
### Actual seven-service rehearsal, 2026-10-08
The matching VM executable for 32317236 passed the full isolated suite:
**2,021 passed, zero failed, five existing ignores**, 532 inputs unchanged.
Its fresh child VM uses the real `Restart=always`, 30-second Podman stop and
45-second systemd stop settings. Manager startup preserved all seven registered
container IDs. The actual authenticated missing-plan update refusal retained
all seven IDs and cleared Updating. The subsequent controller recorded the
frontend exit 0, narrowly qualified idle-worker exit 137, and empty-business
API wrapper exit 1, including its operation-owned runtime restart override.
The database barrier then correctly refused an invalid table assumption:
IndeeHub's actual history is `public.typeorm_migrations`, not `public.migrations`.
Both compiled configuration files in exact original API image
`364a8d5dd4114349b9c09ac0b16b00aa066195294ae41399d72a85122db7b5a9`
and a read-only database existence query confirmed this. The helper now uses
that configured table, requires it in both compatibility snapshots and gives
no row-change exemption to an unrelated table called `migrations`.
**48 pure controller tests pass.** No history table or database row was edited.
Recovery also exposed that the native no-external-overrides guard rejects the
controller's own runtime restart fence. API-only exact ownership recognition
is checkpointed in 884ea492, with regression cases; its combined Rust validation
and executable remain pending. It does not admit arbitrary drop-ins.
A separate candidate-helper rehearsal, with the manager stopped and real
lifecycle flock retained, passed actual database commitments/dump and clean
MinIO/Redis stops. It then refused relay exit 137. The original relay image
`061516573b143b44e331f960036a6a3dc43c9b256ef8ca71afedbeb2cf797a4b`
uses a shell PID 1 around `nostr-rs-relay`. A disposable, networkless child-SIGINT
probe exited 130 without OOM; this is **not accepted as graceful**. Relay shutdown
remains under investigation. The held operation is
`3b3c564b-cce6-4727-8be8-369a95c479e4` in child fixture
`indeehub-v2-20261007T232717`. PostgreSQL remains intact; all original recovery
images and failure evidence are retained. The manager is stopped and startup
barred. The candidate helper was separate from the installed pinned helper.
Neither full target rollback nor successful update has passed. Yaya is unchanged.
scripts/indeehub-maintenance-controller.py
+5 -4
View File
@@ -7,6 +7,7 @@ import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time,
NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay')
VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data')
DATA = pathlib.Path('/var/lib/archipelago')
MIGRATION_TABLE = 'typeorm_migrations'
QUEUE_COUNTS = frozenset(('active','waiting','paused','delayed','failed','completed'))
def valid_queue_counts(counts):
return isinstance(counts,dict) and set(counts)==QUEUE_COUNTS and all(type(v) is int and v>=0 for v in counts.values())
@@ -69,16 +70,16 @@ SELECT format($query$
$query$,c.relname,c.oid,c.oid,c.oid,c.oid,c.relrowsecurity::text||':'||c.relforcerowsecurity::text,c.relname,c.relname)
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname
\gexec
SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.migrations m;
SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.typeorm_migrations m;
COMMIT;
'''
def verify_database_compatibility(before, after):
require(before.get('operation_id')==after.get('operation_id'),'Data compatibility operation changed')
old=before['tables'];new=after['tables'];require(set(old)<=set(new),'Data compatibility lost original tables')
old=before['tables'];new=after['tables'];require(MIGRATION_TABLE in old and MIGRATION_TABLE in new,'Data compatibility lacks configured migration history table');require(set(old)<=set(new),'Data compatibility lost original tables')
extra=set(new)-set(old);require(extra<=ADDITIVE_TABLES,'Data compatibility contains unreviewed tables')
for name in old:
require(old[name]['schema']==new[name]['schema'],'Data compatibility changed original table schema')
if name!='migrations':
if name!=MIGRATION_TABLE:
require(old[name]['rows']==new[name]['rows'] and old[name]['rows_sha256']==new[name]['rows_sha256'],'Data compatibility changed original rows')
for name in extra:require(new[name]['rows']==0,'Data compatibility contains new application data')
previous=before['migrations'];current=after['migrations']
@@ -458,7 +459,7 @@ class Controller:
require(migrations is None,'Duplicate database migration observation');migrations=row['migration_rows']
else:
name=row.pop('table');require(name not in tables and re.fullmatch('[a-zA-Z_][a-zA-Z0-9_]*',name),'Invalid database table observation');tables[name]=row
require(tables and 'migrations' in tables and isinstance(migrations,list),'Database compatibility observation incomplete')
require(tables and MIGRATION_TABLE in tables and isinstance(migrations,list),'Database compatibility observation incomplete')
return {'operation_id':self.operation,'tables':tables,'migrations':migrations}
def verify_restored_data(self):
baseline=self.record.get('database_before')
tests/regression/test_indeehub_maintenance_controller.py
+22 -3
View File
@@ -447,19 +447,38 @@ console.log('process identity cases passed');'''
def test_rollback_compatibility_binds_operation_preserves_rows_and_allows_only_empty_additions(self):
import copy
table={'schema':{'columns':['original']},'rows':0,'rows_sha256':'a'*64}
before={'operation_id':self.operation,'tables':{'migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]}
before={'operation_id':self.operation,'tables':{'typeorm_migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]}
after=copy.deepcopy(before)
self.assertEqual(module.verify_database_compatibility(before,after)['original_tables'],2)
names=list(module.ADDITIVE_MIGRATIONS)
after['migrations'] += [{'id':i+2,'timestamp':module.ADDITIVE_MIGRATIONS[name],'name':name} for i,name in enumerate(names)]
after['tables']['migrations']['rows']=4;after['tables']['migrations']['rows_sha256']='b'*64
after['tables']['typeorm_migrations']['rows']=4;after['tables']['typeorm_migrations']['rows_sha256']='b'*64
for name in module.ADDITIVE_TABLES:after['tables'][name]=copy.deepcopy(table)
self.assertEqual(len(module.verify_database_compatibility(before,after)['new_empty_tables']),5)
for mutate in [lambda d:d.update(operation_id=str(uuid.uuid4())),lambda d:d['tables']['contents'].update(rows_sha256='c'*64),lambda d:d['tables']['contents']['schema'].update(columns=['changed']),lambda d:d['tables']['archipelago_publications'].update(rows=1),lambda d:d['migrations'][0].update(name='Altered'),lambda d:d['migrations'][-1].update(name='Unreviewed'),lambda d:d['tables'].update(unreviewed=copy.deepcopy(table))]:
damaged=copy.deepcopy(after);mutate(damaged)
with self.assertRaisesRegex(RuntimeError,'Data compatibility'):module.verify_database_compatibility(before,damaged)
def test_migration_history_uses_configured_table_and_never_exempts_unrelated_migrations(self):
import copy
self.assertEqual(module.MIGRATION_TABLE,'typeorm_migrations')
self.assertIn('FROM public.typeorm_migrations m;',module.DB_COMMITMENTS_SQL)
self.assertNotIn('FROM public.migrations m;',module.DB_COMMITMENTS_SQL)
table={'schema':{},'rows':0,'rows_sha256':'a'*64}
before={'operation_id':self.operation,'tables':{'typeorm_migrations':copy.deepcopy(table),'migrations':copy.deepcopy(table)},'migrations':[]}
after=copy.deepcopy(before);after['tables']['migrations']['rows_sha256']='b'*64
with self.assertRaisesRegex(RuntimeError,'changed original rows'):module.verify_database_compatibility(before,after)
def test_database_compatibility_rejects_missing_configured_history(self):
baseline={'operation_id':self.operation,'tables':{},'migrations':[]}
with self.assertRaisesRegex(RuntimeError,'lacks configured migration history'):module.verify_database_compatibility(baseline,baseline)
def test_database_observation_requires_actual_typeorm_history_table(self):
c=self.controller;table={'table':'migrations','schema':{},'rows':0,'rows_sha256':'a'*64}
def result(argv,timeout,output):return (json.dumps(table)+'\n'+json.dumps({'migration_rows':[]})+'\n').encode()
c.runner=result
with self.assertRaisesRegex(RuntimeError,'observation incomplete'):c.database_commitments()
table['table']='typeorm_migrations'
self.assertIn('typeorm_migrations',c.database_commitments()['tables'])
def test_verified_rollback_records_operation_proof_before_releasing_fence(self):
c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'migrations':table},'migrations':[]}
c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'typeorm_migrations':table},'migrations':[]}
c.record={'operation_id':self.operation,'phase':'Recovering','database_before':baseline};c.save()
c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
module.atomic(c.data/'update-transactions'/'supervised'/(self.operation+'.json'),{'phase':'Restored','target_startup_began':True})