fix(indeehub): bind backup checks to configured migration history
This commit is contained in:
@@ -380,3 +380,41 @@ fingerprints for fresh transactions are being qualified separately; legacy
|
|||||||
records must retain strict comparison. The pre-target writer-preservation fix
|
records must retain strict comparison. The pre-target writer-preservation fix
|
||||||
in 07c7eb0f also awaits combined compilation/tests. No Yaya migration or catalog
|
in 07c7eb0f also awaits combined compilation/tests. No Yaya migration or catalog
|
||||||
activation has occurred.
|
activation has occurred.
|
||||||
|
|
||||||
|
### Actual seven-service rehearsal, 2026-10-08
|
||||||
|
|
||||||
|
The matching VM executable for 32317236 passed the full isolated suite:
|
||||||
|
**2,021 passed, zero failed, five existing ignores**, 532 inputs unchanged.
|
||||||
|
Its fresh child VM uses the real `Restart=always`, 30-second Podman stop and
|
||||||
|
45-second systemd stop settings. Manager startup preserved all seven registered
|
||||||
|
container IDs. The actual authenticated missing-plan update refusal retained
|
||||||
|
all seven IDs and cleared Updating. The subsequent controller recorded the
|
||||||
|
frontend exit 0, narrowly qualified idle-worker exit 137, and empty-business
|
||||||
|
API wrapper exit 1, including its operation-owned runtime restart override.
|
||||||
|
|
||||||
|
The database barrier then correctly refused an invalid table assumption:
|
||||||
|
IndeeHub's actual history is `public.typeorm_migrations`, not `public.migrations`.
|
||||||
|
Both compiled configuration files in exact original API image
|
||||||
|
`364a8d5dd4114349b9c09ac0b16b00aa066195294ae41399d72a85122db7b5a9`
|
||||||
|
and a read-only database existence query confirmed this. The helper now uses
|
||||||
|
that configured table, requires it in both compatibility snapshots and gives
|
||||||
|
no row-change exemption to an unrelated table called `migrations`.
|
||||||
|
**48 pure controller tests pass.** No history table or database row was edited.
|
||||||
|
|
||||||
|
Recovery also exposed that the native no-external-overrides guard rejects the
|
||||||
|
controller's own runtime restart fence. API-only exact ownership recognition
|
||||||
|
is checkpointed in 884ea492, with regression cases; its combined Rust validation
|
||||||
|
and executable remain pending. It does not admit arbitrary drop-ins.
|
||||||
|
|
||||||
|
A separate candidate-helper rehearsal, with the manager stopped and real
|
||||||
|
lifecycle flock retained, passed actual database commitments/dump and clean
|
||||||
|
MinIO/Redis stops. It then refused relay exit 137. The original relay image
|
||||||
|
`061516573b143b44e331f960036a6a3dc43c9b256ef8ca71afedbeb2cf797a4b`
|
||||||
|
uses a shell PID 1 around `nostr-rs-relay`. A disposable, networkless child-SIGINT
|
||||||
|
probe exited 130 without OOM; this is **not accepted as graceful**. Relay shutdown
|
||||||
|
remains under investigation. The held operation is
|
||||||
|
`3b3c564b-cce6-4727-8be8-369a95c479e4` in child fixture
|
||||||
|
`indeehub-v2-20261007T232717`. PostgreSQL remains intact; all original recovery
|
||||||
|
images and failure evidence are retained. The manager is stopped and startup
|
||||||
|
barred. The candidate helper was separate from the installed pinned helper.
|
||||||
|
Neither full target rollback nor successful update has passed. Yaya is unchanged.
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time,
|
|||||||
NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay')
|
NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay')
|
||||||
VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data')
|
VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data')
|
||||||
DATA = pathlib.Path('/var/lib/archipelago')
|
DATA = pathlib.Path('/var/lib/archipelago')
|
||||||
|
MIGRATION_TABLE = 'typeorm_migrations'
|
||||||
QUEUE_COUNTS = frozenset(('active','waiting','paused','delayed','failed','completed'))
|
QUEUE_COUNTS = frozenset(('active','waiting','paused','delayed','failed','completed'))
|
||||||
def valid_queue_counts(counts):
|
def valid_queue_counts(counts):
|
||||||
return isinstance(counts,dict) and set(counts)==QUEUE_COUNTS and all(type(v) is int and v>=0 for v in counts.values())
|
return isinstance(counts,dict) and set(counts)==QUEUE_COUNTS and all(type(v) is int and v>=0 for v in counts.values())
|
||||||
@@ -69,16 +70,16 @@ SELECT format($query$
|
|||||||
$query$,c.relname,c.oid,c.oid,c.oid,c.oid,c.relrowsecurity::text||':'||c.relforcerowsecurity::text,c.relname,c.relname)
|
$query$,c.relname,c.oid,c.oid,c.oid,c.oid,c.relrowsecurity::text||':'||c.relforcerowsecurity::text,c.relname,c.relname)
|
||||||
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname
|
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p') ORDER BY c.relname
|
||||||
\gexec
|
\gexec
|
||||||
SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.migrations m;
|
SELECT jsonb_build_object('migration_rows',coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),'[]'::jsonb)) FROM public.typeorm_migrations m;
|
||||||
COMMIT;
|
COMMIT;
|
||||||
'''
|
'''
|
||||||
def verify_database_compatibility(before, after):
|
def verify_database_compatibility(before, after):
|
||||||
require(before.get('operation_id')==after.get('operation_id'),'Data compatibility operation changed')
|
require(before.get('operation_id')==after.get('operation_id'),'Data compatibility operation changed')
|
||||||
old=before['tables'];new=after['tables'];require(set(old)<=set(new),'Data compatibility lost original tables')
|
old=before['tables'];new=after['tables'];require(MIGRATION_TABLE in old and MIGRATION_TABLE in new,'Data compatibility lacks configured migration history table');require(set(old)<=set(new),'Data compatibility lost original tables')
|
||||||
extra=set(new)-set(old);require(extra<=ADDITIVE_TABLES,'Data compatibility contains unreviewed tables')
|
extra=set(new)-set(old);require(extra<=ADDITIVE_TABLES,'Data compatibility contains unreviewed tables')
|
||||||
for name in old:
|
for name in old:
|
||||||
require(old[name]['schema']==new[name]['schema'],'Data compatibility changed original table schema')
|
require(old[name]['schema']==new[name]['schema'],'Data compatibility changed original table schema')
|
||||||
if name!='migrations':
|
if name!=MIGRATION_TABLE:
|
||||||
require(old[name]['rows']==new[name]['rows'] and old[name]['rows_sha256']==new[name]['rows_sha256'],'Data compatibility changed original rows')
|
require(old[name]['rows']==new[name]['rows'] and old[name]['rows_sha256']==new[name]['rows_sha256'],'Data compatibility changed original rows')
|
||||||
for name in extra:require(new[name]['rows']==0,'Data compatibility contains new application data')
|
for name in extra:require(new[name]['rows']==0,'Data compatibility contains new application data')
|
||||||
previous=before['migrations'];current=after['migrations']
|
previous=before['migrations'];current=after['migrations']
|
||||||
@@ -458,7 +459,7 @@ class Controller:
|
|||||||
require(migrations is None,'Duplicate database migration observation');migrations=row['migration_rows']
|
require(migrations is None,'Duplicate database migration observation');migrations=row['migration_rows']
|
||||||
else:
|
else:
|
||||||
name=row.pop('table');require(name not in tables and re.fullmatch('[a-zA-Z_][a-zA-Z0-9_]*',name),'Invalid database table observation');tables[name]=row
|
name=row.pop('table');require(name not in tables and re.fullmatch('[a-zA-Z_][a-zA-Z0-9_]*',name),'Invalid database table observation');tables[name]=row
|
||||||
require(tables and 'migrations' in tables and isinstance(migrations,list),'Database compatibility observation incomplete')
|
require(tables and MIGRATION_TABLE in tables and isinstance(migrations,list),'Database compatibility observation incomplete')
|
||||||
return {'operation_id':self.operation,'tables':tables,'migrations':migrations}
|
return {'operation_id':self.operation,'tables':tables,'migrations':migrations}
|
||||||
def verify_restored_data(self):
|
def verify_restored_data(self):
|
||||||
baseline=self.record.get('database_before')
|
baseline=self.record.get('database_before')
|
||||||
|
|||||||
@@ -447,19 +447,38 @@ console.log('process identity cases passed');'''
|
|||||||
def test_rollback_compatibility_binds_operation_preserves_rows_and_allows_only_empty_additions(self):
|
def test_rollback_compatibility_binds_operation_preserves_rows_and_allows_only_empty_additions(self):
|
||||||
import copy
|
import copy
|
||||||
table={'schema':{'columns':['original']},'rows':0,'rows_sha256':'a'*64}
|
table={'schema':{'columns':['original']},'rows':0,'rows_sha256':'a'*64}
|
||||||
before={'operation_id':self.operation,'tables':{'migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]}
|
before={'operation_id':self.operation,'tables':{'typeorm_migrations':copy.deepcopy(table),'contents':copy.deepcopy(table)},'migrations':[{'id':1,'timestamp':1,'name':'Original1'}]}
|
||||||
after=copy.deepcopy(before)
|
after=copy.deepcopy(before)
|
||||||
self.assertEqual(module.verify_database_compatibility(before,after)['original_tables'],2)
|
self.assertEqual(module.verify_database_compatibility(before,after)['original_tables'],2)
|
||||||
names=list(module.ADDITIVE_MIGRATIONS)
|
names=list(module.ADDITIVE_MIGRATIONS)
|
||||||
after['migrations'] += [{'id':i+2,'timestamp':module.ADDITIVE_MIGRATIONS[name],'name':name} for i,name in enumerate(names)]
|
after['migrations'] += [{'id':i+2,'timestamp':module.ADDITIVE_MIGRATIONS[name],'name':name} for i,name in enumerate(names)]
|
||||||
after['tables']['migrations']['rows']=4;after['tables']['migrations']['rows_sha256']='b'*64
|
after['tables']['typeorm_migrations']['rows']=4;after['tables']['typeorm_migrations']['rows_sha256']='b'*64
|
||||||
for name in module.ADDITIVE_TABLES:after['tables'][name]=copy.deepcopy(table)
|
for name in module.ADDITIVE_TABLES:after['tables'][name]=copy.deepcopy(table)
|
||||||
self.assertEqual(len(module.verify_database_compatibility(before,after)['new_empty_tables']),5)
|
self.assertEqual(len(module.verify_database_compatibility(before,after)['new_empty_tables']),5)
|
||||||
for mutate in [lambda d:d.update(operation_id=str(uuid.uuid4())),lambda d:d['tables']['contents'].update(rows_sha256='c'*64),lambda d:d['tables']['contents']['schema'].update(columns=['changed']),lambda d:d['tables']['archipelago_publications'].update(rows=1),lambda d:d['migrations'][0].update(name='Altered'),lambda d:d['migrations'][-1].update(name='Unreviewed'),lambda d:d['tables'].update(unreviewed=copy.deepcopy(table))]:
|
for mutate in [lambda d:d.update(operation_id=str(uuid.uuid4())),lambda d:d['tables']['contents'].update(rows_sha256='c'*64),lambda d:d['tables']['contents']['schema'].update(columns=['changed']),lambda d:d['tables']['archipelago_publications'].update(rows=1),lambda d:d['migrations'][0].update(name='Altered'),lambda d:d['migrations'][-1].update(name='Unreviewed'),lambda d:d['tables'].update(unreviewed=copy.deepcopy(table))]:
|
||||||
damaged=copy.deepcopy(after);mutate(damaged)
|
damaged=copy.deepcopy(after);mutate(damaged)
|
||||||
with self.assertRaisesRegex(RuntimeError,'Data compatibility'):module.verify_database_compatibility(before,damaged)
|
with self.assertRaisesRegex(RuntimeError,'Data compatibility'):module.verify_database_compatibility(before,damaged)
|
||||||
|
def test_migration_history_uses_configured_table_and_never_exempts_unrelated_migrations(self):
|
||||||
|
import copy
|
||||||
|
self.assertEqual(module.MIGRATION_TABLE,'typeorm_migrations')
|
||||||
|
self.assertIn('FROM public.typeorm_migrations m;',module.DB_COMMITMENTS_SQL)
|
||||||
|
self.assertNotIn('FROM public.migrations m;',module.DB_COMMITMENTS_SQL)
|
||||||
|
table={'schema':{},'rows':0,'rows_sha256':'a'*64}
|
||||||
|
before={'operation_id':self.operation,'tables':{'typeorm_migrations':copy.deepcopy(table),'migrations':copy.deepcopy(table)},'migrations':[]}
|
||||||
|
after=copy.deepcopy(before);after['tables']['migrations']['rows_sha256']='b'*64
|
||||||
|
with self.assertRaisesRegex(RuntimeError,'changed original rows'):module.verify_database_compatibility(before,after)
|
||||||
|
def test_database_compatibility_rejects_missing_configured_history(self):
|
||||||
|
baseline={'operation_id':self.operation,'tables':{},'migrations':[]}
|
||||||
|
with self.assertRaisesRegex(RuntimeError,'lacks configured migration history'):module.verify_database_compatibility(baseline,baseline)
|
||||||
|
def test_database_observation_requires_actual_typeorm_history_table(self):
|
||||||
|
c=self.controller;table={'table':'migrations','schema':{},'rows':0,'rows_sha256':'a'*64}
|
||||||
|
def result(argv,timeout,output):return (json.dumps(table)+'\n'+json.dumps({'migration_rows':[]})+'\n').encode()
|
||||||
|
c.runner=result
|
||||||
|
with self.assertRaisesRegex(RuntimeError,'observation incomplete'):c.database_commitments()
|
||||||
|
table['table']='typeorm_migrations'
|
||||||
|
self.assertIn('typeorm_migrations',c.database_commitments()['tables'])
|
||||||
def test_verified_rollback_records_operation_proof_before_releasing_fence(self):
|
def test_verified_rollback_records_operation_proof_before_releasing_fence(self):
|
||||||
c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'migrations':table},'migrations':[]}
|
c=self.controller;table={'schema':{},'rows':0,'rows_sha256':'a'*64};baseline={'operation_id':self.operation,'tables':{'typeorm_migrations':table},'migrations':[]}
|
||||||
c.record={'operation_id':self.operation,'phase':'Recovering','database_before':baseline};c.save()
|
c.record={'operation_id':self.operation,'phase':'Recovering','database_before':baseline};c.save()
|
||||||
c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
|
c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
|
||||||
module.atomic(c.data/'update-transactions'/'supervised'/(self.operation+'.json'),{'phase':'Restored','target_startup_began':True})
|
module.atomic(c.data/'update-transactions'/'supervised'/(self.operation+'.json'),{'phase':'Restored','target_startup_began':True})
|
||||||
|
|||||||
Reference in New Issue
Block a user