diff --git a/docs/managed-update-recovery-implementation.md b/docs/managed-update-recovery-implementation.md index bb153fa4..9f59b0c0 100644 --- a/docs/managed-update-recovery-implementation.md +++ b/docs/managed-update-recovery-implementation.md @@ -54,3 +54,24 @@ Qualification required before integration/activation: - Disk-capacity and recovery-image retention checks, backup integrity and a documented recovery path for missing runtime artifacts. - Actual-node controlled deployment and acceptance, preserving persistent data. + +## Resumed qualification — 2026-10-07 + +Backup verification now requires the full database/four-volume artifact set and +rechecks SHA256, including same-size corruption. Nineteen pure controller tests +pass. The owned, network-none PostgreSQL fixture passes unchanged/additive +commitments, rejects four data/schema/history mutations, restores a real custom +dump with matching original commitments, and rejects a truncated dump. It mounts +no live volume and removes only its own container. This does not qualify actual +application writer drain or the complete supervised systemd cutover. + +The updater compiled and its full isolated suite ran: 1,958 passed, one failed, +five ignored. The failure is the old snake_case rental receipt JSON fixture; +`c2c4d915` already corrects that exact test on the release candidate branch. +Do not duplicate or suppress it here. Integrate and rerun the complete candidate +suite before claiming a green backend gate. The earlier interrupted compile +and PostgreSQL timeout remain failed/incomplete attempts, not acceptance. + +Evidence: `/tmp/archy-resumed-20261007-updater-full-backend.log`, +`/tmp/archy-resumed-20261007-indeehub-controller-tests-final.log`, and +`/tmp/archy-resumed-20261007-indeehub-postgres-restore.log`. diff --git a/scripts/indeehub-maintenance-controller.py b/scripts/indeehub-maintenance-controller.py index d38810b8..daee9d4d 100644 --- a/scripts/indeehub-maintenance-controller.py +++ b/scripts/indeehub-maintenance-controller.py @@ -5,6 +5,7 @@ must already be durable. Never unlock ARCHY_UPDATE_LOCK_FD or release another ho """ import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay') +VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data') DATA = pathlib.Path('/var/lib/archipelago') QUEUE_SCRIPT = r'''const {Queue}=require('bullmq'); (async()=>{const q=new Queue('transcode',{connection:{host:process.env.QUEUE_HOST,port:Number(process.env.QUEUE_PORT||6379),password:process.env.QUEUE_PASSWORD,maxRetriesPerRequest:1}}); @@ -192,7 +193,7 @@ class Controller: classification=('idle-worker-terminated-after-queue-drain' if idle_worker else 'empty-business-store-legacy-api-terminated') if str(code)=='143' else 'clean-process-exit' stopped[name].update(confirmed=True,exit_code=int(code),classification=classification,confirmed_at=time.time());self.save() def volume_sources(self): - expected=['indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data'] + expected=VOLUMES rows=json.loads(self.run(['podman','volume','inspect',*expected])) require({row['Name'] for row in rows}==set(expected),'Persistent volume scope changed') return {row['Name']:row['Mountpoint'] for row in rows} @@ -282,8 +283,11 @@ class Controller: # Verification remains possible when API/storage endpoints are stopped. # The native adapter separately validates target/original runtime identity. for name in NAMES:require(self.record.get('stopped',{}).get(name,{}).get('confirmed'),'Original writer stop evidence missing') + expected_artifacts={'database.dump',*(volume+'.tar' for volume in VOLUMES)} + require(set(self.record.get('artifacts',{}))==expected_artifacts,'Backup artifact inventory incomplete or unexpected') for name,record in self.record['artifacts'].items(): path=self.root/'backup'/name;require(path.is_file() and not path.is_symlink() and path.stat().st_size==record['bytes'],'Backup artifact missing or changed') + require(sha(path)==record['sha256'],'Backup artifact checksum changed') return {'operation_id':self.operation,'state':'held'} def release(self, outcome): require(outcome in ('committed','restored','aborted'),'Invalid release outcome') diff --git a/tests/regression/test_indeehub_maintenance_controller.py b/tests/regression/test_indeehub_maintenance_controller.py index d2a12762..96ef50bb 100644 --- a/tests/regression/test_indeehub_maintenance_controller.py +++ b/tests/regression/test_indeehub_maintenance_controller.py @@ -47,6 +47,30 @@ class MaintenanceTests(unittest.TestCase): c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) (c.root/'backup').mkdir();(c.root/'backup'/'database.dump').write_bytes(b'not evidence') with self.assertRaisesRegex(RuntimeError,'Drain not complete'):c.verify() + def completed_backup(self): + c=self.controller + c.record={'operation_id':self.operation,'phase':'Drained','backup_complete':True, + 'stopped':{name:{'confirmed':True} for name in module.NAMES},'artifacts':{}} + c.save();c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation) + (c.root/'backup').mkdir() + for name in ['database.dump',*(v+'.tar' for v in module.VOLUMES)]: + path=c.root/'backup'/name;path.write_bytes(b'original') + c.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':module.sha(path)} + c.save() + return c + def test_complete_backup_checksums_allow_verification(self): + self.assertEqual(self.completed_backup().verify()['state'],'held') + def test_same_size_corruption_keeps_admission_closed(self): + c=self.completed_backup();(c.root/'backup'/'database.dump').write_bytes(b'corrupt!') + with self.assertRaisesRegex(RuntimeError,'checksum changed'):c.verify() + self.assertEqual(c.fence.read_text(),self.operation);self.assertEqual(self.calls,[]) + def test_incomplete_or_unexpected_artifact_inventory_cannot_pass(self): + c=self.completed_backup();original=dict(c.record['artifacts']) + for artifacts in [{}, {k:v for k,v in original.items() if k!='database.dump'}, + {**original,'../foreign':original['database.dump']}]: + c.record['artifacts']=artifacts + with self.assertRaisesRegex(RuntimeError,'inventory'):c.verify() + self.assertEqual(c.fence.read_text(),self.operation) def test_forced_original_exit_never_marks_writer_completed(self): c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared','original_members':module.validate_members(members())};c.save() def command(argv,timeout,output): diff --git a/tests/regression/test_indeehub_maintenance_postgres.py b/tests/regression/test_indeehub_maintenance_postgres.py new file mode 100644 index 00000000..8410c35c --- /dev/null +++ b/tests/regression/test_indeehub_maintenance_postgres.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Exercise rollback commitments on an owned, network-isolated PostgreSQL. + +Requires an already imported image: --image IMAGE. Never mounts node volumes, +publishes ports, or invokes the maintenance entrypoint against installed apps. +""" +import argparse +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import time +import uuid + +MODULE = Path(__file__).resolve().parents[2] / 'scripts/indeehub-maintenance-controller.py' +spec = importlib.util.spec_from_file_location('maintenance', MODULE) +maintenance = importlib.util.module_from_spec(spec) +spec.loader.exec_module(maintenance) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--image', required=True) + args = parser.parse_args() + image = subprocess.check_output( + ['podman', 'image', 'inspect', '--format', '{{.Id}}', args.image], text=True, + ).strip() + name = 'archy-maintenance-sql-' + uuid.uuid4().hex + container = None + try: + container = subprocess.check_output([ + 'podman', 'run', '-d', '--pull=never', '--network=none', '--name', name, + '--tmpfs', '/var/lib/postgresql/data:rw', + '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', '-e', 'POSTGRES_USER=indeedhub', + '-e', 'POSTGRES_DB=indeedhub', image, + ], text=True).strip() + deadline = time.monotonic() + 60 + while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + if time.monotonic() > deadline: + raise RuntimeError('Disposable PostgreSQL did not become ready') + time.sleep(0.5) + + def sql(statement, database='indeedhub'): + return subprocess.check_output([ + 'podman', 'exec', '-i', container, 'psql', '-XqAt', + '--set=ON_ERROR_STOP=1', '-U', 'indeedhub', '-d', database, + ], input=statement.encode(), timeout=60) + + sql('CREATE TABLE migrations(id serial PRIMARY KEY, timestamp bigint NOT NULL, name text NOT NULL);' + "INSERT INTO migrations(timestamp,name) VALUES(1,'Original1');" + 'CREATE TABLE contents(id int PRIMARY KEY, title text NOT NULL);' + "INSERT INTO contents VALUES(1,'retained original');") + with tempfile.TemporaryDirectory(prefix=name) as root: + controller = maintenance.Controller(root, str(uuid.uuid4()), 0) + + database = 'indeedhub' + + def fixture_run(argv, timeout=30, output=None, input_bytes=None): + assert argv[:4] == ['podman', 'exec', '-i', 'indeedhub-postgres'] + assert output is None and input_bytes is not None + return sql(input_bytes.decode(), database) + + controller.run = fixture_run + before = controller.database_commitments() + dump = subprocess.check_output([ + 'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub', + '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl', + ], timeout=60) + sql('CREATE DATABASE restore_check') + restore_command = ['podman', 'exec', '-i', container, 'pg_restore', + '-U', 'indeedhub', '-d', 'restore_check', + '--exit-on-error', '--no-owner', '--no-acl'] + subprocess.run(restore_command, input=dump, check=True, timeout=60) + database = 'restore_check' + maintenance.verify_database_compatibility(before, controller.database_commitments()) + database = 'indeedhub' + rejected_dump = subprocess.run(restore_command, input=dump[:32], timeout=60, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert rejected_dump.returncode != 0, 'Truncated dump incorrectly accepted' + maintenance.verify_database_compatibility(before, controller.database_commitments()) + for table in sorted(maintenance.ADDITIVE_TABLES): + sql(f'CREATE TABLE {table}(id int PRIMARY KEY);') + for migration, timestamp in maintenance.ADDITIVE_MIGRATIONS.items(): + sql(f"INSERT INTO migrations(timestamp,name) VALUES({timestamp},'{migration}');") + proof = maintenance.verify_database_compatibility(before, controller.database_commitments()) + assert len(proof['new_empty_tables']) == 5 + rejected = 0 + for mutation, undo in [ + ("UPDATE contents SET title='changed'", "UPDATE contents SET title='retained original'"), + ('ALTER TABLE contents ADD COLUMN unexpected text', 'ALTER TABLE contents DROP COLUMN unexpected'), + ('INSERT INTO archipelago_publications VALUES(1)', 'DELETE FROM archipelago_publications'), + ("UPDATE migrations SET name='changed' WHERE id=1", "UPDATE migrations SET name='Original1' WHERE id=1"), + ]: + sql(mutation) + try: + maintenance.verify_database_compatibility(before, controller.database_commitments()) + except RuntimeError: + rejected += 1 + else: + raise AssertionError('Changed database incorrectly accepted') + sql(undo) + maintenance.verify_database_compatibility(before, controller.database_commitments()) + print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected, + 'network': 'none', 'live_volumes_mounted': False, + 'custom_dump_restored': True, 'truncated_dump_rejected': True})) + finally: + if container: + subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL) + + +if __name__ == '__main__': + main()