Verify complete maintenance backup hashes and restored database commitments

docs/managed-update-recovery-implementation.md

@@ -54,3 +54,24 @@ Qualification required before integration/activation:
 - Disk-capacity and recovery-image retention checks, backup integrity and a
   documented recovery path for missing runtime artifacts.
 - Actual-node controlled deployment and acceptance, preserving persistent data.
+
+## Resumed qualification — 2026-10-07
+
+Backup verification now requires the full database/four-volume artifact set and
+rechecks SHA256, including same-size corruption. Nineteen pure controller tests
+pass. The owned, network-none PostgreSQL fixture passes unchanged/additive
+commitments, rejects four data/schema/history mutations, restores a real custom
+dump with matching original commitments, and rejects a truncated dump. It mounts
+no live volume and removes only its own container. This does not qualify actual
+application writer drain or the complete supervised systemd cutover.
+
+The updater compiled and its full isolated suite ran: 1,958 passed, one failed,
+five ignored. The failure is the old snake_case rental receipt JSON fixture;
+`c2c4d915` already corrects that exact test on the release candidate branch.
+Do not duplicate or suppress it here. Integrate and rerun the complete candidate
+suite before claiming a green backend gate. The earlier interrupted compile
+and PostgreSQL timeout remain failed/incomplete attempts, not acceptance.
+
+Evidence: `/tmp/archy-resumed-20261007-updater-full-backend.log`,
+`/tmp/archy-resumed-20261007-indeehub-controller-tests-final.log`, and
+`/tmp/archy-resumed-20261007-indeehub-postgres-restore.log`.

scripts/indeehub-maintenance-controller.py

@@ -5,6 +5,7 @@ must already be durable. Never unlock ARCHY_UPDATE_LOCK_FD or release another ho
 """
 import datetime, hashlib, json, os, pathlib, re, shutil, subprocess, sys, time, uuid
 NAMES = ('indeedhub','indeedhub-api','indeedhub-ffmpeg','indeedhub-minio','indeedhub-postgres','indeedhub-redis','indeedhub-relay')
+VOLUMES = ('indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data')
 DATA = pathlib.Path('/var/lib/archipelago')
 QUEUE_SCRIPT = r'''const {Queue}=require('bullmq');
 (async()=>{const q=new Queue('transcode',{connection:{host:process.env.QUEUE_HOST,port:Number(process.env.QUEUE_PORT||6379),password:process.env.QUEUE_PASSWORD,maxRetriesPerRequest:1}});
@@ -192,7 +193,7 @@ class Controller:
         classification=('idle-worker-terminated-after-queue-drain' if idle_worker else 'empty-business-store-legacy-api-terminated') if str(code)=='143' else 'clean-process-exit'
         stopped[name].update(confirmed=True,exit_code=int(code),classification=classification,confirmed_at=time.time());self.save()
     def volume_sources(self):
-        expected=['indeedhub-minio-data','indeedhub-postgres-data','indeedhub-redis-data','indeedhub-relay-data']
+        expected=VOLUMES
         rows=json.loads(self.run(['podman','volume','inspect',*expected]))
         require({row['Name'] for row in rows}==set(expected),'Persistent volume scope changed')
         return {row['Name']:row['Mountpoint'] for row in rows}
@@ -282,8 +283,11 @@ class Controller:
         # Verification remains possible when API/storage endpoints are stopped.
         # The native adapter separately validates target/original runtime identity.
         for name in NAMES:require(self.record.get('stopped',{}).get(name,{}).get('confirmed'),'Original writer stop evidence missing')
+        expected_artifacts={'database.dump',*(volume+'.tar' for volume in VOLUMES)}
+        require(set(self.record.get('artifacts',{}))==expected_artifacts,'Backup artifact inventory incomplete or unexpected')
         for name,record in self.record['artifacts'].items():
             path=self.root/'backup'/name;require(path.is_file() and not path.is_symlink() and path.stat().st_size==record['bytes'],'Backup artifact missing or changed')
+            require(sha(path)==record['sha256'],'Backup artifact checksum changed')
         return {'operation_id':self.operation,'state':'held'}
     def release(self, outcome):
         require(outcome in ('committed','restored','aborted'),'Invalid release outcome')

tests/regression/test_indeehub_maintenance_controller.py

@@ -47,6 +47,30 @@ class MaintenanceTests(unittest.TestCase):
   c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
   (c.root/'backup').mkdir();(c.root/'backup'/'database.dump').write_bytes(b'not evidence')
   with self.assertRaisesRegex(RuntimeError,'Drain not complete'):c.verify()
+ def completed_backup(self):
+  c=self.controller
+  c.record={'operation_id':self.operation,'phase':'Drained','backup_complete':True,
+            'stopped':{name:{'confirmed':True} for name in module.NAMES},'artifacts':{}}
+  c.save();c.fence.parent.mkdir(parents=True);c.fence.write_text(self.operation)
+  (c.root/'backup').mkdir()
+  for name in ['database.dump',*(v+'.tar' for v in module.VOLUMES)]:
+   path=c.root/'backup'/name;path.write_bytes(b'original')
+   c.record['artifacts'][name]={'bytes':path.stat().st_size,'sha256':module.sha(path)}
+  c.save()
+  return c
+ def test_complete_backup_checksums_allow_verification(self):
+  self.assertEqual(self.completed_backup().verify()['state'],'held')
+ def test_same_size_corruption_keeps_admission_closed(self):
+  c=self.completed_backup();(c.root/'backup'/'database.dump').write_bytes(b'corrupt!')
+  with self.assertRaisesRegex(RuntimeError,'checksum changed'):c.verify()
+  self.assertEqual(c.fence.read_text(),self.operation);self.assertEqual(self.calls,[])
+ def test_incomplete_or_unexpected_artifact_inventory_cannot_pass(self):
+  c=self.completed_backup();original=dict(c.record['artifacts'])
+  for artifacts in [{}, {k:v for k,v in original.items() if k!='database.dump'},
+                    {**original,'../foreign':original['database.dump']}]:
+   c.record['artifacts']=artifacts
+   with self.assertRaisesRegex(RuntimeError,'inventory'):c.verify()
+   self.assertEqual(c.fence.read_text(),self.operation)
  def test_forced_original_exit_never_marks_writer_completed(self):
   c=self.controller;c.record={'operation_id':self.operation,'phase':'Prepared','original_members':module.validate_members(members())};c.save()
   def command(argv,timeout,output):

tests/regression/test_indeehub_maintenance_postgres.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Exercise rollback commitments on an owned, network-isolated PostgreSQL.
+
+Requires an already imported image: --image IMAGE. Never mounts node volumes,
+publishes ports, or invokes the maintenance entrypoint against installed apps.
+"""
+import argparse
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+import uuid
+
+MODULE = Path(__file__).resolve().parents[2] / 'scripts/indeehub-maintenance-controller.py'
+spec = importlib.util.spec_from_file_location('maintenance', MODULE)
+maintenance = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(maintenance)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--image', required=True)
+    args = parser.parse_args()
+    image = subprocess.check_output(
+        ['podman', 'image', 'inspect', '--format', '{{.Id}}', args.image], text=True,
+    ).strip()
+    name = 'archy-maintenance-sql-' + uuid.uuid4().hex
+    container = None
+    try:
+        container = subprocess.check_output([
+            'podman', 'run', '-d', '--pull=never', '--network=none', '--name', name,
+            '--tmpfs', '/var/lib/postgresql/data:rw',
+            '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', '-e', 'POSTGRES_USER=indeedhub',
+            '-e', 'POSTGRES_DB=indeedhub', image,
+        ], text=True).strip()
+        deadline = time.monotonic() + 60
+        while subprocess.run(['podman', 'exec', container, 'pg_isready', '-U', 'indeedhub'],
+                             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode:
+            if time.monotonic() > deadline:
+                raise RuntimeError('Disposable PostgreSQL did not become ready')
+            time.sleep(0.5)
+
+        def sql(statement, database='indeedhub'):
+            return subprocess.check_output([
+                'podman', 'exec', '-i', container, 'psql', '-XqAt',
+                '--set=ON_ERROR_STOP=1', '-U', 'indeedhub', '-d', database,
+            ], input=statement.encode(), timeout=60)
+
+        sql('CREATE TABLE migrations(id serial PRIMARY KEY, timestamp bigint NOT NULL, name text NOT NULL);'
+            "INSERT INTO migrations(timestamp,name) VALUES(1,'Original1');"
+            'CREATE TABLE contents(id int PRIMARY KEY, title text NOT NULL);'
+            "INSERT INTO contents VALUES(1,'retained original');")
+        with tempfile.TemporaryDirectory(prefix=name) as root:
+            controller = maintenance.Controller(root, str(uuid.uuid4()), 0)
+
+            database = 'indeedhub'
+
+            def fixture_run(argv, timeout=30, output=None, input_bytes=None):
+                assert argv[:4] == ['podman', 'exec', '-i', 'indeedhub-postgres']
+                assert output is None and input_bytes is not None
+                return sql(input_bytes.decode(), database)
+
+            controller.run = fixture_run
+            before = controller.database_commitments()
+            dump = subprocess.check_output([
+                'podman', 'exec', container, 'pg_dump', '-U', 'indeedhub',
+                '-d', 'indeedhub', '--format=custom', '--no-owner', '--no-acl',
+            ], timeout=60)
+            sql('CREATE DATABASE restore_check')
+            restore_command = ['podman', 'exec', '-i', container, 'pg_restore',
+                               '-U', 'indeedhub', '-d', 'restore_check',
+                               '--exit-on-error', '--no-owner', '--no-acl']
+            subprocess.run(restore_command, input=dump, check=True, timeout=60)
+            database = 'restore_check'
+            maintenance.verify_database_compatibility(before, controller.database_commitments())
+            database = 'indeedhub'
+            rejected_dump = subprocess.run(restore_command, input=dump[:32], timeout=60,
+                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            assert rejected_dump.returncode != 0, 'Truncated dump incorrectly accepted'
+            maintenance.verify_database_compatibility(before, controller.database_commitments())
+            for table in sorted(maintenance.ADDITIVE_TABLES):
+                sql(f'CREATE TABLE {table}(id int PRIMARY KEY);')
+            for migration, timestamp in maintenance.ADDITIVE_MIGRATIONS.items():
+                sql(f"INSERT INTO migrations(timestamp,name) VALUES({timestamp},'{migration}');")
+            proof = maintenance.verify_database_compatibility(before, controller.database_commitments())
+            assert len(proof['new_empty_tables']) == 5
+            rejected = 0
+            for mutation, undo in [
+                ("UPDATE contents SET title='changed'", "UPDATE contents SET title='retained original'"),
+                ('ALTER TABLE contents ADD COLUMN unexpected text', 'ALTER TABLE contents DROP COLUMN unexpected'),
+                ('INSERT INTO archipelago_publications VALUES(1)', 'DELETE FROM archipelago_publications'),
+                ("UPDATE migrations SET name='changed' WHERE id=1", "UPDATE migrations SET name='Original1' WHERE id=1"),
+            ]:
+                sql(mutation)
+                try:
+                    maintenance.verify_database_compatibility(before, controller.database_commitments())
+                except RuntimeError:
+                    rejected += 1
+                else:
+                    raise AssertionError('Changed database incorrectly accepted')
+                sql(undo)
+            maintenance.verify_database_compatibility(before, controller.database_commitments())
+            print(json.dumps({'postgres_commitments': 'passed', 'rejected_mutations': rejected,
+                              'network': 'none', 'live_volumes_mounted': False,
+                              'custom_dump_restored': True, 'truncated_dump_rejected': True}))
+    finally:
+        if container:
+            subprocess.run(['podman', 'rm', '-f', container], check=True, stdout=subprocess.DEVNULL)
+
+
+if __name__ == '__main__':
+    main()