fix(indeehub): bound transient restore readiness retries

docs/managed-update-recovery-implementation.md

@@ -495,3 +495,15 @@ refusal. Bounded private failure diagnostics now preserve the controller's reaso
 without exposing stderr in the public RPC response. The previously recorded
 2,025 Rust tests predate these final helper changes; a matching executable and
 final combined receipt remain required.
+
+The matching c58d1180 VM build passed with unchanged inputs. A second actual
+RPC operation, `7d2ea2ae-7b42-4a2c-9095-f0ef2ab52048`, successfully drained all
+seven recovered-image originals (including the relay's owned lineage admission)
+and completed backup. Private diagnostics identified a **10-second pg_isready
+probe TimeoutExpired**, not a schema mismatch. Pre-target recovery again reached
+Restored with cleanup complete. The readiness loop now treats probe timeout as
+not ready only within its existing 90-second overall budget, caps each attempt
+and sleep by remaining time, and rejects success after the deadline. No mutation
+or pg_restore timeout is retried. **58 pure controller tests pass**, including
+ready-after-timeout, expired-deadline cleanup and late-success refusal. Another
+matching executable/full transaction and final combined suite remain pending.

scripts/indeehub-maintenance-controller.py

@@ -704,12 +704,16 @@ class Controller:
             # TCP readiness waits for the final server, avoiding interrupted restore.
             deadline=time.monotonic()+90
             while True:
+                remaining=deadline-time.monotonic()
+                require(remaining>0,'Backup restore database did not become ready')
                 try:
-                    self.run(['podman','exec',identifier,'pg_isready','-h','127.0.0.1','-U','indeedhub','-d','indeedhub'],timeout=10)
+                    self.run(['podman','exec',identifier,'pg_isready','-h','127.0.0.1','-U','indeedhub','-d','indeedhub'],timeout=min(10,remaining))
-                    break
+                except (subprocess.CalledProcessError,subprocess.TimeoutExpired):
-                except subprocess.CalledProcessError:
+                    remaining=deadline-time.monotonic()
-                    require(time.monotonic()<deadline,'Backup restore database did not become ready')
+                    require(remaining>0,'Backup restore database did not become ready')
-                    time.sleep(0.5)
+                    time.sleep(min(0.5,remaining));continue
+                require(time.monotonic()<deadline,'Backup restore database did not become ready')
+                break
             with (self.root/'backup'/'database.dump').open('rb') as source:
                 self.run(['podman','exec','-i',identifier,'pg_restore','--exit-on-error','--no-owner','--no-acl',
                           '-U','indeedhub','-d','indeedhub'],timeout=1800,input_file=source)

tests/regression/test_indeehub_maintenance_controller.py

@@ -575,4 +575,38 @@ console.log('process identity cases passed');'''
   c.save_failure('acquire',RuntimeError('diagnostic'*1000));path=c.root/'last-failure.private.json';record=json.loads(path.read_text())
   self.assertEqual(record['operation_id'],self.operation);self.assertEqual(record['action'],'acquire');self.assertEqual(record['error_type'],'RuntimeError');self.assertEqual(len(record['message']),4096)
   self.assertEqual(path.stat().st_mode&0o777,0o600);self.assertEqual(c.path.read_bytes(),before)
+ def database_readiness_fixture(self, failures):
+  c=self.completed_backup();c.record.pop('backup_restore_verified');attempts=[]
+  c.inspect=lambda identifier:{'Mounts':[]}
+  c.cleanup_restore_fixture=lambda:c.record.pop('restore_fixture',None)
+  c.database_commitments=lambda identifier:c.record['database_before']
+  def run(argv,**kwargs):
+   if argv[:2]==['podman','create']:return ('d'*64).encode()
+   if argv[:2]==['podman','start']:return b''
+   if argv[:2]==['podman','exec'] and argv[3]=='pg_isready':
+    attempts.append(argv)
+    if failures:raise failures.pop(0)
+    return b''
+   if argv[:3]==['podman','exec','-i']:return b''
+   raise AssertionError(argv)
+  c.run=run
+  return c,attempts
+ def test_database_readiness_probe_timeout_retries_within_existing_deadline(self):
+  from unittest.mock import patch
+  c,attempts=self.database_readiness_fixture([module.subprocess.TimeoutExpired(['pg_isready'],10),module.subprocess.CalledProcessError(1,['pg_isready'])])
+  with patch.object(module.time,'monotonic',side_effect=[0,1,2,3,4,5,6]),patch.object(module.time,'sleep'):
+   c.verify_database_backup()
+  self.assertEqual(len(attempts),3);self.assertEqual(c.record['backup_restore_verified'],c.backup_restore_terms())
+ def test_database_readiness_probe_timeout_never_extends_overall_deadline(self):
+  from unittest.mock import patch
+  c,attempts=self.database_readiness_fixture([module.subprocess.TimeoutExpired(['pg_isready'],10)])
+  with patch.object(module.time,'monotonic',side_effect=[0,1,91]),patch.object(module.time,'sleep'):
+   with self.assertRaisesRegex(RuntimeError,'did not become ready'):c.verify_database_backup()
+  self.assertEqual(len(attempts),1);self.assertNotIn('backup_restore_verified',c.record);self.assertNotIn('restore_fixture',c.record)
+ def test_database_readiness_rejects_success_after_deadline(self):
+  from unittest.mock import patch
+  c,attempts=self.database_readiness_fixture([])
+  with patch.object(module.time,'monotonic',side_effect=[0,89,91]),patch.object(module.time,'sleep'):
+   with self.assertRaisesRegex(RuntimeError,'did not become ready'):c.verify_database_backup()
+  self.assertEqual(len(attempts),1);self.assertNotIn('backup_restore_verified',c.record);self.assertNotIn('restore_fixture',c.record)
 if __name__=='__main__':unittest.main()