Prepare guarded Fleet delivery plan and read-only preflight

scripts/qualification/fleet-ui-guard.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Prepared Fleet UI-only guard. Do not invoke deploy until backend qualification and live acceptance are complete."""
+import hashlib,json,os,pathlib,shutil,shlex,subprocess,sys,tarfile,tempfile,time,urllib.request
+P=pathlib.Path
+mode, expected_host, receipt_path = sys.argv[1:4]
+assert expected_host in ('archi-dev-box','yaya-server')
+assert subprocess.check_output(['hostname'],text=True).strip()==expected_host
+web=P('/opt/archipelago/web-ui');data=P('/var/lib/archipelago')
+def digest(path):
+    with open(path,'rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()
+def run(*args):subprocess.run(args,check=True,stdout=subprocess.DEVNULL)
+def inventory():
+    ids=subprocess.check_output(['podman','ps','-aq'],text=True).split()
+    rows=json.loads(subprocess.check_output(['podman','inspect',*ids])) if ids else []
+    return {c['Name']:{'id':c['Id'],'started_at':c['State']['StartedAt'],'status':c['State']['Status'],'running':c['State']['Running']} for c in rows}
+def preservation():
+    names=[data/'remember_secret',data/'user-stopped.json',data/'user-uninstalled.json',data/'app-catalog.json',data/'node-app-catalog.json',data/'installed-apps.json']
+    for root in (data,web,data/'catalogs',data/'content',data/'qualification'):
+        if root.is_dir():names.extend(p for p in root.glob('*catalog*') if p.is_file())
+    for root in (web/'packages',web/'aiui',web/'catalogs',data/'identity'):
+        if root.is_dir():names.extend(p for p in root.rglob('*') if p.is_file())
+    return {str(p):digest(p) if p.is_file() else None for p in set(names)}
+def manager():
+    def prop(name):return subprocess.check_output(['systemctl','show','archipelago','--value','-p',name],text=True).strip()
+    return {'pid':prop('MainPID'),'active_state':prop('ActiveState'),'sub_state':prop('SubState'),'started_at':prop('ExecMainStartTimestamp')}
+def running_digest():
+    path='/proc/'+manager()['pid']+'/exe'
+    try:return digest(path)
+    except PermissionError:return subprocess.check_output(['sudo','-n','sha256sum',path],text=True).split()[0]
+assert mode=='deploy' and len(sys.argv)==7, 'Use separate read-only preflight helper; deployment requires backend gate receipt'
+before=json.loads(P(receipt_path).read_text());qualification=json.loads(P(sys.argv[4]).read_text());archive=P(sys.argv[5]);backend=json.loads(P(sys.argv[6]).read_text())
+assert before['host']==expected_host
+assert qualification['baseline_tests_passed']==1459 and qualification['baseline_test_files_passed']==177
+assert qualification['delta_focused_tests_passed']==31 and qualification['delta_app_typecheck_passed']
+assert qualification['production_build_passed'] and qualification['source_inputs_unchanged']
+assert qualification['archive_sha256']=='063752912fdcae2d1abf2abaccd53bfad39acce23c9a5d896ce52ba1e1996080'
+assert backend['host']==expected_host and backend['isolated_failed']==0 and backend['isolated_passed']>2006
+assert backend['production_build_passed'] and backend['source_inputs_unchanged']
+assert backend['test_source_manifest_sha256']==backend['build_source_manifest_sha256']
+assert len(backend['test_source_manifest_sha256'])==64
+required_tests={'unsigned_and_legacy_reports_cannot_assign_their_own_trust','collector_cannot_claim_another_record_identity_or_malformed_id','collector_history_suffix_cannot_overwrite_another_nodes_history','fleet_reads_do_not_promote_old_collector_spoofs_or_history'}
+assert required_tests.issubset(set(backend['fleet_provenance_tests_passed']))
+assert backend['contains_collector_provenance'] and backend['fleet_contract_readonly_checks_passed']
+assert backend['deployment_preservation_passed'] and backend['embedded_helpers_match']
+assert backend['running_sha256']==backend['binary_sha256']==before['backend_sha256']==before['running_backend_sha256']==running_digest()
+assert backend['binary_sha256']!='7b85bb0135743200620963ae836867175283d57b6a8534fee2a19b72c3ce77c3', 'Old live backend is not qualified for Fleet labels'
+assert manager()==before['manager'], 'Manager changed after fresh post-backend preflight'
+assert digest(archive)==qualification['archive_sha256']
+assert digest(web/'index.html')==before['index_sha256'],'Dashboard changed after preflight'
+assert digest('/usr/local/bin/archipelago')==before['backend_sha256'],'Backend changed after preflight'
+assert preservation()==before['preserved'],'Catalog or stopped/uninstalled intent changed after preflight'
+assert inventory()==before['containers'],'Container changed after preflight'
+backup=data/'support'/('fleet-ui-'+time.strftime('%Y%m%dT%H%M%SZ',time.gmtime())+'-'+str(os.getpid()))
+run('sudo','-n','install','-d','-m','700',str(backup))
+run('sudo','-n','tar','--exclude=./aiui','--exclude=./packages','-czf',str(backup/'web-ui.tar.gz'),'-C',str(web),'.')
+rollback_args=['tar','--exclude=./packages','--exclude=./aiui','--exclude=./catalogs']
+rollback_args.extend('--exclude=./'+p.name for p in web.glob('*catalog*'))
+rollback_args.extend(['-xzf',str(backup/'web-ui.tar.gz'),'-C',str(web)])
+rollback='#!/bin/sh\nset -eu\n'+shlex.join(rollback_args)+'\n'
+subprocess.run(['sudo','-n','tee',str(backup/'rollback.sh')],input=rollback.encode(),check=True,stdout=subprocess.DEVNULL)
+run('sudo','-n','chmod','700',str(backup/'rollback.sh'))
+staging=P(tempfile.mkdtemp(prefix='archy-fleet-ui-',dir='/var/tmp'));source=staging/'dist'
+try:
+    with tarfile.open(archive) as tar:
+        for member in tar.getmembers():
+            path=P(member.name)
+            assert not path.is_absolute() and '..' not in path.parts and path.parts[0]=='dist'
+            assert member.isfile() or member.isdir()
+        tar.extractall(staging,filter='data')
+    assert digest(source/'index.html')==qualification['index_sha256']
+    # Keep node-specific catalogs and existing standard/versioned package downloads.
+    copy_code="""import pathlib,shutil,sys
+s=pathlib.Path(sys.argv[1]);d=pathlib.Path('/opt/archipelago/web-ui')
+for p in s.rglob('*'):
+ r=p.relative_to(s)
+ if r.parts[0] in ('aiui','packages','catalogs') or (len(r.parts)==1 and 'catalog' in p.name):continue
+ if p.is_dir():(d/r).mkdir(parents=True,exist_ok=True)
+ elif not (p.parent==s and p.name in ('index.html','sw.js','registerSW.js')):shutil.copy2(p,d/r)
+"""
+    run('sudo','-n','python3','-c',copy_code,str(source))
+    for name in ('registerSW.js','sw.js','index.html'):
+        if (source/name).exists():
+            target=str(web/name)
+            run('sudo','-n','install','-m','644',str(source/name),target+'.next')
+            run('sudo','-n','mv',target+'.next',target)
+    assert digest('/usr/local/bin/archipelago')==before['backend_sha256']
+    assert preservation()==before['preserved'],'Catalog/session/intent preservation check failed'
+    assert inventory()==before['containers'],'Container preservation check failed'
+    assert manager()==before['manager'] and running_digest()==backend['binary_sha256']
+    with urllib.request.urlopen('http://127.0.0.1/',timeout=30) as response:served=hashlib.file_digest(response,'sha256').hexdigest()
+    assert served==qualification['index_sha256']
+    with urllib.request.urlopen('http://127.0.0.1:5678/health',timeout=15) as response:assert response.status==200
+    time.sleep(15)
+    assert digest('/usr/local/bin/archipelago')==before['backend_sha256']
+    assert preservation()==before['preserved'],'Delayed catalog/session/intent/package/aiui check failed'
+    assert inventory()==before['containers'],'Delayed container preservation check failed'
+    assert manager()==before['manager'] and running_digest()==backend['binary_sha256']
+    print(json.dumps({'host':expected_host,'deployed':True,'index_sha256':served,'backend_unchanged':True,
+        'catalog_session_intents_packages_aiui_preserved':True,'delayed_stability_seconds':15,'containers_unchanged':True,'rollback':str(backup/'rollback.sh')}))
+except BaseException:
+    run('sudo','-n',str(backup/'rollback.sh'))
+    assert digest(web/'index.html')==before['index_sha256'],'Rollback index mismatch'
+    print(json.dumps({'host':expected_host,'deployed':False,'rolled_back':True,'restored_index_sha256':digest(web/'index.html')}))
+    raise
+finally:shutil.rmtree(staging)

scripts/qualification/fleet-ui-preflight.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Read-only Fleet artifact/live-state preflight; never deploys or restarts."""
+import argparse,hashlib,json,os,pathlib,subprocess,tarfile,urllib.request,datetime
+P=pathlib.Path
+ap=argparse.ArgumentParser();ap.add_argument('--host',required=True,choices=['archi-dev-box','yaya-server']);ap.add_argument('--artifact',required=True,type=P);ap.add_argument('--out',required=True,type=P);ap.add_argument('--previous',type=P);a=ap.parse_args()
+assert subprocess.check_output(['hostname'],text=True).strip()==a.host
+assert not a.out.exists(),'Refuse to overwrite earlier preflight evidence'
+def digest(p):
+ with open(p,'rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()
+def property(name):return subprocess.check_output(['systemctl','show','archipelago','--value','-p',name],text=True).strip()
+q=json.loads((a.artifact/'qualification-final.json').read_text());manifest=json.loads((a.artifact/'dist-inputs.json').read_text());archive=a.artifact/'qualified-ui.tar.gz'
+assert q['production_build_passed'] and q['source_inputs_unchanged']
+assert q['baseline_tests_passed']==1459 and q['baseline_test_files_passed']==177
+assert q['delta_focused_tests_passed']==31 and q['delta_app_typecheck_passed']
+assert digest(archive)==q['archive_sha256']=='063752912fdcae2d1abf2abaccd53bfad39acce23c9a5d896ce52ba1e1996080'
+assert digest(a.artifact/'dist-inputs.json')==q['dist_manifest_sha256']
+assert digest(a.artifact/'dist/index.html')==q['index_sha256']
+seen=set()
+with tarfile.open(archive) as tar:
+ for member in tar:
+  path=P(member.name)
+  assert not path.is_absolute() and '..' not in path.parts and path.parts[0]=='dist'
+  assert member.isdir() or member.isfile(),'Non-regular archive member'
+  if member.isfile():
+   rel=str(path.relative_to('dist'));assert rel in manifest and rel not in seen
+   with tar.extractfile(member) as f:assert hashlib.file_digest(f,'sha256').hexdigest()==manifest[rel]
+   seen.add(rel)
+assert seen==set(manifest)
+web=P('/opt/archipelago/web-ui');data=P('/var/lib/archipelago')
+files={data/x for x in ['remember_secret','user-stopped.json','user-uninstalled.json','installed-apps.json','app-catalog.json','node-app-catalog.json']}
+for root in (data,web,data/'catalogs',data/'content',data/'qualification'):
+ if root.is_dir():files.update(p for p in root.glob('*catalog*') if p.is_file())
+for root in (web/'packages',web/'aiui',web/'catalogs',data/'identity'):
+ if root.is_dir():files.update(p for p in root.rglob('*') if p.is_file())
+preserved={str(p):digest(p) if p.is_file() else None for p in sorted(files)}
+ids=subprocess.check_output(['podman','ps','-aq'],text=True).split()
+rows=json.loads(subprocess.check_output(['podman','inspect',*ids])) if ids else []
+containers={c['Name']:{'id':c['Id'],'started_at':c['State']['StartedAt'],'status':c['State']['Status'],'running':c['State']['Running']} for c in rows}
+pid=property('MainPID');assert pid.isdigit() and int(pid)>0
+exe=P('/proc')/pid/'exe'
+try:running=digest(exe)
+except PermissionError:running=subprocess.check_output(['sudo','-n','sha256sum',str(exe)],text=True).split()[0]
+with urllib.request.urlopen('http://127.0.0.1/',timeout=20) as r:served=hashlib.file_digest(r,'sha256').hexdigest()
+with urllib.request.urlopen('http://127.0.0.1:5678/health',timeout=20) as r:health=r.status
+record={'captured_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),'host':a.host,'read_only':True,'index_sha256':digest(web/'index.html'),'served_index_sha256':served,'backend_sha256':digest('/usr/local/bin/archipelago'),'running_backend_sha256':running,'manager':{'pid':pid,'active_state':property('ActiveState'),'sub_state':property('SubState'),'started_at':property('ExecMainStartTimestamp')},'health_http':health,'preserved':preserved,'containers':containers,'candidate_archive_sha256':q['archive_sha256'],'candidate_index_sha256':q['index_sha256'],'artifact_members_verified':len(seen),'backend_qualified':False,'deploy_ready':False}
+assert served==record['index_sha256'],'Served UI differs from disk'
+assert running==record['backend_sha256'],'Running backend differs from installed binary'
+assert health==200
+fd=os.open(a.out,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
+with os.fdopen(fd,'w') as f:json.dump(record,f,indent=2);f.write('\n')
+summary={k:record[k] for k in ['captured_at','host','read_only','index_sha256','backend_sha256','running_backend_sha256','health_http','artifact_members_verified','backend_qualified','deploy_ready']};summary.update(preservation_paths=len(preserved),containers=len(containers))
+if a.previous:
+ old=json.loads(a.previous.read_text());summary['drift_from_previous']={
+ 'ui_changed':old.get('index_sha256')!=record['index_sha256'],
+ 'backend_changed':old.get('backend_sha256')!=record['backend_sha256'],
+ 'preservation_paths_changed':len([p for p in set(old.get('preserved',{}))|set(preserved) if old.get('preserved',{}).get(p)!=preserved.get(p)]),
+ 'container_id_or_start_changes':len([n for n in set(old.get('containers',{}))|set(containers) if (old.get('containers',{}).get(n) if isinstance(old.get('containers',{}).get(n),list) else [old.get('containers',{}).get(n,{}).get('id'),old.get('containers',{}).get(n,{}).get('started_at')])!=([containers[n]['id'],containers[n]['started_at']] if n in containers else None)])}
+print(json.dumps(summary))

scripts/qualification/test-fleet-ui-guard-refusal.py

@@ -0,0 +1,19 @@
+import json,pathlib,tempfile,runpy,unittest,sys
+from unittest.mock import patch
+class GuardRefusal(unittest.TestCase):
+ def test_backend_gate_refuses_before_any_service_or_file_mutation(self):
+  good={'host':'archi-dev-box','isolated_failed':0,'isolated_passed':2011,'production_build_passed':True,'source_inputs_unchanged':True,'test_source_manifest_sha256':'a'*64,'build_source_manifest_sha256':'a'*64,'fleet_provenance_tests_passed':['unsigned_and_legacy_reports_cannot_assign_their_own_trust','collector_cannot_claim_another_record_identity_or_malformed_id','collector_history_suffix_cannot_overwrite_another_nodes_history','fleet_reads_do_not_promote_old_collector_spoofs_or_history'],'contains_collector_provenance':True,'fleet_contract_readonly_checks_passed':True,'deployment_preservation_passed':True,'embedded_helpers_match':True}
+  q={'baseline_tests_passed':1459,'baseline_test_files_passed':177,'delta_focused_tests_passed':31,'delta_app_typecheck_passed':True,'production_build_passed':True,'source_inputs_unchanged':True,'archive_sha256':'063752912fdcae2d1abf2abaccd53bfad39acce23c9a5d896ce52ba1e1996080'}
+  cases=[{'isolated_failed':1},{'build_source_manifest_sha256':'b'*64},{'fleet_provenance_tests_passed':[]},{'fleet_contract_readonly_checks_passed':False},{'embedded_helpers_match':False}]
+  with tempfile.TemporaryDirectory() as d:
+   r=pathlib.Path(d);(r/'before').write_text(json.dumps({'host':'archi-dev-box'}));(r/'q').write_text(json.dumps(q))
+   for delta in cases:
+    with self.subTest(delta=delta):
+     (r/'backend').write_text(json.dumps(good|delta))
+     def check(args,**kwargs):
+      if args==['hostname']:return 'archi-dev-box\n'
+      raise RuntimeError('Unexpected live command before refusal')
+     with patch('subprocess.check_output',side_effect=check),patch('subprocess.run',side_effect=RuntimeError('Mutation attempted')) as mutation,patch.object(sys,'argv',['guard','deploy','archi-dev-box',str(r/'before'),str(r/'q'),str(r/'archive'),str(r/'backend')]):
+      with self.assertRaises(AssertionError):runpy.run_path(str(pathlib.Path(__file__).with_name('fleet-ui-guard.py')),run_name='__main__')
+      mutation.assert_not_called()
+unittest.main()