382 lines
21 KiB
Python
382 lines
21 KiB
Python
"""Admit the first public-worker generation after a historical updater restart.
|
|
|
|
Historical installers already replaced the code pair before starting the new
|
|
daemon. Before that daemon runs host bootstrap, this installed release verifies
|
|
both published releases and freezes authenticated old code plus the still-live
|
|
runtime/configuration preimages. App-writable backup bytes alone are never
|
|
accepted as executable recovery authority.
|
|
"""
|
|
|
|
import urllib.error
|
|
|
|
PUBLIC_RELEASE_BASE = 'https://source.archipelago-foundation.org/lfg2025/archy/releases/download'
|
|
LEGACY_MARKER = Path('/var/lib/archipelago/update-pending-verify.json')
|
|
LEGACY_BINARY = Path('/var/lib/archipelago/update-backup/archipelago')
|
|
LEGACY_FRONTEND = Path('/opt/archipelago/web-ui.bak')
|
|
|
|
|
|
def public_download(url, path, size=None, sha256=None):
|
|
from urllib.parse import urlsplit
|
|
parsed = urlsplit(url)
|
|
if (parsed.scheme != 'https' or parsed.netloc != 'source.archipelago-foundation.org'
|
|
or not parsed.path.startswith('/lfg2025/archy/releases/download/')
|
|
or parsed.query or parsed.fragment):
|
|
raise ValueError('legacy admission requires the fixed public release origin')
|
|
limit = size if size is not None else MAX_DOCUMENT
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
digest = hashlib.sha256(); copied = 0; deadline = time.monotonic() + 180
|
|
with opener.open(url, timeout=30) as source, path.open('xb') as target:
|
|
final = urllib.parse.urlsplit(source.geturl())
|
|
if final.scheme != 'https' or final.netloc != parsed.netloc:
|
|
raise ValueError('public admission download changed authenticated origin')
|
|
while block := source.read(1024 * 1024):
|
|
if time.monotonic() > deadline:
|
|
raise TimeoutError('public admission download deadline exceeded')
|
|
copied += len(block)
|
|
if copied > limit:
|
|
raise ValueError('public admission download exceeds signed size')
|
|
digest.update(block); target.write(block)
|
|
target.flush(); os.fchmod(target.fileno(), 0o444); os.fsync(target.fileno())
|
|
if (size is not None and copied != size) or (sha256 is not None and digest.hexdigest() != sha256):
|
|
raise ValueError('public admission download differs from signed identity')
|
|
|
|
|
|
def adoption_discard(path, reason='incomplete authenticated cache'):
|
|
"""Record and remove one owned, regular incomplete fixed-name cache file."""
|
|
os.close(trusted_open(path.parent, directory=True))
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
try:
|
|
info = os.fstat(fd)
|
|
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_nlink != 1:
|
|
raise ValueError('untrusted incomplete admission file')
|
|
current = path.lstat()
|
|
if (current.st_dev, current.st_ino) != (info.st_dev, info.st_ino):
|
|
raise ValueError('incomplete admission file changed')
|
|
with os.fdopen(os.dup(fd), 'rb') as stream:
|
|
observed_hash = hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
finally:
|
|
os.close(fd)
|
|
# One bounded metadata record, never another copy of the large failed
|
|
# archive. It is replaced in place on the next failure in this fixed tx.
|
|
record = json.dumps({'path': path.name, 'size_bytes': info.st_size,
|
|
'sha256': observed_hash, 'reason': reason}, sort_keys=True).encode()
|
|
evidence = path.parent / 'cache-discard.json'
|
|
evidence_fd = os.open(evidence, os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
try:
|
|
evidence_info = os.fstat(evidence_fd)
|
|
if not stat.S_ISREG(evidence_info.st_mode) or evidence_info.st_uid != 0 or evidence_info.st_nlink != 1:
|
|
raise ValueError('untrusted admission cache evidence')
|
|
os.ftruncate(evidence_fd, 0)
|
|
with os.fdopen(os.dup(evidence_fd), 'wb') as stream:
|
|
stream.write(record)
|
|
stream.flush()
|
|
os.fchmod(evidence_fd, 0o600)
|
|
os.fsync(evidence_fd)
|
|
finally:
|
|
os.close(evidence_fd)
|
|
path.unlink()
|
|
sync_directory(path.parent)
|
|
|
|
|
|
def adoption_manifest(path, url):
|
|
"""A cached document is useful only after the installed verifier accepts it."""
|
|
if os.path.lexists(path):
|
|
try:
|
|
raw = secure_read(path)
|
|
return raw, authenticate(raw)
|
|
except (ValueError, OSError, subprocess.CalledProcessError):
|
|
adoption_discard(path, 'cached manifest failed installed verification')
|
|
public_download(url, path)
|
|
raw = secure_read(path)
|
|
return raw, authenticate(raw)
|
|
|
|
|
|
def adoption_download(url, path, size, sha256, *, fallback_url=None):
|
|
"""Reuse only a root-custody file with the exact signed byte identity."""
|
|
if os.path.lexists(path):
|
|
try:
|
|
fd = trusted_open(path)
|
|
except ValueError:
|
|
# An interrupted exclusive-create download may still have its
|
|
# creation mode. It is never reusable without root custody.
|
|
pass
|
|
else:
|
|
with os.fdopen(fd, 'rb') as stream:
|
|
info = os.fstat(stream.fileno())
|
|
if info.st_size == size and hashlib.file_digest(stream, 'sha256').hexdigest() == sha256:
|
|
return
|
|
adoption_discard(path, 'cached component differs from signed identity')
|
|
try:
|
|
public_download(url, path, size, sha256)
|
|
except urllib.error.HTTPError as error:
|
|
if fallback_url is None or error.code not in (404, 410):
|
|
raise
|
|
if os.path.lexists(path):
|
|
adoption_discard(path, 'incomplete original URL download')
|
|
public_download(fallback_url, path, size, sha256)
|
|
# Keep fixture doubles and future download implementations bound to the
|
|
# signed identity as well; a failed check leaves one fixed retry path.
|
|
fd = trusted_open(path)
|
|
with os.fdopen(fd, 'rb') as stream:
|
|
info = os.fstat(stream.fileno())
|
|
if info.st_size != size or hashlib.file_digest(stream, 'sha256').hexdigest() != sha256:
|
|
raise ValueError('public admission download differs from signed identity')
|
|
|
|
|
|
def require_admission_space(components, old_frontend, old_binary_size):
|
|
"""Refuse bulk copies while the actual root filesystem is already full."""
|
|
compressed = sum(component['size_bytes'] for component in components)
|
|
compressed += old_frontend['size_bytes'] + old_binary_size
|
|
required = max(512 * 1024**2, 3 * compressed + 256 * 1024**2)
|
|
root = os.statvfs('/')
|
|
if root.f_bavail * root.f_frsize < required:
|
|
raise ValueError('insufficient root filesystem space for legacy adoption')
|
|
|
|
|
|
def require_admission_extract_space(tx, new_version):
|
|
"""Budget authenticated tar members before writing expanded staging trees."""
|
|
old_bytes, old_entries = validate_archive(tx / 'previous-frontend.tar.gz', False)
|
|
new_bytes, new_entries = validate_archive(tx / f'archipelago-frontend-{new_version}.tar.gz')
|
|
# old-expected, old-frontend and rollback headroom; new-frontend plus a
|
|
# second forward copy. Count directory/inode overhead, not gzip ratios.
|
|
required = (3 * (old_bytes + 4096 * old_entries)
|
|
+ 2 * (new_bytes + 4096 * new_entries) + 512 * 1024**2)
|
|
root = os.statvfs('/')
|
|
if root.f_bavail * root.f_frsize < required:
|
|
raise ValueError('insufficient root filesystem space for authenticated frontend expansion')
|
|
|
|
|
|
def adoption_reset_tree(path):
|
|
if os.path.lexists(path):
|
|
os.close(trusted_open(path, directory=True))
|
|
# Extracted/snapshotted directories are sealed 0555. Make only this
|
|
# fixed transaction staging tree removable before rebuilding it.
|
|
for directory, _, _ in os.walk(path, topdown=False, followlinks=False):
|
|
if Path(directory).is_symlink():
|
|
raise ValueError('linked admission staging directory')
|
|
Path(directory).chmod(0o700)
|
|
shutil.rmtree(path)
|
|
sync_directory(path.parent)
|
|
|
|
|
|
def legacy_routing_bytes(path):
|
|
"""Pin the one active nginx file a legacy app user could have generated.
|
|
|
|
This is not a general exception to root custody. The file's ancestors
|
|
remain root-owned; runtime_prepare validates it with nginx before use.
|
|
"""
|
|
import pwd
|
|
if path != Path(RUNTIME_OBJECTS['routing'][0]):
|
|
raise ValueError('unexpected legacy routing path')
|
|
os.close(trusted_open(path.parent, directory=True))
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
with os.fdopen(fd, 'rb') as stream:
|
|
before = os.fstat(stream.fileno())
|
|
owner = pwd.getpwnam('archipelago').pw_uid
|
|
if (not stat.S_ISREG(before.st_mode) or before.st_uid not in (0, owner)
|
|
or before.st_mode & (0o022 | 0o6000)
|
|
or not 0 < before.st_size <= 1024**2):
|
|
raise ValueError('untrusted legacy routing file')
|
|
raw = stream.read(1024**2 + 1)
|
|
after = os.fstat(stream.fileno())
|
|
current = path.lstat()
|
|
identity = lambda info: (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
|
|
if len(raw) != before.st_size or identity(before) != identity(after) or identity(before) != identity(current):
|
|
raise ValueError('legacy routing changed during capture')
|
|
return raw
|
|
|
|
|
|
def historical_frontend_modes(path):
|
|
# Historical apply_update normalizes EVERY file to0644 and directory to0755.
|
|
# Reproduce that byte/mode domain before comparing the retained old tree.
|
|
for directory, _, files in os.walk(path):
|
|
for filename in files:
|
|
durable_mode(Path(directory) / filename, 0o444)
|
|
durable_mode(Path(directory), 0o555)
|
|
|
|
|
|
def retain_legacy_marker(tx, state):
|
|
expected = state.get('legacy_marker_sha256')
|
|
if expected is None or not os.path.lexists(LEGACY_MARKER):
|
|
return
|
|
original = LEGACY_MARKER.lstat()
|
|
if not stat.S_ISREG(original.st_mode) or not 0 < original.st_size <= MAX_DOCUMENT:
|
|
raise ValueError('invalid legacy verification marker')
|
|
if digest_file(LEGACY_MARKER) != expected:
|
|
raise ValueError('legacy verification marker changed; preserve it for reconciliation')
|
|
retained = tx / 'legacy-verification-marker.json'
|
|
if os.path.lexists(retained):
|
|
if digest_file(retained) != expected:
|
|
# A crash may have interrupted the fixed-name copy. The unchanged
|
|
# original remains authoritative until its complete copy is fsynced.
|
|
adoption_discard(retained, 'interrupted retained marker copy')
|
|
if not os.path.lexists(retained):
|
|
fd = os.open(LEGACY_MARKER, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
try:
|
|
before = os.fstat(fd)
|
|
if not stat.S_ISREG(before.st_mode) or not 0 < before.st_size <= MAX_DOCUMENT:
|
|
raise ValueError('invalid legacy verification marker')
|
|
if copy_file_fd(fd, retained) != expected:
|
|
raise ValueError('legacy marker changed while retaining its preimage')
|
|
finally:
|
|
os.close(fd)
|
|
if digest_file(retained) != expected:
|
|
raise ValueError('retained legacy verification marker differs')
|
|
sync_directory(tx)
|
|
# The old data mount and new root-custody mount can differ. Remove the
|
|
# original only after the fixed retained copy and directory are durable.
|
|
current = LEGACY_MARKER.lstat()
|
|
marker_identity = lambda info: (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
|
|
if marker_identity(current) != marker_identity(original) or digest_file(LEGACY_MARKER) != expected:
|
|
raise ValueError('legacy verification marker changed before removal')
|
|
LEGACY_MARKER.unlink()
|
|
sync_directory(LEGACY_MARKER.parent)
|
|
|
|
|
|
def adopt_legacy_pair(bundle):
|
|
require_legacy_service()
|
|
if os.path.lexists(RUNTIME_RECORD):
|
|
raise ValueError('public runtime already has an owner; no legacy adoption')
|
|
with owner_lock():
|
|
_, previous = current()
|
|
if previous and previous['phase'] not in TERMINAL:
|
|
# A caller restart may observe its already queued independent job.
|
|
if previous.get('adoption'):
|
|
command('/usr/bin/systemctl', 'start', '--no-block', previous['worker_unit'])
|
|
return previous
|
|
raise ValueError('different public transaction pending')
|
|
check_owner_dropins(previous)
|
|
if not manager_adoptable():
|
|
raise ValueError('legacy admission requires the actual starting manager')
|
|
fd = os.open(LEGACY_MARKER, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
with os.fdopen(fd, 'rb') as stream:
|
|
info = os.fstat(stream.fileno())
|
|
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_DOCUMENT:
|
|
raise ValueError('invalid legacy verification marker')
|
|
marker_raw = stream.read(MAX_DOCUMENT + 1)
|
|
if len(marker_raw) > MAX_DOCUMENT:
|
|
raise ValueError('oversized legacy verification marker')
|
|
marker = json.loads(marker_raw, object_pairs_hook=pairs)
|
|
old_version, new_version = marker['previous_version'], marker['new_version']
|
|
if version(old_version) >= version(new_version):
|
|
raise ValueError('legacy marker is not an upgrade')
|
|
marker_sha256 = hashlib.sha256(marker_raw).hexdigest()
|
|
# The legacy marker and target release identify one admission attempt
|
|
# across restarts. Never allocate a new cache directory for a retry.
|
|
tx_id = hashlib.sha256(('legacy-adoption-v1:' + marker_sha256 + ':' + new_version).encode()).hexdigest()[:32]
|
|
tx = TRANSACTIONS / tx_id
|
|
tx.mkdir(mode=0o700, exist_ok=True)
|
|
os.close(trusted_open(tx, directory=True))
|
|
if os.path.lexists(tx / 'state.json'):
|
|
# A journal without the current pointer needs operator recovery,
|
|
# not a second scheduling pass over the same protected preimages.
|
|
raise ValueError('orphaned legacy admission journal requires reconciliation')
|
|
current_doc = tx / 'signed-manifest.json'
|
|
raw, installed = adoption_manifest(current_doc, f'{PUBLIC_RELEASE_BASE}/v{new_version}/manifest.json')
|
|
document = json.loads(raw, object_pairs_hook=pairs)
|
|
if installed != new_version or document.get('version') != new_version:
|
|
raise ValueError('legacy marker does not identify this installed release')
|
|
components = validate_document(document, old_version)
|
|
if document.get('application_update') != RUNTIME_POLICY:
|
|
raise ValueError('first-generation migration requires the signed runtime transaction profile')
|
|
identity = hashlib.sha256(json.dumps([new_version, sorted([
|
|
[c['name'], c['new_version'], c['sha256'], c['size_bytes']] for c in components])],
|
|
separators=(',', ':'), ensure_ascii=True).encode()).hexdigest()
|
|
previous_doc = tx / 'previous-signed-manifest.json'
|
|
old_raw, _ = adoption_manifest(previous_doc,
|
|
f'{PUBLIC_RELEASE_BASE}/v{new_version}/legacy-manifest-{old_version}.json')
|
|
old = json.loads(old_raw, object_pairs_hook=pairs)
|
|
if old.get('version') != old_version or old.get('signed_by') != RELEASE_ROOT or 'os_update' in old:
|
|
raise ValueError('previous release is not an authenticated application baseline')
|
|
old_binary = [c for c in old['components'] if c['name'] == 'archipelago']
|
|
old_frontend = [c for c in old['components'] if c['name'] == f'archipelago-frontend-{old_version}.tar.gz']
|
|
if len(old_binary) != 1 or len(old_frontend) != 1:
|
|
raise ValueError('previous signed code pair missing')
|
|
if LEGACY_BINARY.stat().st_size != old_binary[0]['size_bytes'] or digest_file(LEGACY_BINARY) != old_binary[0]['sha256']:
|
|
raise ValueError('untrusted historical backend backup; refusing executable recovery')
|
|
component = old_frontend[0]
|
|
if (not re.fullmatch('[0-9a-f]{64}', component['sha256'])
|
|
or type(component['size_bytes']) is not int or not 0 < component['size_bytes'] <= MAX_COMPONENT):
|
|
raise ValueError('invalid previous frontend declaration')
|
|
require_admission_space(components, component, old_binary[0]['size_bytes'])
|
|
# The signed historical archive is needed for recovery. Check it first
|
|
# so a missing historical release cannot waste the incoming bulk copy.
|
|
adoption_download(component['download_url'], tx / 'previous-frontend.tar.gz',
|
|
component['size_bytes'], component['sha256'],
|
|
fallback_url=f'{PUBLIC_RELEASE_BASE}/v{new_version}/legacy-frontend-{old_version}.tar.gz')
|
|
old_binary_copy = tx / 'old-binary'
|
|
if os.path.lexists(old_binary_copy):
|
|
try:
|
|
fd = trusted_open(old_binary_copy)
|
|
except ValueError:
|
|
reusable = False
|
|
else:
|
|
with os.fdopen(fd, 'rb') as stream:
|
|
reusable = (os.fstat(stream.fileno()).st_size == old_binary[0]['size_bytes']
|
|
and hashlib.file_digest(stream, 'sha256').hexdigest() == old_binary[0]['sha256'])
|
|
if not reusable:
|
|
adoption_discard(old_binary_copy, 'historical binary preimage copy differs')
|
|
if not old_binary_copy.exists():
|
|
fd = os.open(LEGACY_BINARY, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
try:
|
|
copy_file_fd(fd, old_binary_copy, True)
|
|
finally:
|
|
os.close(fd)
|
|
if digest_file(old_binary_copy) != old_binary[0]['sha256']:
|
|
raise ValueError('historical backend changed while freezing its preimage')
|
|
# Historical apply_update removes staging before its detached restart.
|
|
# Reuse only exact authenticated archives in this one root-owned tx.
|
|
for incoming in components:
|
|
destination = tx / ('new-binary' if incoming['name'] == 'archipelago' else incoming['name'])
|
|
adoption_download(incoming['download_url'], destination,
|
|
incoming['size_bytes'], incoming['sha256'])
|
|
durable_mode(tx / 'new-binary', 0o555)
|
|
require_admission_extract_space(tx, new_version)
|
|
adoption_reset_tree(tx / 'new-frontend')
|
|
extract(tx / f'archipelago-frontend-{new_version}.tar.gz', tx / 'new-frontend')
|
|
historical_frontend_modes(tx / 'new-frontend')
|
|
adoption_reset_tree(tx / 'old-expected')
|
|
extract(tx / 'previous-frontend.tar.gz', tx / 'old-expected', require_declaration=False)
|
|
historical_frontend_modes(tx / 'old-expected')
|
|
# Retain the signed template when it is unchanged. Some prior daemons
|
|
# instead carry accepted root-owned routing into the frontend runtime;
|
|
# permit only that exact alternative, never an arbitrary local edit.
|
|
relative = Path('archipelago-runtime/image-recipe/configs/nginx-archipelago.conf')
|
|
routing = Path(RUNTIME_OBJECTS['routing'][0])
|
|
retained, expected = LEGACY_FRONTEND / relative, tx / 'old-expected' / relative
|
|
if (retained.is_file() and expected.is_file() and routing.exists()
|
|
and digest_file(retained) != digest_file(expected)):
|
|
atomic(expected, legacy_routing_bytes(routing), 0o444)
|
|
# Companion may have been independently updated to this same signed
|
|
# release's APK. Accept only the old or incoming signed artifact bytes.
|
|
for name in ('archipelago-companion.apk', 'archipelago-companion.json'):
|
|
relative = Path('packages') / name
|
|
retained, expected, incoming = LEGACY_FRONTEND / relative, tx / 'old-expected' / relative, tx / 'new-frontend' / relative
|
|
if retained.is_file() and incoming.is_file() and digest_file(retained) == digest_file(incoming):
|
|
atomic(expected, secure_read(incoming, MAX_COMPONENT), 0o444)
|
|
if tree(LEGACY_FRONTEND) != tree(tx / 'old-expected'):
|
|
raise ValueError('historical frontend backup contains unverified changes; preserve for review')
|
|
adoption_reset_tree(tx / 'old-frontend')
|
|
tree(LEGACY_FRONTEND, tx / 'old-frontend')
|
|
if tree(tx / 'old-frontend') != tree(tx / 'old-expected'):
|
|
raise ValueError('historical frontend changed while freezing its preimage')
|
|
# Historical apply preserves separately installed static UI extras.
|
|
# Their frozen old image is now authenticated; reproduce that merge
|
|
# before checking the pair it installed. No runtime script is exempt.
|
|
for name in ('aiui', 'archipelago-companion.apk'):
|
|
old_extra, incoming = tx / 'old-frontend' / name, tx / 'new-frontend' / name
|
|
if old_extra.exists() and not incoming.exists():
|
|
runtime_copy(old_extra, incoming)
|
|
historical_frontend_modes(tx / 'new-frontend')
|
|
initial = pair_identity(BINARY, FRONTEND)
|
|
if initial != pair_identity(tx / 'new-binary', tx / 'new-frontend'):
|
|
raise ValueError('historical installer did not install the exact signed incoming pair')
|
|
state = {'schema': 1, 'id': tx.name, 'release': identity, 'version': new_version,
|
|
'old_version': old_version, 'new_version': new_version, 'was_active': True,
|
|
'adoption': True, 'initial': initial, 'runtime_requested': True,
|
|
'legacy_marker_sha256': hashlib.sha256(marker_raw).hexdigest(),
|
|
'old': pair_identity(tx / 'old-binary', tx / 'old-frontend'),
|
|
'new': pair_identity(tx / 'new-binary', tx / 'new-frontend')}
|
|
custody_frontend_parent()
|
|
return schedule_transaction(tx, state, bundle)
|