lfg2025/archy
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 10 Wiki Activity

fix(apps): preserve state across runtime repairs and restore Gitea SSH

Browse Source
This commit is contained in:
archipelago
2026-09-30 10:46:38 -04:00
parent 7d767c8cb0
commit acf544500f
13 changed files with 250 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/container/app_catalog.rs
+4 -4
View File
@@ -118,7 +118,7 @@ fn selected_manifest(entry: AppCatalogEntry) -> Option<serde_json::Value> {
// Never let an unknown future requirement become an unsafe partial match.
for variant in entry.manifest_variants.into_iter().rev() {
if !variant.requires.is_empty() && variant.requires.iter().all(|capability| {
capability == "network-migration-backup-v1"
capability == "runtime-migration-backup-v1"
}) {
return Some(variant.manifest);
}
@@ -583,8 +583,8 @@ mod tests {
fn catalog_migration_variant_is_compatible_with_old_and_future_daemons() {
let raw = serde_json::json!({
"version": "2.45.0", "manifest": {"app": {"id": "portainer", "container": {}}},
"manifest_variants": [{"requires": ["network-migration-backup-v1"],
"manifest": {"app": {"id": "portainer", "container": {"network": "slirp4netns"}, "backup_on_network_change": true}}}]
"manifest_variants": [{"requires": ["runtime-migration-backup-v1"],
"manifest": {"app": {"id": "portainer", "container": {"network": "slirp4netns"}, "backup_before_runtime_change": true}}}]
});
#[derive(Deserialize)]
struct OldEntry { manifest: serde_json::Value }
@@ -593,7 +593,7 @@ mod tests {
let current: AppCatalogEntry = serde_json::from_value(raw.clone()).unwrap();
let chosen = selected_manifest(current).unwrap();
assert_eq!(chosen["app"]["container"]["network"], "slirp4netns");
assert_eq!(chosen["app"]["backup_on_network_change"], true);
assert_eq!(chosen["app"]["backup_before_runtime_change"], true);
let mut future = raw;
future["manifest_variants"][0]["requires"].as_array_mut().unwrap().push(serde_json::json!("unknown-next-capability"));
let chosen = selected_manifest(serde_json::from_value(future).unwrap()).unwrap();
core/archipelago/src/container/migration_backup.rs
+10 -10
View File
@@ -1,15 +1,15 @@
//! Consistent, private snapshots for declaratively opted-in network migrations.
//! Consistent, private snapshots for declaratively opted-in runtime migrations.
use anyhow::{bail, Context, Result};
use archipelago_container::AppManifest;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
pub fn enabled(manifest: &AppManifest) -> Result<bool> {
match manifest.app.extensions.get("backup_on_network_change") {
match manifest.app.extensions.get("backup_before_runtime_change") {
None => Ok(false),
Some(value) => value
.as_bool()
.context("backup_on_network_change must be boolean"),
.context("backup_before_runtime_change must be boolean"),
}
}
@@ -24,12 +24,12 @@ fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathB
continue;
}
if volume.volume_type != "bind" {
bail!("network migration backup requires bind-mounted persistent state");
bail!("runtime migration backup requires bind-mounted persistent state");
}
let path = Path::new(&volume.source);
let relative = path
.strip_prefix(data_dir)
.context("network migration state must be inside the node data directory")?;
.context("runtime migration state must be inside the node data directory")?;
if relative.starts_with("migration-backups") {
bail!("migration backup cannot include its own archive directory");
}
@@ -38,7 +38,7 @@ fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathB
.components()
.any(|c| !matches!(c, std::path::Component::Normal(_)))
{
bail!("invalid network migration state path");
bail!("invalid runtime migration state path");
}
sources.push(relative.to_path_buf());
}
@@ -51,7 +51,7 @@ fn relative_sources(manifest: &AppManifest, data_dir: &Path) -> Result<Vec<PathB
}
}
if roots.is_empty() {
bail!("network migration backup has no persistent state mounts");
bail!("runtime migration backup has no persistent state mounts");
}
Ok(roots)
}
@@ -83,11 +83,11 @@ async fn snapshot_with_command(
.file_type()
.is_symlink()
{
bail!("network migration state mount is a symlink; explicit backup required");
bail!("runtime migration state mount is a symlink; explicit backup required");
}
let canonical = tokio::fs::canonicalize(&path).await?;
if !canonical.starts_with(&canonical_root) {
bail!("network migration state path resolves outside node data directory");
bail!("runtime migration state path resolves outside node data directory");
}
}
let root = data_dir.join("migration-backups");
@@ -129,7 +129,7 @@ async fn snapshot_with_command(
tokio::fs::File::open(&partial).await?.sync_all().await?;
tokio::fs::rename(&partial, &archive).await?;
let metadata = serde_json::json!({"app": manifest.app.id, "version": manifest.app.version,
"network": manifest.app.container.network, "sources": sources});
"network": manifest.app.container.network, "capabilities": manifest.app.security.capabilities, "sources": sources});
tokio::fs::write(
dir.join("metadata.json"),
serde_json::to_vec_pretty(&metadata)?,
core/archipelago/src/container/prod_orchestrator.rs
+72 -11
View File
@@ -99,6 +99,13 @@ fn rootless_network_mode_drifted(expected: Option<&str>, actual: &str) -> bool {
&& actual.trim().split(':').next() != expected
}
fn missing_declared_capability(expected: &[String], actual: &[String]) -> bool {
expected.iter().any(|required| {
let required = required.strip_prefix("CAP_").unwrap_or(required);
!actual.iter().any(|cap| cap.strip_prefix("CAP_").unwrap_or(cap) == required)
})
}
fn uses_pasta_network(manifest: &AppManifest) -> bool {
manifest.app.container.network.as_deref() == Some("pasta")
}
@@ -2472,6 +2479,7 @@ impl ProdContainerOrchestrator {
.await
{
tracing::info!(app_id = %app_id, container = %name, "container published-port drift detected — recreating");
self.backup_runtime_change(&name, &resolved_manifest).await?;
let _ = self.runtime.stop_container(&name).await;
let _ = self.runtime.remove_container(&name).await;
self.install_fresh(lm).await?;
@@ -2507,7 +2515,7 @@ impl ProdContainerOrchestrator {
return Ok(ReconcileAction::NoOp);
}
tracing::info!(app_id = %app_id, container = %name, "container env drift detected — recreating");
self.backup_network_change(&name, &resolved_manifest).await?;
self.backup_runtime_change(&name, &resolved_manifest).await?;
let _ = self.runtime.stop_container(&name).await;
let _ = self.runtime.remove_container(&name).await;
self.install_fresh(lm).await?;
@@ -2564,7 +2572,7 @@ impl ProdContainerOrchestrator {
.await
{
tracing::info!(app_id = %app_id, container = %name, "stopped container env/port drift detected — recreating");
self.backup_network_change(&name, &resolved_manifest).await?;
self.backup_runtime_change(&name, &resolved_manifest).await?;
let _ = self.runtime.remove_container(&name).await;
self.install_fresh(lm).await?;
return Ok(ReconcileAction::Installed);
@@ -2621,6 +2629,7 @@ impl ProdContainerOrchestrator {
self.prepare_for_start(&resolved_manifest).await?;
if self.container_env_drifted(&name, &resolved_manifest).await {
tracing::info!(app_id = %app_id, container = %name, "created container env drift detected — recreating");
self.backup_runtime_change(&name, &resolved_manifest).await?;
let _ = self.runtime.remove_container(&name).await;
self.install_fresh(lm).await?;
return Ok(ReconcileAction::Installed);
@@ -3128,11 +3137,13 @@ impl ProdContainerOrchestrator {
quadlet::network_aliases_changed(&old_body, &new_body);
let restart_for_exec_change = quadlet::exec_changed(&old_body, &new_body);
let restart_for_health_change = quadlet::health_cmd_changed(&old_body, &new_body);
let restart_for_security_change = quadlet::security_changed(&old_body, &new_body);
let needs_restart = restart_required
|| restart_for_port_change
|| restart_for_network_alias_change
|| restart_for_exec_change
|| restart_for_health_change;
|| restart_for_health_change
|| restart_for_security_change;
// Record the obligation BEFORE replacing the unit. A failed reload or
// restart must not become a no-op on the next tick just because the
// generated file already matches the manifest.
@@ -3140,8 +3151,8 @@ impl ProdContainerOrchestrator {
if pending.is_pending() {
self.ensure_resolved_source_available(lm).await?;
}
if restart_for_network_alias_change {
self.backup_network_change(name, &resolved).await?;
if needs_restart {
self.backup_runtime_change(name, &resolved).await?;
}
let changed = quadlet::write_if_changed(&unit, &unit_dir)
.await
@@ -3870,18 +3881,21 @@ impl ProdContainerOrchestrator {
Ok(())
}
async fn backup_network_change(&self, name: &str, manifest: &AppManifest) -> Result<()> {
async fn backup_runtime_change(&self, name: &str, manifest: &AppManifest) -> Result<()> {
if !crate::container::migration_backup::enabled(manifest)? {
return Ok(());
}
// Only back up an actual network migration, not ordinary env drift.
// A persistent disk/permission failure must not repeatedly stop a
// working old service. Reuse the reconciler's bounded repair budget.
if !self.should_attempt_repair(name).await {
anyhow::bail!("runtime migration retry budget exhausted; original service retained, inspect backup failure before retrying");
}
// Called only before a known runtime change. No app-specific commands;
// opted-in manifests identify their persistent state through bind mounts.
let output = tokio::process::Command::new("podman")
.args(["inspect", name, "--format", "{{.HostConfig.NetworkMode}}"])
.output().await.context("inspect network before migration backup")?;
let present = if output.status.success() {
if !rootless_network_mode_drifted(manifest.app.container.network.as_deref(), &String::from_utf8_lossy(&output.stdout)) {
return Ok(());
}
true
} else {
// A crash after gracefully stopping a --rm Quadlet container can
@@ -3934,9 +3948,29 @@ impl ProdContainerOrchestrator {
return true;
}
// Generated-unit drift handles managed services; preserve deliberate
// systemd drop-in overrides instead of recreating them every tick.
let unmanaged = !quadlet::unit_exists(name).await;
// Podman's effective bounding set, not Docker-compatible CapAdd (which
// can be empty even when Quadlet supplied capabilities).
if unmanaged && !manifest.app.security.capabilities.is_empty() {
if let Ok(output) = tokio::process::Command::new("podman")
.args(["inspect", name, "--format", "{{json .BoundingCaps}}"])
.output().await
{
if output.status.success() {
if let Ok(actual) = serde_json::from_slice::<Vec<String>>(&output.stdout) {
if missing_declared_capability(&manifest.app.security.capabilities, &actual) {
return true;
}
}
}
}
}
// Quadlet handles declarative Network= drift above. Legacy rootless
// Podman containers need the same convergence when no unit owns them.
if matches!(manifest.app.container.network.as_deref(), Some("slirp4netns" | "pasta")) {
if unmanaged && matches!(manifest.app.container.network.as_deref(), Some("slirp4netns" | "pasta")) {
if let Ok(output) = tokio::process::Command::new("podman")
.args(["inspect", name, "--format", "{{.HostConfig.NetworkMode}}"])
.output()
@@ -4993,6 +5027,33 @@ mod tests {
/// recovered when its siblings have live containers (the stack is
/// installed), and left alone when the whole stack is gone or the app
/// is not a stack member at all.
#[tokio::test]
async fn gitea_fresh_url_seed_preserves_operator_config_and_reports_write_failure() {
let manifest = AppManifest::parse(include_str!("../../../../apps/gitea/manifest.yml")).unwrap();
let seed = &manifest.app.files[0];
assert!(!seed.overwrite);
let content = seed.content.replace("{{HOST_IP}}", "192.0.2.1");
assert!(content.contains("ROOT_URL = http://192.0.2.1:3001/"));
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("fresh/app.ini");
assert_eq!(ensure_rendered_file(path.to_str().unwrap(), &content, seed.overwrite).await.unwrap(), HookOutcome::Rewritten);
assert!(tokio::fs::read_to_string(&path).await.unwrap().contains("ROOT_URL"));
let custom = "[server]\nROOT_URL = https://git.example.test/\n[database]\nDB_TYPE = postgres\n";
tokio::fs::write(&path, custom).await.unwrap();
assert_eq!(ensure_rendered_file(path.to_str().unwrap(), &content, seed.overwrite).await.unwrap(), HookOutcome::Unchanged);
assert_eq!(tokio::fs::read_to_string(&path).await.unwrap(), custom);
let impossible = path.join("app.ini");
assert!(ensure_rendered_file(impossible.to_str().unwrap(), &content, seed.overwrite).await.is_err());
}
#[test]
fn ssh_sandbox_capability_repair_uses_bounding_set_and_preserves_extra_overrides() {
let required = vec!["CHOWN".into(), "SYS_CHROOT".into()];
assert!(missing_declared_capability(&required, &["CAP_CHOWN".into()]));
assert!(!missing_declared_capability(&required, &["CAP_CHOWN".into(), "CAP_SYS_CHROOT".into()]));
assert!(!missing_declared_capability(&required, &["CHOWN".into(), "SYS_CHROOT".into(), "CAP_KILL".into()]));
}
#[test]
fn explicit_rootless_network_change_converges_without_guessing_defaults() {
assert!(rootless_network_mode_drifted(Some("slirp4netns"), "pasta"));
core/archipelago/src/container/quadlet.rs
+22
View File
@@ -991,6 +991,16 @@ pub fn publish_ports_changed(old_body: &str, new_body: &str) -> bool {
old_ports != new_ports
}
pub fn security_changed(old_body: &str, new_body: &str) -> bool {
["AddCapability=", "DropCapability=", "NoNewPrivileges=", "ReadOnly=", "User="]
.iter().any(|directive| {
let mut old = directive_values(old_body, directive);
let mut new = directive_values(new_body, directive);
old.sort(); new.sort();
old != new
})
}
pub fn network_aliases_changed(old_body: &str, new_body: &str) -> bool {
let old_network = directive_values(old_body, "Network=");
let new_network = directive_values(new_body, "Network=");
@@ -1989,6 +1999,18 @@ app:
assert!(pending.complete().await.is_err());
}
#[test]
fn gitea_ssh_sandbox_capability_is_applied_as_a_runtime_change() {
let manifest = AppManifest::parse(include_str!("../../../../apps/gitea/manifest.yml")).unwrap();
manifest.validate().unwrap();
let new = QuadletUnit::from_manifest(&manifest, "gitea").render();
assert!(new.contains("AddCapability=SYS_CHROOT\n"));
let old = new.replace("AddCapability=SYS_CHROOT\n", "");
assert!(security_changed(&old, &new));
assert!(!security_changed(&new, &new));
assert!(!security_changed("AddCapability=CHOWN\nAddCapability=SETUID\n", "AddCapability=SETUID\nAddCapability=CHOWN\n"));
}
#[test]
fn network_aliases_changed_detects_network_mode_drift() {
let old = "[Container]\nNetwork=slirp4netns\n";
core/container/src/manifest.rs
+7
View File
@@ -1074,6 +1074,12 @@ impl AppManifest {
// `..` copy sources). See docs/manifest-hooks-design.md.
self.app.hooks.validate()?;
if let Some(value) = self.app.extensions.get("backup_before_runtime_change") {
if value.as_bool().is_none() {
return Err(ManifestError::Invalid("backup_before_runtime_change must be boolean".into()));
}
}
Ok(())
}
}
@@ -1111,6 +1117,7 @@ fn validate_security(policy: &SecurityPolicy) -> Result<(), ManifestError> {
"SETGID",
"SETUID",
"SYS_ADMIN",
"SYS_CHROOT",
];
let mut seen = HashSet::new();
for cap in &policy.capabilities {