lfg2025/archy
1
0
Fork 0
Code Issues 25 Pull Requests 4 Actions Packages Projects Releases 74 Wiki Activity

backend: harden rootless app lifecycle orchestration

Browse Source
This commit is contained in:
archipelago
2026-06-11 00:24:32 -04:00
parent 09ec64932f
commit c393b96da3
56 changed files with 7543 additions and 1994 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/archipelago/src/api/rpc/package/config.rs
+3 -20
View File
@@ -3,7 +3,7 @@ use crate::port_allocator::PortAllocator;
use anyhow::{Context, Result};
use std::time::Duration;
const PODMAN_LIST_TIMEOUT: Duration = Duration::from_secs(15);
const PODMAN_LIST_TIMEOUT: Duration = Duration::from_secs(60);
fn is_platform_managed_app(app_id: &str) -> bool {
matches!(
@@ -31,7 +31,6 @@ fn is_platform_managed_app(app_id: &str) -> bool {
| "fedimint"
| "fedimint-gateway"
| "indeedhub"
| "saleor"
| "immich"
)
}
@@ -501,15 +500,6 @@ pub(super) fn all_container_names(package_id: &str) -> Vec<String> {
"netbird-dashboard".into(),
"netbird-server".into(),
],
"saleor" => vec![
"saleor-db".into(),
"saleor-cache".into(),
"saleor-api".into(),
"saleor-worker".into(),
"saleor-jaeger".into(),
"saleor-mailpit".into(),
"saleor".into(),
],
"nostr-vpn" => vec![
"nostr-vpn".into(),
"archy-nostr-vpn".into(),
@@ -599,7 +589,6 @@ pub(super) fn get_data_dirs_for_app(package_id: &str) -> Vec<String> {
format!("{}/penpot-postgres", base),
],
"netbird" => vec![format!("{}/netbird", base)],
"saleor" => vec![format!("{}/saleor", base), format!("{}/saleor-db", base)],
_ => vec![format!("{}/{}", base, package_id)],
}
}
@@ -977,6 +966,7 @@ pub(super) async fn get_app_config(
vec![
"/var/lib/archipelago/portainer:/data".to_string(),
"/run/user/1000/podman/podman.sock:/var/run/docker.sock".to_string(),
"/var/lib/archipelago/portainer/compose:/data/compose".to_string(),
],
vec![],
None,
@@ -1006,7 +996,7 @@ pub(super) async fn get_app_config(
Some(vec![
"sh".to_string(),
"-c".to_string(),
"tailscaled --tun=userspace-networking & sleep 2; tailscale web --listen 0.0.0.0:8240 & wait".to_string(),
"tailscaled --tun=userspace-networking & for i in $(seq 1 30); do [ -S /var/run/tailscale/tailscaled.sock ] && break; sleep 1; done; tailscale web --listen 0.0.0.0:8240 & wait".to_string(),
]),
),
"fedimint" => (
@@ -1079,13 +1069,6 @@ pub(super) async fn get_app_config(
None,
None,
),
"saleor" => (
vec!["9010:80".to_string(), "8000:8000".to_string()],
vec!["/var/lib/archipelago/saleor:/app/media".to_string()],
vec![],
None,
None,
),
"nostr-rs-relay" => (
vec!["18081:8080".to_string()],
vec!["/var/lib/archipelago/nostr-rs-relay:/usr/src/app/db".to_string()],
core/archipelago/src/api/rpc/package/dependencies.rs
-9
View File
@@ -289,15 +289,6 @@ pub(super) fn startup_order(package_id: &str) -> &'static [&'static str] {
&["archy-btcpay-db", "archy-nbxplorer", "btcpay-server"]
}
"netbird" => &["netbird-server", "netbird-dashboard", "netbird"],
"saleor" => &[
"saleor-db",
"saleor-cache",
"saleor-jaeger",
"saleor-mailpit",
"saleor-api",
"saleor-worker",
"saleor",
],
"penpot" | "penpot-frontend" => &[
"penpot-postgres",
"penpot-valkey",
core/archipelago/src/api/rpc/package/install.rs
+280 -93
View File
@@ -13,11 +13,12 @@ use crate::api::rpc::RpcHandler;
use crate::data_model::InstallPhase;
use crate::update::host_sudo;
use anyhow::{Context, Result};
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::time::{timeout, Duration};
use tracing::{debug, info, warn};
const INSTALL_LOG: &str = "/var/log/archipelago/container-installs.log";
const IMAGE_INSPECT_TIMEOUT: Duration = Duration::from_secs(10);
/// Append a timestamped line to the persistent install log.
pub(in crate::api::rpc) async fn install_log(msg: &str) {
@@ -34,6 +35,36 @@ pub(in crate::api::rpc) async fn install_log(msg: &str) {
}
}
async fn local_podman_image_exists(image: &str) -> Result<bool> {
let mut cmd = tokio::process::Command::new("podman");
cmd.args(["image", "inspect", image]);
cmd.kill_on_drop(true);
let output = timeout(IMAGE_INSPECT_TIMEOUT, cmd.output())
.await
.with_context(|| {
format!(
"podman image inspect {} timed out after {}s",
image,
IMAGE_INSPECT_TIMEOUT.as_secs()
)
})?
.with_context(|| format!("Failed to execute podman image inspect {}", image))?;
match output.status.code() {
Some(0) => Ok(true),
Some(1) => Ok(false),
Some(code) => Err(anyhow::anyhow!(
"podman image inspect {} exited with {}: {}",
image,
code,
String::from_utf8_lossy(&output.stderr).trim()
)),
None => Err(anyhow::anyhow!(
"podman image inspect {} terminated by signal",
image
)),
}
}
pub(super) async fn patch_indeedhub_nostr_provider() {
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
@@ -244,10 +275,6 @@ impl RpcHandler {
if package_id == "netbird" {
return self.install_netbird_stack().await;
}
if package_id == "saleor" {
return self.install_saleor_stack().await;
}
// Dependency checks. Prefer the scanner's cached package state so a
// congested Podman API does not turn an already-running dependency into
// a false install failure. Fall back to a bounded direct Podman probe
@@ -447,6 +474,7 @@ impl RpcHandler {
Ok(container_name) => {
self.set_install_phase(package_id, InstallPhase::WaitingHealthy)
.await;
ensure_host_port_listener(package_id, &container_name, &[]).await?;
crate::api::rpc::package::runtime::reconcile_companions_for(package_id)
.await;
install_log(&format!(
@@ -652,10 +680,6 @@ impl RpcHandler {
self.write_lnd_conf(&rpc_user, &rpc_pass).await?;
}
if package_id == "portainer" {
ensure_user_podman_socket().await?;
}
// Pre-install: SearXNG settings.yml (required or container exits immediately)
if package_id == "searxng" {
let searx_dir = "/var/lib/archipelago/searxng";
@@ -748,16 +772,10 @@ impl RpcHandler {
.await;
debug!("Running container with args: {:?}", run_args);
// Build command with optional custom command/args
let mut cmd = tokio::process::Command::new("podman");
cmd.args(&run_args);
if let Some(custom_cmd) = custom_command {
cmd.arg(custom_cmd);
} else if let Some(args) = custom_args {
cmd.args(args);
}
let mut run_output = cmd.output().await.context("Failed to run container")?;
let command_tail = install_command_tail(custom_command.as_deref(), custom_args.as_ref());
let mut run_output = podman_run_for_install(package_id, &run_args, &command_tail)
.await
.context("Failed to run container")?;
if !run_output.status.success() {
let stderr = String::from_utf8_lossy(&run_output.stderr).to_string();
@@ -766,7 +784,9 @@ impl RpcHandler {
.args(["rm", "-f", container_name])
.output()
.await;
run_output = cmd.output().await.context("Failed to rerun container")?;
run_output = podman_run_for_install(package_id, &run_args, &command_tail)
.await
.context("Failed to rerun container")?;
}
}
@@ -922,12 +942,7 @@ impl RpcHandler {
let is_local_image = docker_image.starts_with("localhost/");
let has_local_fallback = if !is_local_image {
let local_tag = format!("localhost/{}:latest", package_id);
let check = tokio::process::Command::new("podman")
.args(["images", "-q", &local_tag])
.output()
.await
.ok();
check.is_some_and(|o| !String::from_utf8_lossy(&o.stdout).trim().is_empty())
local_podman_image_exists(&local_tag).await.unwrap_or(false)
} else {
false
};
@@ -942,14 +957,9 @@ impl RpcHandler {
);
} else {
// Local image — verify it exists
let images_output = tokio::process::Command::new("podman")
.args(["images", "-q", docker_image])
.output()
if !local_podman_image_exists(docker_image)
.await
.context("Failed to check local image")?;
if String::from_utf8_lossy(&images_output.stdout)
.trim()
.is_empty()
.context("Failed to check local image")?
{
return Err(anyhow::anyhow!(
"Local image {} not found. Build the image first \
@@ -1139,12 +1149,10 @@ impl RpcHandler {
}
// Verify image exists locally after pull.
let verify = tokio::process::Command::new("podman")
.args(["images", "-q", docker_image])
.output()
if !local_podman_image_exists(docker_image)
.await
.context("Failed to verify pulled image")?;
if String::from_utf8_lossy(&verify.stdout).trim().is_empty() {
.context("Failed to verify pulled image")?
{
return Err(anyhow::anyhow!(
"Image {} not found locally after pull",
docker_image
@@ -1278,11 +1286,13 @@ impl RpcHandler {
// set `prune=N` in bitcoin.conf themselves after install.
let bitcoin_conf = format!(
"\
# rpcauth: salted hash only — no plaintext password in config or CLI\n\
# rpcauth: salted hash only - no plaintext password in config or CLI\n\
{}\n\
server=1\n\
rpcallowip=0.0.0.0/0\n\
listen=1\n\
rpcthreads=16\n\
rpcworkqueue=256\n\
printtoconsole=1\n",
rpcauth_line
);
@@ -1871,29 +1881,34 @@ autopilot.active=false\n",
.unwrap_or_default();
super::validation::validate_app_id(app_id)?;
match app_id {
"saleor" => {
let password =
tokio::fs::read_to_string("/var/lib/archipelago/secrets/saleor-admin-password")
.await
.unwrap_or_default()
.trim()
.to_string();
if password.is_empty() {
return Ok(serde_json::json!({ "credentials": [] }));
}
Ok(serde_json::json!({
"title": "Saleor admin login",
"description": "Saleor opens to its own dashboard login. Use this generated admin account to sign in.",
"credentials": [
{ "label": "Email", "value": "admin@example.com", "sensitive": false },
{ "label": "Password", "value": password, "sensitive": true }
]
}))
}
_ => Ok(serde_json::json!({ "credentials": [] })),
if app_id == "filebrowser" {
let password =
tokio::fs::read_to_string("/var/lib/archipelago/secrets/filebrowser/password")
.await
.map(|p| p.trim().to_string())
.unwrap_or_else(|_| "admin".to_string());
return Ok(serde_json::json!({
"title": "File Browser credentials",
"description": "Use these credentials when File Browser asks you to sign in.",
"credentials": [
{ "label": "Username", "value": "admin" },
{ "label": "Password", "value": password, "sensitive": true }
]
}));
}
if app_id == "photoprism" {
return Ok(serde_json::json!({
"title": "PhotoPrism credentials",
"description": "Use these credentials when PhotoPrism asks you to sign in.",
"credentials": [
{ "label": "Username", "value": "admin" },
{ "label": "Password", "value": "archipelago", "sensitive": true }
]
}));
}
Ok(serde_json::json!({ "credentials": [] }))
}
}
@@ -1914,10 +1929,128 @@ async fn cleanup_stale_package_ports(package_id: &str) {
cleanup_stale_pasta_port("8444").await;
}
"nextcloud" => cleanup_stale_pasta_port("8085").await,
"portainer" => cleanup_stale_pasta_port("9000").await,
_ => {}
}
}
fn install_command_tail(
custom_cmd: Option<&str>,
custom_args: Option<&Vec<String>>,
) -> Vec<String> {
if let Some(cmd) = custom_cmd {
vec![cmd.to_string()]
} else if let Some(args) = custom_args {
args.clone()
} else {
Vec::new()
}
}
async fn podman_run_for_install(
package_id: &str,
run_args: &[&str],
command_tail: &[String],
) -> Result<std::process::Output> {
if should_scope_podman_run(package_id) {
match podman_create_then_scoped_start(package_id, run_args, command_tail).await {
Ok(output) => return Ok(output),
Err(err) => {
tracing::warn!(package_id, error = %err, "scoped podman create/start failed; falling back to direct podman run");
}
}
}
let mut cmd = tokio::process::Command::new("podman");
cmd.args(run_args);
cmd.args(command_tail);
cmd.output().await.context("Failed to run podman")
}
async fn podman_create_then_scoped_start(
package_id: &str,
run_args: &[&str],
command_tail: &[String],
) -> Result<std::process::Output> {
let container_name = run_args
.windows(2)
.find_map(|pair| (pair[0] == "--name").then_some(pair[1]))
.unwrap_or(package_id);
let mut create_args = Vec::with_capacity(run_args.len() + command_tail.len());
for (idx, arg) in run_args.iter().enumerate() {
if idx == 0 && *arg == "run" {
create_args.push("create".to_string());
} else if *arg != "-d" {
create_args.push((*arg).to_string());
}
}
create_args.extend(command_tail.iter().cloned());
let mut create = tokio::process::Command::new("podman");
create.args(&create_args);
let create_output = create
.output()
.await
.context("Failed to run podman create")?;
if !create_output.status.success() {
return Ok(create_output);
}
let mut scoped_start = tokio::process::Command::new("systemd-run");
scoped_start.args([
"--user",
"--scope",
"--quiet",
"--collect",
"podman",
"start",
container_name,
]);
match scoped_start.output().await {
Ok(output) if output.status.success() => Ok(create_output),
Ok(output) => {
tracing::warn!(
package_id,
container = container_name,
stderr = %String::from_utf8_lossy(&output.stderr).trim(),
"scoped podman start after create failed; trying direct podman start"
);
let mut direct_start = tokio::process::Command::new("podman");
direct_start.args(["start", container_name]);
let direct_output = direct_start
.output()
.await
.context("Failed to run fallback podman start")?;
if direct_output.status.success() {
Ok(create_output)
} else {
Ok(direct_output)
}
}
Err(err) => Err(err).context("Failed to run scoped podman start"),
}
}
fn should_scope_podman_run(package_id: &str) -> bool {
matches!(
package_id,
"botfights"
| "filebrowser"
| "gitea"
| "grafana"
| "homeassistant"
| "home-assistant"
| "jellyfin"
| "nginx-proxy-manager"
| "nostr-rs-relay"
| "photoprism"
| "portainer"
| "searxng"
| "uptime-kuma"
| "vaultwarden"
)
}
async fn cleanup_start_conflict(package_id: &str, stderr: &str) -> bool {
if stderr.contains("name is already in use") || stderr.contains("name \"") {
return true;
@@ -1968,6 +2101,12 @@ async fn cleanup_start_conflict(package_id: &str, stderr: &str) -> bool {
cleanup_stale_pasta_port("8085").await;
true
}
"portainer"
if stderr.contains("pasta failed") || stderr.contains("address already in use") =>
{
cleanup_stale_pasta_port("9000").await;
true
}
_ => false,
}
}
@@ -2026,7 +2165,7 @@ async fn ensure_host_port_listener(
return Ok(());
};
if wait_for_host_port(port, 10).await {
if wait_for_host_port(package_id, port, 10).await {
return Ok(());
}
@@ -2052,7 +2191,7 @@ async fn ensure_host_port_listener(
));
}
if wait_for_host_port(port, 60).await {
if wait_for_host_port(package_id, port, 60).await {
install_log(&format!(
"INSTALL REPAIR OK: {} — host port {} is listening after restart",
package_id, port
@@ -2084,31 +2223,6 @@ fn published_host_port(container_name: &str) -> Option<u16> {
})
}
async fn ensure_user_podman_socket() -> Result<()> {
let socket_path = "/run/user/1000/podman/podman.sock";
if tokio::fs::try_exists(socket_path).await.unwrap_or(false) {
return Ok(());
}
let status = tokio::process::Command::new("systemctl")
.args(["--user", "restart", "podman.socket"])
.status()
.await
.context("spawn systemctl --user restart podman.socket")?;
if !status.success() {
anyhow::bail!("systemctl --user restart podman.socket exited {status}");
}
for _ in 0..20 {
if tokio::fs::try_exists(socket_path).await.unwrap_or(false) {
return Ok(());
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
anyhow::bail!("podman socket {socket_path} did not appear after restart")
}
fn required_host_port(package_id: &str) -> Option<u16> {
match package_id {
"grafana" => Some(3000),
@@ -2118,17 +2232,21 @@ fn required_host_port(package_id: &str) -> Option<u16> {
"gitea" => Some(3001),
"nextcloud" => Some(8085),
"nginx-proxy-manager" => Some(8081),
"portainer" => Some(9000),
_ => None,
}
}
async fn wait_for_host_port(port: u16, timeout_secs: u64) -> bool {
async fn wait_for_host_port(package_id: &str, port: u16, timeout_secs: u64) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
loop {
if tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok()
{
let ready = match package_id {
"uptime-kuma" => http_host_port_ready(port, "/").await,
_ => tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok(),
};
if ready {
return true;
}
@@ -2140,6 +2258,36 @@ async fn wait_for_host_port(port: u16, timeout_secs: u64) -> bool {
}
}
async fn http_host_port_ready(port: u16, path: &str) -> bool {
let Ok(Ok(mut stream)) = tokio::time::timeout(
Duration::from_secs(3),
tokio::net::TcpStream::connect(("127.0.0.1", port)),
)
.await
else {
return false;
};
let request = format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n");
if stream.write_all(request.as_bytes()).await.is_err() {
return false;
}
let mut buf = [0u8; 128];
let Ok(Ok(n)) = tokio::time::timeout(Duration::from_secs(3), stream.read(&mut buf)).await
else {
return false;
};
if n == 0 {
return false;
}
let head = String::from_utf8_lossy(&buf[..n]);
head.starts_with("HTTP/1.1 2")
|| head.starts_with("HTTP/1.1 3")
|| head.starts_with("HTTP/1.0 2")
|| head.starts_with("HTTP/1.0 3")
}
/// Resolve the host gateway IP for --add-host flag.
/// Resolve the default gateway IP from the routing table for --add-host flag.
/// Explicit IP avoids issues with "host-gateway" in rootless Podman.
@@ -2235,6 +2383,18 @@ set -eu
conf=/var/lib/archipelago/bitcoin/bitcoin.conf
[ -f "$conf" ] || exit 0
changed=0
tmp=$(mktemp)
awk -F= '
/^(server|txindex|rpcbind|rpcallowip|rpcport|listen|bind|dbcache|rpcthreads|rpcworkqueue)=/ {
if (seen[$1]++) next
}
{ print }
' "$conf" > "$tmp"
if ! cmp -s "$conf" "$tmp"; then
cat "$tmp" > "$conf"
changed=1
fi
rm -f "$tmp"
ensure_line() {
line="$1"
key="${line%%=*}"
@@ -2246,6 +2406,8 @@ ensure_line() {
ensure_line server=1
ensure_line rpcallowip=0.0.0.0/0
ensure_line listen=1
ensure_line rpcthreads=16
ensure_line rpcworkqueue=256
[ "$changed" -eq 0 ] && exit 0
exit 2
"#;
@@ -2272,6 +2434,7 @@ fn should_try_orchestrator_install(package_id: &str, orchestrator_available: boo
fn orchestrator_install_app_id(package_id: &str) -> &str {
match package_id {
"electrs" | "mempool-electrs" => "electrumx",
"home-assistant" => "homeassistant",
_ => package_id,
}
}
@@ -2299,6 +2462,16 @@ fn uses_orchestrator_install_flow(package_id: &str) -> bool {
| "archy-btcpay-db"
| "archy-nbxplorer"
| "btcpay-server"
| "homeassistant"
| "home-assistant"
| "nextcloud"
| "vaultwarden"
| "jellyfin"
| "photoprism"
| "uptime-kuma"
| "gitea"
| "portainer"
| "meshtastic"
)
}
@@ -2336,6 +2509,16 @@ mod tests {
"archy-btcpay-db",
"archy-nbxplorer",
"btcpay-server",
"homeassistant",
"home-assistant",
"nextcloud",
"vaultwarden",
"jellyfin",
"photoprism",
"uptime-kuma",
"gitea",
"portainer",
"meshtastic",
] {
assert!(uses_orchestrator_install_flow(app));
assert!(should_try_orchestrator_install(app, true));
@@ -2364,6 +2547,10 @@ mod tests {
assert_eq!(orchestrator_install_app_id("bitcoin-core"), "bitcoin-core");
assert_eq!(orchestrator_install_app_id("electrs"), "electrumx");
assert_eq!(orchestrator_install_app_id("mempool-electrs"), "electrumx");
assert_eq!(
orchestrator_install_app_id("home-assistant"),
"homeassistant"
);
assert_eq!(orchestrator_install_app_id("lnd"), "lnd");
}
core/archipelago/src/api/rpc/package/runtime.rs
+667 -92
View File
@@ -2,15 +2,18 @@ use super::config::{
get_app_capabilities, get_containers_for_app, get_data_dirs_for_app, get_health_check_args,
get_memory_limit, is_valid_docker_image,
};
use super::dependencies::ordered_containers_for_start;
use super::dependencies::{ordered_containers_for_start, startup_order};
use super::install::install_log;
use super::validation::validate_app_id;
use crate::api::rpc::RpcHandler;
use crate::data_model::PackageState;
use anyhow::{Context, Result};
use archipelago_container::AppManifest;
use std::path::Path;
use std::process::Output;
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tracing::warn;
const PODMAN_CONTROL_TIMEOUT: Duration = Duration::from_secs(30);
@@ -53,7 +56,11 @@ impl RpcHandler {
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
validate_app_id(package_id)?;
let to_start = ordered_containers_for_start(package_id).await?;
let to_start = if self.orchestrator.is_some() && uses_single_orchestrator_app(package_id) {
vec![orchestrator_app_id(package_id).to_string()]
} else {
ordered_containers_for_start(package_id).await?
};
if to_start.is_empty() {
tracing::warn!("package.start {}: no containers found", package_id);
return Err(anyhow::anyhow!("No containers found for {}", package_id));
@@ -124,7 +131,16 @@ impl RpcHandler {
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
validate_app_id(package_id)?;
let containers = get_containers_for_app(package_id).await?;
let single_orchestrator_app =
self.orchestrator.is_some() && uses_single_orchestrator_app(package_id);
let mut containers = if single_orchestrator_app {
vec![orchestrator_app_id(package_id).to_string()]
} else {
get_containers_for_app(package_id).await?
};
if !single_orchestrator_app {
containers.reverse();
}
if containers.is_empty() {
tracing::warn!("package.stop {}: no containers found", package_id);
return Err(anyhow::anyhow!("No containers found for {}", package_id));
@@ -190,7 +206,13 @@ impl RpcHandler {
.ok_or_else(|| anyhow::anyhow!("Missing package id"))?;
validate_app_id(package_id)?;
let containers = get_containers_for_app(package_id).await?;
let single_orchestrator_app =
self.orchestrator.is_some() && uses_single_orchestrator_app(package_id);
let containers = if single_orchestrator_app {
vec![orchestrator_app_id(package_id).to_string()]
} else {
get_containers_for_app(package_id).await?
};
if containers.is_empty() {
tracing::warn!("package.restart {}: no containers found", package_id);
return Err(anyhow::anyhow!("No containers found for {}", package_id));
@@ -206,7 +228,11 @@ impl RpcHandler {
let package_id_owned = package_id.to_string();
let companion_app_id = package_id_owned.clone();
let to_restart = ordered_containers_for_start(package_id).await?;
let to_restart = if single_orchestrator_app {
vec![orchestrator_app_id(package_id).to_string()]
} else {
ordered_containers_for_start(package_id).await?
};
let state_manager = Arc::clone(&self.state_manager);
let orchestrator = self.orchestrator.clone();
let pre_state =
@@ -323,7 +349,9 @@ impl RpcHandler {
match rm_out {
Ok(o) if o.status.success() => removed += 1,
Ok(o) => {
// If normal rm fails (e.g., still running), force as fallback
// If normal rm fails (e.g., still running/stopping/removing),
// force with targeted cleanup fallbacks. This is deliberately
// container-scoped; never prune the store during uninstall.
let stderr = String::from_utf8_lossy(&o.stderr);
tracing::warn!(
"Uninstall {}: rm {} failed ({}), trying force",
@@ -331,28 +359,36 @@ impl RpcHandler {
name,
stderr.trim()
);
let force_rm = podman_control(&["rm", "-f", name]).await;
match force_rm {
Ok(o2) if o2.status.success() => removed += 1,
_ => {
let msg = format!("Failed to remove {}: {}", name, stderr.trim());
match force_remove_runtime_container(name).await {
Ok(()) => removed += 1,
Err(e) => {
let msg =
format!("Failed to remove {}: {}; {}", name, stderr.trim(), e);
tracing::error!("Uninstall {}: {}", package_id, msg);
errors.push(msg);
}
}
}
Err(e) => {
let msg = format!("Failed to remove {}: {}", name, e);
tracing::error!("Uninstall {}: {}", package_id, msg);
errors.push(msg);
}
Err(e) => match force_remove_runtime_container(name).await {
Ok(()) => removed += 1,
Err(force_err) => {
let msg = format!("Failed to remove {}: {}; {}", name, e, force_err);
tracing::error!("Uninstall {}: {}", package_id, msg);
errors.push(msg);
}
},
}
}
self.set_uninstall_stage(package_id, "Cleaning up volumes")
.await;
// Clean up dangling volumes associated with removed containers
let _ = podman_control(&["volume", "prune", "-f"]).await;
// Avoid global Podman volume prune on production nodes: store-wide
// Podman cleanup commands can block app health under load. App data is
// removed explicitly below when preserve_data=false.
tracing::info!(
package_id = %package_id,
"Skipping global podman volume prune during uninstall"
);
// Clean up app-specific networks (only if no other containers use them)
let app_networks: Vec<&str> = match package_id {
@@ -600,9 +636,25 @@ async fn do_package_start(to_start: &[String]) -> Result<()> {
if i > 0 {
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
if let Err(e) = ensure_startable_container_state(name).await {
tracing::error!(container = %name, error = %e, "container is not startable");
errors.push(format!("{}: {}", name, e));
continue;
}
match inspect_runtime_container_state(name).await {
Ok(Some(state)) if state == "running" => {
tracing::debug!(container = %name, "container already running during package start");
continue;
}
Ok(_) => {}
Err(e) => {
tracing::warn!(container = %name, error = %e, "failed to re-inspect before package start")
}
}
repair_before_package_start(name).await;
wait_before_package_start(name).await;
tracing::info!("Starting container: {}", name);
let out = podman_control(&["start", name])
let out = podman_start_container(name)
.await
.context(format!("Failed to exec podman start {}", name))?;
if !out.status.success() {
@@ -669,6 +721,7 @@ async fn do_orchestrator_package_start(
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
repair_before_package_start(name).await;
wait_before_package_start(name).await;
match orchestrator.start(name).await {
Ok(()) => wait_after_orchestrator_start(name).await,
Err(e) if is_unknown_app_id_error(&e) => {
@@ -681,10 +734,13 @@ async fn do_orchestrator_package_start(
}
}
}
if errors.is_empty() {
Ok(())
} else {
if !errors.is_empty() {
Err(anyhow::anyhow!("Start failed: {}", errors.join("; ")))
} else {
for name in to_start {
ensure_runtime_host_port_listener(name).await?;
}
Ok(())
}
}
@@ -703,6 +759,137 @@ async fn podman_control(args: &[&str]) -> Result<Output> {
podman_with_timeout(args, podman_control_timeout(args)).await
}
async fn force_remove_runtime_container(container_name: &str) -> Result<()> {
for args in [
vec!["rm", "-f", container_name],
vec!["rm", "-f", "--time", "0", container_name],
] {
let output = podman_control(&args).await?;
if output.status.success()
|| is_missing_container_error(&String::from_utf8_lossy(&output.stderr))
{
return Ok(());
}
}
let _ = podman_control(&["container", "cleanup", container_name]).await;
let output = podman_control(&["rm", "-f", container_name]).await?;
if output.status.success()
|| is_missing_container_error(&String::from_utf8_lossy(&output.stderr))
{
return Ok(());
}
Err(anyhow::anyhow!(
"force remove failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
))
}
async fn force_stop_runtime_container(container_name: &str) -> Result<()> {
for args in [
vec!["stop", "-t", "0", container_name],
vec!["kill", container_name],
] {
let output = podman_control(&args).await?;
if output.status.success()
|| is_missing_container_error(&String::from_utf8_lossy(&output.stderr))
{
return Ok(());
}
}
for _ in 0..15 {
match inspect_runtime_container_state(container_name).await? {
None => return Ok(()),
Some(state) if matches!(state.as_str(), "exited" | "stopped" | "configured") => {
return Ok(())
}
Some(_) => tokio::time::sleep(Duration::from_secs(2)).await,
}
}
Err(anyhow::anyhow!(
"container did not reach stopped state after force stop"
))
}
async fn ensure_startable_container_state(container_name: &str) -> Result<()> {
let Some(state) = inspect_runtime_container_state(container_name).await? else {
return Ok(());
};
match state.as_str() {
"configured" | "created" | "exited" | "stopped" | "running" | "paused" => Ok(()),
"removing" => {
wait_for_container_absent_or_startable(container_name, Duration::from_secs(60)).await
}
other => Err(anyhow::anyhow!(
"container is in unsupported state before start: {}",
other
)),
}
}
async fn wait_for_container_absent_or_startable(
container_name: &str,
timeout: Duration,
) -> Result<()> {
let deadline = std::time::Instant::now() + timeout;
loop {
match inspect_runtime_container_state(container_name).await? {
None => return Ok(()),
Some(state)
if matches!(
state.as_str(),
"configured" | "created" | "exited" | "stopped" | "running" | "paused"
) =>
{
return Ok(())
}
Some(state) if state == "removing" && std::time::Instant::now() < deadline => {
tokio::time::sleep(Duration::from_secs(2)).await;
}
Some(state) if state == "removing" => {
force_remove_runtime_container(container_name).await?;
return Ok(());
}
Some(state) => {
return Err(anyhow::anyhow!(
"container is in unsupported state before start: {}",
state
))
}
}
}
}
async fn inspect_runtime_container_state(container_name: &str) -> Result<Option<String>> {
let output = podman_with_timeout(
&["inspect", container_name, "--format", "{{.State.Status}}"],
Duration::from_secs(10),
)
.await?;
if output.status.success() {
return Ok(Some(
String::from_utf8_lossy(&output.stdout).trim().to_string(),
));
}
let stderr = String::from_utf8_lossy(&output.stderr);
if is_missing_container_error(&stderr) {
Ok(None)
} else {
Err(anyhow::anyhow!("inspect failed: {}", stderr.trim()))
}
}
fn is_missing_container_error(stderr: &str) -> bool {
stderr.contains("no such container")
|| stderr.contains("no container with name")
|| stderr.contains("does not exist")
|| stderr.contains("not found")
}
fn podman_control_timeout(args: &[&str]) -> Duration {
args.windows(2)
.find_map(|pair| {
@@ -714,6 +901,13 @@ fn podman_control_timeout(args: &[&str]) -> Duration {
.unwrap_or(PODMAN_CONTROL_TIMEOUT)
}
fn podman_start_timeout(container_name: &str) -> Duration {
match container_name {
"immich_server" | "netbird-server" => Duration::from_secs(120),
_ => PODMAN_CONTROL_TIMEOUT,
}
}
async fn podman_with_timeout(args: &[&str], timeout: Duration) -> Result<Output> {
let mut cmd = tokio::process::Command::new("podman");
cmd.args(args);
@@ -732,12 +926,48 @@ async fn command_with_timeout(
.with_context(|| format!("Failed to exec {}", description))
}
async fn podman_start_container(container_name: &str) -> Result<Output> {
if !runtime_host_ports(container_name).is_empty() {
let mut cmd = tokio::process::Command::new("systemd-run");
cmd.args([
"--user",
"--scope",
"--quiet",
"--collect",
"podman",
"start",
])
.arg(container_name);
let scoped = command_with_timeout(
cmd,
podman_start_timeout(container_name),
&format!("systemd-run --user --scope podman start {container_name}"),
)
.await;
if scoped.as_ref().is_ok_and(|out| out.status.success()) {
return scoped;
}
if let Err(err) = &scoped {
tracing::warn!(
container = %container_name,
error = %err,
"scoped podman start failed; falling back to direct podman start"
);
}
}
podman_with_timeout(
&["start", container_name],
podman_start_timeout(container_name),
)
.await
}
async fn do_orchestrator_package_stop(
orchestrator: &dyn crate::container::traits::ContainerOrchestrator,
containers: &[String],
) -> Result<()> {
let mut errors = Vec::new();
for name in containers.iter().rev() {
for name in containers {
match orchestrator.stop(name).await {
Ok(()) => {}
Err(e) if is_unknown_app_id_error(&e) => {
@@ -758,6 +988,44 @@ async fn do_orchestrator_package_stop(
}
}
fn orchestrator_app_id(package_id: &str) -> &str {
match package_id {
"electrs" | "mempool-electrs" => "electrumx",
"home-assistant" => "homeassistant",
_ => package_id,
}
}
fn uses_single_orchestrator_app(package_id: &str) -> bool {
startup_order(package_id).is_empty()
&& matches!(
package_id,
"bitcoin-ui"
| "electrs-ui"
| "lnd-ui"
| "bitcoin-core"
| "bitcoin-knots"
| "lnd"
| "fedimint"
| "fedimint-gateway"
| "filebrowser"
| "electrumx"
| "electrs"
| "mempool-electrs"
| "homeassistant"
| "home-assistant"
| "nextcloud"
| "vaultwarden"
| "jellyfin"
| "photoprism"
| "uptime-kuma"
| "gitea"
| "portainer"
| "meshtastic"
| "botfights"
)
}
async fn do_orchestrator_package_restart(
orchestrator: &dyn crate::container::traits::ContainerOrchestrator,
to_restart: &[String],
@@ -770,22 +1038,72 @@ async fn do_orchestrator_package_restart(
async fn do_package_stop(containers: &[String]) -> Result<()> {
let mut errors = Vec::new();
for name in containers {
match inspect_runtime_container_state(name).await {
Ok(None) => {
tracing::debug!(container = %name, "container already absent during stop");
continue;
}
Ok(Some(state)) if matches!(state.as_str(), "exited" | "stopped" | "configured") => {
tracing::debug!(container = %name, state = %state, "container already stopped");
continue;
}
Ok(Some(_)) => {}
Err(e) => {
tracing::warn!(container = %name, error = %e, "failed to inspect before stop")
}
}
tracing::info!(
"Stopping container: {} (timeout: {}s)",
name,
stop_timeout_secs(name)
);
let out = podman_control(&["stop", "-t", stop_timeout_secs(name), name])
.await
.context(format!("Failed to exec podman stop {}", name))?;
let out = match podman_control(&["stop", "-t", stop_timeout_secs(name), name]).await {
Ok(out) => out,
Err(e) => {
tracing::warn!(
container = %name,
error = %e,
"podman stop errored, trying force stop"
);
match force_stop_runtime_container(name).await {
Ok(()) => {
tracing::info!(container = %name, "force stop after stop error succeeded");
continue;
}
Err(force_err) => {
tracing::error!(
"Failed to stop {}: {}; force stop failed: {}",
name,
e,
force_err
);
errors.push(format!("{}: {}; force stop failed: {}", name, e, force_err));
continue;
}
}
}
};
if !out.status.success() {
let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
if is_missing_companion_ok(name, &stderr) {
tracing::debug!(container = %name, "companion already absent during stop");
continue;
}
tracing::error!("Failed to stop {}: {}", name, stderr);
errors.push(format!("{}: {}", name, stderr));
tracing::warn!("Failed to stop {}: {}, trying force stop", name, stderr);
match force_stop_runtime_container(name).await {
Ok(()) => {
tracing::info!(container = %name, "force stop after stop failure succeeded")
}
Err(e) => {
tracing::error!(
"Failed to stop {}: {}; force stop failed: {}",
name,
stderr,
e
);
errors.push(format!("{}: {}; force stop failed: {}", name, stderr, e));
}
}
}
}
if !errors.is_empty() {
@@ -801,6 +1119,7 @@ async fn do_package_restart(containers: &[String]) -> Result<()> {
for name in containers {
tracing::info!("Restarting container: {}", name);
repair_before_package_start(name).await;
wait_before_package_start(name).await;
let out = podman_control(&["restart", "-t", stop_timeout_secs(name), name])
.await
.context(format!("Failed to exec podman restart {}", name))?;
@@ -818,7 +1137,8 @@ async fn do_package_restart(containers: &[String]) -> Result<()> {
);
// Fallback: stop then start
let _ = podman_control(&["stop", "-t", stop_timeout_secs(name), name]).await;
let start_out = podman_control(&["start", name])
wait_before_package_start(name).await;
let start_out = podman_start_container(name)
.await
.context(format!("Failed to exec podman start {}", name))?;
if !start_out.status.success() {
@@ -855,22 +1175,158 @@ fn is_unknown_app_id_error(err: &anyhow::Error) -> bool {
async fn repair_before_package_start(container_name: &str) {
match container_name {
"btcpay-server" | "archy-nbxplorer" => repair_btcpay_dirs().await,
"indeedhub-postgres" | "indeedhub-redis" | "indeedhub-minio" | "indeedhub-relay"
| "indeedhub-api" | "indeedhub-ffmpeg" | "indeedhub" => repair_indeedhub_network().await,
"indeedhub" => repair_indeedhub_network().await,
"immich_server" => repair_immich_dirs().await,
"netbird" => repair_netbird_network().await,
"grafana" => {
repair_grafana_dirs().await;
cleanup_stale_pasta_port("3000").await;
}
"vaultwarden" => cleanup_stale_pasta_port("8082").await,
"homeassistant" | "home-assistant" => cleanup_stale_pasta_port("8123").await,
"nextcloud" => {
repair_nextcloud_dirs().await;
cleanup_stale_pasta_port("8085").await;
}
"nginx-proxy-manager" => repair_nginx_proxy_manager_container().await,
"gitea" => cleanup_gitea_stale_ports().await,
_ => {}
}
cleanup_runtime_host_ports(container_name).await;
}
async fn wait_before_package_start(container_name: &str) {
match container_name {
"indeedhub" => wait_for_indeedhub_dependency_dns().await,
"immich_server" => wait_for_immich_dependencies().await,
"netbird" => wait_for_netbird_dependency_dns().await,
_ => {}
}
}
async fn wait_for_indeedhub_dependency_dns() {
for _ in 0..30 {
if indeedhub_frontend_dependencies_running().await {
super::stacks::repair_indeedhub_network_aliases().await;
break;
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
for _ in 0..30 {
let ready = podman_with_timeout(
&["exec", "indeedhub-minio", "getent", "hosts", "minio"],
Duration::from_secs(5),
)
.await
.map(|out| out.status.success())
.unwrap_or(false);
if ready {
return;
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
}
async fn indeedhub_frontend_dependencies_running() -> bool {
for container in ["indeedhub-minio", "indeedhub-redis", "indeedhub-api"] {
if !container_is_running(container).await {
return false;
}
}
true
}
async fn container_is_running(container: &str) -> bool {
let Ok(output) = podman_with_timeout(
&["inspect", container, "--format", "{{.State.Status}}"],
Duration::from_secs(5),
)
.await
else {
return false;
};
output.status.success() && String::from_utf8_lossy(&output.stdout).trim() == "running"
}
async fn wait_for_netbird_dependency_dns() {
for _ in 0..30 {
if container_is_running("netbird-server").await
&& container_is_running("netbird-dashboard").await
{
super::stacks::repair_netbird_network_aliases().await;
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
return;
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
}
async fn wait_for_immich_dependencies() {
for _ in 0..60 {
if immich_postgres_ready().await && immich_redis_ready().await {
return;
}
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
}
async fn immich_postgres_ready() -> bool {
if container_health_is_healthy("immich_postgres").await {
return true;
}
let Ok(output) = podman_with_timeout(
&[
"exec",
"immich_postgres",
"pg_isready",
"-U",
"postgres",
"-d",
"immich",
],
Duration::from_secs(5),
)
.await
else {
return false;
};
output.status.success()
}
async fn immich_redis_ready() -> bool {
if container_health_is_healthy("immich_redis").await {
return true;
}
let Ok(output) = podman_with_timeout(
&["exec", "immich_redis", "valkey-cli", "ping"],
Duration::from_secs(5),
)
.await
else {
return false;
};
output.status.success() && String::from_utf8_lossy(&output.stdout).contains("PONG")
}
async fn container_health_is_healthy(container: &str) -> bool {
let Ok(output) = podman_with_timeout(
&[
"inspect",
container,
"--format",
"{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}",
],
Duration::from_secs(5),
)
.await
else {
return false;
};
output.status.success() && String::from_utf8_lossy(&output.stdout).trim() == "healthy"
}
async fn repair_netbird_network() {
super::stacks::repair_netbird_network_aliases().await;
}
async fn repair_nginx_proxy_manager_container() {
@@ -1009,11 +1465,11 @@ async fn recreate_nginx_proxy_manager_container() -> Result<()> {
}
async fn ensure_runtime_host_port_listener(container_name: &str) -> Result<()> {
let Some(port) = runtime_required_host_port(container_name) else {
let Some(port) = runtime_host_ports(container_name).into_iter().next() else {
return Ok(());
};
if wait_for_runtime_host_port(port, 10).await {
if wait_for_runtime_host_port(container_name, port, 10).await {
return Ok(());
}
@@ -1035,7 +1491,7 @@ async fn ensure_runtime_host_port_listener(container_name: &str) -> Result<()> {
));
}
if wait_for_runtime_host_port(port, 60).await {
if wait_for_runtime_host_port(container_name, port, 60).await {
install_log(&format!(
"START REPAIR OK: {} — host port {} is listening after restart",
container_name, port
@@ -1051,27 +1507,99 @@ async fn ensure_runtime_host_port_listener(container_name: &str) -> Result<()> {
))
}
fn runtime_required_host_port(container_name: &str) -> Option<u16> {
match container_name {
"grafana" => Some(3000),
"homeassistant" | "home-assistant" => Some(8123),
"searxng" => Some(8888),
"uptime-kuma" => Some(3002),
"vaultwarden" => Some(8082),
"gitea" => Some(3001),
"nextcloud" => Some(8085),
"nginx-proxy-manager" => Some(8081),
_ => None,
fn runtime_host_ports(container_name: &str) -> Vec<u16> {
let manifest_ports = manifest_host_ports(container_name);
if !manifest_ports.is_empty() {
return with_legacy_extra_ports(container_name, manifest_ports);
}
let ports = match container_name {
"grafana" => vec![3000],
"homeassistant" | "home-assistant" => vec![8123],
"jellyfin" => vec![8096],
"searxng" => vec![8888],
"uptime-kuma" => vec![3002],
"vaultwarden" => vec![8082],
"gitea" => vec![3001, 2222, 3000],
"nextcloud" => vec![8085],
"nginx-proxy-manager" => vec![8081, 8084, 8444],
_ => Vec::new(),
};
ports
}
fn with_legacy_extra_ports(container_name: &str, mut ports: Vec<u16>) -> Vec<u16> {
if container_name == "gitea" && !ports.contains(&3000) {
ports.push(3000);
}
if container_name == "nginx-proxy-manager" {
for port in [8084, 8444] {
if !ports.contains(&port) {
ports.push(port);
}
}
}
ports
}
fn manifest_host_ports(container_name: &str) -> Vec<u16> {
for apps_dir in manifest_apps_dirs() {
let Ok(entries) = std::fs::read_dir(apps_dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path().join("manifest.yml");
let Ok(contents) = std::fs::read_to_string(&path) else {
continue;
};
let Ok(manifest) = AppManifest::parse(&contents) else {
continue;
};
if manifest_container_name(&manifest) == container_name {
return manifest.app.ports.iter().map(|p| p.host).collect();
}
}
}
Vec::new()
}
fn manifest_apps_dirs() -> Vec<std::path::PathBuf> {
let mut dirs = Vec::new();
if let Ok(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
dirs.push(Path::new(&manifest_dir).join("../../apps"));
}
dirs.extend([
Path::new("apps").to_path_buf(),
Path::new("/opt/archipelago/apps").to_path_buf(),
Path::new("/opt/archipelago/web-ui/archipelago-runtime/apps").to_path_buf(),
]);
dirs
}
fn manifest_container_name(manifest: &AppManifest) -> String {
if let Some(v) = manifest.app.extensions.get("container_name") {
if let Some(s) = v.as_str() {
if !s.is_empty() {
return s.to_string();
}
}
}
match manifest.app.id.as_str() {
"bitcoin-ui" | "electrs-ui" | "lnd-ui" => format!("archy-{}", manifest.app.id),
id => id.to_string(),
}
}
async fn wait_for_runtime_host_port(port: u16, timeout_secs: u64) -> bool {
async fn wait_for_runtime_host_port(container_name: &str, port: u16, timeout_secs: u64) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs);
loop {
if tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok()
{
let ready = match container_name {
"uptime-kuma" => http_host_port_ready(port, "/").await,
_ => tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok(),
};
if ready {
return true;
}
@@ -1083,6 +1611,37 @@ async fn wait_for_runtime_host_port(port: u16, timeout_secs: u64) -> bool {
}
}
async fn http_host_port_ready(port: u16, path: &str) -> bool {
let Ok(Ok(mut stream)) = tokio::time::timeout(
std::time::Duration::from_secs(3),
tokio::net::TcpStream::connect(("127.0.0.1", port)),
)
.await
else {
return false;
};
let request = format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n");
if stream.write_all(request.as_bytes()).await.is_err() {
return false;
}
let mut buf = [0u8; 128];
let Ok(Ok(n)) =
tokio::time::timeout(std::time::Duration::from_secs(3), stream.read(&mut buf)).await
else {
return false;
};
if n == 0 {
return false;
}
let head = String::from_utf8_lossy(&buf[..n]);
head.starts_with("HTTP/1.1 2")
|| head.starts_with("HTTP/1.1 3")
|| head.starts_with("HTTP/1.0 2")
|| head.starts_with("HTTP/1.0 3")
}
async fn repair_btcpay_dirs() {
let _ = tokio::process::Command::new("sudo")
.args([
@@ -1157,6 +1716,27 @@ async fn repair_nextcloud_dirs() {
}
}
async fn repair_immich_dirs() {
let _ = tokio::process::Command::new("sudo")
.args(["mkdir", "-p", "/var/lib/archipelago/immich"])
.output()
.await;
let podman_chown = podman_control(&[
"unshare",
"chown",
"-R",
"0:0",
"/var/lib/archipelago/immich",
])
.await;
if !podman_chown.as_ref().is_ok_and(|o| o.status.success()) {
let _ = tokio::process::Command::new("sudo")
.args(["chown", "-R", "1000:1000", "/var/lib/archipelago/immich"])
.output()
.await;
}
}
async fn repair_btcpay_database_password() {
let Ok(db_pass) =
tokio::fs::read_to_string("/var/lib/archipelago/secrets/btcpay-db-password").await
@@ -1205,25 +1785,28 @@ async fn cleanup_start_conflict(container_name: &str, stderr: &str) {
return;
}
if container_name == "gitea" {
cleanup_gitea_stale_ports().await;
let ports = runtime_host_ports(container_name);
if !ports.is_empty() {
cleanup_ports(&ports).await;
return;
}
}
match container_name {
"grafana" => cleanup_stale_pasta_port("3000").await,
"homeassistant" | "home-assistant" => cleanup_stale_pasta_port("8123").await,
"vaultwarden" => cleanup_stale_pasta_port("8082").await,
"nextcloud" => cleanup_stale_pasta_port("8085").await,
"nginx-proxy-manager" => cleanup_nginx_proxy_manager_ports().await,
_ => {}
async fn cleanup_runtime_host_ports(container_name: &str) {
let ports = runtime_host_ports(container_name);
if !ports.is_empty() {
cleanup_ports(&ports).await;
}
}
async fn cleanup_nginx_proxy_manager_ports() {
cleanup_stale_pasta_port("8081").await;
cleanup_stale_pasta_port("8084").await;
cleanup_stale_pasta_port("8444").await;
cleanup_ports(&[8081, 8084, 8444]).await;
}
async fn cleanup_ports(ports: &[u16]) {
for port in ports {
cleanup_stale_pasta_port(&port.to_string()).await;
}
}
async fn cleanup_stale_pasta_port(port: &str) {
@@ -1249,31 +1832,6 @@ async fn cleanup_stale_pasta_port(port: &str) {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
async fn cleanup_gitea_stale_ports() {
for port in ["3001", "2222", "3000"] {
let kill_listener = format!(
"ss -ltnp 'sport = :{}' 2>/dev/null | sed -n 's/.*pid=\\([0-9]*\\).*/\\1/p' | xargs -r kill 2>/dev/null || true",
port
);
let _ = tokio::process::Command::new("sh")
.args(["-c", &kill_listener])
.output()
.await;
let pattern = format!("pasta.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
let pattern = format!("rootlessport.*{}", port);
let _ = tokio::process::Command::new("pkill")
.args(["-f", &pattern])
.output()
.await;
}
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
pub(super) fn is_missing_companion_ok(name: &str, stderr: &str) -> bool {
matches!(
name,
@@ -1352,3 +1910,20 @@ pub(super) fn orchestrator_uninstall_app_ids(package_id: &str) -> Vec<String> {
_ => vec![package_id.to_string()],
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn runtime_host_ports_are_manifest_derived_for_public_apps() {
assert_eq!(runtime_host_ports("photoprism"), vec![2342]);
assert_eq!(runtime_host_ports("jellyfin"), vec![8096]);
assert_eq!(runtime_host_ports("uptime-kuma"), vec![3002]);
}
#[test]
fn runtime_host_ports_preserve_legacy_extra_ports() {
assert_eq!(runtime_host_ports("gitea"), vec![3001, 2222, 3000]);
}
}
core/archipelago/src/api/rpc/package/stacks.rs
+746 -1209
View File
File diff suppressed because it is too large Load Diff
core/archipelago/src/api/rpc/package/update.rs
+32 -14
View File
@@ -16,6 +16,8 @@ use anyhow::{Context, Result};
use tokio::io::{AsyncBufReadExt, BufReader};
use tracing::{error, info, warn};
const PODMAN_UPDATE_PULL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
impl RpcHandler {
/// Update a package to the version pinned in image-versions.sh.
/// This is a manual operation — the user clicks "Update" in the UI.
@@ -327,6 +329,7 @@ impl RpcHandler {
if archipelago_container::image_uses_insecure_registry(image) {
cmd.arg("--tls-verify=false");
}
cmd.kill_on_drop(true);
let mut child = cmd
.arg(image)
.stdout(std::process::Stdio::piped())
@@ -334,23 +337,38 @@ impl RpcHandler {
.spawn()
.context("Failed to start image pull")?;
if let Some(stderr) = child.stderr.take() {
let progress_task = if let Some(stderr) = child.stderr.take() {
let reader = BufReader::new(stderr);
let mut lines = reader.lines();
let pkg_id = package_id.to_string();
let state_mgr = self.state_manager.clone();
while let Ok(Some(line)) = lines.next_line().await {
if let Some((downloaded, total)) = parse_pull_progress(&line) {
Self::update_install_progress(&state_mgr, &pkg_id, downloaded, total).await;
Some(tokio::spawn(async move {
while let Ok(Some(line)) = lines.next_line().await {
if let Some((downloaded, total)) = parse_pull_progress(&line) {
Self::update_install_progress(&state_mgr, &pkg_id, downloaded, total).await;
}
}
}
}
}))
} else {
None
};
let status = child
.wait()
.await
.context("Failed to wait for image pull")?;
let status = match tokio::time::timeout(PODMAN_UPDATE_PULL_TIMEOUT, child.wait()).await {
Ok(result) => result.context("Failed to wait for image pull")?,
Err(_) => {
let _ = child.kill().await;
return Err(anyhow::anyhow!(
"podman pull {} timed out after {}s",
image,
PODMAN_UPDATE_PULL_TIMEOUT.as_secs()
));
}
};
if let Some(task) = progress_task {
let _ = task.await;
}
if !status.success() {
return Err(anyhow::anyhow!("podman pull {} failed", image));
}
@@ -430,7 +448,6 @@ fn should_try_orchestrator_update(package_id: &str, orchestrator_available: bool
fn orchestrator_update_app_id(package_id: &str) -> &str {
match package_id {
"bitcoin-knots" => "bitcoin-core",
"electrs" | "mempool-electrs" => "electrumx",
_ => package_id,
}
@@ -459,8 +476,8 @@ fn candidate_app_ids_for_container(container_name: &str) -> Vec<String> {
match container_name {
"bitcoin-knots" | "bitcoin-core" => {
push("bitcoin-core");
push("bitcoin-knots");
push("bitcoin-core");
}
"archy-bitcoin-ui" => push("bitcoin-ui"),
"archy-lnd-ui" => push("lnd-ui"),
@@ -525,7 +542,7 @@ mod tests {
fn container_name_candidates_cover_common_aliases() {
assert_eq!(
candidate_app_ids_for_container("bitcoin-knots"),
vec!["bitcoin-core", "bitcoin-knots"]
vec!["bitcoin-knots", "bitcoin-core"]
);
assert_eq!(
candidate_app_ids_for_container("archy-bitcoin-ui"),
@@ -543,7 +560,8 @@ mod tests {
#[test]
fn update_aliases_map_to_manifest_app_ids() {
assert_eq!(orchestrator_update_app_id("bitcoin-knots"), "bitcoin-core");
assert_eq!(orchestrator_update_app_id("bitcoin-knots"), "bitcoin-knots");
assert_eq!(orchestrator_update_app_id("bitcoin-core"), "bitcoin-core");
assert_eq!(orchestrator_update_app_id("electrs"), "electrumx");
assert_eq!(orchestrator_update_app_id("mempool-electrs"), "electrumx");
assert_eq!(orchestrator_update_app_id("fedimint"), "fedimint");