fix: harden node upgrades and prepare 1.9.0-alpha

This commit is contained in:
archipelago
2026-10-05 12:43:49 -04:00
parent 138a541d01
commit daac47cac4
129 changed files with 9910 additions and 794 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "archipelago"
version = "1.8.22-alpha"
version = "1.9.0-alpha"
edition = "2021"
license.workspace = true
description = "Archipelago Bitcoin Node OS - Native backend"
+1 -1
View File
@@ -136,7 +136,7 @@ impl RpcHandler {
/// ~30% of UI calls error out even though the node is perfectly healthy.
/// With retry + backoff, the UI sees a uniform slow-but-successful
/// response instead of intermittent failures.
async fn bitcoin_rpc_call<T: serde::de::DeserializeOwned>(
pub(in crate::api::rpc) async fn bitcoin_rpc_call<T: serde::de::DeserializeOwned>(
&self,
client: &reqwest::Client,
method: &str,
+36 -29
View File
@@ -73,6 +73,34 @@ fn paid_content_response(bytes: &[u8], mime: &str, paid_sats: u64) -> serde_json
})
}
// Resolve known purchases BEFORE any mint/spend. Missing bytes or an unreadable
// index require recovery; neither is authorization to charge the buyer again.
async fn existing_paid_content(
data_dir: &std::path::Path,
onion: &str,
content_id: &str,
filename: Option<&str>,
) -> Result<Option<serde_json::Value>> {
let owned = crate::content_owned::list_owned_checked(data_dir)
.await
.context("Could not verify previous purchases; no new payment was sent")?;
let Some(item) = owned.iter().find(|o| {
o.onion == onion
&& (o.content_id == content_id
|| filename.is_some_and(|f| {
!f.is_empty() && o.filename.trim_start_matches('/') == f.trim_start_matches('/')
}))
}) else {
return Ok(None);
};
let (mime, bytes) = crate::content_owned::read_owned(data_dir, &item.onion, &item.content_id)
.await.context("This purchase is recorded, but its cached file is unavailable. No new payment was sent. Restore the cached file or contact the seller.")?;
let mut response = paid_content_response(&bytes, &mime, 0);
response["already_owned"] = serde_json::json!(true);
response["filename"] = serde_json::json!(item.filename);
Ok(Some(response))
}
// Updated clients open the persisted file through the Range-capable HTTP
// endpoint. Avoid putting two base64 copies of a large video in a JSON reply.
// Keep older clients compatible until both sides have upgraded.
@@ -517,36 +545,15 @@ impl RpcHandler {
// by exact (onion, content_id) and by (onion, filename) — the latter
// catches duplicate ids pointing at the same file on the same
// seller. The owned copy is served from the local cache instead.
if let Some(cached) = existing_paid_content(
&self.config.data_dir,
onion,
content_id,
params.get("filename").and_then(|v| v.as_str()),
)
.await?
{
let filename = params.get("filename").and_then(|v| v.as_str());
let owned = crate::content_owned::list_owned(&self.config.data_dir).await;
let already = owned.iter().find(|o| {
o.onion == onion
&& (o.content_id == content_id
|| filename.is_some_and(|f| {
!f.is_empty()
&& o.filename.trim_start_matches('/') == f.trim_start_matches('/')
}))
});
if let Some(o) = already {
tracing::info!(
onion,
content_id,
owned_as = %o.content_id,
"paid download: already owned — serving cached copy, NOT paying again"
);
if let Some((mime, bytes)) =
crate::content_owned::read_owned(&self.config.data_dir, &o.onion, &o.content_id)
.await
{
let mut result = paid_content_response(&bytes, &mime, 0);
result["already_owned"] = serde_json::json!(true);
result["filename"] = serde_json::json!(o.filename);
return Ok(result);
}
// Cache record exists but bytes are gone — fall through and
// repurchase rather than stranding the user.
}
return Ok(cached);
}
// `method` pins the backend the user confirmed in the UI ("cashu" |
@@ -71,3 +71,68 @@ fn seller_errors_are_bounded_printable_and_identified_as_peer_text() {
);
}
}
#[tokio::test]
async fn known_purchase_never_becomes_a_new_spend_when_cache_or_index_is_unavailable() {
let dir = tempfile::tempdir().unwrap();
assert!(
existing_paid_content(dir.path(), "seller.onion", "id", None)
.await
.unwrap()
.is_none()
);
crate::content_owned::record_purchase(
dir.path(),
"seller.onion",
"id",
"file.txt",
"text/plain",
b"paid",
1,
"cashu",
"now",
)
.await
.unwrap();
for (id, filename) in [("id", None), ("duplicate-id", Some("/file.txt"))] {
let cached = existing_paid_content(dir.path(), "seller.onion", id, filename)
.await
.unwrap()
.unwrap();
assert_eq!(cached["paid_sats"], 0);
assert_eq!(cached["already_owned"], true);
assert_eq!(cached["data"], "cGFpZA==");
}
assert!(
existing_paid_content(dir.path(), "different.onion", "id", None)
.await
.unwrap()
.is_none()
);
tokio::fs::remove_file(dir.path().join("purchased-content/seller.onion/id"))
.await
.unwrap();
assert!(
existing_paid_content(dir.path(), "seller.onion", "id", None)
.await
.unwrap_err()
.to_string()
.contains("No new payment")
);
tokio::fs::write(dir.path().join("purchased-content/owned.json"), b"damaged")
.await
.unwrap();
assert!(
existing_paid_content(dir.path(), "seller.onion", "other-id", None)
.await
.unwrap_err()
.to_string()
.contains("no new payment")
);
assert_eq!(
tokio::fs::read(dir.path().join("purchased-content/owned.json"))
.await
.unwrap(),
b"damaged"
);
}
@@ -132,6 +132,9 @@ impl RpcHandler {
"lnd.newaddress" => self.handle_lnd_newaddress().await,
"lnd.sendcoins" => self.handle_lnd_sendcoins(params).await,
"lnd.estimatefee" => self.handle_lnd_estimatefee(params).await,
"lnd.bump-quote" => self.handle_lnd_bump_quote(params).await,
"lnd.bump-submit" => self.handle_lnd_bump_submit(params).await,
"lnd.bump-status" => self.handle_lnd_bump_status(params).await,
"lnd.createinvoice" => self.handle_lnd_createinvoice(params).await,
"lnd.invoicestatus" => self.handle_lnd_invoicestatus(params).await,
"lnd.payinvoice" => self.handle_lnd_payinvoice(params).await,
+10 -25
View File
@@ -272,28 +272,8 @@ impl RpcHandler {
.and_then(|v| v.as_bool())
.unwrap_or(false);
// Fee control: either a confirmation target or an explicit fee rate
let target_conf = params.get("target_conf").and_then(|v| v.as_i64());
let sat_per_vbyte = params.get("sat_per_vbyte").and_then(|v| v.as_i64());
if target_conf.is_some() && sat_per_vbyte.is_some() {
return Err(anyhow::anyhow!(
"Invalid fee parameters: specify either target_conf or sat_per_vbyte, not both"
));
}
if let Some(tc) = target_conf {
if !(1..=1008).contains(&tc) {
return Err(anyhow::anyhow!(
"Invalid target_conf: must be between 1 and 1008 blocks"
));
}
}
if let Some(rate) = sat_per_vbyte {
if !(1..=5000).contains(&rate) {
return Err(anyhow::anyhow!(
"Invalid sat_per_vbyte: must be between 1 and 5000"
));
}
}
// Omitted fees target the next block; explicit slower/custom choices win.
let (target_conf, sat_per_vbyte) = super::fee_policy::fee_options(&params)?;
info!(
peer = pubkey,
@@ -574,7 +554,7 @@ impl RpcHandler {
/// LND's CloseChannel REST endpoint takes fee selection as query parameters.
/// With neither parameter LND uses a lax target; keep legacy clients on our
/// explicit Standard target rather than silently accepting that default.
/// explicit next-block target rather than silently accepting that default.
fn close_channel_fee_query(params: &serde_json::Value) -> Result<Vec<(&'static str, String)>> {
let force = match params.get("force") {
None | Some(serde_json::Value::Null) => false,
@@ -609,7 +589,12 @@ fn close_channel_fee_query(params: &serde_json::Value) -> Result<Vec<(&'static s
if let Some(rate) = rate {
query.push(("sat_per_vbyte", rate.to_string()));
} else {
query.push(("target_conf", target.unwrap_or(6).to_string()));
query.push((
"target_conf",
target
.unwrap_or(super::fee_policy::DEFAULT_TARGET as u64)
.to_string(),
));
}
}
Ok(query)
@@ -645,7 +630,7 @@ mod close_fee_tests {
}
assert_eq!(
close_channel_fee_query(&serde_json::json!({})).unwrap(),
vec![("force", "false".into()), ("target_conf", "6".into())]
vec![("force", "false".into()), ("target_conf", "1".into())]
);
assert_eq!(
close_channel_fee_query(&serde_json::json!({"force":true})).unwrap(),
@@ -0,0 +1,835 @@
//! WalletKit BumpFee is CPFP for new wallet outputs, RBF only for sweeper inputs.
//! Never feed an ordinary payment input to it and call that a replacement.
use super::LND_REST_BASE_URL;
use crate::api::rpc::RpcHandler;
use anyhow::{bail, ensure, Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::{collections::HashMap, path::Path, sync::LazyLock};
use tokio::{io::AsyncWriteExt, sync::Mutex};
static QUOTES: LazyLock<Mutex<HashMap<String, Quote>>> = LazyLock::new(Default::default);
// Serialize check/register/persist across dashboard clients. The create_new receipt
// additionally survives process restarts and prevents retries of ambiguous results.
static SUBMIT: Mutex<()> = Mutex::const_new(());
const QUOTE_SECONDS: u64 = 60;
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
struct Plan {
txid: String,
method: String,
input_txid: String,
input_index: u32,
parent_txid: String,
recipient_sats: u64,
rate_sat_vb: u64,
current_fee_sats: u64,
additional_fee_sats: u64,
total_fee_sats: u64,
budget_sats: u64,
input_sats: u64,
parent_vsize: u64,
sweep_vsize_bound: u64,
tip: String,
}
#[derive(Clone, Serialize, Deserialize)]
struct Quote {
quote_id: String,
expires_at: u64,
custom_rate: Option<u64>,
#[serde(flatten)]
plan: Plan,
}
#[derive(Serialize, Deserialize)]
struct Operation {
quote: Quote,
status: String,
message: String,
}
fn now() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn number(v: &Value) -> Result<u64> {
v.as_u64()
.or_else(|| v.as_str().and_then(|s| s.parse().ok()))
.context("Missing or invalid wallet amount")
}
fn txid_param(p: &Value) -> Result<String> {
let s = p["txid"].as_str().context("Missing transaction ID")?;
ensure!(
s.len() == 64 && s.bytes().all(|c| c.is_ascii_hexdigit()),
"Invalid transaction ID"
);
Ok(s.to_ascii_lowercase())
}
fn btc_sats(v: &Value) -> Result<u64> {
let n = v.as_f64().context("Missing Bitcoin fee")? * 100_000_000.0;
ensure!(
n.is_finite() && n >= 0.0 && n <= 2_100_000_000_000_000.0,
"Invalid Bitcoin fee"
);
Ok(n.round() as u64)
}
fn outpoint_matches(v: &Value, txid: &str, index: u32) -> bool {
v["txid_str"].as_str() == Some(txid) && v["output_index"].as_u64() == Some(index as u64)
}
fn array<'a>(v: &'a Value, key: &str) -> Result<&'a Vec<Value>> {
v[key]
.as_array()
.with_context(|| format!("Missing wallet field: {key}"))
}
fn sweep_size(output: &Value) -> Result<u64> {
// One native input, one wallet taproot output, including signature rounding.
match output["output_type"].as_str() {
Some("SCRIPT_TYPE_WITNESS_V1_TAPROOT") => Ok(112),
Some("SCRIPT_TYPE_WITNESS_V0_PUBKEY_HASH") => Ok(123),
_ => bail!("This output type is not supported for fee bumping yet"),
}
}
fn fee_budget(
rate: u64,
parent_size: u64,
parent_fee: u64,
size: u64,
old_fee: u64,
relay: u64,
input: u64,
) -> Result<u64> {
ensure!(
(1..=5000).contains(&rate),
"Fee rate must be a whole number from 1 to 5000 sat/vB"
);
ensure!(
parent_size <= 100_000 && size <= 100_000 && relay <= 5000,
"Unsupported package size or relay fee"
);
let required = rate * (parent_size + size);
let mut budget = required.saturating_sub(parent_fee).max(relay * size);
if old_fee > 0 {
budget = budget.max(old_fee + relay * size + 1);
}
ensure!(budget > old_fee, "Choose a higher fee rate");
// Conservative dust buffer; never attach unrelated wallet inputs to fund fees.
ensure!(
budget.checked_add(1000).is_some_and(|v| v <= input),
"Not enough wallet change for this fee; choose a lower rate"
);
Ok(budget)
}
async fn lnd(
client: &reqwest::Client,
macaroon: &str,
path: &str,
body: Option<Value>,
) -> Result<Value> {
let url = format!("{LND_REST_BASE_URL}{path}");
let req = match body {
Some(v) => client.post(url).json(&v),
None => client.get(url),
};
let response = req
.header("Grpc-Metadata-macaroon", macaroon)
.send()
.await?;
let status = response.status();
let value: Value = response.json().await.context("Invalid LND response")?;
ensure!(
status.is_success() && value.get("code").is_none(),
"{}",
value["message"].as_str().unwrap_or("LND request failed")
);
Ok(value)
}
fn validate_quote(quote: &Quote, fresh: &Plan, timestamp: u64) -> Result<()> {
ensure!(
quote.expires_at > timestamp && quote.plan == *fresh,
"Transaction or fees changed; review a fresh quote"
);
Ok(())
}
fn bump_body(plan: &Plan) -> Value {
json!({"outpoint":{"txid_str":plan.input_txid,"output_index":plan.input_index},
"sat_per_vbyte":plan.rate_sat_vb.to_string(), "budget":plan.budget_sats.to_string(),
"deadline_delta":1, "immediate":true})
}
async fn reserve(path: &Path, op: &Operation) -> Result<()> {
let parent = path.parent().context("Invalid operation path")?;
tokio::fs::create_dir_all(parent).await?;
let mut f = tokio::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)
.await
.context("A bump already exists for this transaction; check its status")?;
f.write_all(&serde_json::to_vec(op)?).await?;
f.sync_all().await?;
// Sync directory entry too: a crash must not make a submitted operation vanish.
tokio::fs::File::open(parent).await?.sync_all().await?;
Ok(())
}
// Only a recorded Archy CPFP with one owned input and no external outputs may
// be folded into a payment. Labels and a fee-sized delta alone are not evidence.
fn fee_child_matches(tx: &Value, plan: &Plan) -> bool {
let input = format!("{}:{}", plan.input_txid, plan.input_index);
let amount = tx["amount"]
.as_i64()
.or_else(|| tx["amount"].as_str()?.parse().ok());
let fee = number(&tx["total_fees"])
.ok()
.and_then(|n| i64::try_from(n).ok());
tx["tx_hash"]
.as_str()
.is_some_and(|id| id.len() == 64 && id.bytes().all(|c| c.is_ascii_hexdigit()))
&& amount
.zip(fee)
.is_some_and(|(amount, fee)| fee > 0 && amount == -fee)
&& tx["previous_outpoints"].as_array().is_some_and(|inputs| {
inputs.len() == 1
&& inputs[0]["outpoint"] == input
&& inputs[0]["is_our_output"] == true
})
&& tx["output_details"].as_array().is_some_and(|outputs| {
!outputs.is_empty() && outputs.iter().all(|o| o["is_our_address"] == true)
})
}
impl RpcHandler {
pub(super) async fn group_fee_bump_history(
&self,
raw: &[Value],
normalized: &mut Vec<Value>,
client: &reqwest::Client,
) {
let mut hidden = std::collections::HashSet::new();
for parent in normalized.iter_mut() {
if parent["direction"] != "outgoing" {
continue;
}
let Some(id) = parent["tx_hash"].as_str().map(str::to_owned) else {
continue;
};
if id.len() != 64 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
continue;
}
let path = self
.config
.data_dir
.join("wallet/fee-bumps")
.join(format!("{id}.json"));
let Ok(bytes) = tokio::fs::read(path).await else {
continue;
};
let Ok(op) = serde_json::from_slice::<Operation>(&bytes) else {
continue;
};
let plan = &op.quote.plan;
if plan.method != "cpfp"
|| plan.txid != id
|| plan.parent_txid != id
|| plan.input_txid != id
{
continue;
}
let candidates: Vec<_> = raw
.iter()
.filter(|tx| fee_child_matches(tx, plan))
.collect();
let mut active = Vec::new();
for child in &candidates {
let child_id = child["tx_hash"].as_str().unwrap();
if child["num_confirmations"].as_i64().unwrap_or(0) > 0
|| self
.bitcoin_rpc_call::<Value>(client, "getmempoolentry", &[json!(child_id)])
.await
.is_ok()
{
active.push(*child);
}
}
// Ambiguous or unavailable chain state must not hide wallet history.
if active.len() != 1 {
continue;
}
let current = active[0];
parent["bump_fee_sats"] = json!(number(&current["total_fees"]).unwrap());
parent["fee_bump_txid"] = current["tx_hash"].clone();
parent["fee_bump_confirmations"] = current["num_confirmations"].clone();
parent["fee_bump_history"] = json!(candidates.iter().map(|child| {
let child_id = child["tx_hash"].as_str().unwrap();
hidden.insert(child_id.to_owned());
json!({"tx_hash":child_id,"fee_sats":number(&child["total_fees"]).unwrap(),
"status":if child["tx_hash"] != current["tx_hash"] { "replaced" }
else if child["num_confirmations"].as_i64().unwrap_or(0) > 0 { "confirmed" } else { "mempool" }})
}).collect::<Vec<_>>());
}
normalized.retain(|tx| !tx["tx_hash"].as_str().is_some_and(|id| hidden.contains(id)));
}
async fn bump_plan(&self, txid: &str, custom_rate: Option<u64>) -> Result<Plan> {
let (client, macaroon) = self.lnd_client().await?;
let info = lnd(&client, &macaroon, "/v1/getinfo", None).await?;
ensure!(
info["synced_to_chain"] == true,
"Wait for the wallet to finish syncing"
);
let version = info["version"]
.as_str()
.context("LND version is unavailable")?;
let mut parts = version.trim_start_matches('v').split('.');
let major: u32 = parts
.next()
.unwrap_or("")
.parse()
.context("Invalid LND version")?;
let minor: u32 = parts
.next()
.unwrap_or("")
.parse()
.context("Invalid LND version")?;
ensure!(
major > 0 || minor >= 21,
"This fee-bump interface requires LND 0.21 or newer"
);
let history = lnd(&client, &macaroon, "/v1/transactions", None).await?;
let txs = array(&history, "transactions")?;
let tx = txs
.iter()
.find(|t| t["tx_hash"] == txid)
.context("Transaction is not in this wallet")?;
ensure!(
tx["num_confirmations"].as_i64() == Some(0),
"This transaction is no longer pending"
);
let entry: Value = self
.bitcoin_rpc_call(&client, "getmempoolentry", &[json!(txid)])
.await
.context("Transaction is not currently in the node's mempool")?;
ensure!(
number(&entry["descendantcount"])? == 1,
"This transaction already has a child; open the child's Bump options instead"
);
let pending = lnd(&client, &macaroon, "/v2/wallet/sweeps/pending", None).await?;
let sweeps = array(&pending, "pending_sweeps")?;
let published = lnd(
&client,
&macaroon,
"/v2/wallet/sweeps?verbose=false&start_height=-1",
None,
)
.await?;
let is_sweep = published["transaction_ids"]["transaction_ids"]
.as_array()
.is_some_and(|ids| ids.iter().any(|id| id == txid));
let outputs = array(tx, "output_details")?;
ensure!(
tx["amount"]
.as_str()
.and_then(|v| v.parse::<i64>().ok())
.or_else(|| tx["amount"].as_i64())
.is_some_and(|v| v < 0),
"Bump is available for outgoing payments and wallet fee sweeps"
);
let (
method,
input_txid,
input_index,
input_sats,
parent_txid,
parent_size,
parent_fee,
old_fee,
size,
recipient_sats,
) = if is_sweep {
// Only a simple wallet CPFP sweep is replaceable here. Anchor/HTLC,
// batched sweeps and arbitrary signed payments need different previews.
let raw: Value = self
.bitcoin_rpc_call(&client, "getrawtransaction", &[json!(txid), json!(true)])
.await?;
let inputs = array(&raw, "vin")?;
ensure!(
inputs.len() == 1 && outputs.len() == 1 && outputs[0]["is_our_address"] == true,
"RBF for batched or channel sweeps is not supported here yet"
);
let input_txid = inputs[0]["txid"]
.as_str()
.context("Missing sweep input")?
.to_string();
let index = u32::try_from(number(&inputs[0]["vout"])?)?;
ensure!(
sweeps.len() == 1 && outpoint_matches(&sweeps[0]["outpoint"], &input_txid, index),
"RBF is unavailable while other wallet sweeps are active"
);
let parent = txs
.iter()
.find(|t| t["tx_hash"] == input_txid)
.context("Sweep parent is unavailable")?;
let parent_output = array(parent, "output_details")?
.iter()
.find(|o| {
number(&o["output_index"]).ok() == Some(index as u64)
&& o["is_our_address"] == true
})
.context("RBF requires a wallet-owned change input")?;
let parent_entry: Value = self
.bitcoin_rpc_call(&client, "getmempoolentry", &[json!(input_txid)])
.await
.context("Only unconfirmed CPFP sweep replacements are supported here")?;
ensure!(
number(&parent_entry["ancestorcount"])? == 1
&& number(&parent_entry["descendantcount"])? == 2,
"Complex sweep package cannot be quoted safely"
);
let recipients = recipient_amount(parent)?;
(
"rbf",
input_txid.clone(),
index,
number(&parent_output["amount"])?,
input_txid,
number(&parent_entry["vsize"])?,
btc_sats(&parent_entry["fees"]["base"])?,
btc_sats(&entry["fees"]["base"])?,
sweep_size(parent_output)?.max(number(&entry["vsize"])?),
recipients,
)
} else {
ensure!(
sweeps.is_empty(),
"Another wallet sweep is active; wait for it before creating a CPFP bump"
);
ensure!(
number(&entry["ancestorcount"])? == 1,
"Fee bumping a chain of unconfirmed payments is not supported yet"
);
let unspent = lnd(
&client,
&macaroon,
"/v2/wallet/utxos",
Some(json!({"unconfirmed_only":true})),
)
.await?;
let utxos = array(&unspent, "utxos")?;
let leases = lnd(
&client,
&macaroon,
"/v2/wallet/utxos/leases",
Some(json!({})),
)
.await?;
let locked = array(&leases, "locked_utxos")?;
let output = outputs
.iter()
.filter(|o| o["is_our_address"] == true && sweep_size(o).is_ok())
.filter(|o| {
number(&o["output_index"]).ok().is_some_and(|i| {
utxos
.iter()
.any(|u| outpoint_matches(&u["outpoint"], txid, i as u32))
&& !locked
.iter()
.any(|u| outpoint_matches(&u["outpoint"], txid, i as u32))
})
})
.max_by_key(|o| number(&o["amount"]).unwrap_or(0))
.context(
"RBF is unavailable for this payment. CPFP needs spendable wallet-owned change",
)?;
let index = u32::try_from(number(&output["output_index"])?)?;
let available: Value = self
.bitcoin_rpc_call(
&client,
"gettxout",
&[json!(txid), json!(index), json!(true)],
)
.await?;
ensure!(
available.is_object()
&& number(&available["confirmations"])? == 0
&& btc_sats(&available["value"])? == number(&output["amount"])?,
"Change is no longer available"
);
(
"cpfp",
txid.to_string(),
index,
number(&output["amount"])?,
txid.to_string(),
number(&entry["vsize"])?,
btc_sats(&entry["fees"]["base"])?,
0,
sweep_size(output)?,
recipient_amount(tx)?,
)
};
let mempool: Value = self
.bitcoin_rpc_call(&client, "getmempoolinfo", &[])
.await?;
let relay = btc_sats(&mempool["incrementalrelayfee"])?
.div_ceil(1000)
.max(1);
let floor = btc_sats(&mempool["mempoolminfee"])?
.max(btc_sats(&mempool["minrelaytxfee"])?)
.div_ceil(1000)
.max(1);
let rate = match custom_rate {
Some(rate) => {
ensure!(
rate >= floor,
"Custom rate is below the current mempool minimum"
);
rate
}
None => {
let estimate = lnd(&client, &macaroon, "/v2/wallet/estimatefee/1", None).await?;
number(&estimate["sat_per_kw"])?.div_ceil(250).max(floor)
}
};
let budget = fee_budget(
rate,
parent_size,
parent_fee,
size,
old_fee,
relay.max(floor),
input_sats,
)?;
let tip: String = self
.bitcoin_rpc_call(&client, "getbestblockhash", &[])
.await?;
Ok(Plan {
txid: txid.to_string(),
method: method.into(),
input_txid,
input_index,
parent_txid,
recipient_sats,
rate_sat_vb: rate,
current_fee_sats: parent_fee + old_fee,
additional_fee_sats: budget - old_fee,
total_fee_sats: parent_fee + budget,
budget_sats: budget,
input_sats,
parent_vsize: parent_size,
sweep_vsize_bound: size,
tip,
})
}
pub(in crate::api::rpc) async fn handle_lnd_bump_quote(
&self,
params: Option<Value>,
) -> Result<Value> {
let p = params.unwrap_or_default();
let txid = txid_param(&p)?;
let path = self
.config
.data_dir
.join("wallet/fee-bumps")
.join(format!("{txid}.json"));
ensure!(
!path.try_exists()?,
"A bump was already submitted for this transaction. Check its status"
);
let custom = p
.get("sat_per_vbyte")
.map(|v| v.as_u64().context("Custom rate must be a whole number"))
.transpose()?;
if let Some(rate) = custom {
ensure!(
(1..=5000).contains(&rate),
"Custom rate must be 1–5000 sat/vB"
);
}
let plan = self.bump_plan(&txid, custom).await?;
let quote = Quote {
quote_id: uuid::Uuid::new_v4().to_string(),
expires_at: now() + QUOTE_SECONDS,
custom_rate: custom,
plan,
};
let mut quotes = QUOTES.lock().await;
quotes.retain(|_, q| q.expires_at > now());
ensure!(quotes.len() < 128, "Too many fee quotes; try again shortly");
quotes.insert(quote.quote_id.clone(), quote.clone());
Ok(serde_json::to_value(quote)?)
}
pub(in crate::api::rpc) async fn handle_lnd_bump_submit(
&self,
params: Option<Value>,
) -> Result<Value> {
let p = params.unwrap_or_default();
let txid = txid_param(&p)?;
let _guard = SUBMIT.lock().await;
let path = self
.config
.data_dir
.join("wallet/fee-bumps")
.join(format!("{txid}.json"));
if path.try_exists()? {
return self
.handle_lnd_bump_status(Some(json!({"txid":txid})))
.await;
}
let id = p["quote_id"]
.as_str()
.context("A reviewed fee quote is required")?;
let quote = QUOTES
.lock()
.await
.get(id)
.cloned()
.context("Quote expired; review the fee again")?;
ensure!(
quote.plan.txid == txid && quote.expires_at > now(),
"Quote expired; review the fee again"
);
let fresh = self.bump_plan(&txid, quote.custom_rate).await?;
validate_quote(&quote, &fresh, now())?;
let op = Operation {
quote: quote.clone(),
status: "unknown".into(),
message: "Submission recorded; checking the wallet. Do not submit another bump.".into(),
};
reserve(&path, &op).await?;
QUOTES.lock().await.remove(id);
let (client, macaroon) = self.lnd_client().await?;
// At a one-block deadline LND may spend ALL this explicitly previewed
// budget. It is always below input value, so no extra funding is requested.
let result = lnd(
&client,
&macaroon,
"/v2/wallet/bumpfee",
Some(bump_body(&fresh)),
)
.await;
// Keep the write-ahead record even for an RPC error: a lost response can
// conceal an accepted bump. Status reconciles from wallet/mempool evidence.
match result {
Ok(_) => Ok(
json!({"status":"registered", "message":"Bump registered with the wallet. Waiting for broadcast.", "quote":quote}),
),
Err(_) => Ok(
json!({"status":"unknown", "message":"The wallet response was not confirmed. Check status; do not submit again.", "quote":quote}),
),
}
}
pub(in crate::api::rpc) async fn handle_lnd_bump_status(
&self,
params: Option<Value>,
) -> Result<Value> {
let txid = txid_param(&params.unwrap_or_default())?;
let path = self
.config
.data_dir
.join("wallet/fee-bumps")
.join(format!("{txid}.json"));
let bytes = match tokio::fs::read(path).await {
Ok(b) => b,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Ok(json!({"status":"none"}))
}
Err(e) => return Err(e.into()),
};
let op: Operation = serde_json::from_slice(&bytes)
.context("Bump receipt needs recovery; do not resubmit")?;
let (client, macaroon) = self.lnd_client().await?;
let history = lnd(&client, &macaroon, "/v1/transactions", None).await?;
let plan = &op.quote.plan;
let input = format!("{}:{}", plan.input_txid, plan.input_index);
let mut candidates: Vec<&Value> = array(&history, "transactions")?
.iter()
.filter(|t| {
t["tx_hash"] != txid
&& t["output_details"].as_array().is_some_and(|outputs| {
!outputs.is_empty() && outputs.iter().all(|o| o["is_our_address"] == true)
})
&& t["previous_outpoints"]
.as_array()
.is_some_and(|inputs| inputs.iter().any(|i| i["outpoint"] == input))
})
.collect();
candidates.sort_by_key(|t| std::cmp::Reverse(number(&t["time_stamp"]).unwrap_or(0)));
for t in candidates {
let id = t["tx_hash"]
.as_str()
.context("Missing bump transaction ID")?;
let confirmed = t["num_confirmations"].as_i64().unwrap_or(0) > 0;
let accepted = if confirmed {
false
} else {
self.bitcoin_rpc_call::<Value>(&client, "getmempoolentry", &[json!(id)])
.await
.is_ok()
};
if confirmed || accepted {
return Ok(json!({"status":if confirmed {"confirmed"} else {"mempool"},
"message":if confirmed {"Fee bump confirmed."} else {"Fee bump accepted in the node's mempool; awaiting confirmation."},
"bump_txid":id,"confirmations":t["num_confirmations"],"actual_sweep_fee_sats":number(&t["total_fees"])?,"quote":op.quote}));
}
}
let pending = lnd(&client, &macaroon, "/v2/wallet/sweeps/pending", None).await?;
let registered = array(&pending, "pending_sweeps")?.iter().any(|s| {
outpoint_matches(&s["outpoint"], &plan.input_txid, plan.input_index)
&& number(&s["budget"]).ok() == Some(plan.budget_sats)
&& number(&s["requested_sat_per_vbyte"]).ok() == Some(plan.rate_sat_vb)
});
Ok(
json!({"status":if registered {"registered"} else {"unknown"},
"message":if registered {"Bump registered; waiting for a verified broadcast."} else {"Submission outcome is unknown. Do not submit again; check wallet status."}, "quote":op.quote}),
)
}
}
fn recipient_amount(tx: &Value) -> Result<u64> {
array(tx, "output_details")?
.iter()
.filter(|o| o["is_our_address"] == false)
.try_fold(0u64, |sum, o| {
sum.checked_add(number(&o["amount"])?)
.context("Recipient amount overflow")
})
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_plan() -> Plan {
serde_json::from_value(json!({"txid":"a","method":"cpfp","input_txid":"a","input_index":0,"parent_txid":"a","recipient_sats":161650,"rate_sat_vb":3,"current_fee_sats":144,"additional_fee_sats":618,"total_fee_sats":762,"budget_sats":618,"input_sats":21126,"parent_vsize":142,"sweep_vsize_bound":112,"tip":"tip"})).unwrap()
}
#[test]
fn history_requires_owned_simple_fee_only_child() {
let plan = sample_plan();
let tx = json!({"tx_hash":"b".repeat(64),"amount":"-200","total_fees":"200",
"previous_outpoints":[{"outpoint":"a:0","is_our_output":true}],
"output_details":[{"is_our_address":true}]});
assert!(fee_child_matches(&tx, &plan));
for bad in [
json!({"amount":"-201"}),
json!({"amount":"200"}),
json!({"total_fees":"0"}),
json!({"previous_outpoints":[{"outpoint":"a:1","is_our_output":true}]}),
json!({"previous_outpoints":[{"outpoint":"a:0","is_our_output":false}]}),
json!({"previous_outpoints":[{"outpoint":"a:0","is_our_output":true},{"outpoint":"c:0","is_our_output":true}]}),
json!({"output_details":[{"is_our_address":false}]}),
json!({"output_details":[]}),
json!({"tx_hash":"../../invalid"}),
] {
let mut changed = tx.clone();
for (key, value) in bad.as_object().unwrap() {
changed[key] = value.clone();
}
assert!(!fee_child_matches(&changed, &plan), "{bad}");
}
}
#[test]
fn stale_quotes_cannot_silently_change_approved_fee_or_transaction() {
let plan = sample_plan();
let q = Quote {
quote_id: "q".into(),
expires_at: 100,
custom_rate: None,
plan: plan.clone(),
};
assert!(validate_quote(&q, &plan, 99).is_ok());
assert!(validate_quote(&q, &plan, 100).is_err());
let mut changed = plan.clone();
changed.budget_sats += 1;
assert!(validate_quote(&q, &changed, 99).is_err());
changed = plan.clone();
changed.input_index += 1;
assert!(validate_quote(&q, &changed, 99).is_err());
changed = plan.clone();
changed.recipient_sats -= 1;
assert!(validate_quote(&q, &changed, 99).is_err());
changed = plan.clone();
changed.tip = "new block".into();
assert!(validate_quote(&q, &changed, 99).is_err());
}
#[test]
fn mutation_always_has_explicit_budget_and_does_not_send_a_second_payment() {
assert_eq!(
bump_body(&sample_plan()),
json!({"outpoint":{"txid_str":"a","output_index":0},"sat_per_vbyte":"3","budget":"618","deadline_delta":1,"immediate":true})
);
let mut rbf = sample_plan();
rbf.method = "rbf".into();
rbf.txid = "child".into();
// RBF uses the already-registered input, not the child's output.
assert_eq!(bump_body(&rbf)["outpoint"]["txid_str"], "a");
}
#[test]
fn outpoint_ownership_and_recipient_exclude_wallet_change() {
assert!(outpoint_matches(
&json!({"txid_str":"a","output_index":2}),
"a",
2
));
assert!(!outpoint_matches(
&json!({"txid_str":"b","output_index":2}),
"a",
2
));
assert!(!outpoint_matches(
&json!({"txid_str":"a","output_index":3}),
"a",
2
));
assert_eq!(recipient_amount(&json!({"output_details":[{"is_our_address":true,"amount":"21126"},{"is_our_address":false,"amount":"161650"}]})).unwrap(), 161650);
assert!(recipient_amount(
&json!({"output_details":[{"is_our_address":false,"amount":"bad"}]})
)
.is_err());
}
#[test]
fn cpfp_budget_covers_parent_and_preserves_change() {
assert_eq!(fee_budget(3, 142, 144, 112, 0, 1, 21126).unwrap(), 618);
assert!(fee_budget(5000, 142, 144, 112, 0, 1, 21126).is_err());
assert!(fee_budget(0, 142, 144, 112, 0, 1, 21126).is_err());
}
#[test]
fn rbf_pays_incremental_relay_cost_and_counts_only_extra_cost() {
let fee = fee_budget(3, 142, 144, 112, 650, 1, 21126).unwrap();
assert_eq!(fee, 763);
assert_eq!(fee - 650, 113);
}
#[test]
fn unsupported_outputs_and_malformed_ids_fail_closed() {
assert!(sweep_size(&json!({"output_type":"SCRIPT_TYPE_WITNESS_V0_SCRIPT_HASH"})).is_err());
assert!(txid_param(&json!({"txid":"../../file"})).is_err());
assert!(number(&json!(-1)).is_err());
assert!(btc_sats(&json!(-0.1)).is_err());
assert_eq!(btc_sats(&json!(0.00000650)).unwrap(), 650);
}
#[tokio::test]
async fn receipt_prevents_duplicate_submission_after_restart() {
let dir = std::env::temp_dir().join(uuid::Uuid::new_v4().to_string());
let path = dir.join("receipt.json");
let plan: Plan = serde_json::from_value(json!({"txid":"a","method":"cpfp","input_txid":"a","input_index":0,"parent_txid":"a","recipient_sats":1000,"rate_sat_vb":3,"current_fee_sats":144,"additional_fee_sats":618,"total_fee_sats":762,"budget_sats":618,"input_sats":21126,"parent_vsize":142,"sweep_vsize_bound":112,"tip":"tip"})).unwrap();
let op = Operation {
quote: Quote {
quote_id: "q".into(),
expires_at: now() + 60,
custom_rate: None,
plan,
},
status: "unknown".into(),
message: "pending".into(),
};
reserve(&path, &op).await.unwrap();
assert!(reserve(&path, &op).await.is_err());
let restored: Operation =
serde_json::from_slice(&tokio::fs::read(&path).await.unwrap()).unwrap();
assert_eq!(restored.quote.plan.budget_sats, 618);
tokio::fs::remove_dir_all(dir).await.unwrap();
}
}
@@ -0,0 +1,120 @@
//! Explicit on-chain fee choices retain priority; omitted choices target the next block.
use anyhow::{ensure, Context, Result};
use serde_json::Value;
pub(super) const DEFAULT_TARGET: i64 = 1;
pub(super) fn estimated_sat_per_vbyte(value: &Value) -> Result<u64> {
let per_kw = value["sat_per_kw"]
.as_u64()
.or_else(|| value["sat_per_kw"].as_str().and_then(|s| s.parse().ok()))
.context("Next-block fee estimate is unavailable")?;
let rate = per_kw.div_ceil(250);
ensure!(
(1..=5000).contains(&rate),
"Next-block fee estimate is outside supported bounds; choose an explicit fee"
);
Ok(rate)
}
pub(super) fn fee_options(params: &Value) -> Result<(Option<i64>, Option<i64>)> {
let integer = |key: &str, max: i64| -> Result<Option<i64>> {
match params.get(key) {
None | Some(Value::Null) => Ok(None),
Some(value) => {
let n = value
.as_i64()
.with_context(|| format!("{key} must be a positive whole number"))?;
ensure!((1..=max).contains(&n), "{key} must be between 1 and {max}");
Ok(Some(n))
}
}
};
let target = integer("target_conf", 1008)?;
let rate = integer("sat_per_vbyte", 5000)?;
ensure!(
target.is_none() || rate.is_none(),
"Specify either target_conf or sat_per_vbyte, not both"
);
Ok((
if rate.is_none() {
Some(target.unwrap_or(DEFAULT_TARGET))
} else {
None
},
rate,
))
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn estimates_round_up_and_missing_or_extreme_estimates_fail_closed() {
assert_eq!(
estimated_sat_per_vbyte(&json!({"sat_per_kw":"501"})).unwrap(),
3
);
assert_eq!(
estimated_sat_per_vbyte(&json!({"sat_per_kw":250})).unwrap(),
1
);
for v in [
json!({}),
json!({"sat_per_kw":0}),
json!({"sat_per_kw":-1}),
json!({"sat_per_kw":1250001}),
] {
assert!(estimated_sat_per_vbyte(&v).is_err());
}
}
#[test]
fn next_block_default_preserves_explicit_slower_and_custom_choices() {
assert_eq!(fee_options(&json!({})).unwrap(), (Some(1), None));
assert_eq!(
fee_options(&json!({"target_conf":null})).unwrap(),
(Some(1), None)
);
for target in [1, 3, 6, 144, 1008] {
assert_eq!(
fee_options(&json!({"target_conf":target})).unwrap(),
(Some(target), None)
);
}
for rate in [1, 17, 5000] {
assert_eq!(
fee_options(&json!({"sat_per_vbyte":rate})).unwrap(),
(None, Some(rate))
);
}
}
#[test]
fn malformed_explicit_fees_never_silently_become_fast() {
for value in [
json!(0),
json!(-1),
json!(1.5),
json!("6"),
json!(true),
json!({}),
json!(1009),
] {
assert!(fee_options(&json!({"target_conf":value})).is_err());
}
for value in [
json!(0),
json!(-1),
json!(1.5),
json!("6"),
json!(true),
json!(5001),
] {
assert!(fee_options(&json!({"sat_per_vbyte":value})).is_err());
}
assert!(fee_options(&json!({"target_conf":1,"sat_per_vbyte":2})).is_err());
}
}
+2
View File
@@ -1,4 +1,6 @@
mod channels;
mod fee_bump;
mod fee_policy;
mod info;
mod macaroons;
mod payments;
@@ -402,6 +402,9 @@ impl RpcHandler {
}));
}
self.group_fee_bump_history(raw_txs, &mut transactions, &client)
.await;
// Sort by timestamp descending (most recent first)
transactions.sort_by(|a, b| {
let ta = a.get("time_stamp").and_then(|v| v.as_i64()).unwrap_or(0);
+26 -37
View File
@@ -124,28 +124,8 @@ impl RpcHandler {
return Err(anyhow::anyhow!("Invalid Bitcoin address format"));
}
// Fee control: either a confirmation target or an explicit fee rate
let target_conf = params.get("target_conf").and_then(|v| v.as_i64());
let sat_per_vbyte = params.get("sat_per_vbyte").and_then(|v| v.as_i64());
if target_conf.is_some() && sat_per_vbyte.is_some() {
return Err(anyhow::anyhow!(
"Invalid fee parameters: specify either target_conf or sat_per_vbyte, not both"
));
}
if let Some(tc) = target_conf {
if !(1..=1008).contains(&tc) {
return Err(anyhow::anyhow!(
"Invalid target_conf: must be between 1 and 1008 blocks"
));
}
}
if let Some(rate) = sat_per_vbyte {
if !(1..=5000).contains(&rate) {
return Err(anyhow::anyhow!(
"Invalid sat_per_vbyte: must be between 1 and 5000"
));
}
}
// Omitted fees target the next block; explicit slower/custom choices win.
let (target_conf, sat_per_vbyte) = super::fee_policy::fee_options(&params)?;
info!(
addr = addr,
@@ -238,15 +218,12 @@ impl RpcHandler {
if !(546..=21_000_000 * 100_000_000).contains(&amount) {
return Err(anyhow::anyhow!("Invalid amount"));
}
let target_conf = params
.get("target_conf")
.and_then(|v| v.as_i64())
.unwrap_or(6);
if !(1..=1008).contains(&target_conf) {
return Err(anyhow::anyhow!(
"Invalid target_conf: must be between 1 and 1008 blocks"
));
}
let (target_conf, custom_rate) = super::fee_policy::fee_options(&params)?;
anyhow::ensure!(
custom_rate.is_none(),
"Fee estimation requires a confirmation target"
);
let target_conf = target_conf.unwrap_or(super::fee_policy::DEFAULT_TARGET);
let (client, macaroon_hex) = self.lnd_client().await?;
@@ -782,10 +759,24 @@ impl RpcHandler {
total_amount += amount;
}
let sat_per_vbyte = params
.get("fee_rate_sat_per_vbyte")
.and_then(|v| v.as_u64())
.unwrap_or(10);
let (_, explicit_rate) = super::fee_policy::fee_options(&serde_json::json!({
"sat_per_vbyte": params.get("fee_rate_sat_per_vbyte")
}))?;
let (client, macaroon_hex) = self.lnd_client().await?;
let sat_per_vbyte = if let Some(rate) = explicit_rate {
rate as u64
} else {
let response = client
.get(format!("{LND_REST_BASE_URL}/v2/wallet/estimatefee/1"))
.header("Grpc-Metadata-macaroon", &macaroon_hex)
.send()
.await
.context("Cannot estimate the next-block fee")?
.error_for_status()
.context("Next-block fee estimate rejected")?;
let estimate: serde_json::Value = response.json().await?;
super::fee_policy::estimated_sat_per_vbyte(&estimate)?
};
info!(
total_amount = total_amount,
@@ -793,8 +784,6 @@ impl RpcHandler {
"Creating PSBT for hardware wallet signing"
);
let (client, macaroon_hex) = self.lnd_client().await?;
let fund_body = serde_json::json!({
"raw": {
"outputs": lnd_outputs,
@@ -221,6 +221,10 @@ impl RpcHandler {
params: Option<serde_json::Value>,
) -> Result<serde_json::Value> {
let params = params.ok_or_else(|| anyhow::anyhow!("Missing params"))?;
if let Some(job) = self.flash_job.read().await.as_ref() {
anyhow::ensure!(job.snapshot().await.done,
"A firmware flash is in progress; wait before reconnecting or changing radio settings");
}
let mut config = mesh::load_config(&self.config.data_dir).await?;
@@ -325,7 +329,16 @@ impl RpcHandler {
{
let service_arc = Arc::clone(&self.mesh_service);
let config_for_apply = config.clone();
let flash_jobs = Arc::clone(&self.flash_job);
tokio::spawn(async move {
// Serialize against flash registration. If a flash started
// after this RPC saved settings, its completion applies them.
let flash_guard = flash_jobs.read().await;
if let Some(job) = flash_guard.as_ref() {
if !job.snapshot().await.done {
return;
}
}
let mut service = service_arc.write().await;
if let Some(svc) = service.as_mut() {
if let Err(e) = svc.configure(config_for_apply).await {
+3 -1
View File
@@ -110,7 +110,8 @@ impl RpcHandler {
// `mesh.probe-device` call (e.g. the hot-swap modal's own re-probe)
// from opening the identical port at the same time and corrupting
// both operations' handshakes.
if let Some(job) = self.flash_job.read().await.as_ref() {
let flash_guard = self.flash_job.read().await;
if let Some(job) = flash_guard.as_ref() {
anyhow::ensure!(
job.snapshot().await.done,
"A firmware flash is in progress — refusing to probe the serial port until it finishes"
@@ -131,6 +132,7 @@ impl RpcHandler {
}
}
let probe = mesh::listener::probe_device(&path).await?;
drop(flash_guard);
Ok(serde_json::to_value(probe)?)
}
@@ -985,28 +985,26 @@ pub(super) async fn get_app_config(
)
}
"nginx-proxy-manager" => {
let storage = crate::container::npm::resolve_storage().await?;
let admin_port = allocator
.allocate_or_get(app_id, 8081, 81)
.await
.unwrap_or(8081);
let http_port = allocator
.allocate_or_get("nginx-proxy-manager-http", 8084, 80)
.allocate_or_get("nginx-proxy-manager-http", 8088, 80)
.await
.unwrap_or(8084);
.unwrap_or(8088);
let https_port = allocator
.allocate_or_get("nginx-proxy-manager-https", 8444, 443)
.await
.unwrap_or(8444);
(
vec![
format!("{}:81", admin_port),
format!("{}:80", http_port),
format!("{}:443", https_port),
],
vec![
"/var/lib/archipelago/nginx-proxy-manager/data:/data".to_string(),
"/var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt".to_string(),
format!("127.0.0.1:{}:81", admin_port),
format!("127.0.0.1:{}:80", http_port),
format!("127.0.0.1:{}:443", https_port),
],
storage.bind_mounts(),
vec![],
None,
None,
+18 -95
View File
@@ -693,7 +693,11 @@ impl RpcHandler {
// These standalone web UIs have repeatedly lost host listeners
// under Podman's rootless pasta backend while staying healthy internally.
// Use slirp4netns/rootlessport for this standalone web UI.
run_args.push("--network=slirp4netns:allow_host_loopback=true");
run_args.push(if package_id == "nginx-proxy-manager" {
"--network=slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"
} else {
"--network=slirp4netns:allow_host_loopback=true"
});
} else if needs_archy_net(package_id) {
// Create archy-net if it doesn't exist (idempotent — "already exists" is fine)
match tokio::process::Command::new("podman")
@@ -1568,88 +1572,8 @@ autopilot.active=false\n",
super::pine_ha::restart_home_assistant_if_running().await;
}
}
if package_id == "filebrowser" {
// Generate a random password (32 bytes, hex-encoded)
let mut buf = [0u8; 32];
rand::RngCore::fill_bytes(&mut rand::rngs::OsRng, &mut buf);
let password = hex::encode(buf);
let client = match reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(10))
.build()
{
Ok(c) => c,
Err(e) => {
tracing::warn!("Failed to create HTTP client for FileBrowser hook: {}", e);
return;
}
};
// Retry loop: FileBrowser may take time to initialize its SQLite database
let mut password_changed = false;
for attempt in 0..6u32 {
let delay = if attempt == 0 { 5 } else { 10 };
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
// Try to log in with default credentials
let login_res = client
.post("http://127.0.0.1:8083/api/login")
.json(&serde_json::json!({"username": "admin", "password": "admin"}))
.send()
.await;
let token = match login_res {
Ok(resp) if resp.status().is_success() => match resp.text().await {
Ok(t) => t.trim_matches('"').to_string(),
Err(_) => continue,
},
_ => {
debug!("FileBrowser not ready (attempt {}/6)", attempt + 1);
continue;
}
};
// Change admin password
let change_res = client
.put("http://127.0.0.1:8083/api/users/1")
.header("X-Auth", &token)
.json(&serde_json::json!({"password": password}))
.send()
.await;
match change_res {
Ok(resp) if resp.status().is_success() => {
let secret_dir = "/var/lib/archipelago/secrets/filebrowser";
if let Err(e) = tokio::fs::create_dir_all(secret_dir).await {
tracing::warn!("Failed to create filebrowser secrets dir: {}", e);
}
let pw_path = format!("{}/password", secret_dir);
if let Err(e) = tokio::fs::write(&pw_path, &password).await {
tracing::warn!("Failed to write filebrowser password: {}", e);
}
// Set restrictive permissions on the password file
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(
&pw_path,
std::fs::Permissions::from_mode(0o600),
);
}
info!("FileBrowser admin password secured (default credentials replaced)");
password_changed = true;
break;
}
_ => continue,
}
}
if !password_changed {
tracing::warn!(
"FileBrowser password could not be changed after 6 attempts — \
default credentials (admin/admin) remain active"
);
}
}
// File Browser credentials are provisioned and verified before the
// server starts. Never attempt a default-password change after launch.
// Auto-configure Tor hidden service for protocol services (LND, ElectrumX, Bitcoin)
{
@@ -1912,10 +1836,10 @@ autopilot.active=false\n",
}
pub(in crate::api::rpc) async fn handle_filebrowser_token(&self) -> Result<serde_json::Value> {
let secret_path = "/var/lib/archipelago/secrets/filebrowser/password";
let password = tokio::fs::read_to_string(secret_path)
.await
.unwrap_or_else(|_| "admin".to_string());
let credentials = crate::container::filebrowser::cloud_credentials(std::path::Path::new(
"/var/lib/archipelago/secrets/filebrowser",
))
.await?;
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(10))
@@ -1924,7 +1848,7 @@ autopilot.active=false\n",
let resp = client
.post("http://127.0.0.1:8083/api/login")
.json(&serde_json::json!({"username": "admin", "password": password}))
.json(&serde_json::json!({"username": credentials.username, "password": credentials.password}))
.send()
.await
.context("Failed to connect to FileBrowser")?;
@@ -1954,17 +1878,16 @@ autopilot.active=false\n",
super::validation::validate_app_id(app_id)?;
if app_id == "filebrowser" {
let password =
tokio::fs::read_to_string("/var/lib/archipelago/secrets/filebrowser/password")
.await
.map(|p| p.trim().to_string())
.unwrap_or_else(|_| "admin".to_string());
let credentials = crate::container::filebrowser::cloud_credentials(
std::path::Path::new("/var/lib/archipelago/secrets/filebrowser"),
)
.await?;
return Ok(serde_json::json!({
"title": "File Browser credentials",
"description": "Use these credentials when File Browser asks you to sign in.",
"credentials": [
{ "label": "Username", "value": "admin" },
{ "label": "Password", "value": password, "sensitive": true }
{ "label": "Username", "value": credentials.username },
{ "label": "Password", "value": credentials.password, "sensitive": true }
]
}));
}
+210 -61
View File
@@ -1576,41 +1576,110 @@ async fn repair_netbird_network() {
}
async fn repair_nginx_proxy_manager_container() {
repair_nginx_proxy_manager_dirs().await;
// Quadlet owns managed containers; its backed-up reconciliation applies
// port and mount changes. Never remove a systemd-owned container here.
if crate::container::quadlet::unit_exists("nginx-proxy-manager").await {
return;
}
// Serialize repair so a second caller cannot overlap a replacement.
static REPAIR: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
let _repair = REPAIR.lock().await;
if !nginx_proxy_manager_has_legacy_admin_port().await {
return;
}
install_log(
"START REPAIR: nginx-proxy-manager - recreating stale container using host port 8081",
)
.await;
let _ = podman_control(&["rm", "-f", "nginx-proxy-manager"]).await;
crate::container::ghost_reaper::reap_for_app("nginx-proxy-manager").await;
if let Err(err) = recreate_nginx_proxy_manager_container().await {
tracing::warn!(error = %err, "failed to recreate stale nginx-proxy-manager container");
if let Err(error) = repair_legacy_nginx_proxy_manager().await {
tracing::warn!(error = %error, "NPM legacy repair failed; persistent state preserved");
}
}
async fn repair_nginx_proxy_manager_dirs() {
let _ = tokio::process::Command::new("sudo")
.args([
"mkdir",
"-p",
"/var/lib/archipelago/nginx-proxy-manager/data/letsencrypt-acme-challenge/.well-known/acme-challenge",
"/var/lib/archipelago/nginx-proxy-manager/letsencrypt",
])
.output()
.await;
let _ = tokio::process::Command::new("sudo")
.args([
"chown",
"-R",
"1000:1000",
"/var/lib/archipelago/nginx-proxy-manager",
])
.output()
.await;
const NPM_PREVIOUS_CONTAINER: &str = "archy-npm-upgrade-previous";
async fn restore_failed_npm_repair() -> Result<()> {
let removed = podman_control(&["rm", "-f", "--ignore", "nginx-proxy-manager"]).await?;
anyhow::ensure!(
removed.status.success(),
"cannot remove failed NPM replacement; previous container retained"
);
let renamed =
podman_control(&["rename", NPM_PREVIOUS_CONTAINER, "nginx-proxy-manager"]).await?;
anyhow::ensure!(
renamed.status.success(),
"cannot restore previous NPM container name"
);
let started = podman_control(&["start", "nginx-proxy-manager"]).await?;
anyhow::ensure!(
started.status.success(),
"previous NPM container restored but failed to start"
);
Ok(())
}
async fn repair_legacy_nginx_proxy_manager() -> Result<()> {
let previous = podman_control(&["container", "exists", NPM_PREVIOUS_CONTAINER]).await?;
anyhow::ensure!(previous.status.code() == Some(1),
"NPM previous-container slot is occupied or cannot be inspected; preserve it and review interrupted repair before proceeding");
let inspection = podman_control(&[
"inspect",
"nginx-proxy-manager",
"--format",
"{{json .Config.Env}}",
])
.await?;
anyhow::ensure!(
inspection.status.success(),
"cannot preserve NPM environment before repair"
);
let environment: Vec<String> =
serde_json::from_slice(&inspection.stdout).context("invalid original NPM environment")?;
let environment = npm_repair_environment(&environment)?;
let storage = crate::container::npm::resolve_storage().await?;
let mut manifest: archipelago_container::AppManifest = serde_yaml::from_str(include_str!(
"../../../../../../apps/nginx-proxy-manager/manifest.yml"
))?;
storage.apply(&mut manifest)?;
let stopped = podman_control(&["stop", "--time", "30", "nginx-proxy-manager"]).await?;
anyhow::ensure!(
stopped.status.success(),
"could not stop NPM for a consistent backup"
);
if let Err(error) = crate::container::migration_backup::snapshot(
&manifest,
std::path::Path::new("/var/lib/archipelago"),
None,
)
.await
{
let _ = podman_control(&["start", "nginx-proxy-manager"]).await;
return Err(error);
}
// Keep the original runtime definition for rollback, including its operator
// options. Never delete it before the replacement has become ready.
let renamed = podman_control(&["rename", "nginx-proxy-manager", NPM_PREVIOUS_CONTAINER]).await;
if !renamed.as_ref().is_ok_and(|out| out.status.success()) {
let _ = podman_control(&["start", "nginx-proxy-manager"]).await;
anyhow::bail!("could not retain legacy NPM runtime; state backup preserved, inspect both container names before retrying");
}
let replacement = async {
recreate_nginx_proxy_manager_container(&storage, &environment).await?;
anyhow::ensure!(
wait_for_runtime_host_port("nginx-proxy-manager", 8081, 180).await,
"replacement NPM admin listener did not become ready"
);
Ok::<_, anyhow::Error>(())
}
.await;
if let Err(error) = replacement {
restore_failed_npm_repair()
.await
.context("restoring previous NPM after replacement failure")?;
return Err(error);
}
let removed = podman_control(&["rm", NPM_PREVIOUS_CONTAINER]).await?;
anyhow::ensure!(
removed.status.success(),
"replacement ready but previous NPM cleanup failed; rollback container retained"
);
Ok(())
}
async fn nginx_proxy_manager_has_legacy_admin_port() -> bool {
@@ -1645,36 +1714,75 @@ async fn nginx_proxy_manager_has_legacy_admin_port() -> bool {
ports.contains(":81->81/tcp") || ports.contains(":8443->443/tcp")
}
async fn recreate_nginx_proxy_manager_container() -> Result<()> {
tokio::process::Command::new("sudo")
.args([
"mkdir",
"-p",
"/var/lib/archipelago/nginx-proxy-manager/data/letsencrypt-acme-challenge/.well-known/acme-challenge",
"/var/lib/archipelago/nginx-proxy-manager/letsencrypt",
])
.output()
.await
.context("failed to create nginx-proxy-manager data directories")?;
let _ = tokio::process::Command::new("sudo")
.args([
"chown",
"-R",
"1000:1000",
"/var/lib/archipelago/nginx-proxy-manager",
])
.output()
.await;
fn npm_repair_environment(values: &[String]) -> Result<Vec<(String, String)>> {
values.iter().map(|value| {
let (key, value) = value.split_once('=').context("invalid NPM environment entry")?;
anyhow::ensure!(!key.is_empty() && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'),
"unsupported NPM environment name; original container preserved");
anyhow::ensure!(!value.contains(['\n', '\r', '\0']),
"NPM environment requires explicit migration of a multiline value; original container preserved");
Ok((key.to_owned(), value.to_owned()))
}).collect()
}
let image = crate::container::image_versions::pinned_image_for_app("nginx-proxy-manager")
.unwrap_or_else(|| "docker.io/jc21/nginx-proxy-manager:latest".to_string());
struct NpmRepairEnvironmentFile(std::path::PathBuf);
impl NpmRepairEnvironmentFile {
fn create(environment: &[(String, String)]) -> Result<Self> {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let path = std::env::temp_dir().join(format!(".archy-npm-env-{}", uuid::Uuid::new_v4()));
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&path)?;
let guard = Self(path);
for (key, value) in environment {
writeln!(file, "{key}={value}")?;
}
file.sync_all()?;
Ok(guard)
}
}
impl Drop for NpmRepairEnvironmentFile {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
async fn recreate_nginx_proxy_manager_container(
storage: &crate::container::npm::Storage,
environment: &[(String, String)],
) -> Result<()> {
// Existing directories and ownership came from the active runtime. Never
// create a second database tree or recursively rewrite data permissions.
for directory in [&storage.data, &storage.certificates] {
anyhow::ensure!(
std::path::Path::new(directory).is_dir(),
"NPM persistent directory is missing"
);
}
// This repair changes connectivity, not NPM's application version. Keep
// the exact old image so rollback never starts an older binary against a
// database that an incidental mutable-tag update may have migrated.
let image_output =
podman_control(&["inspect", NPM_PREVIOUS_CONTAINER, "--format", "{{.Image}}"]).await?;
anyhow::ensure!(
image_output.status.success(),
"cannot resolve original NPM image for repair"
);
let image = String::from_utf8(image_output.stdout)?.trim().to_string();
anyhow::ensure!(!image.is_empty(), "original NPM image is missing");
let mut args = vec![
"run".to_string(),
"-d".to_string(),
"--name".to_string(),
"nginx-proxy-manager".to_string(),
"--restart=unless-stopped".to_string(),
"--network=slirp4netns:allow_host_loopback=true".to_string(),
"--network=slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24".to_string(),
"--cap-drop=ALL".to_string(),
"--security-opt=no-new-privileges:true".to_string(),
"--pids-limit=4096".to_string(),
@@ -1682,24 +1790,32 @@ async fn recreate_nginx_proxy_manager_container() -> Result<()> {
args.extend(get_app_capabilities("nginx-proxy-manager"));
args.extend([
"-p".to_string(),
"8081:81".to_string(),
"127.0.0.1:8081:81".to_string(),
"-p".to_string(),
"8084:80".to_string(),
"127.0.0.1:8088:80".to_string(),
"-p".to_string(),
"8444:443".to_string(),
"127.0.0.1:8444:443".to_string(),
"-v".to_string(),
"/var/lib/archipelago/nginx-proxy-manager/data:/data".to_string(),
format!("{}:/data", storage.data),
"-v".to_string(),
"/var/lib/archipelago/nginx-proxy-manager/letsencrypt:/etc/letsencrypt".to_string(),
format!("{}:/etc/letsencrypt", storage.certificates),
"--memory".to_string(),
get_memory_limit("nginx-proxy-manager").to_string(),
"--cpus=2".to_string(),
]);
args.extend(get_health_check_args("nginx-proxy-manager", ""));
// Keep values out of argv/logs AND out of Podman's host environment
// (a container's PATH or LD_PRELOAD must never alter the host command).
let env_file = NpmRepairEnvironmentFile::create(environment)?;
args.extend([
"--env-file".to_string(),
env_file.0.to_string_lossy().into_owned(),
]);
args.push(image);
let refs = args.iter().map(String::as_str).collect::<Vec<_>>();
let output = podman_control(&refs).await?;
let mut command = tokio::process::Command::new("podman");
command.args(&args);
let output = command_with_timeout(command, Duration::from_secs(120), "NPM replacement").await?;
if !output.status.success() {
anyhow::bail!(
"podman run nginx-proxy-manager failed: {}",
@@ -1767,7 +1883,7 @@ fn runtime_host_ports(container_name: &str) -> Vec<u16> {
"vaultwarden" => vec![8082],
"gitea" => vec![3001, 2222, 3000],
"nextcloud" => vec![8085],
"nginx-proxy-manager" => vec![8081, 8084, 8444],
"nginx-proxy-manager" => vec![8081, 8088, 8444],
_ => Vec::new(),
};
ports
@@ -1778,7 +1894,7 @@ fn with_legacy_extra_ports(container_name: &str, mut ports: Vec<u16>) -> Vec<u16
ports.push(3000);
}
if container_name == "nginx-proxy-manager" {
for port in [8084, 8444] {
for port in [8088, 8444] {
if !ports.contains(&port) {
ports.push(port);
}
@@ -1843,6 +1959,7 @@ async fn wait_for_runtime_host_port(container_name: &str, port: u16, timeout_sec
loop {
let ready = match container_name {
"uptime-kuma" => http_host_port_ready(port, "/").await,
"nginx-proxy-manager" => http_host_port_ready(port, "/api/").await,
_ => tokio::net::TcpStream::connect(("127.0.0.1", port))
.await
.is_ok(),
@@ -2151,6 +2268,38 @@ pub(super) fn orchestrator_uninstall_app_ids(package_id: &str) -> Vec<String> {
#[cfg(test)]
mod tests {
#[test]
fn npm_environment_backup_is_private_exact_and_removed_on_drop() {
use std::os::unix::fs::PermissionsExt;
let values = vec![
"DB_PASSWORD=fixture=a b".to_string(),
"PATH=/container/only".to_string(),
];
let parsed = super::npm_repair_environment(&values).unwrap();
let host_path = std::env::var_os("PATH");
let path = {
let file = super::NpmRepairEnvironmentFile::create(&parsed).unwrap();
assert_eq!(
std::fs::metadata(&file.0).unwrap().permissions().mode() & 0o777,
0o600
);
assert_eq!(
std::fs::read_to_string(&file.0).unwrap(),
"DB_PASSWORD=fixture=a b\nPATH=/container/only\n"
);
assert_eq!(std::env::var_os("PATH"), host_path);
file.0.clone()
};
assert!(!path.exists());
for value in [
"INVALID",
"=empty key",
"KEY=value\nINJECTED=true",
"--env=value",
] {
assert!(super::npm_repair_environment(&[value.to_string()]).is_err());
}
}
use super::*;
#[tokio::test]
+89 -1
View File
@@ -142,11 +142,40 @@ const NGINX_FEDIMINT_SNIPPET_INSERT: &str = "proxy_pass http://127.0.0.1:8175/;\
/// catalog refresh/reconciliation. Replacing the app tree in the background
/// could let a reload observe its temporary empty state and forget disk-only apps.
pub async fn ensure_runtime_assets_ready() {
// Install the guard before any startup path can reload an older dashboard
// vhost. The canonical OTA/ISO config contains the same guard inline.
if Path::new(NGINX_CONF_PATH).exists() || Path::new(NGINX_ENABLED_CONF_PATH).exists() {
match host_sudo(&[
"python3",
"-c",
include_str!("../../../scripts/dashboard-public-guard.py"),
])
.await
{
Ok(status) if status.success() => debug!("Dashboard public source guard verified"),
Ok(status) => warn!("Dashboard public source guard needs attention: {status}"),
Err(error) => warn!("Dashboard public source guard could not run: {error}"),
}
}
match run_runtime_assets().await {
Ok(changed) if changed => info!("Runtime assets synchronized from OTA payload"),
Ok(_) => debug!("No OTA runtime payload to synchronize"),
Err(e) => warn!("Runtime asset bootstrap failed (non-fatal): {:#}", e),
}
// A binary-only qualification or OTA rollback can precede the matching
// script payload. Install the exact embedded helper before Quadlet
// reconciliation can introduce its required ExecStartPre command.
if let Err(error) = write_root_if_needed(
"/opt/archipelago/scripts/filebrowser-credentials.py",
include_str!("../../../scripts/filebrowser-credentials.py"),
)
.await
{
warn!("File Browser credential helper installation failed: {error:#}");
}
if let Err(error) = run_npm_bridge_bootstrap().await {
warn!("NPM public routing bootstrap needs attention: {error:#}");
}
// Repair the narrowly recognized legacy NPM tunnel override before app
// reconciliation. The embedded script ships in both OTA and ISO binaries.
// It preserves native wallet services and refuses unknown custom routing.
@@ -176,6 +205,52 @@ pub async fn ensure_runtime_assets_ready() {
}
}
async fn run_npm_bridge_bootstrap() -> Result<()> {
if !Path::new("/opt/archipelago/scripts").is_dir() {
return Ok(());
}
let mut units_changed = false;
for (path, content) in [
(
"/opt/archipelago/scripts/npm-public-bridge.py",
include_str!("../../../scripts/npm-public-bridge.py"),
),
(
"/opt/archipelago/scripts/dashboard-public-guard.py",
include_str!("../../../scripts/dashboard-public-guard.py"),
),
(
"/etc/systemd/system/archipelago-npm-bridge.service",
include_str!("../../../image-recipe/configs/archipelago-npm-bridge.service"),
),
(
"/etc/systemd/system/archipelago-npm-bridge.timer",
include_str!("../../../image-recipe/configs/archipelago-npm-bridge.timer"),
),
] {
let changed = write_root_if_needed(path, content).await?;
units_changed |= changed && (path.ends_with(".service") || path.ends_with(".timer"));
}
if units_changed {
anyhow::ensure!(
host_sudo(&["systemctl", "daemon-reload"]).await?.success(),
"NPM bridge daemon reload failed"
);
}
anyhow::ensure!(
host_sudo(&[
"systemctl",
"enable",
"--now",
"archipelago-npm-bridge.timer"
])
.await?
.success(),
"NPM bridge timer could not start"
);
Ok(())
}
/// Entry point called from main startup. Never returns an error to the caller —
/// failing to bootstrap host artifacts must not prevent the backend from serving.
pub async fn ensure_doctor_installed() {
@@ -432,6 +507,17 @@ async fn run_runtime_assets() -> Result<bool> {
if !status.success() {
anyhow::bail!("install nginx-archipelago.conf exited with {}", status);
}
let acme_status = host_sudo(&[
"python3",
"-c",
include_str!("../../../scripts/npm-public-bridge.py"),
"--acme-only",
])
.await?;
anyhow::ensure!(
acme_status.success(),
"active NPM ACME root migration failed"
);
changed = true;
}
@@ -448,6 +534,8 @@ async fn run_runtime_assets() -> Result<bool> {
"archipelago-doctor.service",
"archipelago-doctor.timer",
"archipelago-host-secrets-audit.service",
"archipelago-npm-bridge.service",
"archipelago-npm-bridge.timer",
] {
let src = configs.join(unit);
if src.exists() {
@@ -475,7 +563,7 @@ async fn run_runtime_assets() -> Result<bool> {
// or directory"). Skipped when byte-identical; a running daemon is
// unaffected (install replaces the inode) and picks the new binary up
// on its next spawn.
for tool in ["archy-reticulum-daemon", "archy-rnodeconf"] {
for tool in ["archy-reticulum-daemon", "archy-rnodeconf", "archy-esptool"] {
let src = runtime_dir.join("radio-tools").join(tool);
if !src.exists() {
continue;
+145 -4
View File
@@ -118,10 +118,12 @@ fn selected_manifest(entry: AppCatalogEntry) -> Option<serde_json::Value> {
// Never let an unknown future requirement become an unsafe partial match.
for variant in entry.manifest_variants.into_iter().rev() {
if !variant.requires.is_empty()
&& variant
.requires
.iter()
.all(|capability| capability == "runtime-migration-backup-v1")
&& variant.requires.iter().all(|capability| {
matches!(
capability.as_str(),
"runtime-migration-backup-v1" | "npm-legacy-host-gateway-v1"
)
})
{
return Some(variant.manifest);
}
@@ -468,6 +470,12 @@ pub struct CatalogRefresh {
/// changed. Best-effort: a fetch failure leaves the existing cache untouched
/// (origin-always-wins; updates simply aren't refreshed this cycle).
pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<CatalogRefresh> {
// Explicit operator-only qualification of a signed candidate on selected
// nodes. Never change fleet mirrors or fall back to an older public catalog
// while a candidate is selected. Normal signature enforcement still applies.
if let Some(path) = std::env::var_os("ARCHY_APP_CATALOG_CANDIDATE") {
return refresh_candidate_catalog(data_dir, Path::new(&path)).await;
}
let mirrors = crate::update::load_mirrors(data_dir)
.await
.unwrap_or_default();
@@ -514,6 +522,49 @@ pub async fn refresh_catalog(data_dir: &Path) -> anyhow::Result<CatalogRefresh>
Err(last_err.unwrap_or_else(|| anyhow::anyhow!("no catalog mirrors reachable")))
}
async fn refresh_candidate_catalog(data_dir: &Path, path: &Path) -> anyhow::Result<CatalogRefresh> {
anyhow::ensure!(
path.is_absolute(),
"candidate catalog path must be absolute"
);
let metadata = tokio::fs::metadata(path)
.await
.context("inspect candidate catalog")?;
anyhow::ensure!(
metadata.is_file() && metadata.len() <= 4 * 1024 * 1024,
"candidate catalog must be a file no larger than 4 MiB"
);
let body = tokio::fs::read_to_string(path)
.await
.context("read candidate catalog")?;
anyhow::ensure!(
body.len() <= 4 * 1024 * 1024,
"candidate catalog exceeds 4 MiB"
);
let raw: serde_json::Value = serde_json::from_str(&body)?;
anyhow::ensure!(
matches!(
crate::trust::verify_detached(&raw)?,
crate::trust::SignatureStatus::Verified { anchored: true, .. }
),
"candidate catalog requires a signature anchored to the release root"
);
let catalog: AppCatalog = serde_json::from_value(raw)?;
let changed = write_cache(data_dir, &body)?;
if changed {
*CACHE.lock().unwrap() = None;
}
info!(
apps = catalog.apps.len(),
changed,
"app-catalog: using explicitly selected signed candidate; public catalog refresh paused"
);
Ok(CatalogRefresh {
apps: catalog.apps.len(),
changed,
})
}
async fn fetch_one(client: &reqwest::Client, url: &str) -> anyhow::Result<(AppCatalog, String)> {
let resp = client.get(url).send().await?;
if !resp.status().is_success() {
@@ -582,6 +633,77 @@ fn write_cache(data_dir: &Path, body: &str) -> anyhow::Result<bool> {
mod tests {
use super::*;
fn signed_candidate(key_byte: u8) -> serde_json::Value {
// Same test anchor as trust::signed_doc tests; never a production key.
let anchor = ed25519_dalek::SigningKey::from_bytes(&[7u8; 32]);
std::env::set_var(
"ARCHY_RELEASE_ROOT_PUBKEY",
hex::encode(anchor.verifying_key().to_bytes()),
);
let key = ed25519_dalek::SigningKey::from_bytes(&[key_byte; 32]);
let mut value = serde_json::json!({"schema":1,"apps":{"demo":{"version":"2"}},"future_field":{"retain":true}});
let (sig, did) = crate::trust::signed_doc::sign_detached(&key, &value).unwrap();
value["signature"] = sig.into();
value["signed_by"] = did.into();
value
}
#[tokio::test]
async fn candidate_catalog_preserves_signed_bytes_and_is_idempotent() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("candidate.json");
let body = serde_json::to_string_pretty(&signed_candidate(7)).unwrap();
std::fs::write(&path, &body).unwrap();
let first = refresh_candidate_catalog(dir.path(), &path).await.unwrap();
assert!(first.changed);
assert_eq!(first.apps, 1);
assert_eq!(
std::fs::read_to_string(dir.path().join(APP_CATALOG_FILE)).unwrap(),
body
);
assert!(
!refresh_candidate_catalog(dir.path(), &path)
.await
.unwrap()
.changed
);
}
#[tokio::test]
async fn rejected_candidate_never_replaces_previous_catalog() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("candidate.json");
let previous = "previous cached bytes";
write_cache(dir.path(), previous).unwrap();
let mut tampered = signed_candidate(7);
tampered["apps"]["demo"]["version"] = "tampered".into();
for body in [
"malformed".into(),
r#"{"schema":1,"apps":{}}"#.into(),
signed_candidate(11).to_string(),
tampered.to_string(),
" ".repeat(4 * 1024 * 1024 + 1),
] {
std::fs::write(&path, body).unwrap();
assert!(refresh_candidate_catalog(dir.path(), &path).await.is_err());
assert_eq!(
std::fs::read_to_string(dir.path().join(APP_CATALOG_FILE)).unwrap(),
previous
);
}
std::fs::remove_file(&path).unwrap();
assert!(refresh_candidate_catalog(dir.path(), &path).await.is_err());
assert!(
refresh_candidate_catalog(dir.path(), Path::new("relative.json"))
.await
.is_err()
);
assert_eq!(
std::fs::read_to_string(dir.path().join(APP_CATALOG_FILE)).unwrap(),
previous
);
}
#[test]
fn catalog_migration_variant_is_compatible_with_old_and_future_daemons() {
let raw = serde_json::json!({
@@ -608,6 +730,25 @@ mod tests {
assert!(chosen["app"]["container"].get("network").is_none());
}
#[test]
fn npm_gateway_variant_requires_explicit_runtime_support() {
let mut raw = serde_json::json!({
"version": "2.12.1", "manifest": {"network":"pasta"},
"manifest_variants": [{"requires":["runtime-migration-backup-v1", "npm-legacy-host-gateway-v1"],
"manifest":{"network":"slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"}}]
});
assert_eq!(
selected_manifest(serde_json::from_value(raw.clone()).unwrap()).unwrap()["network"],
"slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"
);
// A runtime missing any required capability must retain the base.
raw["manifest_variants"][0]["requires"][1] = serde_json::json!("unknown-gateway-v2");
assert_eq!(
selected_manifest(serde_json::from_value(raw).unwrap()).unwrap()["network"],
"pasta"
);
}
#[test]
fn parses_and_ignores_unknown_fields() {
let json = r#"{
@@ -24,6 +24,7 @@ fn canonical_package_id(name: &str) -> &str {
"immich-postgres" => "immich_postgres",
"immich-redis" => "immich_redis",
"mempool-web" | "mempool-frontend" => "mempool",
"btcpay" | "btcpayserver" => "btcpay-server",
name => name,
}
}
@@ -445,6 +446,15 @@ mod lifecycle_regression_tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[test]
fn btcpay_aliases_share_one_package_without_promoting_dependencies() {
for name in ["btcpay", "btcpayserver", "btcpay-server", "archy-btcpay"] {
assert_eq!(canonical_package_id(name), "btcpay-server");
}
assert_eq!(canonical_package_id("archy-btcpay-db"), "btcpay-db");
assert_eq!(canonical_package_id("archy-nbxplorer"), "nbxplorer");
}
#[test]
fn immich_dependency_aliases_share_the_hidden_component_ids() {
for id in [
+147 -2
View File
@@ -17,6 +17,84 @@ pub const DEFAULT_CONFIG_PATH: &str = "/var/lib/archipelago/filebrowser-data/.fi
const DEFAULT_CONFIG_JSON: &str =
"{\"port\":80,\"baseURL\":\"\",\"address\":\"0.0.0.0\",\"database\":\"/data/filebrowser.db\",\"root\":\"/srv\",\"log\":\"stdout\"}\n";
/// One atomically published record shared by setup, Cloud and credentials UI.
/// Deliberately has no Debug implementation: the password must never be logged.
#[derive(serde::Deserialize)]
pub struct CloudCredentials {
pub schema: u32,
pub username: String,
pub password: String,
}
pub async fn cloud_credentials(directory: &Path) -> Result<CloudCredentials> {
let path = directory.join("credentials.json");
match fs::read(&path).await {
Ok(bytes) => {
let value: CloudCredentials = serde_json::from_slice(&bytes)
.context("Invalid private File Browser credential record")?;
let suffix = value.username.strip_prefix("archy-").unwrap_or("");
anyhow::ensure!(
value.schema == 1
&& suffix.len() == 32
&& suffix.bytes().all(|c| c.is_ascii_hexdigit())
&& value.password.len() == 64
&& value.password.bytes().all(|c| c.is_ascii_hexdigit()),
"Invalid managed File Browser credentials"
);
Ok(value)
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
// Compatibility during staged upgrades only. Never invent admin/admin
// when a secret is missing; the pre-start provisioner repairs legacy DBs.
let password = fs::read_to_string(directory.join("password"))
.await
.context("File Browser secure Cloud login has not been provisioned")?;
let password = password.trim().to_owned();
anyhow::ensure!(
!password.is_empty() && password != "admin",
"File Browser default credentials must be migrated before Cloud login"
);
Ok(CloudCredentials {
schema: 0,
username: "admin".into(),
password,
})
}
Err(error) => Err(error).context("Cannot read private File Browser credentials"),
}
}
/// Prepare a stopped server using the same helper used by Quadlet and the ISO.
pub async fn prepare_credentials(
paths: &EnsurePaths,
secret_dir: &Path,
image: &str,
runtime: &str,
) -> Result<()> {
let output = tokio::process::Command::new("python3")
.args([
"-c",
include_str!("../../../../scripts/filebrowser-credentials.py"),
"--image",
image,
"--runtime",
runtime,
"--data-dir",
&paths.data_dir.to_string_lossy(),
"--srv-root",
&paths.srv_root.to_string_lossy(),
"--secrets-dir",
&secret_dir.to_string_lossy(),
])
.kill_on_drop(true)
.output()
.await
.context("Running File Browser credential setup")?;
anyhow::ensure!(output.status.success(),
"File Browser secure login setup failed; existing state and private rollback backup retained");
Ok(())
}
#[derive(Debug, Clone)]
pub struct EnsurePaths {
pub srv_root: PathBuf,
@@ -82,7 +160,18 @@ async fn create_dir_all_or_sudo(path: &std::path::Path) -> Result<()> {
async fn write_config_atomically(paths: &EnsurePaths) -> Result<()> {
let tmp = paths.config_path.with_extension("tmp");
match fs::write(&tmp, DEFAULT_CONFIG_JSON).await {
let legacy = paths.data_dir.join("database.db").exists();
let canonical = paths.data_dir.join("filebrowser.db").exists();
anyhow::ensure!(
!(legacy && canonical),
"Multiple File Browser databases need explicit config selection"
);
let config = if legacy {
DEFAULT_CONFIG_JSON.replace("/data/filebrowser.db", "/data/database.db")
} else {
DEFAULT_CONFIG_JSON.to_string()
};
match fs::write(&tmp, &config).await {
Ok(()) => {
fs::rename(&tmp, &paths.config_path)
.await
@@ -99,7 +188,7 @@ async fn write_config_atomically(paths: &EnsurePaths) -> Result<()> {
let script = format!(
"set -eu\ncat > '{}' <<'FILEBROWSERCONF'\n{}FILEBROWSERCONF\n",
shell_quote(&paths.config_path.to_string_lossy()),
DEFAULT_CONFIG_JSON
config
);
let status = host_sudo(&["sh", "-lc", &script])
.await
@@ -312,6 +401,62 @@ async fn write_via_userns(dir: PathBuf, name: String, bytes: Vec<u8>) -> Result<
mod tests {
use super::*;
#[tokio::test]
async fn cloud_credentials_use_unique_record_and_never_default_password() {
let dir = tempfile::tempdir().unwrap();
assert!(cloud_credentials(dir.path()).await.is_err());
fs::write(dir.path().join("password"), "admin")
.await
.unwrap();
assert!(cloud_credentials(dir.path()).await.is_err());
fs::write(dir.path().join("password"), "legacy-unique-password")
.await
.unwrap();
assert_eq!(cloud_credentials(dir.path()).await.unwrap().schema, 0);
let value = serde_json::json!({"schema":1,"username":format!("archy-{}", "a".repeat(32)),"password":"b".repeat(64)});
fs::write(dir.path().join("credentials.json"), value.to_string())
.await
.unwrap();
let loaded = cloud_credentials(dir.path()).await.unwrap();
assert_eq!(loaded.username, value["username"].as_str().unwrap());
assert_eq!(loaded.password, value["password"].as_str().unwrap());
fs::write(dir.path().join("credentials.json"), "{}")
.await
.unwrap();
assert!(
cloud_credentials(dir.path()).await.is_err(),
"damaged managed record must not fall back to old credentials"
);
}
#[tokio::test]
async fn missing_config_preserves_legacy_database_and_refuses_ambiguity() {
let tmp = tempfile::tempdir().unwrap();
let paths = EnsurePaths {
srv_root: tmp.path().join("srv"),
data_dir: tmp.path().join("data"),
config_path: tmp.path().join("data/.filebrowser.json"),
};
fs::create_dir_all(&paths.data_dir).await.unwrap();
fs::write(paths.data_dir.join("database.db"), b"legacy fixture")
.await
.unwrap();
ensure_config(&paths).await.unwrap();
let config: serde_json::Value =
serde_json::from_slice(&fs::read(&paths.config_path).await.unwrap()).unwrap();
assert_eq!(config["database"], "/data/database.db");
fs::remove_file(&paths.config_path).await.unwrap();
fs::write(paths.data_dir.join("filebrowser.db"), b"other fixture")
.await
.unwrap();
assert!(ensure_config(&paths).await.is_err());
assert!(!paths.config_path.exists());
assert_eq!(
fs::read(paths.data_dir.join("database.db")).await.unwrap(),
b"legacy fixture"
);
}
#[tokio::test]
async fn ensure_config_creates_dirs_and_file() {
let tmp = tempfile::TempDir::new().unwrap();
+1
View File
@@ -13,6 +13,7 @@ pub mod image_policy;
pub mod image_versions;
pub mod lnd;
pub mod migration_backup;
pub mod npm;
pub mod prod_orchestrator;
pub mod quadlet;
pub mod registry;
+115
View File
@@ -0,0 +1,115 @@
//! Preserve NPM's active persistent mounts across installer and runtime paths.
use anyhow::{bail, Context, Result};
use archipelago_container::AppManifest;
use serde::Deserialize;
use std::path::Path;
use std::time::Duration;
#[derive(Debug, Deserialize)]
pub struct Storage {
pub data: String,
pub certificates: String,
}
impl Storage {
pub fn bind_mounts(&self) -> Vec<String> {
vec![
format!("{}:/data", self.data),
format!("{}:/etc/letsencrypt", self.certificates),
]
}
pub fn apply(&self, manifest: &mut AppManifest) -> Result<()> {
for (target, source) in [
("/data", &self.data),
("/etc/letsencrypt", &self.certificates),
] {
let matching: Vec<_> = manifest
.app
.volumes
.iter_mut()
.filter(|volume| volume.target == target)
.collect();
if matching.len() != 1 {
bail!("NPM requires one persistent mount per storage target");
}
let volume = matching.into_iter().next().unwrap();
if volume.volume_type != "bind" {
bail!("NPM persistent state requires bind mounts");
}
volume.source.clone_from(source);
}
Ok(())
}
}
fn parse_storage(bytes: &[u8]) -> Result<Storage> {
let storage: Storage =
serde_json::from_slice(bytes).context("invalid NPM storage resolution")?;
for value in [&storage.data, &storage.certificates] {
if !Path::new(value).is_absolute() || value.chars().any(|c| c.is_control() || c == ':') {
bail!("invalid NPM persistent storage path");
}
}
Ok(storage)
}
pub async fn resolve_storage() -> Result<Storage> {
// Verify live mounts/database before any caller can stop or recreate NPM.
// Remember only validated mount paths so later recreation, after the inspect
// record is removed, still uses the operator's original storage.
let mut command = tokio::process::Command::new("python3");
command
.args([
"-c",
include_str!("../../../../scripts/npm-public-bridge.py"),
"--resolve",
"--remember",
"--prepare-realip",
])
.kill_on_drop(true);
let output = tokio::time::timeout(Duration::from_secs(75), command.output())
.await
.context("NPM storage resolution timed out; existing state preserved")??;
if !output.status.success() {
bail!("NPM storage resolution failed; inspect mount/database ambiguity or permissions before migration");
}
parse_storage(&output.stdout)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resolved_mounts_preserve_custom_and_legacy_paths() {
for data in [
"/var/lib/archipelago/nginx-proxy-manager/data",
"/srv/operator npm",
] {
let bytes = serde_json::to_vec(
&serde_json::json!({"data": data, "certificates": "/srv/certificates"}),
)
.unwrap();
let storage = parse_storage(&bytes).unwrap();
assert_eq!(
storage.bind_mounts(),
[
format!("{data}:/data"),
"/srv/certificates:/etc/letsencrypt".into(),
]
);
}
}
#[test]
fn invalid_resolution_cannot_become_a_container_mount() {
for data in ["relative", "/srv/data:ro", "/srv/data\nother"] {
let bytes =
serde_json::to_vec(&serde_json::json!({"data": data, "certificates": "/certs"}))
.unwrap();
assert!(parse_storage(&bytes).is_err());
}
assert!(parse_storage(b"{}").is_err());
}
}
@@ -92,16 +92,71 @@ fn is_restart_sensitive_app(app_id: &str) -> bool {
fn is_builtin_network_mode(network: &str) -> bool {
matches!(
network,
"host" | "bridge" | "none" | "slirp4netns" | "pasta"
"host"
| "bridge"
| "none"
| "slirp4netns"
| "slirp4netns:allow_host_loopback=true"
| "slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"
| "pasta"
)
}
// Only an explicitly selected rootless mode establishes drift. An omitted
// network delegates to Podman and must not recreate unrelated installed apps.
fn rootless_network_mode_drifted(expected: Option<&str>, actual: &str) -> bool {
matches!(expected, Some("slirp4netns" | "pasta"))
&& !actual.trim().is_empty()
&& actual.trim().split(':').next() != expected
let actual = actual.trim();
if actual.is_empty() {
return false;
}
match expected {
Some(
expected @ ("slirp4netns:allow_host_loopback=true"
| "slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"),
) => {
let (mode, options) = actual.split_once(':').unwrap_or((actual, ""));
let required = expected.split_once(':').unwrap().1;
mode != "slirp4netns"
|| required.split(',').any(|wanted| {
let key = wanted.split_once('=').unwrap().0;
!options.split(',').any(|option| option == wanted)
|| options.split(',').any(|option| {
option.split_once('=').is_some_and(|(name, _)| name == key)
&& option != wanted
})
})
}
Some(mode @ ("slirp4netns" | "pasta")) => actual.split(':').next() != Some(mode),
_ => false,
}
}
// API-created containers may omit rootless options from HostConfig.NetworkMode.
// generate spec retains them; inspect alone would cause an endless repair loop.
fn rootless_network_from_spec(bytes: &[u8]) -> Option<String> {
let spec: serde_json::Value = serde_json::from_slice(bytes).ok()?;
let mode = spec.get("netns")?.get("nsmode")?.as_str()?;
if !matches!(mode, "slirp4netns" | "pasta") {
return None;
}
let options = spec
.get("network_options")
.and_then(|options| options.get(mode));
match options {
None => Some(mode.to_string()),
Some(options) => {
let options = options
.as_array()?
.iter()
.map(|value| value.as_str())
.collect::<Option<Vec<_>>>()?;
if options.is_empty() {
Some(mode.to_string())
} else {
Some(format!("{mode}:{}", options.join(",")))
}
}
}
}
fn missing_declared_capability(expected: &[String], actual: &[String]) -> bool {
@@ -175,6 +230,11 @@ fn manifest_dependency_app_ids(manifest: &AppManifest) -> Vec<String> {
}
fn host_port_wait_timeout_secs(manifest: &AppManifest) -> u64 {
// First NPM initialization generates keys and migrates its database before
// exposing nginx. Its readiness budget must not depend on the network driver.
if manifest.app.id == "nginx-proxy-manager" {
return 180;
}
if manifest.app.id == "uptime-kuma" {
return 420;
}
@@ -2425,6 +2485,49 @@ impl ProdContainerOrchestrator {
}
match status.state {
ContainerState::Running => {
// Legacy runtime path: migrate credentials once without
// recreating accounts or changing operator passwords.
// Quadlet installations receive the same helper through
// the required ExecStartPre drift/restart above.
if app_id == "filebrowser" && !self.use_quadlet_backends && !cfg!(test) {
let secrets = self.secrets_dir.join("filebrowser");
let managed = filebrowser::cloud_credentials(&secrets)
.await
.map(|value| value.schema == 1)
.unwrap_or(false);
if !managed {
if !self.should_attempt_repair(&name).await {
return Ok(ReconcileAction::Left(
"filebrowser-credential-repair-budget-exhausted".into(),
));
}
let unit_managed = self.runtime.cli_name() == "podman"
&& quadlet::unit_exists(&name).await;
let service = format!("{name}.service");
if unit_managed {
quadlet::stop_service(&service).await?;
} else {
self.runtime.stop_container(&name).await?;
}
let prepared = filebrowser::prepare_credentials(
&self.filebrowser_paths,
&secrets,
&status.image,
self.runtime.cli_name(),
)
.await;
// Restore service availability even when setup
// rolled back. Preserve the setup failure itself.
let started = if unit_managed {
quadlet::restart_service(&service).await
} else {
self.runtime.start_container(&name).await
};
prepared?;
started?;
return Ok(ReconcileAction::Started);
}
}
// Zombie guard: podman can report a container "running"
// after its process has died (conmon SIGKILLed in a
// cgroup cascade on archipelago restart, etc.). Such a
@@ -2971,6 +3074,18 @@ impl ProdContainerOrchestrator {
self.ensure_manifest_files(manifest).await?;
self.apply_data_uid(manifest).await?;
self.run_post_data_uid_hooks(&manifest.app.id).await?;
if manifest.app.id == "filebrowser" && !self.use_quadlet_backends && !cfg!(test) {
let image = manifest.app.container.image.as_deref().ok_or_else(|| {
anyhow::anyhow!("File Browser needs a pinned image for credential setup")
})?;
filebrowser::prepare_credentials(
&self.filebrowser_paths,
&self.secrets_dir.join("filebrowser"),
image,
self.runtime.cli_name(),
)
.await?;
}
Ok(())
}
@@ -3068,6 +3183,11 @@ impl ProdContainerOrchestrator {
container = %name,
"Phase 3.3 migration: replacing pre-Quadlet container with systemd-managed unit"
);
// Resolve active persistent mounts before the old inspect record is
// removed. NPM may use a legacy or operator-selected data directory.
let mut resolved = lm.manifest.clone();
self.resolve_dynamic_env(&mut resolved).await?;
self.backup_runtime_change(name, &resolved).await?;
// Stop+remove the old container record. Volumes survive (host
// bind mounts are not touched by podman rm).
self.runtime
@@ -3077,8 +3197,6 @@ impl ProdContainerOrchestrator {
// Re-render the manifest with dynamic env baked in, then go
// through the same install path a fresh install would.
let mut resolved = lm.manifest.clone();
self.resolve_dynamic_env(&mut resolved).await?;
self.install_via_quadlet(&resolved, name)
.await
.with_context(|| format!("Phase 3.3: re-install {name} via Quadlet"))?;
@@ -3797,6 +3915,11 @@ impl ProdContainerOrchestrator {
}
async fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> {
if manifest.app.id == "nginx-proxy-manager" {
crate::container::npm::resolve_storage()
.await?
.apply(manifest)?;
}
// Idempotency guard: partitioning already ran on this instance.
// Re-running would re-taint against an environment that no longer
// contains the composite entries and silently drop them. Callers
@@ -4068,7 +4191,12 @@ impl ProdContainerOrchestrator {
if unmanaged
&& matches!(
manifest.app.container.network.as_deref(),
Some("slirp4netns" | "pasta")
Some(
"slirp4netns"
| "slirp4netns:allow_host_loopback=true"
| "slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"
| "pasta"
)
)
{
if let Ok(output) = tokio::process::Command::new("podman")
@@ -4076,13 +4204,36 @@ impl ProdContainerOrchestrator {
.output()
.await
{
if output.status.success()
&& rootless_network_mode_drifted(
if output.status.success() {
let mut actual = String::from_utf8_lossy(&output.stdout).trim().to_string();
if matches!(
manifest.app.container.network.as_deref(),
&String::from_utf8_lossy(&output.stdout),
)
{
return true;
Some(
"slirp4netns:allow_host_loopback=true"
| "slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"
)
) && actual == "slirp4netns"
{
let spec = tokio::process::Command::new("podman")
.args(["generate", "spec", name])
.output()
.await;
actual = match spec {
Ok(spec) if spec.status.success() => {
rootless_network_from_spec(&spec.stdout).unwrap_or_default()
}
_ => String::new(),
};
if actual.is_empty() {
tracing::warn!(app = %name, "Could not verify rootless network options; retaining the existing container");
}
}
if rootless_network_mode_drifted(
manifest.app.container.network.as_deref(),
&actual,
) {
return true;
}
}
}
}
@@ -5242,8 +5393,68 @@ mod tests {
));
}
#[test]
fn npm_legacy_gateway_converges_and_rejects_conflicting_network_options() {
let expected = Some("slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24");
assert!(is_builtin_network_mode(expected.unwrap()));
for actual in [
"pasta",
"slirp4netns:allow_host_loopback=true",
"slirp4netns:allow_host_loopback=true,cidr=10.0.2.0/24",
"slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24,cidr=10.0.2.0/24",
] {
assert!(rootless_network_mode_drifted(expected, actual), "{actual}");
}
let actual = "slirp4netns:cidr=169.254.1.0/24,allow_host_loopback=true,mtu=65520";
assert!(!rootless_network_mode_drifted(expected, actual));
let spec = br#"{"netns":{"nsmode":"slirp4netns"},"network_options":{"slirp4netns":["cidr=169.254.1.0/24","allow_host_loopback=true"]}}"#;
assert!(!rootless_network_mode_drifted(
expected,
&rootless_network_from_spec(spec).unwrap()
));
}
#[test]
fn npm_initialization_budget_survives_network_migration() {
let mut manifest = AppManifest::parse(include_str!(
"../../../../apps/nginx-proxy-manager/manifest.yml"
))
.unwrap();
for network in ["pasta", "slirp4netns:allow_host_loopback=true"] {
manifest.app.container.network = Some(network.to_string());
assert_eq!(host_port_wait_timeout_secs(&manifest), 180);
}
}
#[test]
fn api_created_rootless_options_do_not_cause_repeated_recreation() {
let expected = Some("slirp4netns:allow_host_loopback=true");
let spec = br#"{"netns":{"nsmode":"slirp4netns"},"network_options":{"slirp4netns":["allow_host_loopback=true"]}}"#;
let actual = rootless_network_from_spec(spec).unwrap();
assert!(!rootless_network_mode_drifted(expected, &actual));
let plain = rootless_network_from_spec(br#"{"netns":{"nsmode":"slirp4netns"}}"#).unwrap();
assert!(rootless_network_mode_drifted(expected, &plain));
assert!(rootless_network_from_spec(b"invalid").is_none());
}
#[test]
fn explicit_rootless_network_change_converges_without_guessing_defaults() {
let npm = Some("slirp4netns:allow_host_loopback=true");
assert!(is_builtin_network_mode(npm.unwrap()));
for actual in [
"pasta",
"bridge",
"slirp4netns",
"slirp4netns:allow_host_loopback=false",
] {
assert!(rootless_network_mode_drifted(npm, actual), "{actual}");
}
for actual in [
"slirp4netns:allow_host_loopback=true",
"slirp4netns:mtu=65520,allow_host_loopback=true",
] {
assert!(!rootless_network_mode_drifted(npm, actual), "{actual}");
}
assert!(rootless_network_mode_drifted(Some("slirp4netns"), "pasta"));
assert!(rootless_network_mode_drifted(Some("slirp4netns"), "bridge"));
assert!(!rootless_network_mode_drifted(
+81 -1
View File
@@ -68,6 +68,10 @@ pub enum NetworkMode {
/// Rootless slirp4netns networking. Podman rejects network aliases with
/// this mode, so render only Network=slirp4netns.
Slirp4netns,
/// Permit explicit host aliases as well as LAN upstreams for NPM.
Slirp4netnsHostLoopback,
/// Preserve existing NPM upstreams using pasta's former host gateway.
Slirp4netnsLegacyGateway,
/// Rootless pasta networking. This is more reliable than slirp4netns for
/// host port forwarding on long-running web apps.
Pasta,
@@ -229,6 +233,15 @@ impl QuadletUnit {
NetworkMode::Host => {
let _ = writeln!(s, "Network=host");
}
NetworkMode::Slirp4netnsHostLoopback => {
let _ = writeln!(s, "Network=slirp4netns:allow_host_loopback=true");
}
NetworkMode::Slirp4netnsLegacyGateway => {
let _ = writeln!(
s,
"Network=slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"
);
}
NetworkMode::Slirp4netns => {
let _ = writeln!(s, "Network=slirp4netns");
}
@@ -348,6 +361,26 @@ impl QuadletUnit {
}
let _ = writeln!(s);
let _ = writeln!(s, "[Service]");
if self.name == "filebrowser" {
// Runs while the managed server is stopped, before it can expose
// a default account. The helper verifies real login and root access.
let mut argv = vec![
"/usr/bin/python3".to_string(),
"/opt/archipelago/scripts/filebrowser-credentials.py".to_string(),
"--image".to_string(),
self.image.clone(),
];
for (target, flag) in [("/data", "--data-dir"), ("/srv", "--srv-root")] {
if let Some(mount) = self
.bind_mounts
.iter()
.find(|m| m.container == Path::new(target))
{
argv.extend([flag.to_string(), mount.host.display().to_string()]);
}
}
let _ = writeln!(s, "ExecStartPre={}", shell_join(&argv));
}
// Dependency-gated apps may legitimately keep their container entrypoint
// in a wait loop before the actual daemon binds ports. Fedimint waits
// for Bitcoin IBD to finish before execing fedimintd; systemd's default
@@ -447,6 +480,12 @@ impl QuadletUnit {
other if !other.is_empty() && other != "isolated" => NetworkMode::Bridge(other.into()),
_ => match app.container.network.as_deref() {
Some("slirp4netns") => NetworkMode::Slirp4netns,
Some("slirp4netns:allow_host_loopback=true") => {
NetworkMode::Slirp4netnsHostLoopback
}
Some("slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24") => {
NetworkMode::Slirp4netnsLegacyGateway
}
Some("pasta") => NetworkMode::Pasta,
Some(n) if !n.is_empty() && n != "host" => NetworkMode::Bridge(n.into()),
_ => NetworkMode::Default,
@@ -1071,7 +1110,8 @@ pub fn exec_changed(old_body: &str, new_body: &str) -> bool {
// Entrypoint= and Exec= together define what the container runs, so a drift
// in either must recreate the container (e.g. when this renderer first
// splits a folded `Exec=sh -lc ...` into `Entrypoint=sh` + `Exec=-lc ...`).
directive_values(old_body, "Exec=") != directive_values(new_body, "Exec=")
directive_values(old_body, "ExecStartPre=") != directive_values(new_body, "ExecStartPre=")
|| directive_values(old_body, "Exec=") != directive_values(new_body, "Exec=")
|| directive_values(old_body, "Entrypoint=") != directive_values(new_body, "Entrypoint=")
}
@@ -1142,6 +1182,28 @@ pub async fn is_active(service: &str) -> bool {
#[cfg(test)]
mod tests {
#[test]
fn filebrowser_prestart_credentials_are_a_required_runtime_change() {
let unit = super::QuadletUnit {
name: "filebrowser".into(),
image: "registry.example/filebrowser:v2.63.23".into(),
..Default::default()
};
let rendered = unit.render();
assert!(rendered.contains("ExecStartPre=/usr/bin/python3 /opt/archipelago/scripts/filebrowser-credentials.py --image registry.example/filebrowser:v2.63.23"));
let old = rendered
.lines()
.filter(|line| !line.starts_with("ExecStartPre="))
.collect::<Vec<_>>()
.join("\n");
assert!(super::exec_changed(&old, &rendered));
assert!(!super::exec_changed(&rendered, &rendered));
let other = super::QuadletUnit {
name: "other-app".into(),
..unit
};
assert!(!other.render().contains("filebrowser-credentials.py"));
}
use super::*;
use tempfile::tempdir;
@@ -1709,6 +1771,24 @@ app:
assert!(!s.contains("--network-alias"));
}
#[test]
fn npm_network_preserves_same_node_upstreams_without_aliases() {
let m = AppManifest::parse(include_str!(
"../../../../apps/nginx-proxy-manager/manifest.yml"
))
.unwrap();
let rendered = QuadletUnit::from_manifest(&m, "nginx-proxy-manager").render();
assert_eq!(
rendered
.lines()
.filter(|line| line.starts_with("Network="))
.collect::<Vec<_>>(),
vec!["Network=slirp4netns:allow_host_loopback=true,cidr=169.254.1.0/24"]
);
assert!(!rendered.contains("NetworkAlias="));
assert!(!rendered.contains("--network-alias"));
}
#[test]
fn from_manifest_pasta_omits_network_alias() {
let yaml = r#"
+5
View File
@@ -162,6 +162,11 @@ pub async fn record_purchase(
save_index(data_dir, &index).await
}
/// Payment decisions must not interpret an unreadable index as no purchases.
pub async fn list_owned_checked(data_dir: &Path) -> Result<Vec<OwnedItem>> {
Ok(load_index_checked(data_dir).await?.items)
}
/// Every item this node owns.
pub async fn list_owned(data_dir: &Path) -> Vec<OwnedItem> {
load_index(data_dir).await.items
+395 -69
View File
@@ -7,7 +7,8 @@
//! to a full chip erase before write.
//!
//! MeshCore and Meshtastic are flashed the same way: download a released
//! image, `esptool erase_flash`, then `esptool write_flash 0x0 <image>`.
//! image, verify it, then run `write_flash --erase-all 0x0 <image>` with
//! the packaged esptool executable.
//! Reticulum/RNode is different: `archy-rnodeconf --autoinstall` owns the
//! whole fetch+erase+flash+EEPROM-bootstrap sequence itself (confirmed live
//! via `archy-rnodeconf --help` — there is no raw esptool path exposed for
@@ -49,32 +50,18 @@ impl FlashBoard {
}
}
/// Map a detected USB vid:pid to a known flashable board, using the same
/// table as `image-recipe/configs/99-mesh-radio.rules`. CP2102 (10c4:ea60)
/// is confirmed there as Heltec V3's USB-UART bridge chip, and is safe to
/// auto-match since that vid:pid is bridge-chip-specific.
///
/// Heltec V4 is NOT auto-matchable and deliberately has no entry here: it
/// was confirmed live (real hardware, 2026-07-23) to use the ESP32-S3's
/// built-in native-USB JTAG/serial peripheral, reporting vid:pid 303a:1001
/// with product string "USB JTAG/serial debug unit" — that descriptor is
/// baked into the chip's ROM and is IDENTICAL across every ESP32-S3 board
/// with native USB enabled, not just Heltec V4. Adding `303a:1001 =>
/// HeltecV4` here would silently misidentify any other native-USB ESP32-S3
/// board (a T3-S3, a bare devkit, etc.) as a V4 and risk writing the wrong
/// board's image. Callers (the RPC layer / frontend) must let the user pick
/// the board manually whenever this returns `None`.
pub fn resolve_flash_board(info: &DetectedDeviceInfo) -> Option<FlashBoard> {
match (info.vid.as_deref(), info.pid.as_deref()) {
(Some("10c4"), Some("ea60")) => Some(FlashBoard::HeltecV3),
_ => None,
}
/// Generic CP2102 and native ESP32-S3 USB IDs identify adapters/chips, not
/// board wiring. Require an explicit board until a board-specific identity is
/// available; guessing a Heltec model can write incompatible firmware.
pub fn resolve_flash_board(_info: &DetectedDeviceInfo) -> Option<FlashBoard> {
None
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum FlashStage {
Downloading,
Preparing,
Erasing,
Writing,
Autoinstalling,
@@ -101,11 +88,10 @@ const LOG_TAIL_MAX: usize = 200;
/// start opening the port (which itself toggles DTR/RTS) again.
const POST_FLASH_SETTLE_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
/// Absolute ceiling on a whole flash job (download + erase + write, or
/// autoinstall), regardless of what it's doing internally. Last-resort
/// safety net so a hang anywhere can't wedge the single-flash-job guard
/// forever — generous enough to never trigger on a legitimately slow
/// multi-hundred-MB transfer.
/// Deadline for preparation and warning threshold for the active flasher.
/// A download can be cancelled safely. An active write retains ownership until
/// its subprocess exits, even after this threshold: reporting an aborted job
/// while a detached writer continues would let a retry corrupt the device.
const MAX_JOB_DURATION: std::time::Duration = std::time::Duration::from_secs(15 * 60);
/// How long to wait for MeshService::stop() to release the serial port
@@ -247,6 +233,16 @@ fn firmware_cache_dir(data_dir: &Path) -> PathBuf {
data_dir.join("mesh").join("firmware-cache")
}
async fn invalidate_radio_settings_marker(data_dir: &Path) -> Result<()> {
// A full-chip flash erased the device's preferences. A previous host-side
// marker is no longer evidence that this radio has the requested settings.
match tokio::fs::remove_file(data_dir.join("meshcore-radio-params.json")).await {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error).context("Firmware written, but old radio-settings marker could not be cleared; reconnect is paused"),
}
}
/// No blanket `.timeout()` here on purpose: reqwest's request timeout covers
/// the *entire* request including streaming the response body, which would
/// kill a legitimate large download partway through (Meshtastic's esp32s3
@@ -314,6 +310,10 @@ pub async fn list_firmware(family: DeviceType) -> Result<Vec<String>> {
struct GithubAsset {
name: String,
browser_download_url: String,
#[serde(default)]
size: Option<u64>,
#[serde(default)]
digest: Option<String>,
}
#[derive(serde::Deserialize)]
@@ -322,8 +322,24 @@ struct GithubRelease {
assets: Vec<GithubAsset>,
}
async fn register_flash_job(handle: &FlashJobHandle, job: &Arc<FlashJob>) -> Result<()> {
// Keep checking and registering under one lock: concurrent RPCs must
// never start two writers on the same device.
let mut existing = handle.try_write().map_err(|_| anyhow::anyhow!(
"The radio is being inspected or reconfigured; wait for that operation to finish before flashing"
))?;
if let Some(current) = existing.as_ref() {
if !current.snapshot().await.done {
anyhow::bail!("A firmware flash is already in progress on this node");
}
}
*existing = Some(Arc::clone(job));
Ok(())
}
/// Start a flash job in the background. Returns as soon as the job has been
/// registered and the listener released — callers poll `FlashJobHandle` via
/// registered — preparation and listener shutdown run in the background.
/// Callers poll `FlashJobHandle` via
/// `mesh.flash-status` for progress. Only one job may be in flight at a time.
pub async fn start_flash_job(
handle: &FlashJobHandle,
@@ -333,21 +349,44 @@ pub async fn start_flash_job(
board: FlashBoard,
family: DeviceType,
) -> Result<()> {
{
let existing = handle.read().await;
if let Some(job) = existing.as_ref() {
if !job.snapshot().await.done {
anyhow::bail!("A firmware flash is already in progress on this node");
}
}
// Missing/corrupt tooling must fail before stopping a working radio or
// registering a job that can never reach the serial device.
if matches!(family, DeviceType::Meshtastic | DeviceType::Meshcore) {
preflight_esptool().await?;
}
let job = FlashJob::new(board, family, path.clone());
*handle.write().await = Some(Arc::clone(&job));
register_flash_job(handle, &job).await?;
let bg_job = Arc::clone(&job);
let bg_service = Arc::clone(mesh_service);
let task = tokio::spawn(async move {
// Downloads and checksum checks do not need exclusive serial access.
// Keep a working listener alive if upstream/download verification fails.
let prepared_image = if matches!(family, DeviceType::Meshcore | DeviceType::Meshtastic) {
match tokio::time::timeout(
MAX_JOB_DURATION,
fetch_esptool_image(board, family, &data_dir, &bg_job),
)
.await
{
Ok(Ok(image)) => Some(image),
result => {
let error = match result {
Ok(Err(error)) => error,
_ => anyhow::anyhow!(
"Firmware download exceeded the time limit; radio was not changed"
),
};
bg_job.fail(&error).await;
return;
}
}
} else {
None
};
// Cancellation is safe during download. Once listener shutdown starts,
// allow that bounded operation to finish instead of orphaning its task.
bg_job.set_stage(FlashStage::Preparing).await;
// esptool/archy-rnodeconf need exclusive serial access — release
// the listener's hold on the port before touching it. This USED
// TO run synchronously in start_flash_job before the job was even
@@ -404,17 +443,31 @@ pub async fn start_flash_job(
// subsequent mesh.flash-device call failed with "already in
// progress" until the service was restarted). Generous enough that
// a legitimately slow multi-hundred-MB transfer still completes.
let result = match tokio::time::timeout(
MAX_JOB_DURATION,
run_flash(board, family, &data_dir, &path, &bg_job),
)
.await
{
let flash = run_flash(
board,
family,
&data_dir,
&path,
prepared_image.as_deref(),
&bg_job,
);
tokio::pin!(flash);
let result = match tokio::time::timeout(MAX_JOB_DURATION, &mut flash).await {
Ok(inner) => inner,
Err(_) => Err(anyhow::anyhow!(
"Flash job exceeded the {}-minute ceiling — aborted",
MAX_JOB_DURATION.as_secs() / 60
)),
Err(_) if bg_job.snapshot().await.stage == FlashStage::Downloading => Err(
anyhow::anyhow!("Firmware download exceeded the time limit; radio was not written"),
),
Err(_) => {
// Dropping this future does NOT stop its flash subprocess.
// Keep the job busy until it exits, rather than allowing a
// second writer while the first one still owns the device.
bg_job.push_log("Flashing is taking longer than expected; waiting for the active tool to exit before allowing another operation").await;
flash.await
}
};
let result = match result {
Ok(()) => invalidate_radio_settings_marker(&data_dir).await,
error => error,
};
let succeeded = result.is_ok();
@@ -423,7 +476,6 @@ pub async fn start_flash_job(
bg_job
.push_log("Flash completed successfully".to_string())
.await;
bg_job.finish().await;
info!(path = %path, board = ?board, family = %family, "LoRa firmware flash succeeded");
}
Err(e) => {
@@ -434,7 +486,6 @@ pub async fn start_flash_job(
// erase_flash failed", no actual esptool stderr).
warn!(path = %path, error = %format!("{e:#}"), "LoRa firmware flash failed");
bg_job.push_log(format!("ERROR: {e:#}")).await;
bg_job.fail(e).await;
}
}
@@ -450,6 +501,9 @@ pub async fn start_flash_job(
}
if !succeeded {
if let Err(error) = &result {
bg_job.fail(error).await;
}
// Deliberately do NOT auto-restart the listener here. A failed
// flash means we can't vouch for the board's state — reopening
// the port immediately (esptool/rnodeconf's own reset sequence
@@ -499,6 +553,7 @@ pub async fn start_flash_job(
Err(e) => warn!(error = %e, "Failed to load mesh config after flash"),
}
}
bg_job.finish().await;
});
*job.abort_handle.write().await = Some(task.abort_handle());
@@ -510,12 +565,13 @@ async fn run_flash(
family: DeviceType,
data_dir: &Path,
path: &str,
prepared_image: Option<&Path>,
job: &Arc<FlashJob>,
) -> Result<()> {
match family {
DeviceType::Meshtastic | DeviceType::Meshcore => {
let image = fetch_esptool_image(board, family, data_dir, job).await?;
esptool_erase_and_write(path, &image, job).await
let image = prepared_image.context("Firmware was not prepared before serial access")?;
esptool_erase_and_write(path, image, job).await
}
DeviceType::Reticulum => {
let lora_region = super::load_config(data_dir)
@@ -664,15 +720,65 @@ async fn fetch_meshcore_image(
anyhow::anyhow!("No matching MeshCore image in release {}", release.tag_name)
})?;
anyhow::ensure!(
Path::new(&asset.name)
.file_name()
.and_then(|name| name.to_str())
== Some(asset.name.as_str()),
"Invalid firmware asset filename"
);
let out_path = cache.join(&asset.name);
if tokio::fs::metadata(&out_path).await.is_ok() {
job.push_log(format!("Using cached {}", asset.name)).await;
return Ok(out_path);
if verify_meshcore_asset(&out_path, asset).await.is_ok() {
job.push_log(format!("Using verified cached {}", asset.name))
.await;
return Ok(out_path);
}
tokio::fs::remove_file(&out_path)
.await
.context("Removing invalid cached firmware")?;
job.push_log("Cached firmware failed verification; downloading a fresh copy")
.await;
}
download_to_file(client, &asset.browser_download_url, &out_path, job).await?;
if let Err(error) = verify_meshcore_asset(&out_path, asset).await {
// A partial/unverified download must not become next attempt's cache.
let _ = tokio::fs::remove_file(&out_path).await;
return Err(error);
}
Ok(out_path)
}
async fn verify_meshcore_asset(path: &Path, asset: &GithubAsset) -> Result<()> {
use sha2::{Digest, Sha256};
let size = asset
.size
.context("MeshCore release did not provide firmware size")?;
anyhow::ensure!(
(1..=16 * 1024 * 1024).contains(&size),
"Invalid MeshCore firmware size"
);
anyhow::ensure!(
tokio::fs::metadata(path).await?.len() == size,
"Firmware size mismatch; radio was not written"
);
let expected = asset
.digest
.as_deref()
.and_then(|value| value.strip_prefix("sha256:"))
.context("MeshCore release did not provide a SHA-256 checksum")?;
anyhow::ensure!(
expected.len() == 64 && expected.bytes().all(|byte| byte.is_ascii_hexdigit()),
"Invalid MeshCore release checksum"
);
let actual = hex::encode(Sha256::digest(tokio::fs::read(path).await?));
anyhow::ensure!(
actual.eq_ignore_ascii_case(expected),
"Firmware checksum mismatch; radio was not written"
);
Ok(())
}
async fn download_to_file(
client: &reqwest::Client,
url: &str,
@@ -722,7 +828,8 @@ async fn download_to_file(
}
}
}
file.flush().await.ok();
file.flush().await.context("Flushing firmware download")?;
file.sync_all().await.context("Saving firmware download")?;
tokio::fs::rename(&tmp, dest)
.await
.context("Finalizing firmware download")?;
@@ -773,19 +880,10 @@ async fn esptool_erase_and_write(path: &str, image: &Path, job: &Arc<FlashJob>)
/// Building global args separately from subcommand args keeps this correct
/// by construction instead of relying on call-site ordering.
///
/// Normal stub-loader mode (no --no-stub) needs the esp32s3 stub flasher
/// blob at /usr/lib/python3/dist-packages/esptool/targets/stub_flasher/
/// stub_flasher_32s3.json — Debian's `esptool` package (4.7.0+dfsg-0.1)
/// ships without it (stripped for DFSG compliance: the prebuilt blob has no
/// buildable-from-source path Debian could verify), so scripts/self-update.sh
/// fetches the exact same file from the matching upstream esptool release
/// tag and installs it alongside the apt package (see the esptool install
/// step there). --no-stub (talk directly to the ROM bootloader, skip the
/// stub) was tried first and works for connecting, but the ROM bootloader
/// doesn't implement a full-chip-erase opcode at all — only the stub does —
/// so --no-stub broke our "always erase before write" default outright
/// rather than just being slower. Restoring the real stub file is the
/// correct fix, not routing around its absence.
/// Normal stub-loader mode is required for full-chip erase. The packaged
/// archy-esptool includes and self-tests Espressif's ESP32-S3 stub. Debian's
/// stripped esptool package alone did not provide that resource, so changing
/// flags to --no-stub cannot repair this operation.
fn esptool_global_args<'a>(path: &'a str, baud: Option<&'a str>) -> Vec<&'a str> {
let mut args = vec!["--chip", ESPTOOL_CHIP, "--port", path];
if let Some(b) = baud {
@@ -795,24 +893,96 @@ fn esptool_global_args<'a>(path: &'a str, baud: Option<&'a str>) -> Vec<&'a str>
args
}
fn esptool_executable() -> Result<PathBuf> {
let mut candidates = vec![PathBuf::from("/usr/local/bin/archy-esptool")];
if let Some(paths) = std::env::var_os("PATH") {
for directory in std::env::split_paths(&paths) {
for name in ["esptool", "esptool.py"] {
candidates.push(directory.join(name));
}
}
}
executable_from_candidates(candidates)
}
fn executable_from_candidates(candidates: impl IntoIterator<Item = PathBuf>) -> Result<PathBuf> {
use std::os::unix::fs::PermissionsExt;
candidates.into_iter().find(|path| {
path.is_file() && path.metadata().is_ok_and(|metadata| metadata.permissions().mode() & 0o111 != 0)
}).ok_or_else(|| anyhow::anyhow!(
"Radio flashing tools are missing. Install the complete Archipelago update and retry; the radio has not been changed."
))
}
async fn preflight_esptool() -> Result<PathBuf> {
let executable = esptool_executable()?;
check_esptool(&executable).await?;
Ok(executable)
}
async fn check_esptool(executable: &Path) -> Result<()> {
let mut command = Command::new(executable);
command
.arg(
if executable
.file_name()
.is_some_and(|name| name == "archy-esptool")
{
"--archy-self-test"
} else {
"version"
},
)
.kill_on_drop(true);
let output = tokio::time::timeout(std::time::Duration::from_secs(20), command.output())
.await
.context("Radio flashing tool did not respond; the radio has not been changed")?
.context("Radio flashing tool could not start; check its installation and permissions")?;
anyhow::ensure!(
output.status.success(),
"Radio flashing tool self-check failed; reinstall the complete update before retrying"
);
Ok(())
}
fn retryable_flash_error(error: &anyhow::Error) -> bool {
if error
.chain()
.any(|cause| cause.downcast_ref::<std::io::Error>().is_some())
{
return false;
}
let detail = format!("{error:#}").to_lowercase();
[
"failed to connect",
"timed out waiting",
"invalid head of packet",
"serial data stream stopped",
]
.iter()
.any(|message| detail.contains(message))
}
async fn esptool_with_retry(path: &str, subcommand: &[&str], job: &Arc<FlashJob>) -> Result<()> {
let mut cmd = Command::new("esptool");
let executable = preflight_esptool().await?;
let mut cmd = Command::new(&executable);
cmd.args(esptool_global_args(path, None));
cmd.args(subcommand);
match run_streamed(cmd, None, job).await {
Ok(()) => Ok(()),
Err(first_err) => {
Err(first_err) if retryable_flash_error(&first_err) => {
job.push_log(format!(
"First attempt failed ({first_err:#}); retrying once at {ESPTOOL_FALLBACK_BAUD} baud"
))
.await;
let mut retry = Command::new("esptool");
let mut retry = Command::new(&executable);
retry.args(esptool_global_args(path, Some(ESPTOOL_FALLBACK_BAUD)));
retry.args(subcommand);
run_streamed(retry, None, job)
.await
.context(format!("retry also failed (first attempt: {first_err:#})"))
}
Err(error) => Err(error),
}
}
@@ -990,3 +1160,159 @@ async fn run_streamed(mut cmd: Command, stdin: Option<Vec<u8>>, job: &Arc<FlashJ
}
Ok(())
}
#[cfg(test)]
mod flashing_regression_tests {
use super::*;
#[tokio::test]
async fn full_flash_invalidates_only_radio_settings_marker() {
let dir = tempfile::tempdir().unwrap();
let marker = dir.path().join("meshcore-radio-params.json");
tokio::fs::write(&marker, b"old settings").await.unwrap();
let other = dir.path().join("mesh-config.json");
tokio::fs::write(&other, b"preserved").await.unwrap();
invalidate_radio_settings_marker(dir.path()).await.unwrap();
assert!(!marker.exists());
assert_eq!(tokio::fs::read(&other).await.unwrap(), b"preserved");
invalidate_radio_settings_marker(dir.path()).await.unwrap();
tokio::fs::create_dir(&marker).await.unwrap();
assert!(invalidate_radio_settings_marker(dir.path()).await.is_err());
}
use std::os::unix::fs::PermissionsExt;
#[tokio::test]
async fn meshcore_cache_requires_release_size_and_checksum() {
use sha2::{Digest, Sha256};
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("fixture.bin");
tokio::fs::write(&file, b"firmware fixture").await.unwrap();
let mut asset = GithubAsset {
name: "fixture.bin".into(),
browser_download_url: "https://example.invalid/fixture.bin".into(),
size: Some(16),
digest: Some(format!(
"sha256:{}",
hex::encode(Sha256::digest(b"firmware fixture"))
)),
};
assert!(verify_meshcore_asset(&file, &asset).await.is_ok());
tokio::fs::write(&file, b"tampered fixture").await.unwrap();
assert!(verify_meshcore_asset(&file, &asset).await.is_err());
tokio::fs::write(&file, b"short").await.unwrap();
assert!(verify_meshcore_asset(&file, &asset).await.is_err());
tokio::fs::write(&file, b"firmware fixture").await.unwrap();
asset.digest = None;
assert!(verify_meshcore_asset(&file, &asset).await.is_err());
}
#[test]
fn generic_usb_ids_do_not_select_firmware() {
for (vid, pid) in [("10c4", "ea60"), ("303a", "1001")] {
let info = DetectedDeviceInfo {
path: "/dev/fixture".into(),
vid: Some(vid.into()),
pid: Some(pid.into()),
product: None,
manufacturer: None,
plugged_at: None,
};
assert_eq!(resolve_flash_board(&info), None);
}
}
#[test]
fn absent_nonexecutable_and_directory_candidates_are_rejected() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("flasher");
std::fs::write(&file, "#!/bin/sh\nexit 0\n").unwrap();
std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap();
assert!(executable_from_candidates([
dir.path().join("absent"),
file.clone(),
dir.path().to_path_buf()
])
.is_err());
std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o700)).unwrap();
assert_eq!(executable_from_candidates([file.clone()]).unwrap(), file);
}
#[tokio::test]
async fn flasher_preflight_reports_missing_interpreter_and_failed_self_check() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("archy-esptool");
for script in ["#!/missing/python\n", "#!/bin/sh\nexit 7\n"] {
std::fs::write(&file, script).unwrap();
std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o700)).unwrap();
assert!(check_esptool(&file).await.is_err());
}
std::fs::write(&file, "#!/bin/sh\n[ \"$1\" = --archy-self-test ]\n").unwrap();
assert!(check_esptool(&file).await.is_ok());
}
#[tokio::test]
async fn flash_registration_excludes_other_writers_and_active_probes() {
let handle = new_job_handle();
let first = FlashJob::new(
FlashBoard::HeltecV3,
DeviceType::Meshcore,
"/dev/fixture".into(),
);
let second = FlashJob::new(
FlashBoard::HeltecV3,
DeviceType::Meshcore,
"/dev/fixture".into(),
);
let probe = handle.read().await;
assert!(register_flash_job(&handle, &first).await.is_err());
drop(probe);
let (a, b) = tokio::join!(
register_flash_job(&handle, &first),
register_flash_job(&handle, &second)
);
assert_eq!(usize::from(a.is_ok()) + usize::from(b.is_ok()), 1);
handle.read().await.as_ref().unwrap().finish().await;
let next = FlashJob::new(
FlashBoard::HeltecV3,
DeviceType::Meshcore,
"/dev/fixture".into(),
);
assert!(register_flash_job(&handle, &next).await.is_ok());
}
#[test]
fn retries_only_known_serial_transport_failures() {
for kind in [
std::io::ErrorKind::NotFound,
std::io::ErrorKind::PermissionDenied,
] {
assert!(!retryable_flash_error(
&anyhow::Error::from(std::io::Error::from(kind))
.context("Failed to start subprocess")
));
}
for message in [
"Wrong chip",
"Invalid image",
"module missing",
"permission denied",
"port is busy",
] {
assert!(!retryable_flash_error(&anyhow::anyhow!(message)));
}
assert!(retryable_flash_error(&anyhow::anyhow!(
"Failed to connect to ESP32-S3: timed out waiting for packet header"
)));
assert_eq!(
esptool_global_args("/dev/fixture", Some("115200")),
[
"--chip",
"esp32s3",
"--port",
"/dev/fixture",
"--baud",
"115200"
]
);
}
}
+54 -5
View File
@@ -1092,6 +1092,14 @@ impl MeshService {
let (new_tx, new_rx) = tokio::sync::mpsc::channel(32);
*self.state.cmd_tx.write().await = new_tx;
self.cmd_rx = Some(new_rx);
{
let mut status = self.state.status.write().await;
status.device_connected = false;
status.device_path = None;
status.firmware_version = None;
status.self_node_id = None;
status.peer_count = 0;
}
info!("Mesh service stopped");
}
@@ -2321,7 +2329,10 @@ impl MeshService {
}
}
let was_enabled = self.config.enabled;
let listener_running = self
.listener_handle
.as_ref()
.is_some_and(|handle| !handle.is_finished());
let needs_session_restart = session_config_changed(&self.config, &config);
self.config = config.clone();
@@ -2335,10 +2346,13 @@ impl MeshService {
.unwrap_or_else(|| "archipelago".to_string());
}
// If enabled state changed, start/stop the listener
if config.enabled && !was_enabled {
// Reconcile desired state with the actual task, not the old enabled
// flag. A failed flash can stop the task while keeping enabled=true;
// explicitly reconnecting with the same settings must restart it.
if config.enabled && !listener_running {
self.stop().await;
self.start()?;
} else if !config.enabled && was_enabled {
} else if !config.enabled {
self.stop().await;
// Clear connected state
let mut status = self.state.status.write().await;
@@ -2347,7 +2361,7 @@ impl MeshService {
status.firmware_version = None;
status.self_node_id = None;
status.peer_count = 0;
} else if config.enabled && was_enabled && needs_session_restart {
} else if needs_session_restart {
info!("Mesh session config changed — restarting listener to apply");
self.stop().await;
self.start()?;
@@ -2537,6 +2551,41 @@ mod tests {
}
use super::*;
#[tokio::test]
async fn reconnect_with_unchanged_enabled_config_restarts_stopped_listener() {
let dir = tempfile::tempdir().unwrap();
let key = SigningKey::from_bytes(&[7; 32]);
let public = hex::encode(key.verifying_key().to_bytes());
let did = crate::identity::did_key_from_pubkey_hex(&public).unwrap();
let config = MeshConfig {
enabled: true,
..Default::default()
};
save_config(dir.path(), &config).await.unwrap();
let mut service = MeshService::new(dir.path(), &key, &did, &public)
.await
.unwrap();
assert!(service.listener_handle.is_none());
service.configure(config.clone()).await.unwrap();
assert!(service.listener_handle.is_some());
service.stop().await;
assert!(service.config.enabled);
service.configure(config.clone()).await.unwrap();
assert!(service.listener_handle.is_some());
service.stop().await;
service.listener_handle = Some(tokio::spawn(async {}));
tokio::task::yield_now().await;
service.configure(config).await.unwrap();
assert!(!service.listener_handle.as_ref().unwrap().is_finished());
service.stop().await;
let disabled = MeshConfig {
enabled: false,
..Default::default()
};
service.configure(disabled).await.unwrap();
assert!(service.listener_handle.is_none());
}
#[test]
fn session_config_change_detection() {
let base = MeshConfig::default();
+42
View File
@@ -378,6 +378,19 @@ fn decode_mesh_name(bytes: &[u8], fallback: &str) -> String {
/// Parse RESP_DEVICE_INFO (0x0D) response.
/// Returns firmware version string and device capabilities.
pub fn parse_device_info(data: &[u8]) -> Result<(String, u16)> {
// Official companion v3+ binary layout: protocol, half-capacity,
// channels, PIN (4), build date (12), model (40), version (20).
// Verified against companion-v1.17.1 MyMesh.cpp and Heltec V3 readback.
if data
.first()
.is_some_and(|version| (3..=31).contains(version))
{
anyhow::ensure!(data.len() >= 79, "Truncated MeshCore device info");
return Ok((
decode_mesh_name(&data[59..79], "unknown"),
u16::from(data[1]) * 2,
));
}
// Device info format varies by firmware version.
// Minimum: firmware version string (null-terminated) + max_contacts (u16 LE)
if data.is_empty() {
@@ -404,6 +417,15 @@ pub fn parse_self_info(data: &[u8]) -> Result<(u32, String)> {
anyhow::bail!("Self info response too short: {} bytes", data.len());
}
// Current companions send type/power/max-power, public key (32),
// position (8), four preference bytes, RF parameters (10), then name.
if data.len() >= 57 {
let node_id = u32::from_le_bytes(data[3..7].try_into().unwrap());
return Ok((
node_id,
decode_mesh_name(&data[57..], &format!("node-{node_id:08x}")),
));
}
let node_id = u32::from_le_bytes([data[0], data[1], data[2], data[3]]);
// Name follows after fixed fields. A firmware whose fixed-field layout
@@ -966,4 +988,24 @@ mod tests {
fn test_parse_self_info_too_short() {
assert!(parse_self_info(&[0x01, 0x02]).is_err());
}
#[test]
fn current_companion_binary_metadata_is_not_a_name_or_version() {
let mut info = vec![0u8; 81];
info[0] = 13;
info[1] = 150;
info[19..28].copy_from_slice(b"Heltec V3");
info[59..74].copy_from_slice(b"v1.17.1-d929643");
assert_eq!(
parse_device_info(&info).unwrap(),
("v1.17.1-d929643".into(), 300)
);
assert!(parse_device_info(&info[..78]).is_err());
let mut own = vec![0u8; 57];
own[0..3].copy_from_slice(&[1, 22, 22]);
own[3..7].copy_from_slice(&42u32.to_le_bytes());
own[47..51].copy_from_slice(&869618u32.to_le_bytes());
own.extend_from_slice(b"My radio");
assert_eq!(parse_self_info(&own).unwrap(), (42, "My radio".into()));
}
}
+48 -11
View File
@@ -277,6 +277,19 @@ fn terminate_group(child: &Child) {
}
}
/// Own the daemon during its asynchronous handshake too. Cancellation drops
/// the future without executing an error branch; a bare Child would survive
/// and keep the serial port open even though the listener had stopped.
struct StartingDaemon(Option<Child>);
impl Drop for StartingDaemon {
fn drop(&mut self) {
if let Some(child) = self.0.as_ref() {
terminate_group(child);
}
}
}
/// One peer learned via an RNS announce (LXMF delivery destination).
#[derive(Clone)]
struct ReticulumPeer {
@@ -517,10 +530,10 @@ impl ReticulumLink {
let child = cmd
.spawn()
.context("Failed to spawn reticulum-daemon — is it installed/packaged?")?;
let mut starting = StartingDaemon(Some(child));
// Wait for the socket to appear, then for the daemon's "ready" event.
// Runs as a block so every failure path tears the just-spawned daemon
// group down via `terminate_group` (the child has no `kill_on_drop`).
// StartingDaemon tears the group down on errors AND cancellation.
let init = async {
let deadline = tokio::time::Instant::now() + Duration::from_secs(15);
let stream = loop {
@@ -560,20 +573,17 @@ impl ReticulumLink {
dest_hash = %dest_hash_hex,
"Reticulum daemon ready"
);
Ok((write_half, reader, dest_hash, display_name))
};
let (write_half, reader, dest_hash, display_name) = match init.await {
Ok(parts) => parts,
Err(e) => {
terminate_group(&child);
return Err(e);
}
Ok::<_, anyhow::Error>((write_half, reader, dest_hash, display_name))
};
let (write_half, reader, dest_hash, display_name) = init.await?;
let mut link = Self {
device_path: label,
socket_path,
child,
child: starting
.0
.take()
.expect("starting daemon is owned until ready"),
writer: write_half,
reader,
dest_hash,
@@ -1557,6 +1567,33 @@ impl Drop for ReticulumLink {
mod tests {
use super::*;
#[tokio::test]
async fn cancelled_daemon_handshake_terminates_child() {
let (started, ready) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
let child = Command::new("sleep")
.arg("60")
.process_group(0)
.spawn()
.unwrap();
let pid = child.id().unwrap();
let _starting = StartingDaemon(Some(child));
started.send(pid).unwrap();
std::future::pending::<()>().await;
});
let pid = ready.await.unwrap();
assert_eq!(unsafe { libc::kill(pid as i32, 0) }, 0);
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
tokio::time::timeout(Duration::from_secs(3), async {
while unsafe { libc::kill(pid as i32, 0) } == 0 {
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("cancelled handshake left its child alive");
}
#[test]
fn announced_name_precedence() {
// Daemon-decoded LXMF name always wins.
+6 -2
View File
@@ -146,12 +146,16 @@ impl MeshcoreDevice {
}
}
let info = DeviceInfo {
firmware_version: name.clone(),
let mut info = DeviceInfo {
firmware_version: "unknown".to_string(),
node_id,
max_contacts: 100,
device_type: super::types::DeviceType::Meshcore,
};
if let Some((version, capacity)) = self.query_device_info().await {
info.firmware_version = version;
info.max_contacts = capacity;
}
self.device_info = Some(info.clone());
info!("Meshcore initialization complete on {}", self.device_path);
+10 -1
View File
@@ -64,7 +64,16 @@ async fn relay_cannot_substitute_forged_fields_for_a_known_event_id() {
let relay = tokio::spawn(async move {
let (socket, _) = listener.accept().await.unwrap();
let mut socket = accept_async(socket).await.unwrap();
while let Some(Ok(Message::Text(text))) = socket.next().await {
while let Some(message) = socket.next().await {
let text = match message.unwrap() {
Message::Text(text) => text,
Message::Ping(payload) => {
socket.send(Message::Pong(payload)).await.unwrap();
continue;
}
Message::Close(_) => break,
_ => continue,
};
let message: serde_json::Value = serde_json::from_str(&text).unwrap();
if message[0] != "REQ" {
continue;
+2
View File
@@ -83,6 +83,8 @@ impl EndpointRateLimiter {
limits.insert("wallet.send".to_string(), (5usize, 300u64));
limits.insert("wallet.ecash-send".to_string(), (10, 300));
limits.insert("lnd.sendcoins".to_string(), (5, 300));
limits.insert("lnd.bump-submit".to_string(), (5, 300));
limits.insert("lnd.bump-quote".to_string(), (30, 60));
limits.insert("lnd.payinvoice".to_string(), (10, 300));
limits.insert("lnd.openchannel".to_string(), (3, 300));
limits.insert("lnd.closechannel".to_string(), (3, 300));
+137 -1
View File
@@ -1475,6 +1475,48 @@ pub fn is_peer_allowed_path(path: &str) -> bool {
|| path.starts_with("/dwn/")
}
/// The ordinary API listener is a management surface even when contacted
/// directly, without host nginx. Peer traffic has its own path-restricted
/// listener and retains its existing cryptographic authentication.
fn management_peer_is_private(address: std::net::IpAddr) -> bool {
match address {
std::net::IpAddr::V4(ip) => {
ip.is_loopback()
|| ip.is_private()
|| ip.is_link_local()
|| (ip.octets()[0] == 100 && (64..=127).contains(&ip.octets()[1]))
}
std::net::IpAddr::V6(ip) => {
if let Some(mapped) = ip.to_ipv4_mapped() {
management_peer_is_private(std::net::IpAddr::V4(mapped))
} else {
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
}
}
}
}
fn request_surface_allowed(
peer_only: bool,
peer: std::net::IpAddr,
request: &hyper::Request<hyper::Body>,
) -> bool {
if peer_only {
return is_peer_allowed_path(request.uri().path());
}
// Keep purpose-built content/peer HTTP endpoints reachable with their
// existing handler-level checks. The general RPC dispatcher is a management
// surface here; only the dedicated peer listener retains public peer RPC.
if request.uri().path() != "/rpc/v1" && is_peer_allowed_path(request.uri().path()) {
return true;
}
management_peer_is_private(peer)
&& !request
.headers()
.get("x-archipelago-public-ingress")
.is_some_and(|value| !value.as_bytes().is_empty())
}
async fn accept_loop(
handler: Arc<ApiHandler>,
listener: TcpListener,
@@ -1552,7 +1594,7 @@ async fn accept_loop(
// forwarded headers on loopback (nginx) connections.
req.extensions_mut()
.insert(crate::api::rpc::PeerAddr(peer_addr));
if peer_only && !is_peer_allowed_path(req.uri().path()) {
if !request_surface_allowed(peer_only, peer_addr.ip(), &req) {
let resp = hyper::Response::builder()
.status(hyper::StatusCode::NOT_FOUND)
.body(hyper::Body::empty())
@@ -2520,3 +2562,97 @@ mod merge_tests {
);
}
}
#[cfg(test)]
mod management_surface_tests {
use super::*;
#[test]
fn public_api_listener_rejects_management_despite_forged_headers() {
for peer in [
"198.18.0.2",
"2001:db8::2",
"::ffff:198.18.0.2",
"100.63.255.255",
"100.128.0.1",
] {
for path in ["/", "/login", "/assets/index.js", "/rpc/v1", "/ws"] {
for method in ["GET", "POST"] {
let request = hyper::Request::builder()
.uri(path)
.method(method)
.header("host", "127.0.0.1")
.header("x-forwarded-for", "127.0.0.1")
.header("x-real-ip", "192.168.1.10")
.body(hyper::Body::empty())
.unwrap();
assert!(
!request_surface_allowed(false, peer.parse().unwrap(), &request),
"{peer} {method} {path}"
);
}
}
}
}
#[test]
fn private_management_and_restricted_peer_transport_remain_available() {
let mut request = hyper::Request::builder()
.uri("/rpc/v1")
.body(hyper::Body::empty())
.unwrap();
for peer in [
"127.0.0.1",
"10.0.0.2",
"172.16.0.2",
"192.168.1.2",
"169.254.1.2",
"100.64.0.1",
"100.127.255.254",
"::1",
"fd00::1",
"fe80::1",
"::ffff:192.168.1.2",
] {
assert!(request_surface_allowed(
false,
peer.parse().unwrap(),
&request
));
}
request
.headers_mut()
.insert("x-archipelago-public-ingress", "1".parse().unwrap());
assert!(!request_surface_allowed(
false,
"127.0.0.1".parse().unwrap(),
&request
));
// The dedicated peer listener retains its existing signed RPC contract.
assert!(request_surface_allowed(
true,
"198.18.0.2".parse().unwrap(),
&request
));
for path in [
"/content",
"/content/fixture/invoice",
"/blob/fixture",
"/dwn/health",
"/archipelago/node-message",
] {
*request.uri_mut() = path.parse().unwrap();
assert!(request_surface_allowed(
false,
"198.18.0.2".parse().unwrap(),
&request
));
}
*request.uri_mut() = "/login".parse().unwrap();
assert!(!request_surface_allowed(
true,
"198.18.0.2".parse().unwrap(),
&request
));
}
}
+2
View File
@@ -2654,6 +2654,8 @@ mod tests {
#[test]
fn test_is_newer() {
assert!(is_newer("1.9.0-alpha", "1.8.22-alpha"));
assert!(!is_newer("1.9.0-alpha", "1.9.0-alpha"));
assert!(is_newer("1.7.19-alpha", "1.7.18-alpha"));
assert!(is_newer("1.8.0-alpha", "1.7.99-alpha"));
assert!(is_newer("1.7.10-alpha", "1.7.9-alpha")); // numeric, not lexical