feat(netbird): manifest-driven migration via reusable orchestrator primitives
Migrate the netbird stack (server/dashboard/proxy) off ~500 lines of per-app Rust to 3 declarative manifests, adding 4 reusable primitives: - SecretGenKind::Base64 (netbird relay authSecret + sqlite store encryptionKey) - GeneratedCert schema + ensure_manifest_certs (self-signed TLS so the dashboard gets a secure context for OIDC PKCE — issue #15; https proxy on 8087 preserved) - templated GeneratedFile render: {{HOST_IP}}/{{HOST_MDNS}}/{{NETWORK_GATEWAY}} (aardvark resolver for the #15 stale-IP fix) /{{secret:NAME}} (never logged) - legacy create_container now honours port.protocol (3478/udp STUN) install_netbird_stack routes via the orchestrator first (legacy kept as fallback, mirroring indeedhub); launch URL derives https://{host_ip}:8087 from host facts. Legacy Rust deletion deferred to post-live-verify. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
archipelago
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
3c36cf1c40
commit
a8b9b0f5e8
@@ -696,6 +696,16 @@ fn immich_stack_app_ids() -> &'static [&'static str] {
|
||||
&["immich-postgres", "immich-redis", "immich"]
|
||||
}
|
||||
|
||||
fn netbird_stack_app_ids() -> &'static [&'static str] {
|
||||
// Dependency/startup order: the combined management/signal/relay server
|
||||
// first (it owns the base64 relay/store secrets + the sqlite store, and is
|
||||
// the OIDC issuer the others point at), then the dashboard SPA, then the
|
||||
// user-facing TLS proxy ("netbird", which carries the self-signed cert +
|
||||
// the templated nginx.conf and is the launcher). Mirrors the netbird
|
||||
// startup_order in dependencies.rs.
|
||||
&["netbird-server", "netbird-dashboard", "netbird"]
|
||||
}
|
||||
|
||||
fn indeedhub_stack_app_ids() -> &'static [&'static str] {
|
||||
// Dependency order: backends + their generated secrets first, then the api
|
||||
// (owns indeedhub-jwt; reads the db/minio secrets the backends materialised),
|
||||
@@ -1828,6 +1838,23 @@ impl RpcHandler {
|
||||
|
||||
/// Install self-hosted NetBird (dashboard + combined management/signal/relay server).
|
||||
pub(super) async fn install_netbird_stack(&self) -> Result<serde_json::Value> {
|
||||
// Manifest-driven path (#20 phase 4): render the 3-member stack from
|
||||
// apps/netbird-*/manifest.yml via the orchestrator — dedicated
|
||||
// netbird-net + network_aliases, base64 generated_secrets, a self-signed
|
||||
// TLS cert (generated_certs) so the dashboard gets a secure context for
|
||||
// OIDC PKCE (#15), and templated config.yaml/nginx.conf rendered from
|
||||
// host facts + the netbird-net gateway. The manifests use the exact live
|
||||
// container names, so on an existing node this ADOPTS the running stack
|
||||
// rather than recreating it (the sqlite store + base64 keys are
|
||||
// preserved — ensure_generated_secrets no-ops on existing files). Falls
|
||||
// back to the legacy installer below only when the orchestrator doesn't
|
||||
// know these app_ids (manifests not yet deployed to the node).
|
||||
if let Some(orchestrated) =
|
||||
install_stack_via_orchestrator(self, "netbird", netbird_stack_app_ids()).await?
|
||||
{
|
||||
return Ok(orchestrated);
|
||||
}
|
||||
|
||||
if let Some(adopted) = adopt_stack_if_exists(
|
||||
"netbird",
|
||||
"netbird",
|
||||
|
||||
@@ -691,16 +691,37 @@ fn extract_lan_address(ports: &[String]) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
/// netbird's dashboard launch URL: HTTPS on 8087 (the proxy terminates TLS —
|
||||
/// the dashboard needs a secure context for OIDC PKCE, issue #15) at the node's
|
||||
/// primary host IP so it's reachable from the LAN. Manifest-driven netbird no
|
||||
/// longer writes `dashboard.env`, so this is derived from host facts (the same
|
||||
/// `{{HOST_IP}}` the orchestrator bakes into the cert/config); it falls back to
|
||||
/// the static localhost mapping when the host IP can't be read. URL shape is
|
||||
/// identical to the legacy installer's, so the existing https reachability
|
||||
/// wrapper still applies.
|
||||
async fn netbird_configured_launch_url() -> Option<String> {
|
||||
let env = tokio::fs::read_to_string("/var/lib/archipelago/netbird/dashboard.env")
|
||||
if let Some(ip) = first_host_ip().await {
|
||||
return Some(format!("https://{ip}:8087"));
|
||||
}
|
||||
PodmanClient::lan_address_for("netbird")
|
||||
}
|
||||
|
||||
/// First address from `hostname -I` — the node's primary host IP. Mirrors the
|
||||
/// orchestrator's `detect_host_ip` so launch URLs match the cert/config the
|
||||
/// orchestrator renders for `{{HOST_IP}}`.
|
||||
async fn first_host_ip() -> Option<String> {
|
||||
let out = tokio::process::Command::new("hostname")
|
||||
.arg("-I")
|
||||
.output()
|
||||
.await
|
||||
.ok()?;
|
||||
env.lines()
|
||||
.find_map(|line| line.strip_prefix("NETBIRD_MGMT_API_ENDPOINT="))
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
if !out.status.success() {
|
||||
return None;
|
||||
}
|
||||
String::from_utf8_lossy(&out.stdout)
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.map(ToOwned::to_owned)
|
||||
.or_else(|| PodmanClient::lan_address_for("netbird"))
|
||||
}
|
||||
|
||||
async fn reachable_lan_address(app_id: &str, candidate: Option<String>) -> Option<String> {
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
use anyhow::{Context, Result};
|
||||
use archipelago_container::{
|
||||
AppManifest, ContainerRuntime as ContainerRuntimeTrait, ContainerState, ContainerStatus,
|
||||
Dependency, GeneratedFile, HostFacts, ManifestError, ResolvedSource, SecretsProvider,
|
||||
Dependency, HostFacts, ManifestError, ResolvedSource, SecretsProvider,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -1809,6 +1809,9 @@ impl ProdContainerOrchestrator {
|
||||
self.run_pre_start_hooks(&manifest.app.id).await?;
|
||||
self.ensure_bind_mount_sockets(manifest).await?;
|
||||
self.ensure_bind_mount_dirs(manifest).await?;
|
||||
// Certs before files: a templated file may not need the cert, but the
|
||||
// container's bind-mounts expect both present before create_container.
|
||||
self.ensure_manifest_certs(manifest).await?;
|
||||
self.ensure_manifest_files(manifest).await?;
|
||||
self.apply_data_uid(manifest).await?;
|
||||
self.run_post_data_uid_hooks(&manifest.app.id).await?;
|
||||
@@ -2750,7 +2753,14 @@ impl ProdContainerOrchestrator {
|
||||
async fn ensure_manifest_files(&self, manifest: &AppManifest) -> Result<HookOutcome> {
|
||||
let mut outcome = HookOutcome::Unchanged;
|
||||
for file in &manifest.app.files {
|
||||
if ensure_generated_file(file)
|
||||
// Render templated placeholders before comparing/writing so the
|
||||
// idempotency check is against the FINAL bytes (not the template),
|
||||
// otherwise a rendered file would be rewritten every reconcile.
|
||||
let rendered = self
|
||||
.render_file_placeholders(manifest, &file.content)
|
||||
.await
|
||||
.with_context(|| format!("rendering manifest file {}", file.path))?;
|
||||
if ensure_rendered_file(&file.path, &rendered, file.overwrite)
|
||||
.await
|
||||
.with_context(|| format!("ensure manifest file {}", file.path))?
|
||||
== HookOutcome::Rewritten
|
||||
@@ -2760,23 +2770,185 @@ impl ProdContainerOrchestrator {
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Substitute the allow-listed placeholders a manifest `GeneratedFile` may
|
||||
/// carry. Keeps runtime-derived config (netbird's `config.yaml`/`nginx.conf`)
|
||||
/// declarative instead of generated by per-app Rust:
|
||||
/// - `{{HOST_IP}}` / `{{HOST_MDNS}}` — host facts (`hostname -I` / `.local`).
|
||||
/// - `{{NETWORK_GATEWAY}}` — the gateway of the app's podman network, i.e.
|
||||
/// aardvark's DNS address. nginx uses it as an explicit `resolver` so it
|
||||
/// re-resolves container names per request instead of pinning a stale IP
|
||||
/// and 502-ing after a restart/reboot (issue #15). The network is ensured
|
||||
/// to exist first so the gateway is readable on a fresh install (this runs
|
||||
/// before `install_fresh`'s own `ensure_container_network`; both idempotent).
|
||||
/// - `{{secret:NAME}}` — a `0600` secret read from the service-owned secrets
|
||||
/// dir (e.g. netbird's base64 relay/store keys). NEVER logged.
|
||||
async fn render_file_placeholders(
|
||||
&self,
|
||||
manifest: &AppManifest,
|
||||
content: &str,
|
||||
) -> Result<String> {
|
||||
let mut out = content.to_string();
|
||||
if out.contains("{{HOST_IP}}") || out.contains("{{HOST_MDNS}}") {
|
||||
let facts = self.detect_host_facts();
|
||||
out = out
|
||||
.replace("{{HOST_IP}}", &facts.host_ip)
|
||||
.replace("{{HOST_MDNS}}", &facts.host_mdns);
|
||||
}
|
||||
if out.contains("{{NETWORK_GATEWAY}}") {
|
||||
self.ensure_container_network(manifest).await?;
|
||||
let gw = self.network_gateway(manifest).await?;
|
||||
out = out.replace("{{NETWORK_GATEWAY}}", &gw);
|
||||
}
|
||||
out = self.render_secret_placeholders(&out).await?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Replace every `{{secret:NAME}}` with the trimmed contents of
|
||||
/// `<secrets_dir>/NAME`. `NAME` must be a bare filename (the same safety bar
|
||||
/// as `secret_env`). The secret value is never placed in an error or log.
|
||||
async fn render_secret_placeholders(&self, content: &str) -> Result<String> {
|
||||
const OPEN: &str = "{{secret:";
|
||||
let mut out = String::with_capacity(content.len());
|
||||
let mut rest = content;
|
||||
while let Some(start) = rest.find(OPEN) {
|
||||
out.push_str(&rest[..start]);
|
||||
let after = &rest[start + OPEN.len()..];
|
||||
let end = after
|
||||
.find("}}")
|
||||
.ok_or_else(|| anyhow::anyhow!("unterminated {{secret:...}} placeholder"))?;
|
||||
let name = &after[..end];
|
||||
if name.is_empty() || name.contains('/') || name.contains("..") {
|
||||
anyhow::bail!("invalid secret placeholder name '{name}' (must be a bare filename)");
|
||||
}
|
||||
let value = tokio::fs::read_to_string(self.secrets_dir.join(name))
|
||||
.await
|
||||
.map_err(|_| {
|
||||
// Do not surface the path-with-value or io detail beyond the name.
|
||||
anyhow::anyhow!("secret '{name}' referenced by a manifest file is missing")
|
||||
})?;
|
||||
out.push_str(value.trim());
|
||||
rest = &after[end + 2..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The gateway IP of the app's podman network — aardvark's DNS resolver
|
||||
/// address. Mirrors the legacy `netbird_net_resolver_ip`; falls back to
|
||||
/// podman's usual first-pool gateway if the inspect can't be parsed (the
|
||||
/// network was just ensured to exist, so this is a belt-and-braces default).
|
||||
async fn network_gateway(&self, manifest: &AppManifest) -> Result<String> {
|
||||
let network = manifest
|
||||
.app
|
||||
.container
|
||||
.network
|
||||
.as_deref()
|
||||
.filter(|n| !n.is_empty() && !is_builtin_network_mode(n))
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!("{{NETWORK_GATEWAY}} used but app has no dedicated network")
|
||||
})?;
|
||||
let out = tokio::process::Command::new("podman")
|
||||
.args([
|
||||
"network",
|
||||
"inspect",
|
||||
network,
|
||||
"--format",
|
||||
"{{range .Subnets}}{{.Gateway}}{{end}}",
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.with_context(|| format!("inspecting podman network {network} for gateway"))?;
|
||||
let gw = String::from_utf8_lossy(&out.stdout).trim().to_string();
|
||||
if !gw.is_empty() && gw.parse::<std::net::IpAddr>().is_ok() {
|
||||
return Ok(gw);
|
||||
}
|
||||
tracing::warn!(
|
||||
network,
|
||||
"could not read network gateway; falling back to 10.89.0.1"
|
||||
);
|
||||
Ok("10.89.0.1".to_string())
|
||||
}
|
||||
|
||||
/// Materialise manifest-declared self-signed TLS certs before the container
|
||||
/// is created (so a bind-mounted cert path resolves to a real file). Skips an
|
||||
/// entry whose crt+key already exist (idempotent / data-preserving). CN and
|
||||
/// SAN templates are rendered against host facts; when omitted they default
|
||||
/// to the node's host IP plus `127.0.0.1`/`localhost` so the cert is valid
|
||||
/// however the box is reached locally. Mirrors the legacy
|
||||
/// `ensure_netbird_tls_cert` (rsa:2048, 10-year, no per-app Rust).
|
||||
async fn ensure_manifest_certs(&self, manifest: &AppManifest) -> Result<()> {
|
||||
let facts = self.detect_host_facts();
|
||||
let render = |s: &str| {
|
||||
s.replace("{{HOST_IP}}", &facts.host_ip)
|
||||
.replace("{{HOST_MDNS}}", &facts.host_mdns)
|
||||
};
|
||||
for cert in &manifest.app.container.generated_certs {
|
||||
if tokio::fs::metadata(&cert.crt).await.is_ok()
|
||||
&& tokio::fs::metadata(&cert.key).await.is_ok()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if let Some(parent) = Path::new(&cert.crt).parent() {
|
||||
create_dir_all_or_sudo(parent).await?;
|
||||
}
|
||||
if let Some(parent) = Path::new(&cert.key).parent() {
|
||||
create_dir_all_or_sudo(parent).await?;
|
||||
}
|
||||
let cn = render(cert.common_name.as_deref().unwrap_or("{{HOST_IP}}"));
|
||||
let san = if cert.sans.is_empty() {
|
||||
format!("IP:{},IP:127.0.0.1,DNS:localhost", facts.host_ip)
|
||||
} else {
|
||||
cert.sans
|
||||
.iter()
|
||||
.map(|s| render(s))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
};
|
||||
let status = tokio::process::Command::new("openssl")
|
||||
.args([
|
||||
"req",
|
||||
"-x509",
|
||||
"-newkey",
|
||||
"rsa:2048",
|
||||
"-nodes",
|
||||
"-keyout",
|
||||
&cert.key,
|
||||
"-out",
|
||||
&cert.crt,
|
||||
"-days",
|
||||
"3650",
|
||||
"-subj",
|
||||
&format!("/CN={cn}"),
|
||||
"-addext",
|
||||
&format!("subjectAltName={san}"),
|
||||
])
|
||||
.status()
|
||||
.await
|
||||
.with_context(|| format!("running openssl for manifest cert {}", cert.crt))?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("openssl failed to generate manifest cert {}", cert.crt);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_generated_file(file: &GeneratedFile) -> Result<HookOutcome> {
|
||||
let path = Path::new(&file.path);
|
||||
if let Ok(existing) = tokio::fs::read_to_string(path).await {
|
||||
if existing == file.content || !file.overwrite {
|
||||
async fn ensure_rendered_file(path: &str, content: &str, overwrite: bool) -> Result<HookOutcome> {
|
||||
let p = Path::new(path);
|
||||
if let Ok(existing) = tokio::fs::read_to_string(p).await {
|
||||
if existing == content || !overwrite {
|
||||
return Ok(HookOutcome::Unchanged);
|
||||
}
|
||||
} else if path.exists() && !file.overwrite {
|
||||
} else if p.exists() && !overwrite {
|
||||
return Ok(HookOutcome::Unchanged);
|
||||
}
|
||||
|
||||
let parent = path
|
||||
let parent = p
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("generated file path has no parent: {}", file.path))?;
|
||||
.ok_or_else(|| anyhow::anyhow!("generated file path has no parent: {}", path))?;
|
||||
create_dir_all_or_sudo(parent).await?;
|
||||
write_generated_file_atomically(path, &file.content).await?;
|
||||
write_generated_file_atomically(p, content).await?;
|
||||
Ok(HookOutcome::Rewritten)
|
||||
}
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ fn ensure_one(dir: &Path, gs: &GeneratedSecret) -> Result<()> {
|
||||
match gs.kind {
|
||||
SecretGenKind::Hex16 => write_secret(&dir.join(&gs.name), &random_hex(16))?,
|
||||
SecretGenKind::Hex32 => write_secret(&dir.join(&gs.name), &random_hex(32))?,
|
||||
SecretGenKind::Base64 => write_secret(&dir.join(&gs.name), &random_base64(32))?,
|
||||
SecretGenKind::Bcrypt => {
|
||||
let password = random_hex(BCRYPT_PASSWORD_BYTES);
|
||||
let hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST)
|
||||
@@ -92,6 +93,15 @@ fn random_hex(bytes: usize) -> String {
|
||||
hex::encode(buf)
|
||||
}
|
||||
|
||||
/// `bytes` of entropy, standard base64 (with padding). For keys that a service
|
||||
/// base64-decodes to recover the raw bytes (e.g. netbird's store encryptionKey).
|
||||
fn random_base64(bytes: usize) -> String {
|
||||
use base64::Engine as _;
|
||||
let mut buf = vec![0u8; bytes];
|
||||
rand::thread_rng().fill_bytes(&mut buf);
|
||||
base64::engine::general_purpose::STANDARD.encode(buf)
|
||||
}
|
||||
|
||||
/// Atomically write a `0600` secret: a temp file in the same dir (so the rename
|
||||
/// is atomic), fsynced, then renamed over the target.
|
||||
fn write_secret(path: &Path, value: &str) -> Result<()> {
|
||||
|
||||
Reference in New Issue
Block a user