fix(indeedhub): generate per-node encryption root
This commit is contained in:
archipelago
@@ -1559,6 +1559,31 @@ impl RpcHandler {
|
||||
self.set_install_progress("indeedhub", n_images, n_images)
|
||||
.await;
|
||||
|
||||
// The retired installer injected one fleet-wide AES root directly in
|
||||
// the API/worker environment. Detect those consumers before removing
|
||||
// anything, then persist the legacy value exactly once so an upgrade
|
||||
// cannot orphan encrypted data. A genuinely fresh fallback install
|
||||
// receives a random per-node root instead.
|
||||
let mut had_existing_crypto_consumer = false;
|
||||
for name in [
|
||||
"indeedhub-api",
|
||||
"indeedhub-ffmpeg",
|
||||
"indeedhub-build_api_1",
|
||||
"indeedhub-build_ffmpeg-worker_1",
|
||||
] {
|
||||
let status =
|
||||
podman_stack_status(&["container", "exists", name], PODMAN_STACK_PROBE_TIMEOUT)
|
||||
.await?;
|
||||
had_existing_crypto_consumer |= status.success();
|
||||
}
|
||||
let secrets_dir = self.config.data_dir.join("secrets");
|
||||
crate::container::secrets::ensure_indeedhub_aes_master_secret(
|
||||
&secrets_dir,
|
||||
had_existing_crypto_consumer,
|
||||
)
|
||||
.context("preparing IndeedHub encryption root")?;
|
||||
let aes_master = crate::container::secrets::indeedhub_aes_master_secret(&secrets_dir)?;
|
||||
|
||||
// Remove any leftover containers from a previous partial install (or
|
||||
// from the first-boot frontend stub that used to race the installer).
|
||||
// Without this, `podman run --name indeedhub` fails on name conflict
|
||||
@@ -1759,7 +1784,7 @@ impl RpcHandler {
|
||||
"-e".to_string(),
|
||||
"NOSTR_JWT_EXPIRES_IN=7d".to_string(),
|
||||
"-e".to_string(),
|
||||
"AES_MASTER_SECRET=0123456789abcdef0123456789abcdef".to_string(),
|
||||
format!("AES_MASTER_SECRET={aes_master}"),
|
||||
"-e".to_string(),
|
||||
"ENVIRONMENT=production".to_string(),
|
||||
format!("{registry}/indeedhub-api:1.0.0"),
|
||||
@@ -1810,7 +1835,7 @@ impl RpcHandler {
|
||||
"-e".to_string(),
|
||||
"ENVIRONMENT=production".to_string(),
|
||||
"-e".to_string(),
|
||||
"AES_MASTER_SECRET=0123456789abcdef0123456789abcdef".to_string(),
|
||||
format!("AES_MASTER_SECRET={aes_master}"),
|
||||
format!("{registry}/indeedhub-ffmpeg:1.0.0"),
|
||||
],
|
||||
&tmp_env,
|
||||
|
||||
@@ -3565,6 +3565,54 @@ impl ProdContainerOrchestrator {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Materialise IndeedHub's AES root before the generic generated-secret
|
||||
/// pass. Old installers injected one known value directly into the API and
|
||||
/// worker environments, so an upgrade with either consumer still present
|
||||
/// must persist that value before container drift can recreate them. With
|
||||
/// no existing consumer this is a fresh install and receives random bytes.
|
||||
async fn ensure_indeedhub_aes_master(&self, manifest: &AppManifest) -> Result<()> {
|
||||
if manifest.app.id != "indeedhub-api" {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let secret_path = self
|
||||
.secrets_dir
|
||||
.join(crate::container::secrets::INDEEDHUB_AES_SECRET_NAME);
|
||||
let preserve_legacy = if secret_path.exists() {
|
||||
// The secret helper validates the existing file and, critically,
|
||||
// refuses to replace a damaged encryption root.
|
||||
false
|
||||
} else {
|
||||
let consumers = [
|
||||
"indeedhub-api",
|
||||
"indeedhub-ffmpeg",
|
||||
"indeedhub-build_api_1",
|
||||
"indeedhub-build_ffmpeg-worker_1",
|
||||
];
|
||||
self.runtime
|
||||
.list_containers()
|
||||
.await
|
||||
.context("detecting an existing IndeedHub encryption-key consumer")?
|
||||
.iter()
|
||||
.any(|container| {
|
||||
let name = container.name.trim_start_matches('/');
|
||||
consumers.contains(&name)
|
||||
})
|
||||
};
|
||||
|
||||
if crate::container::secrets::ensure_indeedhub_aes_master_secret(
|
||||
&self.secrets_dir,
|
||||
preserve_legacy,
|
||||
)? {
|
||||
tracing::info!(
|
||||
app = "indeedhub-api",
|
||||
path = %secret_path.display(),
|
||||
"Persisted the legacy IndeedHub encryption root for upgrade compatibility"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resolve_dynamic_env(&self, manifest: &mut AppManifest) -> Result<()> {
|
||||
// Idempotency guard: partitioning already ran on this instance.
|
||||
// Re-running would re-taint against an environment that no longer
|
||||
@@ -3573,6 +3621,11 @@ impl ProdContainerOrchestrator {
|
||||
if !manifest.app.container.secret_env_refs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
// IndeedHub's data-encryption root needs an upgrade-aware first pass:
|
||||
// generic generation alone would replace the fleet-wide legacy value
|
||||
// and make previously encrypted data unreadable.
|
||||
self.ensure_indeedhub_aes_master(manifest).await?;
|
||||
|
||||
// Materialise any manifest-declared generated secrets before they're
|
||||
// read below. This is the single chokepoint every install/reconcile
|
||||
// path funnels through, so an app's secrets exist by the time its
|
||||
@@ -5627,6 +5680,52 @@ app:
|
||||
"app:\n id: fedimint-gateway\n name: Fedimint Gateway\n version: 0.10.0\n container:\n image: x:1\n generated_secrets:\n - name: fedimint-gateway-hash\n kind: bcrypt\n secret_env:\n - key: FEDI_HASH\n secret_file: fedimint-gateway-hash\n"
|
||||
}
|
||||
|
||||
fn indeedhub_api_manifest_yaml() -> &'static str {
|
||||
"app:\n id: indeedhub-api\n name: IndeedHub API\n version: 1.0.0\n container:\n image: x:1\n generated_secrets:\n - name: indeedhub-aes-master\n kind: hex16\n secret_env:\n - key: AES_MASTER_SECRET\n secret_file: indeedhub-aes-master\n"
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_indeedhub_consumer_gets_migration_compatible_root() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
rt.set_state("indeedhub-api", ContainerState::Running);
|
||||
let mut orch = orch_with(rt).await;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
orch.set_secrets_dir(tmp.path().to_path_buf());
|
||||
|
||||
let mut manifest = AppManifest::parse(indeedhub_api_manifest_yaml()).unwrap();
|
||||
orch.resolve_dynamic_env(&mut manifest).await.unwrap();
|
||||
let resolved = manifest
|
||||
.app
|
||||
.container
|
||||
.secret_env_refs
|
||||
.iter()
|
||||
.find(|entry| entry.env_key == "AES_MASTER_SECRET")
|
||||
.unwrap();
|
||||
assert_eq!(resolved.value.len(), 32);
|
||||
assert!(tmp.path().join("indeedhub-aes-master").exists());
|
||||
assert!(
|
||||
crate::container::secrets::ensure_indeedhub_aes_master_secret(tmp.path(), true).is_ok(),
|
||||
"the migrated file remains valid and stable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_indeedhub_install_gets_random_root() {
|
||||
let rt = Arc::new(MockRuntime::default());
|
||||
let mut orch = orch_with(rt).await;
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
orch.set_secrets_dir(tmp.path().to_path_buf());
|
||||
|
||||
let mut manifest = AppManifest::parse(indeedhub_api_manifest_yaml()).unwrap();
|
||||
orch.resolve_dynamic_env(&mut manifest).await.unwrap();
|
||||
let first = crate::container::secrets::indeedhub_aes_master_secret(tmp.path()).unwrap();
|
||||
|
||||
let other = tempfile::TempDir::new().unwrap();
|
||||
crate::container::secrets::ensure_indeedhub_aes_master_secret(other.path(), false).unwrap();
|
||||
let second = crate::container::secrets::indeedhub_aes_master_secret(other.path()).unwrap();
|
||||
assert_ne!(first, second, "fresh installs must receive per-node roots");
|
||||
}
|
||||
|
||||
/// FED-07. Rotating a compromised credential leaves the RUNNING container
|
||||
/// holding the old value, so the rotation must flag the app for recreate.
|
||||
/// Without the flag the drift check skips it as restart-sensitive and the
|
||||
|
||||
@@ -140,6 +140,79 @@ fn random_base64(bytes: usize) -> String {
|
||||
/// daemon read `fedimint-gateway-hash`).
|
||||
pub const GATEWAY_HASH_SECRET_NAME: &str = "fedimint-gateway-hash";
|
||||
|
||||
/// Canonical filename for IndeedHub's envelope-encryption root. API and media
|
||||
/// worker must receive the same stable value: changing it after data has been
|
||||
/// encrypted can make that data unreadable.
|
||||
pub const INDEEDHUB_AES_SECRET_NAME: &str = "indeedhub-aes-master";
|
||||
|
||||
/// The fleet-wide value used by the legacy IndeedHub installers. It remains
|
||||
/// here only for the one-way migration of an already-installed stack: those
|
||||
/// nodes must persist the value they have been using before the manifest
|
||||
/// starts reading it from a file. Fresh installs must never receive it.
|
||||
const KNOWN_LEGACY_INDEEDHUB_AES_MASTER: &str = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
/// Ensure IndeedHub has a stable encryption root.
|
||||
///
|
||||
/// `preserve_legacy` is true only when an API/worker container already exists,
|
||||
/// proving this is an upgrade from the installer that shipped the known legacy
|
||||
/// value. In that case we persist that value once so recreating the containers
|
||||
/// does not orphan encrypted data. A fresh installation gets 16 random bytes
|
||||
/// encoded as 32 hex characters.
|
||||
///
|
||||
/// Unlike ordinary generated credentials, an existing-but-empty or unreadable
|
||||
/// encryption root is never self-healed by rotation: replacement could destroy
|
||||
/// access to data, so this fails loudly and leaves the file untouched.
|
||||
/// Returns true only when the legacy migration value was written.
|
||||
pub fn ensure_indeedhub_aes_master_secret(
|
||||
secrets_dir: &Path,
|
||||
preserve_legacy: bool,
|
||||
) -> Result<bool> {
|
||||
fs::create_dir_all(secrets_dir)
|
||||
.with_context(|| format!("creating secrets dir {}", secrets_dir.display()))?;
|
||||
let path = secrets_dir.join(INDEEDHUB_AES_SECRET_NAME);
|
||||
|
||||
if path.exists() {
|
||||
let value = fs::read_to_string(&path).with_context(|| {
|
||||
format!(
|
||||
"reading IndeedHub encryption root {} (refusing to replace it)",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
if value.trim().is_empty() {
|
||||
anyhow::bail!(
|
||||
"IndeedHub encryption root {} is empty; refusing to replace a potentially \
|
||||
data-bearing key",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if preserve_legacy {
|
||||
write_secret(&path, KNOWN_LEGACY_INDEEDHUB_AES_MASTER)?;
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let spec = GeneratedSecret {
|
||||
name: INDEEDHUB_AES_SECRET_NAME.to_string(),
|
||||
kind: SecretGenKind::Hex16,
|
||||
};
|
||||
ensure_one(secrets_dir, &spec)?;
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Read the stable IndeedHub encryption root after it has been materialised.
|
||||
pub fn indeedhub_aes_master_secret(secrets_dir: &Path) -> Result<String> {
|
||||
let path = secrets_dir.join(INDEEDHUB_AES_SECRET_NAME);
|
||||
let value = fs::read_to_string(&path)
|
||||
.with_context(|| format!("reading IndeedHub encryption root {}", path.display()))?;
|
||||
let value = value.trim();
|
||||
if value.is_empty() {
|
||||
anyhow::bail!("IndeedHub encryption root {} is empty", path.display());
|
||||
}
|
||||
Ok(value.to_string())
|
||||
}
|
||||
|
||||
/// Detection-only denylist of bcrypt hashes that shipped as hardcoded
|
||||
/// fallback credentials in this repository before FED-07. `t9YjjxkiktrlYvjajB
|
||||
/// /zgOMDnSNVg4HqrbDqh47u7Jf42whNdxNqC` was substituted for the Fedimint
|
||||
@@ -356,6 +429,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indeedhub_fresh_installs_get_distinct_per_node_encryption_roots() {
|
||||
let dir_a = tempfile::tempdir().unwrap();
|
||||
let dir_b = tempfile::tempdir().unwrap();
|
||||
|
||||
assert!(!ensure_indeedhub_aes_master_secret(dir_a.path(), false).unwrap());
|
||||
assert!(!ensure_indeedhub_aes_master_secret(dir_b.path(), false).unwrap());
|
||||
let value_a = indeedhub_aes_master_secret(dir_a.path()).unwrap();
|
||||
let value_b = indeedhub_aes_master_secret(dir_b.path()).unwrap();
|
||||
|
||||
assert_eq!(value_a.len(), 32);
|
||||
assert!(value_a.chars().all(|c| c.is_ascii_hexdigit()));
|
||||
assert_ne!(value_a, KNOWN_LEGACY_INDEEDHUB_AES_MASTER);
|
||||
assert_ne!(value_a, value_b, "fresh nodes must not share an AES root");
|
||||
let mode = std::fs::metadata(dir_a.path().join(INDEEDHUB_AES_SECRET_NAME))
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777;
|
||||
assert_eq!(mode, 0o600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indeedhub_existing_install_persists_legacy_root_once() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
assert!(ensure_indeedhub_aes_master_secret(dir.path(), true).unwrap());
|
||||
assert_eq!(
|
||||
indeedhub_aes_master_secret(dir.path()).unwrap(),
|
||||
KNOWN_LEGACY_INDEEDHUB_AES_MASTER
|
||||
);
|
||||
assert!(
|
||||
!ensure_indeedhub_aes_master_secret(dir.path(), true).unwrap(),
|
||||
"a second migration pass must be a no-op"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indeedhub_existing_unique_root_is_never_rotated() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
ensure_indeedhub_aes_master_secret(dir.path(), false).unwrap();
|
||||
let before = indeedhub_aes_master_secret(dir.path()).unwrap();
|
||||
|
||||
assert!(!ensure_indeedhub_aes_master_secret(dir.path(), true).unwrap());
|
||||
assert_eq!(before, indeedhub_aes_master_secret(dir.path()).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indeedhub_empty_root_fails_without_overwriting() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join(INDEEDHUB_AES_SECRET_NAME);
|
||||
std::fs::write(&path, "").unwrap();
|
||||
|
||||
let err = ensure_indeedhub_aes_master_secret(dir.path(), true).unwrap_err();
|
||||
assert!(err.to_string().contains("refusing to replace"));
|
||||
assert_eq!(std::fs::read(&path).unwrap(), b"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gateway_credential_fresh_generation_verifies_and_is_0600() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user