feat(01-16): rotate existing installs off the shipped gateway credential (FED-07)

01-11 stopped new installs from ever taking a shipped credential, but did
nothing for the nodes that already did — those gateways still answer to a
password published in this repository.

rotate_compromised_gateway_credential() detects an EXACT match against the
denylist and replaces the pair; absent, unique, or merely unrecognised values
are left alone and return false. That distinction is the point: an operator
who deliberately set their own credential also has an "unrecognised" one, and
rotating it would be the same class of harm as leaving the default in place.

It hangs off resolve_dynamic_env beside ensure_generated_secrets, gated on the
gateway's app id, so an affected node heals on its next reconcile tick. There
is deliberately no teardown here: the new hash changes the resolved secret env,
which changes secret_env_hash, which the drift check reads as a container-label
mismatch — so the platform's own recreate path rebuilds the gateway around its
unchanged data directory, ports, volumes and name.

Rotation is self-terminating (the value written is not on the denylist, so the
next tick is a no-op) and errors propagate rather than being swallowed, because
the atomic write leaves the previous credential intact on failure.

Bcrypt generation was factored out of ensure_one into write_bcrypt_pair, which
both generation and rotation call — 01-11's SUMMARY claimed such a helper
existed but the arm was still inline, and rotation cannot reuse
ensure_gateway_credential because its idempotent fast path returns early
exactly when the file is present, which is the case rotation acts on.

Also fixes cargo fmt drift left by 42652547 in install.rs.

Verified: 6 new tests, secrets suite 16/16; full suite 1008 passed with one
known wall-clock flake (green 4/4 in isolation). NOT verified on a node —
Task 2's blocking checkpoint has not been run, so FED-07 stays open.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

core/archipelago/src/api/rpc/package/install.rs

@@ -598,9 +598,9 @@ impl RpcHandler {
             // ensure_gateway_credential above, so the secret is guaranteed to
             // exist here; re-reading it (rather than threading the value
             // through) keeps one canonical read site in container::secrets.
-            let fedi_hash = crate::container::secrets::gateway_bcrypt_hash(
-                std::path::Path::new("/var/lib/archipelago/secrets"),
-            )?;
+            let fedi_hash = crate::container::secrets::gateway_bcrypt_hash(std::path::Path::new(
+                "/var/lib/archipelago/secrets",
+            ))?;
             configure_fedimint_lnd(
                 &self.config.host_ip,
                 &mut ports,

core/archipelago/src/container/prod_orchestrator.rs

@@ -3244,6 +3244,37 @@ impl ProdContainerOrchestrator {
         // `secret_env` resolves — no per-app code, no host provisioning.
         crate::container::secrets::ensure_generated_secrets(&self.secrets_dir, manifest)?;
 
+        // FED-07 migration. A node installed before the shipped-default
+        // fallbacks were removed is still answering to a credential published
+        // in this repository. Detect that exact value and rotate it, once.
+        //
+        // The rotated hash changes the resolved secret env below, which changes
+        // `secret_env_hash`, which the drift check reads as a label mismatch on
+        // the running container — so the platform's own recreate path rebuilds
+        // it around an unchanged data directory, ports, volumes and container
+        // name. Deliberately no teardown here: a hand-rolled remove-and-run is
+        // the anti-pattern CLAUDE.md names, and it is what would lose the
+        // gateway's state.
+        //
+        // An error propagates rather than being swallowed: the atomic write
+        // leaves the previous credential in place, so surfacing the failure is
+        // strictly better than continuing with a half-rotated gateway.
+        if manifest.app.id == "fedimint-gateway"
+            && crate::container::secrets::rotate_compromised_gateway_credential(&self.secrets_dir)?
+        {
+            // Names a path, never a value — this line crosses into the node's
+            // logs, which are a wider audience than the 0600 secrets dir.
+            tracing::info!(
+                app = "fedimint-gateway",
+                "Rotated the Fedimint gateway admin credential: this node was carrying a publicly \
+                 known default that shipped in the repository (FED-07). The gateway will be \
+                 recreated around its existing data. The new password is readable by the service \
+                 user at {}/{}.pw",
+                self.secrets_dir.display(),
+                crate::container::secrets::GATEWAY_HASH_SECRET_NAME,
+            );
+        }
+
         let mut facts = self.detect_host_facts().await;
         // Only pay the podman cost to detect Knots-vs-Core when this manifest
         // actually templates the Bitcoin node into its env (mempool — B12).

core/archipelago/src/container/secrets.rs

@@ -67,18 +67,29 @@ fn ensure_one(dir: &Path, gs: &GeneratedSecret) -> Result<()> {
         SecretGenKind::Hex16 => write_secret(&dir.join(&gs.name), &random_hex(16))?,
         SecretGenKind::Hex32 => write_secret(&dir.join(&gs.name), &random_hex(32))?,
         SecretGenKind::Base64 => write_secret(&dir.join(&gs.name), &random_base64(32))?,
-        SecretGenKind::Bcrypt => {
-            let password = random_hex(BCRYPT_PASSWORD_BYTES);
-            let hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST)
-                .context("bcrypt-hashing generated password")?;
-            // Primary (server-facing hash) first, then the plaintext sibling.
-            write_secret(&dir.join(&gs.name), &hash)?;
-            write_secret(&dir.join(format!("{}.pw", gs.name)), &password)?;
-        }
+        SecretGenKind::Bcrypt => write_bcrypt_pair(dir, &gs.name)?,
     }
     Ok(())
 }
 
+/// Generate a fresh bcrypt credential pair for `name` under `dir`: the
+/// server-facing hash at `<name>` and its plaintext sibling at `<name>.pw`,
+/// both 0600 through the atomic [`write_secret`].
+///
+/// The single implementation of bcrypt generation on this platform —
+/// [`ensure_one`]'s `Bcrypt` arm and
+/// [`rotate_compromised_gateway_credential`] both go through here, so there is
+/// one place where a credential comes into existence.
+fn write_bcrypt_pair(dir: &Path, name: &str) -> Result<()> {
+    let password = random_hex(BCRYPT_PASSWORD_BYTES);
+    let hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST)
+        .context("bcrypt-hashing generated password")?;
+    // Primary (server-facing hash) first, then the plaintext sibling.
+    write_secret(&dir.join(name), &hash)?;
+    write_secret(&dir.join(format!("{}.pw", name)), &password)?;
+    Ok(())
+}
+
 /// True when `path` exists, is readable by this process, and is non-empty after
 /// trimming. Any error (missing, permission denied, empty) reads as false.
 fn readable_nonempty(path: &Path) -> bool {
@@ -177,6 +188,48 @@ pub fn gateway_bcrypt_hash(secrets_dir: &Path) -> Result<String> {
     Ok(hash.to_string())
 }
 
+/// Detect and rotate a Fedimint gateway credential that is a publicly known
+/// shipped default (FED-07 migration).
+///
+/// Returns `Ok(true)` only when the stored hash was an EXACT match for a
+/// [`KNOWN_DEFAULT_GATEWAY_HASHES`] entry and has been replaced with a freshly
+/// generated pair. An absent, unreadable, or simply unrecognised-but-unique
+/// value returns `Ok(false)` and writes nothing: rotation must never fire on
+/// "anything I did not generate this run", or an operator who deliberately set
+/// their own credential would have it silently replaced.
+///
+/// Generating a credential where none exists is
+/// [`ensure_gateway_credential`]'s job, not this function's.
+///
+/// **Rollback:** the replacement goes through [`write_secret`]'s atomic
+/// temp-file-plus-rename, so a failure part-way through leaves the previous
+/// credential file intact and the gateway keeps working with it. Do NOT
+/// "improve" this into a truncate-in-place or a remove-then-write — that turns
+/// a failed rotation into a gateway configured against a credential nobody
+/// holds.
+///
+/// **Self-terminating:** the value written is freshly generated and therefore
+/// not on the denylist, so the next reconcile tick detects nothing and changes
+/// nothing. Rotation happens at most once per affected node.
+pub fn rotate_compromised_gateway_credential(secrets_dir: &Path) -> Result<bool> {
+    let path = secrets_dir.join(GATEWAY_HASH_SECRET_NAME);
+    let Ok(current) = fs::read_to_string(&path) else {
+        // Absent or unreadable: nothing to rotate. ensure_gateway_credential
+        // owns materialising it.
+        return Ok(false);
+    };
+    if !KNOWN_DEFAULT_GATEWAY_HASHES.contains(&current.trim()) {
+        return Ok(false);
+    }
+    write_bcrypt_pair(secrets_dir, GATEWAY_HASH_SECRET_NAME).with_context(|| {
+        format!(
+            "rotating compromised gateway credential at {}",
+            path.display()
+        )
+    })?;
+    Ok(true)
+}
+
 /// Write an externally computed secret value (0600, atomic). For derived
 /// secrets that aren't random generators — e.g. the btcpay internal-LND
 /// connection string assembled in `container::lnd`.
@@ -290,9 +343,8 @@ mod tests {
         ensure_gateway_credential(dir.path()).unwrap();
 
         let hash = std::fs::read_to_string(dir.path().join(GATEWAY_HASH_SECRET_NAME)).unwrap();
-        let pw =
-            std::fs::read_to_string(dir.path().join(format!("{GATEWAY_HASH_SECRET_NAME}.pw")))
-                .unwrap();
+        let pw = std::fs::read_to_string(dir.path().join(format!("{GATEWAY_HASH_SECRET_NAME}.pw")))
+            .unwrap();
         assert!(bcrypt::verify(pw.trim(), hash.trim()).unwrap());
 
         for f in [
@@ -357,6 +409,123 @@ mod tests {
         assert_ne!(hash_a, hash_b, "two fresh installs must not share a hash");
     }
 
+    // ── FED-07 migration: rotating a shipped default off an existing node ──
+
+    #[test]
+    fn rotates_a_denylisted_gateway_credential() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+
+        assert!(rotate_compromised_gateway_credential(dir.path()).unwrap());
+
+        // The new value is readable through the normal accessor, which means
+        // it is neither empty nor still denylisted.
+        let rotated = gateway_bcrypt_hash(dir.path()).unwrap();
+        assert!(!KNOWN_DEFAULT_GATEWAY_HASHES.contains(&rotated.as_str()));
+
+        // The plaintext sibling was written too and verifies against the hash,
+        // so the operator can actually get back into the gateway.
+        let pw = std::fs::read_to_string(dir.path().join(format!("{GATEWAY_HASH_SECRET_NAME}.pw")))
+            .unwrap();
+        assert!(bcrypt::verify(pw.trim(), rotated.trim()).unwrap());
+
+        for f in [
+            GATEWAY_HASH_SECRET_NAME.to_string(),
+            format!("{GATEWAY_HASH_SECRET_NAME}.pw"),
+        ] {
+            let mode = std::fs::metadata(dir.path().join(&f))
+                .unwrap()
+                .permissions()
+                .mode()
+                & 0o777;
+            assert_eq!(mode, 0o600, "{f} must stay 0600 after rotation");
+        }
+    }
+
+    #[test]
+    fn leaves_a_unique_gateway_credential_alone() {
+        let dir = tempfile::tempdir().unwrap();
+        ensure_gateway_credential(dir.path()).unwrap();
+        let before = gateway_bcrypt_hash(dir.path()).unwrap();
+
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert_eq!(before, gateway_bcrypt_hash(dir.path()).unwrap());
+    }
+
+    #[test]
+    fn leaves_an_unrecognised_credential_alone() {
+        // The adjacency edge that matters: an operator's own hand-set value is
+        // not on the denylist and must survive. Rotation is denylist-exact,
+        // never "anything I did not generate".
+        let dir = tempfile::tempdir().unwrap();
+        let operator_set = "$2y$10$operatorChosenValueThatWeMustNeverTouchAAAAAAAAAAAAAAAAAAAAA";
+        std::fs::write(dir.path().join(GATEWAY_HASH_SECRET_NAME), operator_set).unwrap();
+
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert_eq!(
+            std::fs::read_to_string(dir.path().join(GATEWAY_HASH_SECRET_NAME)).unwrap(),
+            operator_set
+        );
+    }
+
+    #[test]
+    fn no_op_when_no_gateway_credential_exists() {
+        let dir = tempfile::tempdir().unwrap();
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert!(!dir.path().join(GATEWAY_HASH_SECRET_NAME).exists());
+    }
+
+    #[test]
+    fn rotation_is_idempotent() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+
+        assert!(rotate_compromised_gateway_credential(dir.path()).unwrap());
+        let after_first = gateway_bcrypt_hash(dir.path()).unwrap();
+
+        // Second tick: nothing detected, nothing changed. This is what stops a
+        // reconcile loop from recreating the gateway on every pass.
+        assert!(!rotate_compromised_gateway_credential(dir.path()).unwrap());
+        assert_eq!(after_first, gateway_bcrypt_hash(dir.path()).unwrap());
+    }
+
+    #[test]
+    fn rotation_touches_no_other_secret() {
+        let dir = tempfile::tempdir().unwrap();
+        std::fs::write(
+            dir.path().join(GATEWAY_HASH_SECRET_NAME),
+            KNOWN_DEFAULT_GATEWAY_HASHES[0],
+        )
+        .unwrap();
+        let bystanders = [
+            ("mempool-db-password", "mempool-value"),
+            ("immich-db-password", "immich-value"),
+            ("fmcd-password", "fmcd-value"),
+            ("bitcoin-rpc-password", "bitcoin-value"),
+        ];
+        for (name, value) in bystanders {
+            std::fs::write(dir.path().join(name), value).unwrap();
+        }
+
+        assert!(rotate_compromised_gateway_credential(dir.path()).unwrap());
+
+        for (name, value) in bystanders {
+            assert_eq!(
+                std::fs::read_to_string(dir.path().join(name)).unwrap(),
+                value,
+                "{name} must be byte-identical after a gateway rotation"
+            );
+        }
+    }
+
     #[test]
     fn self_heals_unreadable_secret() {
         // Simulate the root-owned case: a present-but-unreadable file. We can't