fix(content): never take a paid buyer's ecash and then fail to deliver

2026-09-18: a peer purchase paid 10 sats, the seller redeemed them, and the
buyer got no file plus a "seller doesn't accept your Cashu mint" error.
Three defects lined up:

1. The seller checked file existence with stat() but only read the file
   AFTER redeeming the payment. Filebrowser-owned 0640 files (uid 100999)
   passed stat but failed fs::read for the archipelago service user.
   serve_content now checks existence and readability BEFORE the payment
   gate, so an unservable file costs the buyer nothing.
2. The HTTP handler mapped every serve_content error to a bare, unlogged
   404. A server-side failure is now a logged 500. (A 404 also makes the
   buyer's Auto transport re-send the request over Tor.)
3. That re-send carried the same single-use token, which the mint had
   already spent, so the seller answered 402. Redemption is now
   idempotent: a token that verified for an item keeps authorising that
   item for 10 minutes (per token, per item; SHA-256 keyed, in-memory,
   concurrent requests serialised, failures never cached).

Buyer side: reclaim_spent_ecash now reports whether the refund worked, and
the error text no longer claims "refunded" when it wasn't, or asserts the
seller rejects the mint when the cause is unknown.

Adds tests for replay, concurrency, failure-not-cached, cross-item, and
unreadable-file-before-payment.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

core/archipelago/src/api/handler/content.rs

@@ -162,11 +162,25 @@ impl ApiHandler {
                     r#"{"error":"This file is shared with the host's federation peers only. Federate with that node (exchange invites) so it recognizes you, then try again."}"#,
                 ),
             )),
-            Ok(content_server::ServeResult::NotFound) | Err(_) => Ok(build_response(
+            Ok(content_server::ServeResult::NotFound) => Ok(build_response(
                 StatusCode::NOT_FOUND,
                 "text/plain",
                 hyper::Body::from("Content not found"),
             )),
+            // A server-side failure is NOT "not found": reporting it as a 404
+            // hid an unreadable file behind a silent, unlogged response, and a
+            // buyer's client re-sends a 404 over another transport. 5xx it, and
+            // say why in the journal.
+            Err(e) => {
+                tracing::warn!(content_id = %content_id, "content request failed: {e:#}");
+                Ok(build_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    "application/json",
+                    hyper::Body::from(
+                        r#"{"error":"The seller could not read this file right now. You have not been charged."}"#,
+                    ),
+                ))
+            }
         }
     }
 

core/archipelago/src/api/rpc/content.rs

@@ -22,9 +22,11 @@ const FILE_CATALOG_PROTOCOL: &str = "https://archipelago.dev/protocols/file-cata
 /// Best-effort reclaim of an ecash payment token that was minted but the sale
 /// didn't complete (seller unreachable or couldn't redeem it), so the buyer
 /// doesn't lose the value. For Fedimint the spender can reissue its own
-/// un-redeemed notes; for Cashu the proofs are received back. Fails silently if
-/// the seller already claimed the token (then the value is genuinely gone).
-async fn reclaim_spent_ecash(data_dir: &std::path::Path, token: &str, backend: &str) {
+/// un-redeemed notes; for Cashu the proofs are received back. Returns whether
+/// the value came back: false if the seller already claimed the token (then
+/// the value is genuinely gone), so callers never tell the buyer they were
+/// refunded when they weren't.
+async fn reclaim_spent_ecash(data_dir: &std::path::Path, token: &str, backend: &str) -> bool {
     let res = match backend {
         "fedimint" => crate::wallet::fedimint_client::reissue_into_any(data_dir, token)
             .await
@@ -32,13 +34,29 @@ async fn reclaim_spent_ecash(data_dir: &std::path::Path, token: &str, backend: &
         _ => ecash::receive_token(data_dir, token).await,
     };
     match res {
-        Ok(sats) => tracing::info!(
-            "paid download: reclaimed {sats} sats of unspent {backend} ecash after a failed sale"
-        ),
-        Err(e) => tracing::warn!(
-            "paid download: could not reclaim {backend} ecash (the peer may have already \
-             claimed it): {e:#}"
-        ),
+        Ok(sats) => {
+            tracing::info!(
+                "paid download: reclaimed {sats} sats of unspent {backend} ecash after a failed sale"
+            );
+            true
+        }
+        Err(e) => {
+            tracing::warn!(
+                "paid download: could not reclaim {backend} ecash (the peer may have already \
+                 claimed it): {e:#}"
+            );
+            false
+        }
+    }
+}
+
+/// What to tell the buyer about their payment after a failed sale.
+fn refund_note(reclaimed: bool) -> &'static str {
+    if reclaimed {
+        "Your ecash was refunded to your wallet."
+    } else {
+        "The seller had already claimed the payment, so it could not be refunded \
+         automatically — contact the seller."
     }
 }
 
@@ -564,9 +582,13 @@ impl RpcHandler {
                 tracing::warn!("paid peer download dial failed for {}: {:#}", onion, e);
                 // The token was already minted/spent — reclaim it so the buyer
                 // doesn't lose the value when the seller was simply unreachable.
-                reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
+                let reclaimed =
+                    reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
                 return Ok(serde_json::json!({
-                    "error": "Could not reach the peer over mesh or Tor — it may be offline. Your ecash was refunded to your wallet. Please try again."
+                    "error": format!(
+                        "Could not reach the peer over mesh or Tor — it may be offline. {} Please try again.",
+                        refund_note(reclaimed)
+                    )
                 }));
             }
         };
@@ -592,15 +614,19 @@ impl RpcHandler {
             );
             // Seller couldn't redeem the token — reclaim it so the buyer keeps
             // their funds (the spent-but-unredeemed-notes case the user hit).
-            reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
+            let reclaimed =
+                reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
+            // The 402 body is generic, so don't assert a cause — a seller that
+            // redeemed the token and then failed to deliver also lands here.
             let hint = match used_backend {
-                "fedimint" => "the seller isn't in the same Fedimint federation as you",
-                _ => "the seller doesn't accept your Cashu mint",
+                "fedimint" => "the seller may not be in the same Fedimint federation as you",
+                _ => "the seller may not accept your Cashu mint",
             };
             return Ok(serde_json::json!({
                 "error": format!(
-                    "Payment rejected by the seller — {hint}. Your ecash was refunded to \
-                     your wallet. Try the other ecash type, or use a shared mint/federation."
+                    "Payment not accepted by the seller — {hint}. {} Try the other ecash \
+                     type, or use a shared mint/federation.",
+                    refund_note(reclaimed)
                 )
             }));
         }
@@ -609,9 +635,10 @@ impl RpcHandler {
             let status = response.status();
             let body = response.text().await.unwrap_or_default();
             tracing::warn!("paid download: seller {onion} returned {status}: {body}");
-            reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
+            let reclaimed =
+                reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
             return Ok(serde_json::json!({
-                "error": format!("Peer returned an error ({status}). Your ecash was refunded to your wallet.")
+                "error": format!("Peer returned an error ({status}). {}", refund_note(reclaimed))
             }));
         }
 

core/archipelago/src/content_server.rs

@@ -5,13 +5,110 @@
 
 use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use std::collections::HashMap;
+use std::future::Future;
 use std::path::{Path, PathBuf};
+use std::sync::{Arc, LazyLock};
+use std::time::{Duration, Instant};
 use tokio::fs;
+use tokio::sync::Mutex;
 use tracing::{debug, warn};
 
 const CATALOG_FILE: &str = "content/catalog.json";
 const CONTENT_DIR: &str = "content/files";
 
+/// How long a redeemed payment token keeps entitling its buyer to re-fetch the
+/// item it paid for. Long enough to cover a buyer's transport fallback (FIPS →
+/// Tor re-sends the same request, token included) and a manual retry; short
+/// enough that the ledger stays tiny and a leaked token isn't a standing pass.
+const REDEMPTION_TTL: Duration = Duration::from_secs(600);
+
+/// One ledger slot per payment token (keyed by its SHA-256 — the raw bearer
+/// token is never held here). The inner mutex serialises verification of the
+/// same token; its value is the content id the token was redeemed for.
+struct RedemptionSlot {
+    created_at: Instant,
+    redeemed_for: Arc<Mutex<Option<String>>>,
+}
+
+static REDEMPTIONS: LazyLock<Mutex<HashMap<String, RedemptionSlot>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Decide whether `token` pays for `content_id`, redeeming it at most once.
+///
+/// Payment tokens are single-use: verifying one swaps its proofs at the mint,
+/// so a second verification of the same token always fails "already spent".
+/// A buyer's HTTP client can legitimately send the same request twice — its
+/// FIPS attempt gets a 404/5xx and it re-sends over Tor — and without this
+/// the seller redeemed the token on the first request, then answered the
+/// retry `402 Payment required`: money taken, file never delivered.
+///
+/// So the first verification that succeeds is remembered (per token, per
+/// item, for [`REDEMPTION_TTL`]) and later requests for the same item present
+/// the same token are authorised without touching the mint again. Concurrent
+/// requests with one token queue on the slot so only one runs `verify`.
+/// A failed verification is not remembered — the slot is dropped so garbage
+/// tokens can't accumulate and a legitimate retry gets a fresh attempt.
+async fn authorize_payment<F, Fut>(token: &str, content_id: &str, verify: F) -> bool
+where
+    F: FnOnce() -> Fut,
+    Fut: Future<Output = bool>,
+{
+    let key = hex::encode(Sha256::digest(token.as_bytes()));
+    let redeemed_for = {
+        let mut ledger = REDEMPTIONS.lock().await;
+        ledger.retain(|_, s| s.created_at.elapsed() < REDEMPTION_TTL);
+        ledger
+            .entry(key.clone())
+            .or_insert_with(|| RedemptionSlot {
+                created_at: Instant::now(),
+                redeemed_for: Arc::new(Mutex::new(None)),
+            })
+            .redeemed_for
+            .clone()
+    };
+
+    let mut state = redeemed_for.lock().await;
+    if state.as_deref() == Some(content_id) {
+        debug!(
+            "Payment token already redeemed for '{}' — serving without re-verifying",
+            content_id
+        );
+        return true;
+    }
+    if verify().await {
+        *state = Some(content_id.to_string());
+        return true;
+    }
+    // Keep a slot that already holds a redemption (this token paid for a
+    // different item); drop one that never verified anything.
+    let never_redeemed = state.is_none();
+    drop(state);
+    if never_redeemed {
+        REDEMPTIONS.lock().await.remove(&key);
+    }
+    false
+}
+
+/// Confirm the node can actually hand the file over: it exists and this
+/// process may read it. Must run BEFORE a payment is redeemed — a paid buyer
+/// who then hits a read error has lost their token for nothing (2026-09-18:
+/// filebrowser-owned `0640` files the node's service user couldn't open; the
+/// stat calls passed, `fs::read` failed after the swap, the buyer got a 404).
+/// Reading a byte (not just opening) also rejects a directory.
+async fn ensure_servable(file_path: &Path) -> Result<()> {
+    use tokio::io::AsyncReadExt;
+    let mut file = fs::File::open(file_path)
+        .await
+        .with_context(|| format!("content file {} is not readable", file_path.display()))?;
+    let mut probe = [0u8; 1];
+    file.read(&mut probe)
+        .await
+        .with_context(|| format!("content file {} cannot be read", file_path.display()))?;
+    Ok(())
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ContentItem {
     pub id: String,
@@ -296,6 +393,31 @@ pub async fn serve_content(
         }
     }
 
+    // Verify the file can be served BEFORE any payment is redeemed. The gate
+    // below swaps the buyer's token at the mint; failing to hand over the file
+    // after that takes their money and delivers nothing.
+    let file_path = content_file_path(data_dir, item);
+    if !file_path.exists() {
+        // The catalog entry survived (it's a separate JSON file) but its
+        // backing file is gone — most likely lost in an unrelated data-dir
+        // reset (a shared filebrowser file, 2026-07-01: two catalog entries
+        // outlived a filebrowser reinstall that wiped the files themselves).
+        // Leaving the entry in place would keep advertising it as available
+        // to every peer forever, each hitting the exact same dead end this
+        // one just did. Prune it so it stops being offered.
+        warn!(
+            content_id = %id,
+            filename = %item.filename,
+            "content catalog entry's file is missing on disk — pruning the stale entry"
+        );
+        prune_missing_content_entry(data_dir, id).await;
+        return Ok(ServeResult::NotFound);
+    }
+    if let Err(e) = ensure_servable(&file_path).await {
+        warn!(content_id = %id, "cannot serve content (payment not taken): {e:#}");
+        return Err(e);
+    }
+
     // Check access control
     if !owner_session {
         match &item.access {
@@ -309,7 +431,10 @@ pub async fn serve_content(
                 if let Some(token) = payment_token {
                     if (method_accepted(&item.access, "ecash")
                         || method_accepted(&item.access, "fedimint"))
-                        && verify_payment_token(data_dir, token, *price_sats).await
+                        && authorize_payment(token, id, || {
+                            verify_payment_token(data_dir, token, *price_sats)
+                        })
+                        .await
                     {
                         authorized = true;
                     }
@@ -336,24 +461,6 @@ pub async fn serve_content(
         }
     }
 
-    let file_path = content_file_path(data_dir, item);
-    if !file_path.exists() {
-        // The catalog entry survived (it's a separate JSON file) but its
-        // backing file is gone — most likely lost in an unrelated data-dir
-        // reset (a shared filebrowser file, 2026-07-01: two catalog entries
-        // outlived a filebrowser reinstall that wiped the files themselves).
-        // Leaving the entry in place would keep advertising it as available
-        // to every peer forever, each hitting the exact same dead end this
-        // one just did. Prune it so it stops being offered.
-        warn!(
-            content_id = %id,
-            filename = %item.filename,
-            "content catalog entry's file is missing on disk — pruning the stale entry"
-        );
-        prune_missing_content_entry(data_dir, id).await;
-        return Ok(ServeResult::NotFound);
-    }
-
     let metadata = fs::metadata(&file_path)
         .await
         .context("Failed to read file metadata")?;
@@ -725,3 +832,182 @@ mod prune_missing_content_tests {
         assert_eq!(reloaded.items[0].id, "present-item");
     }
 }
+
+#[cfg(test)]
+mod paid_delivery_tests {
+    use super::*;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    /// A verifier that counts how often it actually runs.
+    fn counting(
+        calls: &Arc<AtomicUsize>,
+        result: bool,
+    ) -> impl FnOnce() -> std::future::Ready<bool> {
+        let calls = calls.clone();
+        move || {
+            calls.fetch_add(1, Ordering::SeqCst);
+            std::future::ready(result)
+        }
+    }
+
+    #[tokio::test]
+    async fn replayed_token_is_served_without_redeeming_twice() {
+        // The 2026-09-18 incident: the buyer's client re-sent the same request
+        // over Tor after the seller had already redeemed the token, and the
+        // second verification ("already spent") turned into a 402.
+        let calls = Arc::new(AtomicUsize::new(0));
+        assert!(authorize_payment("tok-replay", "item-a", counting(&calls, true)).await);
+        assert!(authorize_payment("tok-replay", "item-a", counting(&calls, true)).await);
+        assert_eq!(calls.load(Ordering::SeqCst), 1, "mint must be hit once");
+    }
+
+    #[tokio::test]
+    async fn concurrent_requests_with_one_token_redeem_once() {
+        // FIPS attempt still in flight when the Tor fallback arrives.
+        let calls = Arc::new(AtomicUsize::new(0));
+        let slow = |calls: Arc<AtomicUsize>| {
+            move || async move {
+                calls.fetch_add(1, Ordering::SeqCst);
+                tokio::time::sleep(Duration::from_millis(100)).await;
+                true
+            }
+        };
+        let (a, b) = tokio::join!(
+            authorize_payment("tok-concurrent", "item-a", slow(calls.clone())),
+            authorize_payment("tok-concurrent", "item-a", slow(calls.clone())),
+        );
+        assert!(a && b, "both requests must be served");
+        assert_eq!(calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn failed_verification_is_not_remembered() {
+        let calls = Arc::new(AtomicUsize::new(0));
+        assert!(!authorize_payment("tok-bad", "item-a", counting(&calls, false)).await);
+        // A retry gets a fresh attempt — and can succeed (e.g. mint was down).
+        assert!(authorize_payment("tok-bad", "item-a", counting(&calls, true)).await);
+        assert_eq!(calls.load(Ordering::SeqCst), 2);
+        let ledger = REDEMPTIONS.lock().await;
+        let key = hex::encode(Sha256::digest(b"tok-bad"));
+        assert!(ledger.contains_key(&key), "successful redemption is kept");
+    }
+
+    #[tokio::test]
+    async fn failed_verification_leaves_no_ledger_entry() {
+        let calls = Arc::new(AtomicUsize::new(0));
+        assert!(!authorize_payment("tok-garbage", "item-a", counting(&calls, false)).await);
+        let key = hex::encode(Sha256::digest(b"tok-garbage"));
+        assert!(
+            !REDEMPTIONS.lock().await.contains_key(&key),
+            "garbage tokens must not accumulate"
+        );
+    }
+
+    #[tokio::test]
+    async fn token_redeemed_for_one_item_does_not_unlock_another() {
+        let calls = Arc::new(AtomicUsize::new(0));
+        assert!(authorize_payment("tok-cross", "item-a", counting(&calls, true)).await);
+        // Item B is verified on its own merits (the real mint would say
+        // "already spent"); it must not ride on item A's redemption…
+        assert!(!authorize_payment("tok-cross", "item-b", counting(&calls, false)).await);
+        assert_eq!(calls.load(Ordering::SeqCst), 2);
+        // …and failing there must not revoke what the token already paid for.
+        assert!(authorize_payment("tok-cross", "item-a", counting(&calls, true)).await);
+        assert_eq!(calls.load(Ordering::SeqCst), 2);
+    }
+
+    fn paid_item(id: &str, filename: &str) -> ContentItem {
+        ContentItem {
+            id: id.to_string(),
+            filename: filename.to_string(),
+            mime_type: "audio/mpeg".to_string(),
+            size_bytes: 4,
+            description: String::new(),
+            access: AccessControl::Paid {
+                price_sats: 10,
+                accepted: vec!["ecash".to_string()],
+            },
+            availability: Availability::AllPeers,
+            added_at: "2026-01-01T00:00:00Z".to_string(),
+        }
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn unreadable_paid_file_errors_before_any_payment_is_redeemed() {
+        // Filebrowser-owned 0640 files the node's service user can't read:
+        // stat() succeeds, read() fails. That must surface as an error BEFORE
+        // the token is verified — never after the swap has taken the money.
+        use std::os::unix::fs::PermissionsExt;
+        let dir = tempfile::tempdir().unwrap();
+        let data_dir = dir.path();
+        save_catalog(
+            data_dir,
+            &ContentCatalog {
+                items: vec![paid_item("locked", "locked.mp3")],
+            },
+        )
+        .await
+        .unwrap();
+        let files = data_dir.join("content").join("files");
+        tokio::fs::create_dir_all(&files).await.unwrap();
+        let file = files.join("locked.mp3");
+        tokio::fs::write(&file, b"data").await.unwrap();
+        std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o000)).unwrap();
+        if std::fs::File::open(&file).is_ok() {
+            return; // running as root: permissions can't be enforced here
+        }
+
+        // A token that would fail verification if it were reached: getting
+        // PaymentRequired here would mean the gate ran before the file check.
+        let result = serve_content(
+            data_dir,
+            "locked",
+            Some("cashuBnot-a-real-token"),
+            None,
+            None,
+            None,
+            false,
+        )
+        .await;
+        assert!(
+            result.is_err(),
+            "unreadable file must be a server error, not 402/404"
+        );
+        let key = hex::encode(Sha256::digest(b"cashuBnot-a-real-token"));
+        assert!(
+            !REDEMPTIONS.lock().await.contains_key(&key),
+            "no redemption may be attempted for an unservable file"
+        );
+    }
+
+    #[tokio::test]
+    async fn readable_paid_file_with_bad_token_still_requires_payment() {
+        let dir = tempfile::tempdir().unwrap();
+        let data_dir = dir.path();
+        save_catalog(
+            data_dir,
+            &ContentCatalog {
+                items: vec![paid_item("ok", "ok.mp3")],
+            },
+        )
+        .await
+        .unwrap();
+        let files = data_dir.join("content").join("files");
+        tokio::fs::create_dir_all(&files).await.unwrap();
+        tokio::fs::write(files.join("ok.mp3"), b"data").await.unwrap();
+
+        let result = serve_content(
+            data_dir,
+            "ok",
+            Some("cashuBnot-a-real-token-2"),
+            None,
+            None,
+            None,
+            false,
+        )
+        .await
+        .unwrap();
+        assert!(matches!(result, ServeResult::PaymentRequired(10)));
+    }
+}