fix(content): never charge for a file the seller can't serve or replay a spent token

After the keyset-id fix, a Minibits paid download still failed and the
buyer lost the sats. What happened, 2026-09-29, amishparadise:

1. The seller redeemed the token, then failed to read the file. It was a
   FileBrowser upload owned by the container subuid (100999) with mode
   0640. The handler mapped that Err to 404.
2. The buyer's FIPS dial treats 404 as "fall back to Tor" and resent the
   request with the same, now spent, token. The seller answered 402, and
   the buyer showed "seller doesn't accept your Cashu mint".

Fixes:
- serve_content checks the file is readable before the paid gate. If it
  isn't, it grants read with `podman unshare chmod a+r`, which matches
  the other shared files. If that also fails it returns Unavailable (503)
  without taking payment.
- The content handler returns 500 on internal errors and logs them,
  instead of a silent 404.
- New PeerRequest::single_delivery(), used for the paid download: the
  FIPS answer is final, FIPS retries only when it never connected, and
  there's no Tor replay once the request may have been delivered.
- The buyer shows the seller's error text for non-402 failures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

core/archipelago/src/api/handler/content.rs

@@ -162,11 +162,28 @@ impl ApiHandler {
                     r#"{"error":"This file is shared with the host's federation peers only. Federate with that node (exchange invites) so it recognizes you, then try again."}"#,
                 ),
             )),
-            Ok(content_server::ServeResult::NotFound) | Err(_) => Ok(build_response(
+            Ok(content_server::ServeResult::Unavailable) => Ok(build_response(
+                StatusCode::SERVICE_UNAVAILABLE,
+                "application/json",
+                hyper::Body::from(
+                    r#"{"error":"The seller's node can't read this file right now. No payment was taken."}"#,
+                ),
+            )),
+            Ok(content_server::ServeResult::NotFound) => Ok(build_response(
                 StatusCode::NOT_FOUND,
                 "text/plain",
                 hyper::Body::from("Content not found"),
             )),
+            // Not a 404: a paid request may already have been charged by the
+            // time this fails, and "not found" hid the real error entirely.
+            Err(e) => {
+                tracing::error!("Serving content {content_id} failed: {e:#}");
+                Ok(build_response(
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    "text/plain",
+                    hyper::Body::from("Failed to serve content"),
+                ))
+            }
         }
     }
 

core/archipelago/src/api/rpc/content.rs

@@ -555,6 +555,9 @@ impl RpcHandler {
         .service(crate::settings::transport::PeerService::PeerFiles)
         .header("X-Federation-DID", local_did)
         .header("X-Payment-Token", token_str.clone())
+        // The token is a bearer instrument the seller redeems on first sight:
+        // a Tor replay after FIPS delivered it can only arrive spent.
+        .single_delivery()
         .timeout(std::time::Duration::from_secs(900))
         .send_get()
         .await
@@ -610,8 +613,12 @@ impl RpcHandler {
             let body = response.text().await.unwrap_or_default();
             tracing::warn!("paid download: seller {onion} returned {status}: {body}");
             reclaim_spent_ecash(&self.config.data_dir, &token_str, used_backend).await;
+            let reason = serde_json::from_str::<serde_json::Value>(&body)
+                .ok()
+                .and_then(|v| v.get("error").and_then(|e| e.as_str()).map(str::to_string))
+                .unwrap_or_else(|| format!("Peer returned an error ({status})."));
             return Ok(serde_json::json!({
-                "error": format!("Peer returned an error ({status}). Your ecash was refunded to your wallet.")
+                "error": format!("{reason} Your ecash was refunded to your wallet.")
             }));
         }
 

core/archipelago/src/content_server.rs

@@ -7,7 +7,7 @@ use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 use tokio::fs;
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 
 const CATALOG_FILE: &str = "content/catalog.json";
 const CONTENT_DIR: &str = "content/files";
@@ -238,6 +238,9 @@ pub enum ServeResult {
     Forbidden,
     /// Content not found.
     NotFound,
+    /// The catalog entry and file exist but this node can't read the file.
+    /// Returned before any payment is taken.
+    Unavailable,
 }
 
 /// Serve a content item by ID with access control and optional range request.
@@ -296,6 +299,37 @@ pub async fn serve_content(
         }
     }
 
+    let file_path = content_file_path(data_dir, item);
+    if !file_path.exists() {
+        // The catalog entry survived (it's a separate JSON file) but its
+        // backing file is gone — most likely lost in an unrelated data-dir
+        // reset (a shared filebrowser file, 2026-07-01: two catalog entries
+        // outlived a filebrowser reinstall that wiped the files themselves).
+        // Leaving the entry in place would keep advertising it as available
+        // to every peer forever, each hitting the exact same dead end this
+        // one just did. Prune it so it stops being offered.
+        warn!(
+            content_id = %id,
+            filename = %item.filename,
+            "content catalog entry's file is missing on disk — pruning the stale entry"
+        );
+        prune_missing_content_entry(data_dir, id).await;
+        return Ok(ServeResult::NotFound);
+    }
+
+    // Confirm the file is readable BEFORE the paid gate below redeems the
+    // buyer's token. Reading it only afterwards meant a permission error
+    // surfaced after the sale: the buyer was charged and got an error
+    // instead of the file (2026-09-29, a FileBrowser upload left 0640).
+    if let Err(e) = ensure_readable(&file_path).await {
+        warn!(
+            content_id = %id,
+            path = %file_path.display(),
+            "shared content file is not readable by this node: {e:#}"
+        );
+        return Ok(ServeResult::Unavailable);
+    }
+
     // Check access control
     if !owner_session {
         match &item.access {
@@ -336,23 +370,6 @@ pub async fn serve_content(
         }
     }
 
-    let file_path = content_file_path(data_dir, item);
-    if !file_path.exists() {
-        // The catalog entry survived (it's a separate JSON file) but its
-        // backing file is gone — most likely lost in an unrelated data-dir
-        // reset (a shared filebrowser file, 2026-07-01: two catalog entries
-        // outlived a filebrowser reinstall that wiped the files themselves).
-        // Leaving the entry in place would keep advertising it as available
-        // to every peer forever, each hitting the exact same dead end this
-        // one just did. Prune it so it stops being offered.
-        warn!(
-            content_id = %id,
-            filename = %item.filename,
-            "content catalog entry's file is missing on disk — pruning the stale entry"
-        );
-        prune_missing_content_entry(data_dir, id).await;
-        return Ok(ServeResult::NotFound);
-    }
 
     let metadata = fs::metadata(&file_path)
         .await
@@ -572,6 +589,38 @@ pub async fn serve_content_preview(data_dir: &Path, id: &str) -> Result<PreviewR
     }
 }
 
+/// Make sure this service can open `path`, granting read access if it can't.
+///
+/// FileBrowser writes uploads as its container user (a rootless subuid such
+/// as 100999), and some arrive 0640 — unreadable by this service, although
+/// most shared files are already 0644. Inside the rootless user namespace
+/// that subuid is ours, so `podman unshare chmod a+r` grants the same read
+/// access the other shared files have, without sudo.
+async fn ensure_readable(path: &Path) -> Result<()> {
+    match fs::File::open(path).await {
+        Ok(_) => return Ok(()),
+        Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {}
+        Err(e) => return Err(e).context("Failed to open content file"),
+    }
+    let out = tokio::process::Command::new("podman")
+        .args(["unshare", "chmod", "a+r"])
+        .arg(path)
+        .output()
+        .await
+        .context("Failed to run podman unshare chmod")?;
+    if !out.status.success() {
+        anyhow::bail!(
+            "podman unshare chmod a+r failed: {}",
+            String::from_utf8_lossy(&out.stderr).trim()
+        );
+    }
+    info!("Granted read access to shared content file {}", path.display());
+    fs::File::open(path)
+        .await
+        .context("Content file still unreadable after chmod")?;
+    Ok(())
+}
+
 /// Verify a payment token covers the required amount.
 /// Accepts both cashuA tokens (real Cashu) and legacy cashuSend_ format.
 /// Swaps proofs at the mint to verify they're unspent before accepting.

core/archipelago/src/fips/dial.rs

@@ -130,10 +130,18 @@ pub fn client_with_timeout(timeout: Duration) -> reqwest::Client {
 /// robust". Only connect/timeout errors are retried (a real HTTP response,
 /// including 4xx/5xx, is returned as-is for the caller to interpret).
 async fn send_with_retry(rb: reqwest::RequestBuilder) -> Result<reqwest::Response, reqwest::Error> {
+    send_with_retry_if(rb, |e| e.is_connect() || e.is_timeout()).await
+}
+
+/// [`send_with_retry`], retrying only on errors `retryable` accepts.
+async fn send_with_retry_if(
+    rb: reqwest::RequestBuilder,
+    retryable: impl Fn(&reqwest::Error) -> bool,
+) -> Result<reqwest::Response, reqwest::Error> {
     let retry = rb.try_clone();
     match rb.send().await {
         Ok(resp) => Ok(resp),
-        Err(e) if (e.is_connect() || e.is_timeout()) && retry.is_some() => {
+        Err(e) if retryable(&e) && retry.is_some() => {
             // Brief pause so the hole-punch packets from the first attempt can
             // traverse before we re-dial onto the warmed path.
             tokio::time::sleep(Duration::from_millis(600)).await;
@@ -350,6 +358,9 @@ pub struct PeerRequest<'a> {
     /// the per-peer FIPS/Tor badge reflects reality. Opt-in because not
     /// every caller has a data dir in scope.
     pub record_data_dir: Option<std::path::PathBuf>,
+    /// The request carries something that must reach the peer at most once
+    /// (a bearer ecash token). See [`PeerRequest::single_delivery`].
+    pub single_delivery: bool,
 }
 
 impl<'a> PeerRequest<'a> {
@@ -363,9 +374,25 @@ impl<'a> PeerRequest<'a> {
             fips_timeout: None,
             service: None,
             record_data_dir: None,
+            single_delivery: false,
         }
     }
 
+    /// Never send this request twice. A paid download carries a bearer ecash
+    /// token that the seller redeems on first sight; replaying it over Tor
+    /// after FIPS already delivered it hands the seller a spent token, so the
+    /// buyer is charged and gets a 402 instead of the file (2026-09-29: FIPS
+    /// answered 404 after the seller redeemed, the Tor retry got 402).
+    ///
+    /// With this set, whatever FIPS answers is final, the FIPS retry fires
+    /// only when the first attempt never connected, and Tor is used only when
+    /// FIPS could not have delivered the request. An attempt that may have
+    /// been delivered but timed out is an error, not a fallback.
+    pub fn single_delivery(mut self) -> Self {
+        self.single_delivery = true;
+        self
+    }
+
     /// Record the transport that serves this request into federation storage
     /// (matched by this request's onion host). Best-effort, off the hot path.
     pub fn record_transport(mut self, data_dir: impl Into<std::path::PathBuf>) -> Self {
@@ -481,7 +508,10 @@ impl<'a> PeerRequest<'a> {
         if matches!(pref, TransportPref::Auto | TransportPref::Fips) {
             match self.try_fips_get().await? {
                 Some(resp) => {
-                    if pref == TransportPref::Fips || !fips_should_fall_back(resp.status()) {
+                    if pref == TransportPref::Fips
+                        || self.single_delivery
+                        || !fips_should_fall_back(resp.status())
+                    {
                         telemetry::record_fips_ok();
                         self.spawn_record(crate::transport::TransportKind::Fips);
                         return Ok((resp, crate::transport::TransportKind::Fips));
@@ -617,8 +647,25 @@ impl<'a> PeerRequest<'a> {
         for (k, v) in &self.headers {
             rb = rb.header(*k, v);
         }
-        match tokio::time::timeout(budget, send_with_retry(rb)).await {
+        let single = self.single_delivery;
+        let attempt = send_with_retry_if(rb, |e| {
+            e.is_connect() || (!single && e.is_timeout())
+        });
+        match tokio::time::timeout(budget, attempt).await {
             Ok(Ok(r)) => Ok(Some(r)),
+            // Anything but a failed connect may have reached the peer.
+            Ok(Err(e)) if single && !e.is_connect() => Err(anyhow::anyhow!(
+                "FIPS GET {} failed after the request may have been delivered \
+                 (not retrying over Tor): {}",
+                self.path,
+                e
+            )),
+            Err(_) if single => Err(anyhow::anyhow!(
+                "FIPS GET {} exceeded its {:?} budget after the request may have \
+                 been delivered (not retrying over Tor)",
+                self.path,
+                budget
+            )),
             Ok(Err(e)) => {
                 telemetry::record_fallback(FallbackReason::ConnectFail);
                 tracing::info!(