Return retryable payment status errors and record NPM release gate

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v1.8.23-alpha (2026-10-01)
+
+- Preserve paid-file Lightning entitlements across restarts and recover settled invoices from LND. Retry delivery without paying again and retain purchased files in the owned cache.
+- Return explicit payment-status errors with safe retry guidance when verification is unavailable.
+- Show compact upload progress across screens, retain the original destination, and cancel active and queued uploads.
+- Make transaction filters transparent and horizontally scrollable on mobile.
+- Keep Immich internal services out of My Apps, avoid false recovery states for healthy stacks, and allow removal of retired catalog apps.
+- Repair the redundant managed Portainer network override that can prevent startup, preserving custom overrides and persistent state.
+- Offer Standard, Medium, Fast and custom fees when cooperatively closing Lightning channels.
+- Add a clear-search icon to My Apps, Services and the App Store on desktop and mobile.
+- Keep Angor Indexer and the optional Angor Relay in the signed app catalog. Full indexing requires a synced, unpruned Bitcoin node and its indexing dependencies.
+
 ## v1.8.22-alpha (2026-09-30)
 
 - Fixed Nginx Proxy Manager launch readiness choosing a proxy listener instead of its admin port after container recreation.

core/archipelago/src/api/handler/content.rs

@@ -336,36 +336,10 @@ impl ApiHandler {
         &self,
         path: &str,
     ) -> Result<Response<hyper::Body>> {
-        let rest = path.strip_prefix("/content/").unwrap_or("");
+        Ok(invoice_status_response(path, |hash, id| async move {
-        let (content_id, payment_hash) = match rest.split_once("/invoice-status/") {
+            self.rpc_handler.settle_content_invoice(&hash, &id).await
-            Some((id, hash)) => (id, hash),
+        })
-            None => {
+        .await)
-                return Ok(build_response(
-                    StatusCode::BAD_REQUEST,
-                    "text/plain",
-                    hyper::Body::from("Invalid request"),
-                ))
-            }
-        };
-        if content_id.is_empty() || !is_valid_app_id(content_id) || payment_hash.is_empty() {
-            return Ok(build_response(
-                StatusCode::BAD_REQUEST,
-                "text/plain",
-                hyper::Body::from("Invalid request"),
-            ));
-        }
-
-        let paid = self
-            .rpc_handler
-            .settle_content_invoice(payment_hash, content_id)
-            .await?;
-
-        let body = serde_json::json!({ "paid": paid });
-        Ok(build_response(
-            StatusCode::OK,
-            "application/json",
-            hyper::Body::from(serde_json::to_vec(&body).unwrap_or_default()),
-        ))
     }
 
     /// Seller side (#46): issue a fresh on-chain address for a paid catalog item
@@ -568,3 +542,109 @@ impl ApiHandler {
         }
     }
 }
+
+/// Keep invalid input and an unavailable wallet inside the HTTP protocol so
+/// buyers can retry delivery without treating a dropped socket as lost payment.
+async fn invoice_status_response<F, Fut>(path: &str, settle: F) -> Response<hyper::Body>
+where
+    F: FnOnce(String, String) -> Fut,
+    Fut: std::future::Future<Output = Result<bool>>,
+{
+    let parsed = path
+        .strip_prefix("/content/")
+        .and_then(|rest| rest.split_once("/invoice-status/"))
+        .filter(|(id, hash)| {
+            !id.is_empty()
+                && is_valid_app_id(id)
+                && hash.len() == 64
+                && hash.bytes().all(|c| c.is_ascii_hexdigit())
+        });
+    let Some((id, hash)) = parsed else {
+        return build_response(
+            StatusCode::BAD_REQUEST,
+            "application/json",
+            hyper::Body::from(r#"{"error":"Invalid content ID or payment hash"}"#),
+        );
+    };
+    match settle(hash.to_ascii_lowercase(), id.to_owned()).await {
+        Ok(paid) => build_response(
+            StatusCode::OK,
+            "application/json",
+            hyper::Body::from(serde_json::json!({"paid": paid}).to_string()),
+        ),
+        Err(_) => {
+            tracing::warn!("Peer-file payment status verification is temporarily unavailable");
+            let mut response = build_response(
+                StatusCode::SERVICE_UNAVAILABLE,
+                "application/json",
+                hyper::Body::from(
+                    r#"{"error":"Payment verification is temporarily unavailable. Retry without paying again."}"#,
+                ),
+            );
+            response.headers_mut().insert(
+                hyper::header::RETRY_AFTER,
+                hyper::header::HeaderValue::from_static("5"),
+            );
+            response
+        }
+    }
+}
+
+#[cfg(test)]
+mod invoice_status_tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn malformed_requests_do_not_query_the_wallet() {
+        for path in [
+            "/bad",
+            "/content//invoice-status/aa",
+            "/content/file/invoice-status/aa",
+            "/content/file/invoice-status/",
+            "/content/file/invoice-status/not-a-hash",
+        ] {
+            let response = invoice_status_response(path, |_, _| async {
+                panic!("Invalid request reached wallet");
+                #[allow(unreachable_code)]
+                Ok(false)
+            })
+            .await;
+            assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+            assert_eq!(response.headers()["content-type"], "application/json");
+            let body = hyper::body::to_bytes(response.into_body()).await.unwrap();
+            assert!(
+                serde_json::from_slice::<serde_json::Value>(&body).unwrap()["error"].is_string()
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn settlement_results_and_failures_have_explicit_http_responses() {
+        let hash = "AB".repeat(32);
+        let path = format!("/content/file/invoice-status/{hash}");
+        for paid in [false, true] {
+            let response = invoice_status_response(&path, |hash, id| async move {
+                assert_eq!(hash, "ab".repeat(32));
+                assert_eq!(id, "file");
+                Ok(paid)
+            })
+            .await;
+            assert_eq!(response.status(), StatusCode::OK);
+            let body = hyper::body::to_bytes(response.into_body()).await.unwrap();
+            assert_eq!(
+                serde_json::from_slice::<serde_json::Value>(&body).unwrap()["paid"],
+                paid
+            );
+        }
+        let response = invoice_status_response(&path, |_, _| async {
+            anyhow::bail!("private wallet details must not escape")
+        })
+        .await;
+        assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(response.headers()["retry-after"], "5");
+        let body = hyper::body::to_bytes(response.into_body()).await.unwrap();
+        let text = String::from_utf8(body.to_vec()).unwrap();
+        assert!(text.contains("without paying again"));
+        assert!(!text.contains("private wallet"));
+    }
+}

docs/TODO.md

@@ -3,6 +3,17 @@
 Working backlog of forward-looking items not yet scoped into a dedicated plan
 doc. See [`ROADMAP.md`](ROADMAP.md) for the curated, public-facing direction.
 
+## NPM certificate release blocker — reported 2026-10-01
+
+- [ ] **NPM first certificate and renewal must work without manual repair.**
+  Shorty's HTTP-01 404 exposed inconsistent active data/ACME paths between the
+  manifest, legacy creation paths and host nginx bridge. Independent reviewer
+  acknowledged the handoff; separate release owner must acknowledge receipt,
+  implement data-safe migration and complete fresh-install, legacy-upgrade,
+  issuance/renewal, restart/reboot and OTA/ISO acceptance. See
+  [repair evidence and required release tests](npm-certificate-handoff-20261001.md).
+  Do not equate the live challenge-route repair with a tested release fix.
+
 ## Framework incident — closed with operator acceptance
 
 - **CLOSED WITH OPERATOR ACCEPTANCE (2026-09-30): Framework LND startup /

docs/npm-certificate-handoff-20261001.md

@@ -0,0 +1,169 @@
+# NPM certificate failure: repair and next-release gate
+
+Status: OPEN for release — affected Angor endpoint repaired and publicly verified;
+durable source correction and release regression acceptance remain pending.
+
+The operator requested immediate repair on Shorty's node and a complete, tested
+fix for subsequent releases. The independent review agent acknowledged receipt
+of the repair/handoff on 2026-10-01. This does **not** establish receipt by the
+owner of another release session. That owner must acknowledge this document and
+record implementation and acceptance before shipping the next OTA or ISO.
+
+## Evidence and immediate repair
+
+The live investigator reported NPM certificate failures at 18:06, 18:07 and
+18:10 UTC on 2026-10-01: the CA received HTTP 404 for its HTTP-01 challenge.
+The running container mounts `/var/lib/archipelago/nginx-proxy-manager` at
+`/data`, but host nginx served the challenge from the obsolete nested
+`/var/lib/archipelago/nginx-proxy-manager/data/letsencrypt-acme-challenge`.
+NPM writes to `/data/letsencrypt-acme-challenge` inside its container. These are
+different host directories. Host nginx owns public ports 80 and 443.
+
+The investigator backed up `/etc/nginx/sites-available/archipelago` as
+`/etc/nginx/sites-available/archipelago.before-angor-acme-1790878342`, corrected
+the default HTTP challenge root to the actual mount's challenge directory,
+passed `nginx -t`, and reloaded nginx. A temporary challenge file written inside
+NPM returned its exact expected body over the public domain's port 80.
+This initial probe proves the repaired challenge route; it alone does not prove issuance,
+certificate attachment, public HTTPS routing, renewal, or release persistence.
+No wallet or channel data is involved in this repair.
+
+## Independently confirmed source inconsistencies
+
+- `apps/nginx-proxy-manager/manifest.yml`: base app directory mounts at `/data`;
+  separate `letsencrypt` directory mounts at `/etc/letsencrypt`; only the admin
+  port is published. NPM's own public listeners are not published by this path.
+- `image-recipe/configs/nginx-archipelago.conf`: default challenge location uses
+  the obsolete nested `data/letsencrypt-acme-challenge` root.
+- `scripts/sync-npm-public-hosts.sh`: both SQLite DB and challenge root use the
+  nested directory. Missing DB exits successfully without syncing any hosts.
+- `scripts/container-doctor.sh::fix_npm_public_hosts`: its independent nested
+  DB existence guard prevents the synchronizer from running on manifest installs.
+- `core/archipelago/src/api/rpc/package/config.rs` and `runtime.rs`: legacy
+  creation/repair still mount the nested `data` directory at `/data`.
+- `scripts/first-boot-containers.sh`: legacy first boot creates and mounts nested
+  data, and publishes different public-listener ports from the manifest path.
+- The synchronizer exports selected NPM fields to host nginx. It checks enabled
+  hosts with a certificate, but does not filter deleted rows, validate inserted
+  configuration values, or preserve all NPM access/custom-location behavior.
+  It restores the old generated file on syntax failure, but not reload failure.
+
+## Required implementation
+
+1. Define one authoritative method for finding the active NPM data mount and
+   certificate store across fresh installs and supported legacy layouts. Do not
+   blindly change mounts and strand the operator's existing DB, hosts or account.
+   Detect ambiguous dual databases explicitly. Back up before any migration;
+   preserve existing certificates, private keys, renewal files, account records,
+   custom settings, permissions and uninstall decisions. Repeated migration must
+   be harmless. Failed migration must leave the original usable state intact.
+2. Make default and named-host HTTP challenge routes use that active directory.
+   Include pre-certificate issuance and renewal under forced HTTPS. Do not expose
+   account/private-key/database directories through nginx.
+3. Correct the synchronizer and doctor gate together, with reliable lifecycle
+   invocation after host/certificate edits and service startup. A certificate
+   created in NPM must actually become the certificate served by public nginx.
+   Avoid requiring users to run a repair command for each host or renewal.
+4. Define how the host bridge preserves NPM routing and security settings.
+   Handle disabled/deleted hosts, host edits, certificate replacement/deletion,
+   multiple domains, custom locations and access restrictions correctly. Do not
+   silently publish a restricted NPM host as an unrestricted host-nginx proxy.
+   Validate DB-derived configuration, serialize concurrent writers, avoid
+   unnecessary reloads, and retain a working config on generation/test/reload
+   failure. Report actionable failure causes instead of apparent success.
+5. Carry the correction through actual OTA migration and fresh ISO paths.
+   Include source, runtime scripts, nginx configuration, and app catalog as
+   applicable. Verify candidate package contents and installed behavior rather
+   than assuming a source edit is shipped by every packaging path.
+
+## Acceptance matrix — all relevant gates require recorded results
+
+Use disposable fixtures/test domains for destructive/error cases and ACME
+staging for repeated issuance/renewal. Avoid production CA retry loops. Backend
+unit tests on installed nodes must use `scripts/test-backend-isolated.sh` per
+`AGENTS.md`. Do not reboot an operator node without the necessary recovery/access
+arrangements and authorization; use a disposable VM for release lifecycle tests.
+
+- [ ] Fresh manifest installation: actual mount, DB path, admin API, public
+      challenge file and first certificate request work without manual repair.
+- [ ] Legacy nested-data upgrade: hosts, accounts, certs, renewal data and
+      custom settings remain intact; the active DB is still the original DB.
+- [ ] Current flat-data upgrade: same preservation assertions; no empty database
+      is initialized and no old nested DB is silently chosen instead.
+- [ ] Ambiguous dual DBs, missing/corrupt DB, backup failure and permission errors
+      fail safely with useful diagnostics; no deletion or identity replacement.
+- [ ] Challenge file created in container is fetched byte-for-byte from public
+      port 80 for an unconfigured domain, named host and forced-HTTPS host.
+- [ ] Staging first issuance completes through the normal NPM UI/API. One live
+      production issuance on the affected node is independently verified with
+      hostname, trust chain, validity and actual served certificate.
+- [ ] Certificate binding and public HTTPS route reach the intended Angor
+      backend; normal API health and a representative read-only indexed request
+      are checked separately from TLS. Backend readiness failures stay explicit.
+- [ ] Staging renewal succeeds through the normal scheduling/renewal path and
+      public nginx reloads the renewed certificate without manual intervention.
+- [ ] Host create/edit/disable/delete, certificate replacement, multi-domain
+      routing, restrictions and custom locations match supported NPM behavior.
+- [ ] Injection/invalid data, concurrent sync, nginx syntax failure and reload
+      failure preserve the previous working service; retries converge safely.
+- [ ] NPM restart, manager restart, host nginx restart, controlled VM reboot and
+      repeated reconciliation preserve public routing and certificates.
+- [ ] Migration is idempotent; upgrade rollback retains original data and
+      usable routing. Existing unrelated public hosts continue working.
+- [ ] Signed OTA and RAW ISO candidate contents contain the same correction;
+      installed OTA legacy/flat layouts and fresh ISO pass the relevant checks.
+- [ ] Release owner acknowledges receipt and records exact commit/artifact IDs,
+      test commands/results, live evidence, remaining limits and release decision.
+
+## Handoff acknowledgements
+
+- 2026-10-01: independent review agent received the parent investigator's live
+  fix details and explicitly acknowledged responsibility for source review and
+  this test/handoff checklist. No production code or node changes by reviewer.
+- 2026-10-01: next-release owner explicitly acknowledged this handoff in
+  `/tmp/npm-release-handoff-ack.txt` and adopted the matrix as a required
+  1.8.23-alpha OTA/raw ISO gate. Received the operator report that Shorty
+  certificate npm-8 is issued and public HTTPS health returns 200. Independent
+  release validation and the durable fleet correction remain pending.
+
+## Final live repair evidence — investigator report, 2026-10-01
+
+The live investigator issued the certificate through NPM's own API using an
+ephemeral in-memory local admin token, without changing passwords or disclosing
+the token. Certificate ID 8 expires at `2026-12-30 17:16:35`; existing proxy host
+ID 2 now uses that certificate with forced HTTPS enabled.
+
+The running container publishes only its admin listener (container 81 to host
+`127.0.0.1:8081`). The investigator therefore added a domain-scoped host-nginx
+configuration at `/etc/nginx/conf.d/angor-indexer-npm.conf`. It serves HTTP 80 and
+HTTPS 443 on IPv4 and IPv6, uses the corrected ACME root and NPM certificate 8,
+and forwards to `http://127.0.0.1:8998`. `nginx -t` and reload passed.
+
+External requests with normal TLS verification confirmed:
+
+- `/health`: HTTP 200, indexed height 969474.
+- `/`: valid mainnet JSON response.
+- `/api/v1/fees/recommended`: valid JSON response.
+- HTTP requests redirect to HTTPS with HTTP 301.
+- CORS preflight `OPTIONS /api/tx` with origin `https://angor.io`: HTTP 204,
+  allowed origin `*`, method POST and header Content-Type. This was a preflight
+  check, not a transaction submission.
+- An existing shop endpoint continued to return HTTPS 302.
+
+The temporary challenge probe was removed. These are investigator-reported live
+checks, not independently repeated node checks by the review agent. They establish
+that the affected endpoint now serves trusted HTTPS and responds as an indexer.
+They do not establish renewal, automatic host bridge updates, restart/reboot or
+packaged-release correctness; the source repair remains the release owner's work.
+
+A separate read-only public probe of the existing Shorty's website failed TLS
+hostname verification. Its configuration was unchanged by this repair, and no
+pre-repair baseline establishes when that mismatch began. The release owner must
+investigate it separately and verify existing-host compatibility; do not attribute
+it to this repair without evidence or silently mark that gate passed.
+
+The investigator located the actual main release session, delivered the handoff,
+and observed its explicit acknowledgement. The owner then recorded receipt in
+`/tmp/npm-release-handoff-ack.txt` and in the acknowledgement section above.
+Publication remains held for the NPM release gate. Unavailable external acceptance
+must be stated explicitly and cannot be silently treated as passed.

docs/post-1.8.22-regressions-20261001.md

@@ -256,3 +256,18 @@ Backend SHA-256:
 
 Physical companion upload diagnosis and remaining release acceptance stay OPEN.
 This is a candidate deployment, not a newly signed OTA or ISO.
+
+## Release authorization — 2026-10-01
+
+The operator authorized preparing the next OTA and raw ISO after completing all
+checks available here. Keep the missing original file/buyer confirmation,
+physical companion upload and full-chain Angor indexing as explicit follow-ups;
+this authorization does not establish that those scenarios passed. Complete
+the known invoice-status HTTP error fix and available automated release gates
+before requesting the offline signatures. Angor Indexer and the optional relay
+are already present in the current signed catalog and remain included.
+
+Shorty's Mempool report was inspected read-only: frontend/API had been running
+for over two weeks, systemd reported zero restarts, and the API was processing
+current blocks. The operator said it looked normal after applying the latest
+update. No Mempool repair or native-service restart was performed in this check.

docs/release-1.8.23-acceptance.md

@@ -0,0 +1,60 @@
+# Archipelago 1.8.23-alpha acceptance
+
+Status: PREPARING. Do not publish until artifact checks and offline signatures pass.
+
+## Scope
+
+Durable Lightning paid-file entitlements and delivery retries, atomic buyer
+ownership, compact persistent upload progress/cancellation, mobile transaction
+filters, Immich inventory and retired-app uninstall, Portainer duplicate-network
+migration, channel-close fee selection, and shared app-search clearing.
+
+Angor Indexer and the optional Angor Relay remain in the signed catalog. They
+were already included in 1.8.22; full-chain acceptance still awaits dev Bitcoin
+initial sync. Do not advertise full-chain verification as complete.
+
+## Validation carried into preparation
+
+- Candidate deployed on dev and Framework with matching backend/dashboard bytes.
+- Native Bitcoin/LND and Framework Immich container IDs/start times preserved.
+- Six live desktop/mobile app-search cases passed.
+- Actual uploads verified exact bytes, original destination after navigation,
+  compact progress geometry and cancellation. These are browser checks, not
+  physical companion acceptance.
+- Framework seller's settled invoice recovered to a mode-0600 entitlement and
+  remained paid across another manager restart; mismatched item rejected.
+- Framework CryptPad stale inventory removed with data preserved; removal
+  persisted after management restart. Dev normal package.uninstall RPC also
+  completed with preserve_data=true and zero cleanup errors for the retired ID.
+- Cooperative-close fee selector tested with intercepted requests; no real
+  channel was closed. Real payment recovery never sent another payment.
+- Shorty's Mempool inspected read-only; frontend/API healthy for over two weeks,
+  systemd zero restarts. Operator reported normal status after their update.
+
+## Follow-ups accepted for release preparation
+
+The operator authorized release preparation after available tests pass.
+
+- Actual failed-purchase delivery still requires buyer/file confirmation. The
+  located invoice's source file is missing from Framework's recorded paths.
+  Seller settlement recovery does not prove buyer delivery.
+- Physical companion upload diagnosis needs the affected device/route.
+- Framework rendered inventory/uninstall checks need dashboard second-factor
+  authentication; SSH/runtime and dev API acceptance are recorded separately.
+- Angor full-chain indexing awaits Bitcoin IBD.
+- Prior Primal automatic-comment and lost-response ecash receipt follow-ups
+  retain their documented boundaries; this release does not claim to fix them.
+
+## Final release checks
+
+Pending full release gates, versioned builds, exact artifact deployment, ISO
+payload/boot acceptance, pinned-root signatures and publication verification.
+No universal or future-failure guarantee is implied by these tests.
+
+## Required NPM certificate gate
+
+The release owner acknowledged `docs/npm-certificate-handoff-20261001.md` in
+`/tmp/npm-release-handoff-ack.txt`. Shorty's npm-8 issuance and HTTPS health 200
+were reported by the operator; durable data-path resolution, safe host routing,
+automatic certificate renewal/reload, and the handoff acceptance matrix remain
+required before publication. Earlier authorization does not waive this new gate.

neode-ui/src/views/settings/AccountInfoSection.vue

@@ -362,6 +362,24 @@ init()
               </button>
             </div>
             <div class="overflow-y-auto flex-1 min-h-0 space-y-6 pr-1">
+              <!-- v1.8.23-alpha -->
+              <div>
+                <div class="flex items-center gap-2 mb-3">
+                  <span class="text-xs font-mono px-2 py-0.5 rounded bg-orange-500/20 text-orange-300">v1.8.23-alpha</span>
+                  <span class="text-xs text-white/40">October 1, 2026</span>
+                </div>
+                <div class="space-y-3 text-sm text-white/80 pl-3 border-l border-white/10">
+                  <p>Preserve paid-file Lightning entitlements across restarts and recover settled invoices from LND. Retry delivery without paying again and retain purchased files in the owned cache.</p>
+                  <p>Return explicit payment-status errors with safe retry guidance when verification is unavailable.</p>
+                  <p>Show compact upload progress across screens, retain the original destination, and cancel active and queued uploads.</p>
+                  <p>Make transaction filters transparent and horizontally scrollable on mobile.</p>
+                  <p>Keep Immich internal services out of My Apps, avoid false recovery states for healthy stacks, and allow removal of retired catalog apps.</p>
+                  <p>Repair the redundant managed Portainer network override that can prevent startup, preserving custom overrides and persistent state.</p>
+                  <p>Offer Standard, Medium, Fast and custom fees when cooperatively closing Lightning channels.</p>
+                  <p>Add a clear-search icon to My Apps, Services and the App Store on desktop and mobile.</p>
+                  <p>Keep Angor Indexer and the optional Angor Relay in the signed app catalog. Full indexing requires a synced, unpruned Bitcoin node and its indexing dependencies.</p>
+                </div>
+              </div>
               <!-- v1.8.22-alpha -->
               <div>
                 <div class="flex items-center gap-2 mb-3">