//! What this node sells through its Routstr Provider app: the inference //! servers Routstr forwards to (this node's own llama-server, or an //! OpenAI-compatible server on the LAN such as a DGX Spark) and, for each, //! which models are offered to mesh peers at what price. //! //! Routstr's own database is the single record of servers, models and prices; //! this module only drives Routstr's admin API, logging in with the admin //! password the orchestrator generated for the app. That password, and any //! API key an operator enters for a server, are sent only to the app on //! loopback and are never returned or logged. Routstr does not hand a key //! back, so the node keeps its own copy (0600, beside the app's other //! secrets) to re-read the server's model list. //! //! Prices are US dollars per million tokens, as Routstr stores them; Routstr //! converts them to sats at its current exchange rate when quoting buyers. use anyhow::{bail, ensure, Context, Result}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::net::IpAddr; use std::path::Path; use std::time::Duration; /// The Routstr app's host-local port (`apps/routstr/manifest.yml`). pub(crate) const ROUTSTR_APP: &str = "http://127.0.0.1:8200"; const ADMIN_SECRET: &str = "routstr-admin-password"; /// This node's own llama-server, as Routstr reaches it on the app network. const LOCAL_SERVER_URL: &str = "http://llama-server:8080/v1"; /// The model alias llama-server serves (`apps/llama-server/manifest.yml`). const LOCAL_MODEL_ID: &str = "archy-llama"; const PROBE_TIMEOUT: Duration = Duration::from_secs(10); const MAX_PROBE_BYTES: usize = 1024 * 1024; const MAX_CONTEXT: u64 = 1_048_576; const MIN_CONTEXT: u64 = 512; /// A sanity ceiling, far above any real price, against a typo of several /// orders of magnitude. const MAX_USD_PER_MILLION: f64 = 1000.0; const MAX_NAME_CHARS: usize = 80; /// A logged-in session with this node's Routstr admin API. pub(crate) struct Admin { client: reqwest::Client, app: String, token: String, /// Where the node keeps API keys of the servers it sells from. keys_dir: std::path::PathBuf, } impl Admin { pub(crate) async fn login(data_dir: &Path) -> Result { Self::login_at(ROUTSTR_APP, &data_dir.join("secrets").join(ADMIN_SECRET)).await } pub(crate) async fn login_at(app: &str, secret: &Path) -> Result { let password = tokio::fs::read_to_string(secret) .await .context("Routstr Provider is not installed")?; let client = reqwest::Client::builder() .no_proxy() .redirect(reqwest::redirect::Policy::none()) .timeout(Duration::from_secs(30)) .build()?; let login: Value = client .post(format!("{app}/admin/api/login")) .json(&json!({ "password": password.trim() })) .send() .await .context("Routstr Provider is not running")? .error_for_status() .context("Routstr Provider refused the node's admin login")? .json() .await?; let token = login .get("token") .and_then(Value::as_str) .context("Routstr Provider returned no admin session")? .to_string(); Ok(Self { client, app: app.to_string(), token, keys_dir: secret .parent() .unwrap_or(Path::new(".")) .join("routstr-servers"), }) } async fn send( &self, method: reqwest::Method, path: &str, body: Option<&Value>, ) -> Result { let mut request = self .client .request(method, format!("{}{path}", self.app)) .bearer_auth(&self.token); if let Some(body) = body { request = request.json(body); } let response = request .send() .await .context("Routstr Provider is not running")?; let status = response.status(); let text = response.text().await.unwrap_or_default(); if !status.is_success() { let detail = serde_json::from_str::(&text) .ok() .and_then(|v| v.get("detail").cloned()) .map(|d| match d { Value::String(s) => s, other => other.to_string(), }) .unwrap_or_else(|| status.to_string()); bail!("Routstr Provider refused the change: {detail}"); } Ok(serde_json::from_str(&text).unwrap_or(Value::Null)) } /// The node's copy of a server's API key. Routstr slugs are lowercase /// letters, digits and dashes; anything else has no key file. fn key_path(&self, server: &str) -> Option { let safe = !server.is_empty() && server.len() <= 64 && server .chars() .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-'); safe.then(|| self.keys_dir.join(format!("{server}.key"))) } async fn server_key(&self, server: &str) -> Option { let path = self.key_path(server)?; let key = tokio::fs::read_to_string(path).await.ok()?; Some(key.trim().to_string()).filter(|k| !k.is_empty()) } pub(crate) async fn get(&self, path: &str) -> Result { self.send(reqwest::Method::GET, path, None).await } async fn post(&self, path: &str, body: &Value) -> Result { self.send(reqwest::Method::POST, path, Some(body)).await } async fn delete(&self, path: &str) -> Result { self.send(reqwest::Method::DELETE, path, None).await } /// Routstr's public info: version and the mints it accepts. pub(crate) async fn info(&self) -> Result { Ok(self .client .get(format!("{}/v1/info", self.app)) .send() .await .context("Routstr Provider is not running")? .error_for_status()? .json() .await?) } /// The public catalog buyers see, with each model's sats quote. async fn catalog(&self) -> Result { Ok(self .client .get(format!("{}/v1/models", self.app)) .send() .await .context("Routstr Provider is not running")? .error_for_status()? .json() .await?) } } /// One model offered to mesh peers. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub(crate) struct Offer { pub model_id: String, pub name: String, pub context_length: u64, pub max_completion_tokens: u64, pub prompt_usd_per_million: f64, pub completion_usd_per_million: f64, pub enabled: bool, /// Sats a buyer sends up front per message (the unused part comes back), /// from Routstr's own catalog. Absent while the model is not offered. #[serde(default, skip_deserializing)] pub quote_sats: Option, } /// A model an inference server reports. #[derive(Clone, Debug, PartialEq, Serialize)] pub(crate) struct ServerModel { pub id: String, /// The most context the server itself allows, when it says. pub context_length: Option, } /// An inference server Routstr forwards to. #[derive(Debug, Serialize)] pub(crate) struct Server { pub id: String, pub url: String, /// This node's own llama-server. pub local: bool, pub has_key: bool, pub offers: Vec, } #[derive(Debug, Serialize)] pub(crate) struct Selling { pub servers: Vec, /// Sats per US dollar Routstr quotes at (fee included), when any priced /// model shows it; lets the screen preview a quote before saving. pub sats_per_usd: Option, } /// A server address an operator may sell from: http(s) on the LAN or beyond, /// never this node itself, its app network, or link-local and metadata /// addresses. Returns the normalised base URL (ending in `/v1` unless a path /// was given). pub(crate) fn validate_server_url(input: &str) -> Result { let url = reqwest::Url::parse(input.trim()) .context("Enter the server's address, e.g. http://192.168.1.50:8000/v1")?; ensure!( matches!(url.scheme(), "http" | "https"), "The address must start with http:// or https://" ); ensure!( url.username().is_empty() && url.password().is_none(), "Put an API key in the key field, not in the address" ); ensure!( url.query().is_none() && url.fragment().is_none(), "The address must not contain ? or #" ); let host = url.host_str().unwrap_or_default(); match host .trim_start_matches('[') .trim_end_matches(']') .parse::() { Ok(ip) => check_ip(ip)?, Err(_) if host.is_empty() => bail!("The address has no host"), Err(_) => { let name = host.trim_end_matches('.').to_ascii_lowercase(); ensure!( name != "localhost" && !name.ends_with(".localhost"), "This node's own address cannot be a server; the local llama-server is listed already" ); // Single-label names are this node's app containers (or depend on // search domains Routstr's container does not share). ensure!( name.contains('.'), "Use the server's IP address or a full name such as spark.local" ); } } let path = url.path().trim_end_matches('/'); let path = if path.is_empty() { "/v1" } else { path }; let port = url.port().map(|p| format!(":{p}")).unwrap_or_default(); Ok(format!("{}://{host}{port}{path}", url.scheme())) } fn check_ip(ip: IpAddr) -> Result<()> { let refused = match ip { IpAddr::V4(v4) => { let [a, b, ..] = v4.octets(); v4.is_loopback() || v4.is_unspecified() || v4.is_link_local() || v4.is_multicast() || v4.is_broadcast() // Podman's app networks on this node (archy-net and the default). || (a == 10 && (b == 88 || b == 89)) } IpAddr::V6(v6) => { if let Some(v4) = v6.to_ipv4_mapped() { return check_ip(IpAddr::V4(v4)); } v6.is_loopback() || v6.is_unspecified() || v6.is_multicast() || (v6.segments()[0] & 0xffc0) == 0xfe80 } }; ensure!( !refused, "That address is this node, its app network, or a link-local address; enter the server's LAN address" ); Ok(()) } /// Ask a server which models it serves (`GET {url}/models`), with the most /// context each allows when the server says so: vLLM's `max_model_len`, a /// `context_length`, or llama.cpp's running `n_ctx` from `/props`. pub(crate) async fn probe(url: &str, api_key: Option<&str>) -> Result> { let client = reqwest::Client::builder() .no_proxy() .redirect(reqwest::redirect::Policy::none()) .timeout(PROBE_TIMEOUT) .build()?; let get = |path: String| { let mut request = client.get(path); if let Some(key) = api_key.filter(|k| !k.is_empty()) { request = request.bearer_auth(key); } request.send() }; let response = get(format!("{url}/models")) .await .context("Could not reach the server; check the address and that it is running")?; ensure!( response.status() != reqwest::StatusCode::UNAUTHORIZED && response.status() != reqwest::StatusCode::FORBIDDEN, "The server refused the API key (HTTP {})", response.status().as_u16() ); ensure!( response.status().is_success(), "The server answered HTTP {} for its model list; is this an OpenAI-compatible /v1 address?", response.status().as_u16() ); let listing: Value = serde_json::from_slice(&read_limited(response).await?).context( "The server's model list is not JSON; is this an OpenAI-compatible /v1 address?", )?; let mut models = parse_models(&listing); ensure!(!models.is_empty(), "The server reports no models"); // llama.cpp reports the context it was started with only in /props. if listing .get("data") .and_then(Value::as_array) .is_some_and(|d| { d.iter() .any(|m| m.get("owned_by").and_then(Value::as_str) == Some("llamacpp")) }) { let root = url.strip_suffix("/v1").unwrap_or(url); if let Ok(response) = get(format!("{root}/props")).await { if let Ok(bytes) = read_limited(response).await { if let Some(n_ctx) = serde_json::from_slice::(&bytes).ok().and_then(|p| { p.pointer("/default_generation_settings/n_ctx") .and_then(Value::as_u64) }) { for model in &mut models { model.context_length = Some(n_ctx); } } } } } Ok(models) } async fn read_limited(mut response: reqwest::Response) -> Result> { let mut out = Vec::new(); while let Some(chunk) = response.chunk().await? { ensure!( out.len() + chunk.len() <= MAX_PROBE_BYTES, "The server's answer is too large" ); out.extend_from_slice(&chunk); } Ok(out) } fn parse_models(listing: &Value) -> Vec { listing .get("data") .and_then(Value::as_array) .into_iter() .flatten() .filter_map(|m| { let id = m.get("id")?.as_str()?.trim(); (!id.is_empty() && id.len() <= 200 && !id.chars().any(char::is_control)).then(|| { let context_length = ["max_model_len", "context_length", "context_window"] .iter() .find_map(|k| m.get(*k).and_then(Value::as_u64)) .or_else(|| m.pointer("/meta/n_ctx").and_then(Value::as_u64)) .filter(|n| *n > 0); ServerModel { id: id.to_string(), context_length, } }) }) .collect() } /// The ID buyers use for a model: Routstr strips anything before the first /// slash (`routstr/upstream/model_paths.py`, `public_model_id`). pub(crate) fn public_id(model_id: &str) -> &str { model_id.split_once('/').map_or(model_id, |(_, rest)| rest) } fn provider_ref(provider: &Value) -> Option { provider .get("slug") .and_then(Value::as_str) .map(String::from) .or_else(|| provider.get("id").map(|id| id.to_string())) } fn is_local(provider: &Value) -> bool { provider.get("base_url").and_then(Value::as_str) == Some(LOCAL_SERVER_URL) } fn offer_from_db(model: &Value) -> Option { let per_million = |k: &str| { model .pointer(&format!("/pricing/{k}")) .and_then(Value::as_f64) .map(|p| (p * 1e6 * 1e6).round() / 1e6) .unwrap_or(0.0) }; let context_length = model.get("context_length").and_then(Value::as_u64)?; Some(Offer { model_id: model.get("id")?.as_str()?.to_string(), name: model .get("name") .and_then(Value::as_str) .unwrap_or_default() .to_string(), context_length, max_completion_tokens: model .pointer("/top_provider/max_completion_tokens") .and_then(Value::as_u64) .unwrap_or(context_length), prompt_usd_per_million: per_million("prompt"), completion_usd_per_million: per_million("completion"), enabled: model .get("enabled") .and_then(Value::as_bool) .unwrap_or(true), quote_sats: None, }) } /// Sats per US dollar in Routstr's catalog, read off any priced model. fn sats_per_usd(catalog: &Value) -> Option { catalog.get("data")?.as_array()?.iter().find_map(|m| { let usd = m .pointer("/pricing/prompt") .and_then(|v| v.as_f64().or_else(|| v.as_str()?.parse().ok()))?; let sats = m.pointer("/sats_pricing/prompt").and_then(Value::as_f64)?; (usd > 0.0 && sats > 0.0).then(|| sats / usd) }) } async fn providers(admin: &Admin) -> Result> { Ok(admin .get("/admin/api/upstream-providers") .await? .as_array() .cloned() .unwrap_or_default()) } async fn find_provider(admin: &Admin, id: &str) -> Result { providers(admin) .await? .into_iter() .find(|p| provider_ref(p).as_deref() == Some(id)) .context("No such server") } async fn db_models(admin: &Admin, provider_id: &str) -> Result> { Ok(admin .get(&format!( "/admin/api/upstream-providers/{provider_id}/models" )) .await? .get("db_models") .and_then(Value::as_array) .cloned() .unwrap_or_default()) } /// Every server and what it offers, with live quotes. pub(crate) async fn selling(admin: &Admin) -> Result { let catalog = admin.catalog().await.unwrap_or(Value::Null); let mut servers = Vec::new(); for provider in providers(admin).await? { let Some(id) = provider_ref(&provider) else { continue; }; let mut offers: Vec = db_models(admin, &id) .await? .iter() .filter_map(offer_from_db) .collect(); for offer in &mut offers { offer.quote_sats = offer .enabled .then(|| { crate::assistant::routstr_peer::quote_sats(&catalog, public_id(&offer.model_id)) }) .flatten(); } servers.push(Server { url: provider .get("base_url") .and_then(Value::as_str) .unwrap_or_default() .to_string(), local: is_local(&provider), has_key: !is_local(&provider) && admin.server_key(&id).await.is_some(), offers, id, }); } // This node's own llama-server first. servers.sort_by_key(|s| !s.local); Ok(Selling { servers, sats_per_usd: sats_per_usd(&catalog), }) } /// Add a LAN (or remote) server after checking it answers with a model list. pub(crate) async fn add_server( admin: &Admin, url: &str, api_key: Option<&str>, ) -> Result<(String, Vec)> { let url = validate_server_url(url)?; let api_key = api_key.map(str::trim).filter(|k| !k.is_empty()); if let Some(key) = api_key { ensure!( key.len() <= 4096 && !key.chars().any(|c| c.is_control() || c.is_whitespace()), "The API key contains spaces or control characters" ); } ensure!( !providers(admin) .await? .iter() .any(|p| p.get("base_url").and_then(Value::as_str) == Some(url.as_str())), "This server is added already" ); let models = probe(&url, api_key).await?; register_server(admin, &url, api_key) .await .map(|id| (id, models)) } /// Record a checked server in Routstr, and its key on the node. async fn register_server(admin: &Admin, url: &str, api_key: Option<&str>) -> Result { let created = admin .post( "/admin/api/upstream-providers", &json!({ "provider_type": "custom", "base_url": url, "api_key": api_key.unwrap_or_default(), "enabled": true, }), ) .await?; let id = provider_ref(&created).context("Routstr Provider returned no server id")?; if let Some(key) = api_key { let stored = match admin.key_path(&id) { Some(path) => { crate::settings::model_provider::write_private(&path, key.as_bytes()).await } None => Err(anyhow::anyhow!("unexpected server id {id}")), }; if let Err(error) = stored { // Without the node's copy the model list could not be re-read. let _ = admin .delete(&format!("/admin/api/upstream-providers/{id}")) .await; return Err(error.context("Could not store the server's API key")); } } tracing::info!(server = %id, %url, "routstr selling: inference server added"); Ok(id) } /// The models a server reports now. The local llama-server is reached only /// by Routstr, so its one model is the entry Routstr already holds. pub(crate) async fn server_models(admin: &Admin, id: &str) -> Result> { let provider = find_provider(admin, id).await?; if is_local(&provider) { return Ok(db_models(admin, id) .await? .iter() .filter(|m| m.get("id").and_then(Value::as_str) == Some(LOCAL_MODEL_ID)) .map(|m| ServerModel { id: LOCAL_MODEL_ID.to_string(), context_length: m.get("context_length").and_then(Value::as_u64), }) .collect()); } let url = provider .get("base_url") .and_then(Value::as_str) .context("The server has no address")?; probe(url, admin.server_key(id).await.as_deref()).await } pub(crate) async fn remove_server(admin: &Admin, id: &str) -> Result<()> { let provider = find_provider(admin, id).await?; ensure!( !is_local(&provider), "This node's own llama-server cannot be removed; stop selling its model instead" ); admin .delete(&format!("/admin/api/upstream-providers/{id}/models")) .await?; admin .delete(&format!("/admin/api/upstream-providers/{id}")) .await?; if let Some(path) = admin.key_path(id) { let _ = tokio::fs::remove_file(path).await; } tracing::info!(server = %id, "routstr selling: inference server removed"); Ok(()) } /// Check one offer against what the server reports. `fixed_context` is the /// local llama-server's context, which only its manifest can change. fn check_offer( offer: &mut Offer, reported: &[ServerModel], fixed_context: Option, ) -> Result<()> { offer.name = offer.name.trim().to_string(); let label = if offer.name.is_empty() { offer.model_id.clone() } else { offer.name.clone() }; let server_model = reported .iter() .find(|m| m.id == offer.model_id) .with_context(|| format!("The server does not serve {}", offer.model_id))?; ensure!( !offer.name.is_empty() && offer.name.chars().count() <= MAX_NAME_CHARS, "{label}: give it a name of 1 to {MAX_NAME_CHARS} characters" ); ensure!( !offer.name.chars().any(char::is_control), "{label}: the name contains control characters" ); if let Some(context) = fixed_context { offer.context_length = context; } ensure!( (MIN_CONTEXT..=MAX_CONTEXT).contains(&offer.context_length), "{label}: context must be between {MIN_CONTEXT} and {MAX_CONTEXT} tokens" ); if let Some(limit) = server_model.context_length { ensure!( offer.context_length <= limit, "{label}: the server allows at most {limit} tokens of context" ); } ensure!( (16..=offer.context_length).contains(&offer.max_completion_tokens), "{label}: the answer limit must be between 16 tokens and the context" ); for price in [ offer.prompt_usd_per_million, offer.completion_usd_per_million, ] { ensure!( price.is_finite() && (0.0..=MAX_USD_PER_MILLION).contains(&price), "{label}: prices must be between $0 and ${MAX_USD_PER_MILLION} per million tokens" ); } ensure!( !offer.enabled || offer.prompt_usd_per_million > 0.0 || offer.completion_usd_per_million > 0.0, "{label}: set a price before selling it" ); Ok(()) } fn model_body(offer: &Offer, url: &str) -> Value { let host = reqwest::Url::parse(url) .ok() .and_then(|u| u.host_str().map(String::from)) .unwrap_or_default(); json!({ "id": offer.model_id, "name": offer.name, "description": if url == LOCAL_SERVER_URL { "Local llama-server model".to_string() } else { format!("Served from {host}") }, "created": 0, "context_length": offer.context_length, "architecture": { "modality": "text->text", "input_modalities": ["text"], "output_modalities": ["text"], "tokenizer": "Other", "instruct_type": null, }, "pricing": { "prompt": offer.prompt_usd_per_million / 1e6, "completion": offer.completion_usd_per_million / 1e6, "request": 0, "image": 0, "web_search": 0, "internal_reasoning": 0, }, "top_provider": { "context_length": offer.context_length, "max_completion_tokens": offer.max_completion_tokens, "is_moderated": false, }, "enabled": offer.enabled, }) } /// Save what a server offers. Models not listed are left as they are. pub(crate) async fn set_offers(admin: &Admin, id: &str, mut offers: Vec) -> Result<()> { let provider = find_provider(admin, id).await?; let url = provider .get("base_url") .and_then(Value::as_str) .unwrap_or_default() .to_string(); let local = is_local(&provider); let reported = server_models(admin, id).await?; for offer in &mut offers { let fixed = local .then(|| reported.iter().find(|m| m.id == offer.model_id)) .flatten() .and_then(|m| m.context_length); check_offer(offer, &reported, fixed)?; } // Buyers name a model only by its public ID; two servers offering the // same one would make Routstr split requests between them. let mut taken = std::collections::HashMap::new(); for other in providers(admin).await? { let Some(other_id) = provider_ref(&other) else { continue; }; if other_id == id { continue; } for model in db_models(admin, &other_id).await? { if model .get("enabled") .and_then(Value::as_bool) .unwrap_or(true) { if let Some(model_id) = model.get("id").and_then(Value::as_str) { taken.insert(public_id(model_id).to_string(), other_id.clone()); } } } } let mut seen = std::collections::HashSet::new(); for offer in offers.iter().filter(|o| o.enabled) { let public = public_id(&offer.model_id); ensure!( !taken.contains_key(public), "{} is already sold from another server; stop selling it there first", offer.name ); ensure!( seen.insert(public.to_string()), "{} is listed twice", offer.name ); } for offer in &offers { admin .post( &format!("/admin/api/upstream-providers/{id}/models"), &model_body(offer, &url), ) .await?; } tracing::info!(server = %id, offers = offers.len(), "routstr selling: offers saved"); Ok(()) } #[cfg(test)] mod tests { use super::*; #[test] fn lan_addresses_are_accepted_and_normalised() { for (input, normalised) in [ ( "http://192.168.88.198:8000/v1", "http://192.168.88.198:8000/v1", ), ( "http://192.168.88.198:8000", "http://192.168.88.198:8000/v1", ), ( "http://192.168.88.198:8000/v1/", "http://192.168.88.198:8000/v1", ), ( " https://spark.local:8443/openai/v1 ", "https://spark.local:8443/openai/v1", ), ( "http://[fd12:3456::1]:8000", "http://[fd12:3456::1]:8000/v1", ), ("http://10.1.2.3:11434/v1", "http://10.1.2.3:11434/v1"), ] { assert_eq!(validate_server_url(input).unwrap(), normalised, "{input}"); } } #[test] fn this_node_and_its_app_network_are_refused() { for input in [ "ftp://192.168.1.5/v1", "file:///etc/passwd", "http://127.0.0.1:8080/v1", "http://localhost:8080/v1", "http://[::1]:8080/v1", "http://0.0.0.0:8080/v1", "http://169.254.169.254/latest", "http://[fe80::1]:8000/v1", "http://[::ffff:127.0.0.1]:8000/v1", "http://10.89.0.20:8080/v1", "http://10.88.0.5:8080/v1", "http://llama-server:8080/v1", "http://bitcoin-knots:8332", "http://user:secret@192.168.1.5:8000/v1", "http://192.168.1.5:8000/v1?x=1", "http://192.168.1.5:8000/v1#x", "not a url", ] { assert!(validate_server_url(input).is_err(), "accepted {input}"); } } #[test] fn model_lists_from_common_servers_are_read() { let vllm = json!({"data": [{"id": "meta-llama/Llama-3.1-8B-Instruct", "max_model_len": 131072}]}); let ollama = json!({"object": "list", "data": [{"id": "qwen2.5:7b", "owned_by": "library"}]}); let odd = json!({"data": [{"id": ""}, {"id": "a\u{7}b"}, {"no": "id"}, {"id": "ok", "context_length": 0}]}); assert_eq!( parse_models(&vllm), [ServerModel { id: "meta-llama/Llama-3.1-8B-Instruct".into(), context_length: Some(131072) }] ); assert_eq!( parse_models(&ollama), [ServerModel { id: "qwen2.5:7b".into(), context_length: None }] ); assert_eq!( parse_models(&odd), [ServerModel { id: "ok".into(), context_length: None }] ); assert!(parse_models(&json!({"error": "no"})).is_empty()); } fn offer() -> Offer { Offer { model_id: "qwen2.5:7b".into(), name: " Qwen 2.5 7B ".into(), context_length: 8192, max_completion_tokens: 1024, prompt_usd_per_million: 0.2, completion_usd_per_million: 0.6, enabled: true, quote_sats: None, } } #[test] fn offers_are_checked_against_the_server() { let reported = [ServerModel { id: "qwen2.5:7b".into(), context_length: Some(32768), }]; let mut ok = offer(); check_offer(&mut ok, &reported, None).unwrap(); assert_eq!(ok.name, "Qwen 2.5 7B"); let bad: [(fn(&mut Offer), &str); 9] = [ (|o| o.model_id = "other".into(), "does not serve"), (|o| o.name = " ".into(), "name"), (|o| o.name = "x".repeat(81), "name"), (|o| o.context_length = 65536, "at most 32768"), (|o| o.context_length = 100, "between 512"), (|o| o.max_completion_tokens = 9000, "answer limit"), (|o| o.prompt_usd_per_million = f64::NAN, "prices"), (|o| o.completion_usd_per_million = 5000.0, "prices"), ( |o| { o.prompt_usd_per_million = 0.0; o.completion_usd_per_million = 0.0 }, "set a price", ), ]; for (spoil, message) in bad { let mut o = offer(); spoil(&mut o); let error = check_offer(&mut o, &reported, None) .unwrap_err() .to_string(); assert!(error.contains(message), "{error} lacks {message}"); } // Unpriced is fine while not sold. let mut off = offer(); off.enabled = false; off.prompt_usd_per_million = 0.0; off.completion_usd_per_million = 0.0; check_offer(&mut off, &reported, None).unwrap(); } #[test] fn the_local_model_keeps_llama_servers_context() { let reported = [ServerModel { id: LOCAL_MODEL_ID.into(), context_length: Some(8192), }]; let mut local = offer(); local.model_id = LOCAL_MODEL_ID.into(); local.context_length = 65536; check_offer(&mut local, &reported, Some(8192)).unwrap(); assert_eq!(local.context_length, 8192); } #[test] fn a_saved_offer_round_trips_through_routstrs_model_record() { let o = Offer { name: "Qwen 2.5 7B".into(), ..offer() }; let mut stored = model_body(&o, "http://192.168.88.198:8000/v1"); assert_eq!(stored["description"], "Served from 192.168.88.198"); assert_eq!(stored["pricing"]["prompt"], json!(0.2 / 1e6)); assert_eq!(stored["top_provider"]["max_completion_tokens"], 1024); stored["sats_pricing"] = json!({"max_cost": 3.0}); assert_eq!(offer_from_db(&stored).unwrap(), o); assert_eq!( model_body(&o, LOCAL_SERVER_URL)["description"], "Local llama-server model" ); } #[test] fn buyers_see_the_id_after_the_first_slash() { assert_eq!(public_id("meta-llama/Llama-3.1-8B"), "Llama-3.1-8B"); assert_eq!(public_id("a/b/c"), "b/c"); assert_eq!(public_id("qwen2.5:7b"), "qwen2.5:7b"); } #[test] fn the_exchange_rate_is_read_off_a_priced_model() { let catalog = json!({"data": [ {"id": "free", "pricing": {"prompt": "0"}, "sats_pricing": {"prompt": 0.0}}, {"id": "archy-llama", "pricing": {"prompt": "0.000001"}, "sats_pricing": {"prompt": 0.00125}}, ]}); assert_eq!(sats_per_usd(&catalog), Some(1250.0)); assert_eq!(sats_per_usd(&json!({"data": []})), None); } // ── Against a fake Routstr admin API and a fake LAN server ────────── use hyper::{Body, Request, Response}; use std::sync::{Arc, Mutex}; #[derive(Default)] struct Fake { /// (provider json, model rows) providers: Vec<(Value, Vec)>, posted_models: Vec, } type Shared = Arc>; /// A JSON server; the handler gets method, path, body and Authorization. async fn serve( handler: impl Fn(&str, &str, Value, &str) -> (u16, Value) + Send + Sync + 'static, ) -> String { let handler = Arc::new(handler); let make = hyper::service::make_service_fn(move |_| { let handler = handler.clone(); async move { Ok::<_, std::convert::Infallible>(hyper::service::service_fn( move |req: Request| { let handler = handler.clone(); async move { let method = req.method().to_string(); let path = req.uri().path().to_string(); let auth = req .headers() .get("authorization") .and_then(|v| v.to_str().ok()) .unwrap_or_default() .to_string(); let bytes = hyper::body::to_bytes(req.into_body()).await.unwrap(); let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null); let (status, out) = handler(&method, &path, body, &auth); Ok::<_, std::convert::Infallible>( Response::builder() .status(status) .header("Content-Type", "application/json") .body(Body::from(out.to_string())) .unwrap(), ) } }, )) } }); let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap()).serve(make); let url = format!("http://{}", server.local_addr()); tokio::spawn(server); url } async fn fake_routstr(state: Shared) -> String { serve(move |method, path, body, _| { let mut st = state.lock().unwrap(); let rest = path.strip_prefix("/admin/api/upstream-providers").map(str::to_string); match (method, path, rest.as_deref()) { ("POST", "/admin/api/login", _) => (200, json!({"token": "t"})), ("GET", "/v1/models", _) => { let data: Vec = st.providers.iter().flat_map(|(_, m)| m.clone()) .filter(|m| m["enabled"] == json!(true)) .map(|m| json!({"id": public_id(m["id"].as_str().unwrap()), "pricing": {"prompt": m["pricing"]["prompt"]}, "sats_pricing": {"prompt": m["pricing"]["prompt"].as_f64().unwrap() * 1250.0, "max_cost": 11.5}})) .collect(); (200, json!({"data": data})) } // Like Routstr, never hand a key back. ("GET", _, Some("")) => (200, Value::Array(st.providers.iter().map(|(p, _)| { let mut p = p.clone(); p.as_object_mut().unwrap().remove("api_key"); p }).collect())), ("POST", _, Some("")) => { let slug = format!("custom-{}", st.providers.len() + 1); let mut p = body.clone(); p["slug"] = json!(slug); st.providers.push((p.clone(), vec![])); (200, p) } (_, _, Some(rest)) => { let rest = rest.trim_start_matches('/'); let (id, tail) = rest.split_once('/').unwrap_or((rest, "")); let Some(i) = st.providers.iter().position(|(p, _)| p["slug"] == json!(id)) else { return (404, json!({"detail": "Provider not found"})); }; match (method, tail) { ("GET", "models") => (200, json!({"db_models": st.providers[i].1, "remote_models": []})), ("POST", "models") => { st.posted_models.push(body.clone()); let models = &mut st.providers[i].1; models.retain(|m| m["id"] != body["id"]); models.push(body); (200, json!({})) } ("DELETE", "models") => { st.providers[i].1.clear(); (200, json!({})) } ("DELETE", "") => { st.providers.remove(i); (200, json!({"ok": true})) } _ => (404, json!({"detail": "no"})), } } _ => (404, json!({"detail": "no"})), } }) .await } /// A llama.cpp-style LAN server that wants `Bearer sk-lan`. async fn fake_lan_server() -> String { serve(|_, path, _, auth| match (path, auth == "Bearer sk-lan") { (_, false) => (401, json!({"error": "no key"})), ("/v1/models", true) => ( 200, json!({"data": [ {"id": "org/Qwen-7B", "owned_by": "llamacpp"}, {"id": "archy-llama", "owned_by": "llamacpp"}, ]}), ), ("/props", true) => ( 200, json!({"default_generation_settings": {"n_ctx": 16384}}), ), _ => (404, json!({})), }) .await } async fn admin_with_local() -> (Admin, Shared, tempfile::TempDir) { let local_model = json!({ "id": LOCAL_MODEL_ID, "name": "Qwen2.5 1.5B Instruct", "context_length": 8192, "pricing": {"prompt": 1e-6, "completion": 2e-6}, "top_provider": {"context_length": 8192, "max_completion_tokens": 1024}, "enabled": true, }); let state: Shared = Arc::new(Mutex::new(Fake { providers: vec![( json!({"id": 1, "slug": "custom", "provider_type": "custom", "base_url": LOCAL_SERVER_URL, "api_key": "llama-secret"}), vec![local_model], )], ..Default::default() })); let app = fake_routstr(state.clone()).await; let dir = tempfile::tempdir().unwrap(); let secret = dir.path().join("pw"); std::fs::write(&secret, "pw\n").unwrap(); (Admin::login_at(&app, &secret).await.unwrap(), state, dir) } fn lan_offer(model_id: &str, context: u64) -> Offer { Offer { model_id: model_id.into(), name: "Qwen 7B on the Spark".into(), context_length: context, max_completion_tokens: 1024, prompt_usd_per_million: 0.2, completion_usd_per_million: 0.6, enabled: true, quote_sats: None, } } #[tokio::test] async fn a_lan_server_is_added_priced_and_listed_without_its_key() { let (admin, state, _dir) = admin_with_local().await; // The fake LAN server listens on 127.0.0.1, which operators may not // add; register it as add_server does after checking the address. let lan = format!("{}/v1", fake_lan_server().await); assert!(probe(&lan, None) .await .unwrap_err() .to_string() .contains("refused the API key")); let models = probe(&lan, Some("sk-lan")).await.unwrap(); assert_eq!( models[0], ServerModel { id: "org/Qwen-7B".into(), context_length: Some(16384) } ); let id = register_server(&admin, &lan, Some("sk-lan")).await.unwrap(); let key_file = admin.key_path(&id).unwrap(); assert_eq!(std::fs::read_to_string(&key_file).unwrap(), "sk-lan"); use std::os::unix::fs::PermissionsExt; assert_eq!( std::fs::metadata(&key_file).unwrap().permissions().mode() & 0o777, 0o600 ); // More context than the server runs with is refused. let error = set_offers(&admin, &id, vec![lan_offer("org/Qwen-7B", 32768)]) .await .unwrap_err(); assert!(error.to_string().contains("at most 16384"), "{error}"); // The local model's public ID is taken. let error = set_offers(&admin, &id, vec![lan_offer("archy-llama", 8192)]) .await .unwrap_err(); assert!( error .to_string() .contains("already sold from another server"), "{error}" ); set_offers(&admin, &id, vec![lan_offer("org/Qwen-7B", 8192)]) .await .unwrap(); let posted = state.lock().unwrap().posted_models.last().cloned().unwrap(); assert_eq!(posted["pricing"]["completion"], json!(0.6 / 1e6)); assert_eq!(posted["top_provider"]["context_length"], 8192); let listed = selling(&admin).await.unwrap(); assert_eq!(listed.servers.len(), 2); assert!( listed.servers[0].local, "the local llama-server comes first" ); assert!( !listed.servers[0].has_key, "llama-server's own key is not the operator's" ); let spark = &listed.servers[1]; assert!(spark.has_key); assert_eq!(spark.offers[0].name, "Qwen 7B on the Spark"); assert_eq!(spark.offers[0].quote_sats, Some(13)); assert_eq!(listed.sats_per_usd, Some(1250.0)); let text = serde_json::to_string(&listed).unwrap(); assert!( !text.contains("sk-lan") && !text.contains("llama-secret"), "a key leaked: {text}" ); remove_server(&admin, &id).await.unwrap(); assert_eq!(selling(&admin).await.unwrap().servers.len(), 1); assert!(!key_file.exists(), "the key outlived its server"); } #[tokio::test] async fn the_local_server_keeps_its_context_and_cannot_be_removed() { let (admin, state, _dir) = admin_with_local().await; let mut offer = lan_offer(LOCAL_MODEL_ID, 65536); offer.name = "Qwen on pa3".into(); offer.prompt_usd_per_million = 2.0; set_offers(&admin, "custom", vec![offer]).await.unwrap(); let posted = state.lock().unwrap().posted_models.last().cloned().unwrap(); assert_eq!(posted["context_length"], 8192); assert_eq!(posted["name"], "Qwen on pa3"); assert_eq!(posted["description"], "Local llama-server model"); let error = set_offers(&admin, "custom", vec![lan_offer("other", 8192)]) .await .unwrap_err(); assert!( error.to_string().contains("does not serve other"), "{error}" ); assert!(remove_server(&admin, "custom").await.is_err()); } #[test] fn key_files_are_named_only_by_routstr_slugs() { let admin = Admin { client: reqwest::Client::new(), app: String::new(), token: String::new(), keys_dir: "/keys".into(), }; assert_eq!( admin.key_path("custom-2"), Some("/keys/custom-2.key".into()) ); for bad in ["", "../x", "a/b", "Custom", "x.y", &"a".repeat(65)] { assert!(admin.key_path(bad).is_none(), "{bad}"); } } #[tokio::test] async fn unreachable_or_refusing_servers_explain_themselves() { let error = probe("http://127.0.0.1:9/v1", None).await.unwrap_err(); assert!( error.to_string().contains("Could not reach the server"), "{error}" ); let refusing = serve(|_, _, _, _| (401, json!({"error": "no key"}))).await; let error = probe(&format!("{refusing}/v1"), None).await.unwrap_err(); assert!(error.to_string().contains("refused the API key"), "{error}"); let empty = serve(|_, _, _, _| (200, json!({"data": []}))).await; let error = probe(&format!("{empty}/v1"), None).await.unwrap_err(); assert!(error.to_string().contains("no models"), "{error}"); } }