Routstr's own announcer drops private addresses, and a FIPS address
(fd00::/8) is one, so a mesh seller could never be listed. The node now
signs a Routstr provider listing (NIP-91, kind 38421) itself:
- Signed with the node's FIPS key; the endpoint is
http://<npub>.fips:5679/routstr, the signer's own mesh address, so a
listing cannot point buyers at someone else's node.
- Tags: d=archipelago-fips, u, mint (Routstr's accepted mints from
/v1/info), version, t=fips, t=archipelago. Content: name, about,
transport "fips", and each model on sale with context, longest answer,
USD per million tokens and the sats sent up front.
- Sent to Routstr's default relays plus the node's own, through the
node's Nostr Tor proxy when set. Each relay is then asked separately,
in parallel, for the event by id; the result is saved and shown.
- Off by default. routstr.selling.set-listing {enabled} publishes or
withdraws (NIP-09 deletion by id and address, then a check of which
relays still hold it). Saving prices or removing a server re-publishes
while the listing is on. routstr.selling.get returns the saved state.
- Settings: an "Announce on Nostr" card in What you sell; publishing asks
for a second press because it is public.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1271 lines
46 KiB
Rust
1271 lines
46 KiB
Rust
//! What this node sells through its Routstr Provider app: the inference
|
|
//! servers Routstr forwards to (this node's own llama-server, or an
|
|
//! OpenAI-compatible server on the LAN such as a DGX Spark) and, for each,
|
|
//! which models are offered to mesh peers at what price.
|
|
//!
|
|
//! Routstr's own database is the single record of servers, models and prices;
|
|
//! this module only drives Routstr's admin API, logging in with the admin
|
|
//! password the orchestrator generated for the app. That password, and any
|
|
//! API key an operator enters for a server, are sent only to the app on
|
|
//! loopback and are never returned or logged. Routstr does not hand a key
|
|
//! back, so the node keeps its own copy (0600, beside the app's other
|
|
//! secrets) to re-read the server's model list.
|
|
//!
|
|
//! Prices are US dollars per million tokens, as Routstr stores them; Routstr
|
|
//! converts them to sats at its current exchange rate when quoting buyers.
|
|
|
|
use anyhow::{bail, ensure, Context, Result};
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::{json, Value};
|
|
use std::net::IpAddr;
|
|
use std::path::Path;
|
|
use std::time::Duration;
|
|
|
|
/// The Routstr app's host-local port (`apps/routstr/manifest.yml`).
|
|
pub(crate) const ROUTSTR_APP: &str = "http://127.0.0.1:8200";
|
|
const ADMIN_SECRET: &str = "routstr-admin-password";
|
|
/// This node's own llama-server, as Routstr reaches it on the app network.
|
|
const LOCAL_SERVER_URL: &str = "http://llama-server:8080/v1";
|
|
/// The model alias llama-server serves (`apps/llama-server/manifest.yml`).
|
|
const LOCAL_MODEL_ID: &str = "archy-llama";
|
|
const PROBE_TIMEOUT: Duration = Duration::from_secs(10);
|
|
const MAX_PROBE_BYTES: usize = 1024 * 1024;
|
|
const MAX_CONTEXT: u64 = 1_048_576;
|
|
const MIN_CONTEXT: u64 = 512;
|
|
/// A sanity ceiling, far above any real price, against a typo of several
|
|
/// orders of magnitude.
|
|
const MAX_USD_PER_MILLION: f64 = 1000.0;
|
|
const MAX_NAME_CHARS: usize = 80;
|
|
|
|
/// A logged-in session with this node's Routstr admin API.
|
|
pub(crate) struct Admin {
|
|
client: reqwest::Client,
|
|
app: String,
|
|
token: String,
|
|
/// Where the node keeps API keys of the servers it sells from.
|
|
keys_dir: std::path::PathBuf,
|
|
}
|
|
|
|
impl Admin {
|
|
pub(crate) async fn login(data_dir: &Path) -> Result<Self> {
|
|
Self::login_at(ROUTSTR_APP, &data_dir.join("secrets").join(ADMIN_SECRET)).await
|
|
}
|
|
|
|
pub(crate) async fn login_at(app: &str, secret: &Path) -> Result<Self> {
|
|
let password = tokio::fs::read_to_string(secret)
|
|
.await
|
|
.context("Routstr Provider is not installed")?;
|
|
let client = reqwest::Client::builder()
|
|
.no_proxy()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.timeout(Duration::from_secs(30))
|
|
.build()?;
|
|
let login: Value = client
|
|
.post(format!("{app}/admin/api/login"))
|
|
.json(&json!({ "password": password.trim() }))
|
|
.send()
|
|
.await
|
|
.context("Routstr Provider is not running")?
|
|
.error_for_status()
|
|
.context("Routstr Provider refused the node's admin login")?
|
|
.json()
|
|
.await?;
|
|
let token = login
|
|
.get("token")
|
|
.and_then(Value::as_str)
|
|
.context("Routstr Provider returned no admin session")?
|
|
.to_string();
|
|
Ok(Self {
|
|
client,
|
|
app: app.to_string(),
|
|
token,
|
|
keys_dir: secret
|
|
.parent()
|
|
.unwrap_or(Path::new("."))
|
|
.join("routstr-servers"),
|
|
})
|
|
}
|
|
|
|
async fn send(
|
|
&self,
|
|
method: reqwest::Method,
|
|
path: &str,
|
|
body: Option<&Value>,
|
|
) -> Result<Value> {
|
|
let mut request = self
|
|
.client
|
|
.request(method, format!("{}{path}", self.app))
|
|
.bearer_auth(&self.token);
|
|
if let Some(body) = body {
|
|
request = request.json(body);
|
|
}
|
|
let response = request
|
|
.send()
|
|
.await
|
|
.context("Routstr Provider is not running")?;
|
|
let status = response.status();
|
|
let text = response.text().await.unwrap_or_default();
|
|
if !status.is_success() {
|
|
let detail = serde_json::from_str::<Value>(&text)
|
|
.ok()
|
|
.and_then(|v| v.get("detail").cloned())
|
|
.map(|d| match d {
|
|
Value::String(s) => s,
|
|
other => other.to_string(),
|
|
})
|
|
.unwrap_or_else(|| status.to_string());
|
|
bail!("Routstr Provider refused the change: {detail}");
|
|
}
|
|
Ok(serde_json::from_str(&text).unwrap_or(Value::Null))
|
|
}
|
|
|
|
/// The node's copy of a server's API key. Routstr slugs are lowercase
|
|
/// letters, digits and dashes; anything else has no key file.
|
|
fn key_path(&self, server: &str) -> Option<std::path::PathBuf> {
|
|
let safe = !server.is_empty()
|
|
&& server.len() <= 64
|
|
&& server
|
|
.chars()
|
|
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-');
|
|
safe.then(|| self.keys_dir.join(format!("{server}.key")))
|
|
}
|
|
|
|
async fn server_key(&self, server: &str) -> Option<String> {
|
|
let path = self.key_path(server)?;
|
|
let key = tokio::fs::read_to_string(path).await.ok()?;
|
|
Some(key.trim().to_string()).filter(|k| !k.is_empty())
|
|
}
|
|
|
|
pub(crate) async fn get(&self, path: &str) -> Result<Value> {
|
|
self.send(reqwest::Method::GET, path, None).await
|
|
}
|
|
|
|
async fn post(&self, path: &str, body: &Value) -> Result<Value> {
|
|
self.send(reqwest::Method::POST, path, Some(body)).await
|
|
}
|
|
|
|
async fn delete(&self, path: &str) -> Result<Value> {
|
|
self.send(reqwest::Method::DELETE, path, None).await
|
|
}
|
|
|
|
/// Routstr's public info: version and the mints it accepts.
|
|
pub(crate) async fn info(&self) -> Result<Value> {
|
|
Ok(self
|
|
.client
|
|
.get(format!("{}/v1/info", self.app))
|
|
.send()
|
|
.await
|
|
.context("Routstr Provider is not running")?
|
|
.error_for_status()?
|
|
.json()
|
|
.await?)
|
|
}
|
|
|
|
/// The public catalog buyers see, with each model's sats quote.
|
|
async fn catalog(&self) -> Result<Value> {
|
|
Ok(self
|
|
.client
|
|
.get(format!("{}/v1/models", self.app))
|
|
.send()
|
|
.await
|
|
.context("Routstr Provider is not running")?
|
|
.error_for_status()?
|
|
.json()
|
|
.await?)
|
|
}
|
|
}
|
|
|
|
/// One model offered to mesh peers.
|
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
|
pub(crate) struct Offer {
|
|
pub model_id: String,
|
|
pub name: String,
|
|
pub context_length: u64,
|
|
pub max_completion_tokens: u64,
|
|
pub prompt_usd_per_million: f64,
|
|
pub completion_usd_per_million: f64,
|
|
pub enabled: bool,
|
|
/// Sats a buyer sends up front per message (the unused part comes back),
|
|
/// from Routstr's own catalog. Absent while the model is not offered.
|
|
#[serde(default, skip_deserializing)]
|
|
pub quote_sats: Option<u64>,
|
|
}
|
|
|
|
/// A model an inference server reports.
|
|
#[derive(Clone, Debug, PartialEq, Serialize)]
|
|
pub(crate) struct ServerModel {
|
|
pub id: String,
|
|
/// The most context the server itself allows, when it says.
|
|
pub context_length: Option<u64>,
|
|
}
|
|
|
|
/// An inference server Routstr forwards to.
|
|
#[derive(Debug, Serialize)]
|
|
pub(crate) struct Server {
|
|
pub id: String,
|
|
pub url: String,
|
|
/// This node's own llama-server.
|
|
pub local: bool,
|
|
pub has_key: bool,
|
|
pub offers: Vec<Offer>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
pub(crate) struct Selling {
|
|
pub servers: Vec<Server>,
|
|
/// Sats per US dollar Routstr quotes at (fee included), when any priced
|
|
/// model shows it; lets the screen preview a quote before saving.
|
|
pub sats_per_usd: Option<f64>,
|
|
}
|
|
|
|
/// A server address an operator may sell from: http(s) on the LAN or beyond,
|
|
/// never this node itself, its app network, or link-local and metadata
|
|
/// addresses. Returns the normalised base URL (ending in `/v1` unless a path
|
|
/// was given).
|
|
pub(crate) fn validate_server_url(input: &str) -> Result<String> {
|
|
let url = reqwest::Url::parse(input.trim())
|
|
.context("Enter the server's address, e.g. http://192.168.1.50:8000/v1")?;
|
|
ensure!(
|
|
matches!(url.scheme(), "http" | "https"),
|
|
"The address must start with http:// or https://"
|
|
);
|
|
ensure!(
|
|
url.username().is_empty() && url.password().is_none(),
|
|
"Put an API key in the key field, not in the address"
|
|
);
|
|
ensure!(
|
|
url.query().is_none() && url.fragment().is_none(),
|
|
"The address must not contain ? or #"
|
|
);
|
|
let host = url.host_str().unwrap_or_default();
|
|
match host
|
|
.trim_start_matches('[')
|
|
.trim_end_matches(']')
|
|
.parse::<IpAddr>()
|
|
{
|
|
Ok(ip) => check_ip(ip)?,
|
|
Err(_) if host.is_empty() => bail!("The address has no host"),
|
|
Err(_) => {
|
|
let name = host.trim_end_matches('.').to_ascii_lowercase();
|
|
ensure!(
|
|
name != "localhost" && !name.ends_with(".localhost"),
|
|
"This node's own address cannot be a server; the local llama-server is listed already"
|
|
);
|
|
// Single-label names are this node's app containers (or depend on
|
|
// search domains Routstr's container does not share).
|
|
ensure!(
|
|
name.contains('.'),
|
|
"Use the server's IP address or a full name such as spark.local"
|
|
);
|
|
}
|
|
}
|
|
let path = url.path().trim_end_matches('/');
|
|
let path = if path.is_empty() { "/v1" } else { path };
|
|
let port = url.port().map(|p| format!(":{p}")).unwrap_or_default();
|
|
Ok(format!("{}://{host}{port}{path}", url.scheme()))
|
|
}
|
|
|
|
fn check_ip(ip: IpAddr) -> Result<()> {
|
|
let refused = match ip {
|
|
IpAddr::V4(v4) => {
|
|
let [a, b, ..] = v4.octets();
|
|
v4.is_loopback()
|
|
|| v4.is_unspecified()
|
|
|| v4.is_link_local()
|
|
|| v4.is_multicast()
|
|
|| v4.is_broadcast()
|
|
// Podman's app networks on this node (archy-net and the default).
|
|
|| (a == 10 && (b == 88 || b == 89))
|
|
}
|
|
IpAddr::V6(v6) => {
|
|
if let Some(v4) = v6.to_ipv4_mapped() {
|
|
return check_ip(IpAddr::V4(v4));
|
|
}
|
|
v6.is_loopback()
|
|
|| v6.is_unspecified()
|
|
|| v6.is_multicast()
|
|
|| (v6.segments()[0] & 0xffc0) == 0xfe80
|
|
}
|
|
};
|
|
ensure!(
|
|
!refused,
|
|
"That address is this node, its app network, or a link-local address; enter the server's LAN address"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// Ask a server which models it serves (`GET {url}/models`), with the most
|
|
/// context each allows when the server says so: vLLM's `max_model_len`, a
|
|
/// `context_length`, or llama.cpp's running `n_ctx` from `/props`.
|
|
pub(crate) async fn probe(url: &str, api_key: Option<&str>) -> Result<Vec<ServerModel>> {
|
|
let client = reqwest::Client::builder()
|
|
.no_proxy()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.timeout(PROBE_TIMEOUT)
|
|
.build()?;
|
|
let get = |path: String| {
|
|
let mut request = client.get(path);
|
|
if let Some(key) = api_key.filter(|k| !k.is_empty()) {
|
|
request = request.bearer_auth(key);
|
|
}
|
|
request.send()
|
|
};
|
|
let response = get(format!("{url}/models"))
|
|
.await
|
|
.context("Could not reach the server; check the address and that it is running")?;
|
|
ensure!(
|
|
response.status() != reqwest::StatusCode::UNAUTHORIZED
|
|
&& response.status() != reqwest::StatusCode::FORBIDDEN,
|
|
"The server refused the API key (HTTP {})",
|
|
response.status().as_u16()
|
|
);
|
|
ensure!(
|
|
response.status().is_success(),
|
|
"The server answered HTTP {} for its model list; is this an OpenAI-compatible /v1 address?",
|
|
response.status().as_u16()
|
|
);
|
|
let listing: Value = serde_json::from_slice(&read_limited(response).await?).context(
|
|
"The server's model list is not JSON; is this an OpenAI-compatible /v1 address?",
|
|
)?;
|
|
let mut models = parse_models(&listing);
|
|
ensure!(!models.is_empty(), "The server reports no models");
|
|
|
|
// llama.cpp reports the context it was started with only in /props.
|
|
if listing
|
|
.get("data")
|
|
.and_then(Value::as_array)
|
|
.is_some_and(|d| {
|
|
d.iter()
|
|
.any(|m| m.get("owned_by").and_then(Value::as_str) == Some("llamacpp"))
|
|
})
|
|
{
|
|
let root = url.strip_suffix("/v1").unwrap_or(url);
|
|
if let Ok(response) = get(format!("{root}/props")).await {
|
|
if let Ok(bytes) = read_limited(response).await {
|
|
if let Some(n_ctx) = serde_json::from_slice::<Value>(&bytes).ok().and_then(|p| {
|
|
p.pointer("/default_generation_settings/n_ctx")
|
|
.and_then(Value::as_u64)
|
|
}) {
|
|
for model in &mut models {
|
|
model.context_length = Some(n_ctx);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(models)
|
|
}
|
|
|
|
async fn read_limited(mut response: reqwest::Response) -> Result<Vec<u8>> {
|
|
let mut out = Vec::new();
|
|
while let Some(chunk) = response.chunk().await? {
|
|
ensure!(
|
|
out.len() + chunk.len() <= MAX_PROBE_BYTES,
|
|
"The server's answer is too large"
|
|
);
|
|
out.extend_from_slice(&chunk);
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
fn parse_models(listing: &Value) -> Vec<ServerModel> {
|
|
listing
|
|
.get("data")
|
|
.and_then(Value::as_array)
|
|
.into_iter()
|
|
.flatten()
|
|
.filter_map(|m| {
|
|
let id = m.get("id")?.as_str()?.trim();
|
|
(!id.is_empty() && id.len() <= 200 && !id.chars().any(char::is_control)).then(|| {
|
|
let context_length = ["max_model_len", "context_length", "context_window"]
|
|
.iter()
|
|
.find_map(|k| m.get(*k).and_then(Value::as_u64))
|
|
.or_else(|| m.pointer("/meta/n_ctx").and_then(Value::as_u64))
|
|
.filter(|n| *n > 0);
|
|
ServerModel {
|
|
id: id.to_string(),
|
|
context_length,
|
|
}
|
|
})
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// The ID buyers use for a model: Routstr strips anything before the first
|
|
/// slash (`routstr/upstream/model_paths.py`, `public_model_id`).
|
|
pub(crate) fn public_id(model_id: &str) -> &str {
|
|
model_id.split_once('/').map_or(model_id, |(_, rest)| rest)
|
|
}
|
|
|
|
fn provider_ref(provider: &Value) -> Option<String> {
|
|
provider
|
|
.get("slug")
|
|
.and_then(Value::as_str)
|
|
.map(String::from)
|
|
.or_else(|| provider.get("id").map(|id| id.to_string()))
|
|
}
|
|
|
|
fn is_local(provider: &Value) -> bool {
|
|
provider.get("base_url").and_then(Value::as_str) == Some(LOCAL_SERVER_URL)
|
|
}
|
|
|
|
fn offer_from_db(model: &Value) -> Option<Offer> {
|
|
let per_million = |k: &str| {
|
|
model
|
|
.pointer(&format!("/pricing/{k}"))
|
|
.and_then(Value::as_f64)
|
|
.map(|p| (p * 1e6 * 1e6).round() / 1e6)
|
|
.unwrap_or(0.0)
|
|
};
|
|
let context_length = model.get("context_length").and_then(Value::as_u64)?;
|
|
Some(Offer {
|
|
model_id: model.get("id")?.as_str()?.to_string(),
|
|
name: model
|
|
.get("name")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_string(),
|
|
context_length,
|
|
max_completion_tokens: model
|
|
.pointer("/top_provider/max_completion_tokens")
|
|
.and_then(Value::as_u64)
|
|
.unwrap_or(context_length),
|
|
prompt_usd_per_million: per_million("prompt"),
|
|
completion_usd_per_million: per_million("completion"),
|
|
enabled: model
|
|
.get("enabled")
|
|
.and_then(Value::as_bool)
|
|
.unwrap_or(true),
|
|
quote_sats: None,
|
|
})
|
|
}
|
|
|
|
/// Sats per US dollar in Routstr's catalog, read off any priced model.
|
|
fn sats_per_usd(catalog: &Value) -> Option<f64> {
|
|
catalog.get("data")?.as_array()?.iter().find_map(|m| {
|
|
let usd = m
|
|
.pointer("/pricing/prompt")
|
|
.and_then(|v| v.as_f64().or_else(|| v.as_str()?.parse().ok()))?;
|
|
let sats = m.pointer("/sats_pricing/prompt").and_then(Value::as_f64)?;
|
|
(usd > 0.0 && sats > 0.0).then(|| sats / usd)
|
|
})
|
|
}
|
|
|
|
async fn providers(admin: &Admin) -> Result<Vec<Value>> {
|
|
Ok(admin
|
|
.get("/admin/api/upstream-providers")
|
|
.await?
|
|
.as_array()
|
|
.cloned()
|
|
.unwrap_or_default())
|
|
}
|
|
|
|
async fn find_provider(admin: &Admin, id: &str) -> Result<Value> {
|
|
providers(admin)
|
|
.await?
|
|
.into_iter()
|
|
.find(|p| provider_ref(p).as_deref() == Some(id))
|
|
.context("No such server")
|
|
}
|
|
|
|
async fn db_models(admin: &Admin, provider_id: &str) -> Result<Vec<Value>> {
|
|
Ok(admin
|
|
.get(&format!(
|
|
"/admin/api/upstream-providers/{provider_id}/models"
|
|
))
|
|
.await?
|
|
.get("db_models")
|
|
.and_then(Value::as_array)
|
|
.cloned()
|
|
.unwrap_or_default())
|
|
}
|
|
|
|
/// Every server and what it offers, with live quotes.
|
|
pub(crate) async fn selling(admin: &Admin) -> Result<Selling> {
|
|
let catalog = admin.catalog().await.unwrap_or(Value::Null);
|
|
let mut servers = Vec::new();
|
|
for provider in providers(admin).await? {
|
|
let Some(id) = provider_ref(&provider) else {
|
|
continue;
|
|
};
|
|
let mut offers: Vec<Offer> = db_models(admin, &id)
|
|
.await?
|
|
.iter()
|
|
.filter_map(offer_from_db)
|
|
.collect();
|
|
for offer in &mut offers {
|
|
offer.quote_sats = offer
|
|
.enabled
|
|
.then(|| {
|
|
crate::assistant::routstr_peer::quote_sats(&catalog, public_id(&offer.model_id))
|
|
})
|
|
.flatten();
|
|
}
|
|
servers.push(Server {
|
|
url: provider
|
|
.get("base_url")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_string(),
|
|
local: is_local(&provider),
|
|
has_key: !is_local(&provider) && admin.server_key(&id).await.is_some(),
|
|
offers,
|
|
id,
|
|
});
|
|
}
|
|
// This node's own llama-server first.
|
|
servers.sort_by_key(|s| !s.local);
|
|
Ok(Selling {
|
|
servers,
|
|
sats_per_usd: sats_per_usd(&catalog),
|
|
})
|
|
}
|
|
|
|
/// Add a LAN (or remote) server after checking it answers with a model list.
|
|
pub(crate) async fn add_server(
|
|
admin: &Admin,
|
|
url: &str,
|
|
api_key: Option<&str>,
|
|
) -> Result<(String, Vec<ServerModel>)> {
|
|
let url = validate_server_url(url)?;
|
|
let api_key = api_key.map(str::trim).filter(|k| !k.is_empty());
|
|
if let Some(key) = api_key {
|
|
ensure!(
|
|
key.len() <= 4096 && !key.chars().any(|c| c.is_control() || c.is_whitespace()),
|
|
"The API key contains spaces or control characters"
|
|
);
|
|
}
|
|
ensure!(
|
|
!providers(admin)
|
|
.await?
|
|
.iter()
|
|
.any(|p| p.get("base_url").and_then(Value::as_str) == Some(url.as_str())),
|
|
"This server is added already"
|
|
);
|
|
let models = probe(&url, api_key).await?;
|
|
register_server(admin, &url, api_key)
|
|
.await
|
|
.map(|id| (id, models))
|
|
}
|
|
|
|
/// Record a checked server in Routstr, and its key on the node.
|
|
async fn register_server(admin: &Admin, url: &str, api_key: Option<&str>) -> Result<String> {
|
|
let created = admin
|
|
.post(
|
|
"/admin/api/upstream-providers",
|
|
&json!({
|
|
"provider_type": "custom",
|
|
"base_url": url,
|
|
"api_key": api_key.unwrap_or_default(),
|
|
"enabled": true,
|
|
}),
|
|
)
|
|
.await?;
|
|
let id = provider_ref(&created).context("Routstr Provider returned no server id")?;
|
|
if let Some(key) = api_key {
|
|
let stored = match admin.key_path(&id) {
|
|
Some(path) => {
|
|
crate::settings::model_provider::write_private(&path, key.as_bytes()).await
|
|
}
|
|
None => Err(anyhow::anyhow!("unexpected server id {id}")),
|
|
};
|
|
if let Err(error) = stored {
|
|
// Without the node's copy the model list could not be re-read.
|
|
let _ = admin
|
|
.delete(&format!("/admin/api/upstream-providers/{id}"))
|
|
.await;
|
|
return Err(error.context("Could not store the server's API key"));
|
|
}
|
|
}
|
|
tracing::info!(server = %id, %url, "routstr selling: inference server added");
|
|
Ok(id)
|
|
}
|
|
|
|
/// The models a server reports now. The local llama-server is reached only
|
|
/// by Routstr, so its one model is the entry Routstr already holds.
|
|
pub(crate) async fn server_models(admin: &Admin, id: &str) -> Result<Vec<ServerModel>> {
|
|
let provider = find_provider(admin, id).await?;
|
|
if is_local(&provider) {
|
|
return Ok(db_models(admin, id)
|
|
.await?
|
|
.iter()
|
|
.filter(|m| m.get("id").and_then(Value::as_str) == Some(LOCAL_MODEL_ID))
|
|
.map(|m| ServerModel {
|
|
id: LOCAL_MODEL_ID.to_string(),
|
|
context_length: m.get("context_length").and_then(Value::as_u64),
|
|
})
|
|
.collect());
|
|
}
|
|
let url = provider
|
|
.get("base_url")
|
|
.and_then(Value::as_str)
|
|
.context("The server has no address")?;
|
|
probe(url, admin.server_key(id).await.as_deref()).await
|
|
}
|
|
|
|
pub(crate) async fn remove_server(admin: &Admin, id: &str) -> Result<()> {
|
|
let provider = find_provider(admin, id).await?;
|
|
ensure!(
|
|
!is_local(&provider),
|
|
"This node's own llama-server cannot be removed; stop selling its model instead"
|
|
);
|
|
admin
|
|
.delete(&format!("/admin/api/upstream-providers/{id}/models"))
|
|
.await?;
|
|
admin
|
|
.delete(&format!("/admin/api/upstream-providers/{id}"))
|
|
.await?;
|
|
if let Some(path) = admin.key_path(id) {
|
|
let _ = tokio::fs::remove_file(path).await;
|
|
}
|
|
tracing::info!(server = %id, "routstr selling: inference server removed");
|
|
Ok(())
|
|
}
|
|
|
|
/// Check one offer against what the server reports. `fixed_context` is the
|
|
/// local llama-server's context, which only its manifest can change.
|
|
fn check_offer(
|
|
offer: &mut Offer,
|
|
reported: &[ServerModel],
|
|
fixed_context: Option<u64>,
|
|
) -> Result<()> {
|
|
offer.name = offer.name.trim().to_string();
|
|
let label = if offer.name.is_empty() {
|
|
offer.model_id.clone()
|
|
} else {
|
|
offer.name.clone()
|
|
};
|
|
let server_model = reported
|
|
.iter()
|
|
.find(|m| m.id == offer.model_id)
|
|
.with_context(|| format!("The server does not serve {}", offer.model_id))?;
|
|
ensure!(
|
|
!offer.name.is_empty() && offer.name.chars().count() <= MAX_NAME_CHARS,
|
|
"{label}: give it a name of 1 to {MAX_NAME_CHARS} characters"
|
|
);
|
|
ensure!(
|
|
!offer.name.chars().any(char::is_control),
|
|
"{label}: the name contains control characters"
|
|
);
|
|
if let Some(context) = fixed_context {
|
|
offer.context_length = context;
|
|
}
|
|
ensure!(
|
|
(MIN_CONTEXT..=MAX_CONTEXT).contains(&offer.context_length),
|
|
"{label}: context must be between {MIN_CONTEXT} and {MAX_CONTEXT} tokens"
|
|
);
|
|
if let Some(limit) = server_model.context_length {
|
|
ensure!(
|
|
offer.context_length <= limit,
|
|
"{label}: the server allows at most {limit} tokens of context"
|
|
);
|
|
}
|
|
ensure!(
|
|
(16..=offer.context_length).contains(&offer.max_completion_tokens),
|
|
"{label}: the answer limit must be between 16 tokens and the context"
|
|
);
|
|
for price in [
|
|
offer.prompt_usd_per_million,
|
|
offer.completion_usd_per_million,
|
|
] {
|
|
ensure!(
|
|
price.is_finite() && (0.0..=MAX_USD_PER_MILLION).contains(&price),
|
|
"{label}: prices must be between $0 and ${MAX_USD_PER_MILLION} per million tokens"
|
|
);
|
|
}
|
|
ensure!(
|
|
!offer.enabled
|
|
|| offer.prompt_usd_per_million > 0.0
|
|
|| offer.completion_usd_per_million > 0.0,
|
|
"{label}: set a price before selling it"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn model_body(offer: &Offer, url: &str) -> Value {
|
|
let host = reqwest::Url::parse(url)
|
|
.ok()
|
|
.and_then(|u| u.host_str().map(String::from))
|
|
.unwrap_or_default();
|
|
json!({
|
|
"id": offer.model_id,
|
|
"name": offer.name,
|
|
"description": if url == LOCAL_SERVER_URL {
|
|
"Local llama-server model".to_string()
|
|
} else {
|
|
format!("Served from {host}")
|
|
},
|
|
"created": 0,
|
|
"context_length": offer.context_length,
|
|
"architecture": {
|
|
"modality": "text->text", "input_modalities": ["text"],
|
|
"output_modalities": ["text"], "tokenizer": "Other", "instruct_type": null,
|
|
},
|
|
"pricing": {
|
|
"prompt": offer.prompt_usd_per_million / 1e6,
|
|
"completion": offer.completion_usd_per_million / 1e6,
|
|
"request": 0, "image": 0, "web_search": 0, "internal_reasoning": 0,
|
|
},
|
|
"top_provider": {
|
|
"context_length": offer.context_length,
|
|
"max_completion_tokens": offer.max_completion_tokens,
|
|
"is_moderated": false,
|
|
},
|
|
"enabled": offer.enabled,
|
|
})
|
|
}
|
|
|
|
/// Save what a server offers. Models not listed are left as they are.
|
|
pub(crate) async fn set_offers(admin: &Admin, id: &str, mut offers: Vec<Offer>) -> Result<()> {
|
|
let provider = find_provider(admin, id).await?;
|
|
let url = provider
|
|
.get("base_url")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
let local = is_local(&provider);
|
|
let reported = server_models(admin, id).await?;
|
|
for offer in &mut offers {
|
|
let fixed = local
|
|
.then(|| reported.iter().find(|m| m.id == offer.model_id))
|
|
.flatten()
|
|
.and_then(|m| m.context_length);
|
|
check_offer(offer, &reported, fixed)?;
|
|
}
|
|
|
|
// Buyers name a model only by its public ID; two servers offering the
|
|
// same one would make Routstr split requests between them.
|
|
let mut taken = std::collections::HashMap::new();
|
|
for other in providers(admin).await? {
|
|
let Some(other_id) = provider_ref(&other) else {
|
|
continue;
|
|
};
|
|
if other_id == id {
|
|
continue;
|
|
}
|
|
for model in db_models(admin, &other_id).await? {
|
|
if model
|
|
.get("enabled")
|
|
.and_then(Value::as_bool)
|
|
.unwrap_or(true)
|
|
{
|
|
if let Some(model_id) = model.get("id").and_then(Value::as_str) {
|
|
taken.insert(public_id(model_id).to_string(), other_id.clone());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let mut seen = std::collections::HashSet::new();
|
|
for offer in offers.iter().filter(|o| o.enabled) {
|
|
let public = public_id(&offer.model_id);
|
|
ensure!(
|
|
!taken.contains_key(public),
|
|
"{} is already sold from another server; stop selling it there first",
|
|
offer.name
|
|
);
|
|
ensure!(
|
|
seen.insert(public.to_string()),
|
|
"{} is listed twice",
|
|
offer.name
|
|
);
|
|
}
|
|
|
|
for offer in &offers {
|
|
admin
|
|
.post(
|
|
&format!("/admin/api/upstream-providers/{id}/models"),
|
|
&model_body(offer, &url),
|
|
)
|
|
.await?;
|
|
}
|
|
tracing::info!(server = %id, offers = offers.len(), "routstr selling: offers saved");
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn lan_addresses_are_accepted_and_normalised() {
|
|
for (input, normalised) in [
|
|
(
|
|
"http://192.168.88.198:8000/v1",
|
|
"http://192.168.88.198:8000/v1",
|
|
),
|
|
(
|
|
"http://192.168.88.198:8000",
|
|
"http://192.168.88.198:8000/v1",
|
|
),
|
|
(
|
|
"http://192.168.88.198:8000/v1/",
|
|
"http://192.168.88.198:8000/v1",
|
|
),
|
|
(
|
|
" https://spark.local:8443/openai/v1 ",
|
|
"https://spark.local:8443/openai/v1",
|
|
),
|
|
(
|
|
"http://[fd12:3456::1]:8000",
|
|
"http://[fd12:3456::1]:8000/v1",
|
|
),
|
|
("http://10.1.2.3:11434/v1", "http://10.1.2.3:11434/v1"),
|
|
] {
|
|
assert_eq!(validate_server_url(input).unwrap(), normalised, "{input}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn this_node_and_its_app_network_are_refused() {
|
|
for input in [
|
|
"ftp://192.168.1.5/v1",
|
|
"file:///etc/passwd",
|
|
"http://127.0.0.1:8080/v1",
|
|
"http://localhost:8080/v1",
|
|
"http://[::1]:8080/v1",
|
|
"http://0.0.0.0:8080/v1",
|
|
"http://169.254.169.254/latest",
|
|
"http://[fe80::1]:8000/v1",
|
|
"http://[::ffff:127.0.0.1]:8000/v1",
|
|
"http://10.89.0.20:8080/v1",
|
|
"http://10.88.0.5:8080/v1",
|
|
"http://llama-server:8080/v1",
|
|
"http://bitcoin-knots:8332",
|
|
"http://user:secret@192.168.1.5:8000/v1",
|
|
"http://192.168.1.5:8000/v1?x=1",
|
|
"http://192.168.1.5:8000/v1#x",
|
|
"not a url",
|
|
] {
|
|
assert!(validate_server_url(input).is_err(), "accepted {input}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn model_lists_from_common_servers_are_read() {
|
|
let vllm =
|
|
json!({"data": [{"id": "meta-llama/Llama-3.1-8B-Instruct", "max_model_len": 131072}]});
|
|
let ollama =
|
|
json!({"object": "list", "data": [{"id": "qwen2.5:7b", "owned_by": "library"}]});
|
|
let odd = json!({"data": [{"id": ""}, {"id": "a\u{7}b"}, {"no": "id"}, {"id": "ok", "context_length": 0}]});
|
|
assert_eq!(
|
|
parse_models(&vllm),
|
|
[ServerModel {
|
|
id: "meta-llama/Llama-3.1-8B-Instruct".into(),
|
|
context_length: Some(131072)
|
|
}]
|
|
);
|
|
assert_eq!(
|
|
parse_models(&ollama),
|
|
[ServerModel {
|
|
id: "qwen2.5:7b".into(),
|
|
context_length: None
|
|
}]
|
|
);
|
|
assert_eq!(
|
|
parse_models(&odd),
|
|
[ServerModel {
|
|
id: "ok".into(),
|
|
context_length: None
|
|
}]
|
|
);
|
|
assert!(parse_models(&json!({"error": "no"})).is_empty());
|
|
}
|
|
|
|
fn offer() -> Offer {
|
|
Offer {
|
|
model_id: "qwen2.5:7b".into(),
|
|
name: " Qwen 2.5 7B ".into(),
|
|
context_length: 8192,
|
|
max_completion_tokens: 1024,
|
|
prompt_usd_per_million: 0.2,
|
|
completion_usd_per_million: 0.6,
|
|
enabled: true,
|
|
quote_sats: None,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn offers_are_checked_against_the_server() {
|
|
let reported = [ServerModel {
|
|
id: "qwen2.5:7b".into(),
|
|
context_length: Some(32768),
|
|
}];
|
|
let mut ok = offer();
|
|
check_offer(&mut ok, &reported, None).unwrap();
|
|
assert_eq!(ok.name, "Qwen 2.5 7B");
|
|
|
|
let bad: [(fn(&mut Offer), &str); 9] = [
|
|
(|o| o.model_id = "other".into(), "does not serve"),
|
|
(|o| o.name = " ".into(), "name"),
|
|
(|o| o.name = "x".repeat(81), "name"),
|
|
(|o| o.context_length = 65536, "at most 32768"),
|
|
(|o| o.context_length = 100, "between 512"),
|
|
(|o| o.max_completion_tokens = 9000, "answer limit"),
|
|
(|o| o.prompt_usd_per_million = f64::NAN, "prices"),
|
|
(|o| o.completion_usd_per_million = 5000.0, "prices"),
|
|
(
|
|
|o| {
|
|
o.prompt_usd_per_million = 0.0;
|
|
o.completion_usd_per_million = 0.0
|
|
},
|
|
"set a price",
|
|
),
|
|
];
|
|
for (spoil, message) in bad {
|
|
let mut o = offer();
|
|
spoil(&mut o);
|
|
let error = check_offer(&mut o, &reported, None)
|
|
.unwrap_err()
|
|
.to_string();
|
|
assert!(error.contains(message), "{error} lacks {message}");
|
|
}
|
|
|
|
// Unpriced is fine while not sold.
|
|
let mut off = offer();
|
|
off.enabled = false;
|
|
off.prompt_usd_per_million = 0.0;
|
|
off.completion_usd_per_million = 0.0;
|
|
check_offer(&mut off, &reported, None).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn the_local_model_keeps_llama_servers_context() {
|
|
let reported = [ServerModel {
|
|
id: LOCAL_MODEL_ID.into(),
|
|
context_length: Some(8192),
|
|
}];
|
|
let mut local = offer();
|
|
local.model_id = LOCAL_MODEL_ID.into();
|
|
local.context_length = 65536;
|
|
check_offer(&mut local, &reported, Some(8192)).unwrap();
|
|
assert_eq!(local.context_length, 8192);
|
|
}
|
|
|
|
#[test]
|
|
fn a_saved_offer_round_trips_through_routstrs_model_record() {
|
|
let o = Offer {
|
|
name: "Qwen 2.5 7B".into(),
|
|
..offer()
|
|
};
|
|
let mut stored = model_body(&o, "http://192.168.88.198:8000/v1");
|
|
assert_eq!(stored["description"], "Served from 192.168.88.198");
|
|
assert_eq!(stored["pricing"]["prompt"], json!(0.2 / 1e6));
|
|
assert_eq!(stored["top_provider"]["max_completion_tokens"], 1024);
|
|
stored["sats_pricing"] = json!({"max_cost": 3.0});
|
|
assert_eq!(offer_from_db(&stored).unwrap(), o);
|
|
assert_eq!(
|
|
model_body(&o, LOCAL_SERVER_URL)["description"],
|
|
"Local llama-server model"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn buyers_see_the_id_after_the_first_slash() {
|
|
assert_eq!(public_id("meta-llama/Llama-3.1-8B"), "Llama-3.1-8B");
|
|
assert_eq!(public_id("a/b/c"), "b/c");
|
|
assert_eq!(public_id("qwen2.5:7b"), "qwen2.5:7b");
|
|
}
|
|
|
|
#[test]
|
|
fn the_exchange_rate_is_read_off_a_priced_model() {
|
|
let catalog = json!({"data": [
|
|
{"id": "free", "pricing": {"prompt": "0"}, "sats_pricing": {"prompt": 0.0}},
|
|
{"id": "archy-llama", "pricing": {"prompt": "0.000001"}, "sats_pricing": {"prompt": 0.00125}},
|
|
]});
|
|
assert_eq!(sats_per_usd(&catalog), Some(1250.0));
|
|
assert_eq!(sats_per_usd(&json!({"data": []})), None);
|
|
}
|
|
|
|
// ── Against a fake Routstr admin API and a fake LAN server ──────────
|
|
|
|
use hyper::{Body, Request, Response};
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
#[derive(Default)]
|
|
struct Fake {
|
|
/// (provider json, model rows)
|
|
providers: Vec<(Value, Vec<Value>)>,
|
|
posted_models: Vec<Value>,
|
|
}
|
|
|
|
type Shared = Arc<Mutex<Fake>>;
|
|
|
|
/// A JSON server; the handler gets method, path, body and Authorization.
|
|
async fn serve(
|
|
handler: impl Fn(&str, &str, Value, &str) -> (u16, Value) + Send + Sync + 'static,
|
|
) -> String {
|
|
let handler = Arc::new(handler);
|
|
let make = hyper::service::make_service_fn(move |_| {
|
|
let handler = handler.clone();
|
|
async move {
|
|
Ok::<_, std::convert::Infallible>(hyper::service::service_fn(
|
|
move |req: Request<Body>| {
|
|
let handler = handler.clone();
|
|
async move {
|
|
let method = req.method().to_string();
|
|
let path = req.uri().path().to_string();
|
|
let auth = req
|
|
.headers()
|
|
.get("authorization")
|
|
.and_then(|v| v.to_str().ok())
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
let bytes = hyper::body::to_bytes(req.into_body()).await.unwrap();
|
|
let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
|
|
let (status, out) = handler(&method, &path, body, &auth);
|
|
Ok::<_, std::convert::Infallible>(
|
|
Response::builder()
|
|
.status(status)
|
|
.header("Content-Type", "application/json")
|
|
.body(Body::from(out.to_string()))
|
|
.unwrap(),
|
|
)
|
|
}
|
|
},
|
|
))
|
|
}
|
|
});
|
|
let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap()).serve(make);
|
|
let url = format!("http://{}", server.local_addr());
|
|
tokio::spawn(server);
|
|
url
|
|
}
|
|
|
|
async fn fake_routstr(state: Shared) -> String {
|
|
serve(move |method, path, body, _| {
|
|
let mut st = state.lock().unwrap();
|
|
let rest = path.strip_prefix("/admin/api/upstream-providers").map(str::to_string);
|
|
match (method, path, rest.as_deref()) {
|
|
("POST", "/admin/api/login", _) => (200, json!({"token": "t"})),
|
|
("GET", "/v1/models", _) => {
|
|
let data: Vec<Value> = st.providers.iter().flat_map(|(_, m)| m.clone())
|
|
.filter(|m| m["enabled"] == json!(true))
|
|
.map(|m| json!({"id": public_id(m["id"].as_str().unwrap()),
|
|
"pricing": {"prompt": m["pricing"]["prompt"]},
|
|
"sats_pricing": {"prompt": m["pricing"]["prompt"].as_f64().unwrap() * 1250.0, "max_cost": 11.5}}))
|
|
.collect();
|
|
(200, json!({"data": data}))
|
|
}
|
|
// Like Routstr, never hand a key back.
|
|
("GET", _, Some("")) => (200, Value::Array(st.providers.iter().map(|(p, _)| {
|
|
let mut p = p.clone();
|
|
p.as_object_mut().unwrap().remove("api_key");
|
|
p
|
|
}).collect())),
|
|
("POST", _, Some("")) => {
|
|
let slug = format!("custom-{}", st.providers.len() + 1);
|
|
let mut p = body.clone();
|
|
p["slug"] = json!(slug);
|
|
st.providers.push((p.clone(), vec![]));
|
|
(200, p)
|
|
}
|
|
(_, _, Some(rest)) => {
|
|
let rest = rest.trim_start_matches('/');
|
|
let (id, tail) = rest.split_once('/').unwrap_or((rest, ""));
|
|
let Some(i) = st.providers.iter().position(|(p, _)| p["slug"] == json!(id)) else {
|
|
return (404, json!({"detail": "Provider not found"}));
|
|
};
|
|
match (method, tail) {
|
|
("GET", "models") => (200, json!({"db_models": st.providers[i].1, "remote_models": []})),
|
|
("POST", "models") => {
|
|
st.posted_models.push(body.clone());
|
|
let models = &mut st.providers[i].1;
|
|
models.retain(|m| m["id"] != body["id"]);
|
|
models.push(body);
|
|
(200, json!({}))
|
|
}
|
|
("DELETE", "models") => { st.providers[i].1.clear(); (200, json!({})) }
|
|
("DELETE", "") => { st.providers.remove(i); (200, json!({"ok": true})) }
|
|
_ => (404, json!({"detail": "no"})),
|
|
}
|
|
}
|
|
_ => (404, json!({"detail": "no"})),
|
|
}
|
|
})
|
|
.await
|
|
}
|
|
|
|
/// A llama.cpp-style LAN server that wants `Bearer sk-lan`.
|
|
async fn fake_lan_server() -> String {
|
|
serve(|_, path, _, auth| match (path, auth == "Bearer sk-lan") {
|
|
(_, false) => (401, json!({"error": "no key"})),
|
|
("/v1/models", true) => (
|
|
200,
|
|
json!({"data": [
|
|
{"id": "org/Qwen-7B", "owned_by": "llamacpp"},
|
|
{"id": "archy-llama", "owned_by": "llamacpp"},
|
|
]}),
|
|
),
|
|
("/props", true) => (
|
|
200,
|
|
json!({"default_generation_settings": {"n_ctx": 16384}}),
|
|
),
|
|
_ => (404, json!({})),
|
|
})
|
|
.await
|
|
}
|
|
|
|
async fn admin_with_local() -> (Admin, Shared, tempfile::TempDir) {
|
|
let local_model = json!({
|
|
"id": LOCAL_MODEL_ID, "name": "Qwen2.5 1.5B Instruct", "context_length": 8192,
|
|
"pricing": {"prompt": 1e-6, "completion": 2e-6},
|
|
"top_provider": {"context_length": 8192, "max_completion_tokens": 1024}, "enabled": true,
|
|
});
|
|
let state: Shared = Arc::new(Mutex::new(Fake {
|
|
providers: vec![(
|
|
json!({"id": 1, "slug": "custom", "provider_type": "custom", "base_url": LOCAL_SERVER_URL, "api_key": "llama-secret"}),
|
|
vec![local_model],
|
|
)],
|
|
..Default::default()
|
|
}));
|
|
let app = fake_routstr(state.clone()).await;
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let secret = dir.path().join("pw");
|
|
std::fs::write(&secret, "pw\n").unwrap();
|
|
(Admin::login_at(&app, &secret).await.unwrap(), state, dir)
|
|
}
|
|
|
|
fn lan_offer(model_id: &str, context: u64) -> Offer {
|
|
Offer {
|
|
model_id: model_id.into(),
|
|
name: "Qwen 7B on the Spark".into(),
|
|
context_length: context,
|
|
max_completion_tokens: 1024,
|
|
prompt_usd_per_million: 0.2,
|
|
completion_usd_per_million: 0.6,
|
|
enabled: true,
|
|
quote_sats: None,
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_lan_server_is_added_priced_and_listed_without_its_key() {
|
|
let (admin, state, _dir) = admin_with_local().await;
|
|
// The fake LAN server listens on 127.0.0.1, which operators may not
|
|
// add; register it as add_server does after checking the address.
|
|
let lan = format!("{}/v1", fake_lan_server().await);
|
|
assert!(probe(&lan, None)
|
|
.await
|
|
.unwrap_err()
|
|
.to_string()
|
|
.contains("refused the API key"));
|
|
let models = probe(&lan, Some("sk-lan")).await.unwrap();
|
|
assert_eq!(
|
|
models[0],
|
|
ServerModel {
|
|
id: "org/Qwen-7B".into(),
|
|
context_length: Some(16384)
|
|
}
|
|
);
|
|
let id = register_server(&admin, &lan, Some("sk-lan")).await.unwrap();
|
|
let key_file = admin.key_path(&id).unwrap();
|
|
assert_eq!(std::fs::read_to_string(&key_file).unwrap(), "sk-lan");
|
|
use std::os::unix::fs::PermissionsExt;
|
|
assert_eq!(
|
|
std::fs::metadata(&key_file).unwrap().permissions().mode() & 0o777,
|
|
0o600
|
|
);
|
|
|
|
// More context than the server runs with is refused.
|
|
let error = set_offers(&admin, &id, vec![lan_offer("org/Qwen-7B", 32768)])
|
|
.await
|
|
.unwrap_err();
|
|
assert!(error.to_string().contains("at most 16384"), "{error}");
|
|
// The local model's public ID is taken.
|
|
let error = set_offers(&admin, &id, vec![lan_offer("archy-llama", 8192)])
|
|
.await
|
|
.unwrap_err();
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("already sold from another server"),
|
|
"{error}"
|
|
);
|
|
|
|
set_offers(&admin, &id, vec![lan_offer("org/Qwen-7B", 8192)])
|
|
.await
|
|
.unwrap();
|
|
let posted = state.lock().unwrap().posted_models.last().cloned().unwrap();
|
|
assert_eq!(posted["pricing"]["completion"], json!(0.6 / 1e6));
|
|
assert_eq!(posted["top_provider"]["context_length"], 8192);
|
|
|
|
let listed = selling(&admin).await.unwrap();
|
|
assert_eq!(listed.servers.len(), 2);
|
|
assert!(
|
|
listed.servers[0].local,
|
|
"the local llama-server comes first"
|
|
);
|
|
assert!(
|
|
!listed.servers[0].has_key,
|
|
"llama-server's own key is not the operator's"
|
|
);
|
|
let spark = &listed.servers[1];
|
|
assert!(spark.has_key);
|
|
assert_eq!(spark.offers[0].name, "Qwen 7B on the Spark");
|
|
assert_eq!(spark.offers[0].quote_sats, Some(13));
|
|
assert_eq!(listed.sats_per_usd, Some(1250.0));
|
|
let text = serde_json::to_string(&listed).unwrap();
|
|
assert!(
|
|
!text.contains("sk-lan") && !text.contains("llama-secret"),
|
|
"a key leaked: {text}"
|
|
);
|
|
|
|
remove_server(&admin, &id).await.unwrap();
|
|
assert_eq!(selling(&admin).await.unwrap().servers.len(), 1);
|
|
assert!(!key_file.exists(), "the key outlived its server");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn the_local_server_keeps_its_context_and_cannot_be_removed() {
|
|
let (admin, state, _dir) = admin_with_local().await;
|
|
let mut offer = lan_offer(LOCAL_MODEL_ID, 65536);
|
|
offer.name = "Qwen on pa3".into();
|
|
offer.prompt_usd_per_million = 2.0;
|
|
set_offers(&admin, "custom", vec![offer]).await.unwrap();
|
|
let posted = state.lock().unwrap().posted_models.last().cloned().unwrap();
|
|
assert_eq!(posted["context_length"], 8192);
|
|
assert_eq!(posted["name"], "Qwen on pa3");
|
|
assert_eq!(posted["description"], "Local llama-server model");
|
|
let error = set_offers(&admin, "custom", vec![lan_offer("other", 8192)])
|
|
.await
|
|
.unwrap_err();
|
|
assert!(
|
|
error.to_string().contains("does not serve other"),
|
|
"{error}"
|
|
);
|
|
assert!(remove_server(&admin, "custom").await.is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn key_files_are_named_only_by_routstr_slugs() {
|
|
let admin = Admin {
|
|
client: reqwest::Client::new(),
|
|
app: String::new(),
|
|
token: String::new(),
|
|
keys_dir: "/keys".into(),
|
|
};
|
|
assert_eq!(
|
|
admin.key_path("custom-2"),
|
|
Some("/keys/custom-2.key".into())
|
|
);
|
|
for bad in ["", "../x", "a/b", "Custom", "x.y", &"a".repeat(65)] {
|
|
assert!(admin.key_path(bad).is_none(), "{bad}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unreachable_or_refusing_servers_explain_themselves() {
|
|
let error = probe("http://127.0.0.1:9/v1", None).await.unwrap_err();
|
|
assert!(
|
|
error.to_string().contains("Could not reach the server"),
|
|
"{error}"
|
|
);
|
|
let refusing = serve(|_, _, _, _| (401, json!({"error": "no key"}))).await;
|
|
let error = probe(&format!("{refusing}/v1"), None).await.unwrap_err();
|
|
assert!(error.to_string().contains("refused the API key"), "{error}");
|
|
let empty = serve(|_, _, _, _| (200, json!({"data": []}))).await;
|
|
let error = probe(&format!("{empty}/v1"), None).await.unwrap_err();
|
|
assert!(error.to_string().contains("no models"), "{error}");
|
|
}
|
|
}
|