feat: initial make-space full-stack release with weighted multi-provider LLM and Dockerfile
This commit is contained in:
commit
37b323c9ac
55 files changed
+17087
No files matched your search
@@ -0,0 +1,746 @@
|
||||
//! OpenAI-compatible chat client with vision input, strict JSON-schema output,
|
||||
//! weighted multi-provider routing (`strata` / `cpa`), and automatic failover.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use base64::Engine;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// Health probe result for a single configured LLM provider.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ProviderHealth {
|
||||
pub provider: ProviderKind,
|
||||
pub endpoint: String,
|
||||
pub model: String,
|
||||
pub weight: u32,
|
||||
pub healthy: bool,
|
||||
pub latency_ms: u64,
|
||||
pub detail: String,
|
||||
}
|
||||
|
||||
/// Identifier for the underlying LLM inference backend.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ProviderKind {
|
||||
/// Local/self-hosted Strata inference engine (Niko1221/Strata).
|
||||
Strata,
|
||||
/// CLIProxyAPI (`cpa`) OpenAI-compatible proxy.
|
||||
#[serde(alias = "cliproxy", alias = "cli-proxy")]
|
||||
Cpa,
|
||||
/// Fallback for any other custom OpenAI-compatible provider name.
|
||||
#[serde(untagged)]
|
||||
Custom(String),
|
||||
}
|
||||
|
||||
impl ProviderKind {
|
||||
pub fn as_str(&self) -> &str {
|
||||
match self {
|
||||
ProviderKind::Strata => "strata",
|
||||
ProviderKind::Cpa => "cpa",
|
||||
ProviderKind::Custom(s) => s.as_str(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ProviderKind {
|
||||
fn default() -> Self {
|
||||
ProviderKind::Cpa
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for a single weighted LLM capacity provider.
|
||||
/// Accepts both uppercase env-style keys (`PROVIDER`, `OPENAI_COMPATIBLE_API_ENDPOINT`,
|
||||
/// `OPENAI_COMPATIBLE_API_KEY`, `MODEL`, `WEIGHT`) and lowercase keys (`provider`,
|
||||
/// `endpoint`, `api_key`, `model`, `weight`).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LlmProviderConfig {
|
||||
#[serde(default, alias = "PROVIDER", alias = "provider")]
|
||||
pub provider: ProviderKind,
|
||||
#[serde(
|
||||
alias = "OPENAI_COMPATIBLE_API_ENDPOINT",
|
||||
alias = "endpoint",
|
||||
alias = "ENDPOINT",
|
||||
alias = "url"
|
||||
)]
|
||||
pub endpoint: String,
|
||||
#[serde(
|
||||
default,
|
||||
alias = "OPENAI_COMPATIBLE_API_KEY",
|
||||
alias = "api_key",
|
||||
alias = "API_KEY",
|
||||
alias = "key",
|
||||
alias = "KEY"
|
||||
)]
|
||||
pub api_key: String,
|
||||
#[serde(alias = "MODEL", alias = "LLM_MODEL", alias = "model")]
|
||||
pub model: String,
|
||||
#[serde(default = "default_weight", alias = "WEIGHT", alias = "weight")]
|
||||
pub weight: u32,
|
||||
}
|
||||
|
||||
fn default_weight() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
impl LlmProviderConfig {
|
||||
/// Display tag such as `strata/qwen3.8-flash-next` or `cpa/gpt-6.1-sol`.
|
||||
pub fn display_model(&self) -> String {
|
||||
format!("{}/{}", self.provider.as_str(), self.model)
|
||||
}
|
||||
|
||||
fn normalized(mut self) -> Self {
|
||||
let ep = self.endpoint.trim().trim_end_matches('/');
|
||||
self.endpoint = ep.strip_suffix("/v1").unwrap_or(ep).to_string();
|
||||
self.api_key = self.api_key.trim().to_string();
|
||||
self.model = self.model.trim().to_string();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Successful JSON response together with the provider/model that served it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChatJsonOutput<T> {
|
||||
pub data: T,
|
||||
pub provider: ProviderKind,
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
impl<T> ChatJsonOutput<T> {
|
||||
/// Formatted tag (`{provider}/{model}`) suitable for storing in `Layout.model`.
|
||||
pub fn model_tag(&self) -> String {
|
||||
format!("{}/{}", self.provider.as_str(), self.model)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LlmClient {
|
||||
client: reqwest::Client,
|
||||
providers: Arc<Vec<LlmProviderConfig>>,
|
||||
/// Latest known health flag per provider (`true` = healthy, initialized to `true`).
|
||||
health_flags: Arc<Vec<AtomicBool>>,
|
||||
/// Weighted schedule table of indices into `providers` (e.g. `[0, 0, 1]` for weights 2, 1).
|
||||
schedule: Arc<Vec<usize>>,
|
||||
cursor: Arc<AtomicUsize>,
|
||||
/// Primary model name kept for backwards compatibility (`{provider}/{model}`).
|
||||
pub model: String,
|
||||
}
|
||||
|
||||
/// One part of a multimodal user message.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Part {
|
||||
Text(String),
|
||||
/// Raw image bytes + mime type; sent as a base64 data URL.
|
||||
Image { bytes: Vec<u8>, mime: String },
|
||||
}
|
||||
|
||||
impl Part {
|
||||
pub fn text(s: impl Into<String>) -> Self {
|
||||
Part::Text(s.into())
|
||||
}
|
||||
|
||||
fn to_json(&self) -> Value {
|
||||
match self {
|
||||
Part::Text(t) => json!({ "type": "text", "text": t }),
|
||||
Part::Image { bytes, mime } => {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
|
||||
json!({ "type": "image_url", "image_url": { "url": format!("data:{mime};base64,{b64}") } })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone)]
|
||||
struct Message {
|
||||
role: &'static str,
|
||||
content: Value,
|
||||
}
|
||||
|
||||
impl LlmClient {
|
||||
/// Construct a single-provider client (weight = 1).
|
||||
pub fn new(endpoint: impl Into<String>, key: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self::from_providers(vec![LlmProviderConfig {
|
||||
provider: ProviderKind::Cpa,
|
||||
endpoint: endpoint.into(),
|
||||
api_key: key.into(),
|
||||
model: model.into(),
|
||||
weight: 1,
|
||||
}])
|
||||
.expect("single provider config is always non-empty")
|
||||
}
|
||||
|
||||
/// Construct a weighted multi-provider client from a list of `LlmProviderConfig`s.
|
||||
/// Providers with `weight == 0` are kept only as failover backups unless all weights are 0.
|
||||
pub fn from_providers(configs: Vec<LlmProviderConfig>) -> anyhow::Result<Self> {
|
||||
let providers: Vec<LlmProviderConfig> = configs
|
||||
.into_iter()
|
||||
.map(LlmProviderConfig::normalized)
|
||||
.filter(|p| !p.endpoint.is_empty() && !p.model.is_empty())
|
||||
.collect();
|
||||
if providers.is_empty() {
|
||||
anyhow::bail!("at least one valid LLM provider must be configured");
|
||||
}
|
||||
|
||||
let schedule = build_interleaved_schedule(&providers);
|
||||
let primary_idx = schedule.first().copied().unwrap_or(0);
|
||||
let model = providers[primary_idx].display_model();
|
||||
let health_flags = providers.iter().map(|_| AtomicBool::new(true)).collect();
|
||||
|
||||
Ok(Self {
|
||||
client: reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(10))
|
||||
.timeout(Duration::from_secs(240))
|
||||
.build()
|
||||
.expect("reqwest client"),
|
||||
providers: Arc::new(providers),
|
||||
health_flags: Arc::new(health_flags),
|
||||
schedule: Arc::new(schedule),
|
||||
cursor: Arc::new(AtomicUsize::new(0)),
|
||||
model,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse a JSON array string of `LlmProviderConfig`s.
|
||||
pub fn from_json_str(raw: &str) -> anyhow::Result<Self> {
|
||||
let configs: Vec<LlmProviderConfig> = serde_json::from_str(raw.trim())
|
||||
.map_err(|e| anyhow::anyhow!("invalid LLM_PROVIDERS JSON: {e}"))?;
|
||||
Self::from_providers(configs)
|
||||
}
|
||||
|
||||
pub fn providers(&self) -> &[LlmProviderConfig] {
|
||||
&self.providers
|
||||
}
|
||||
|
||||
/// Run provider-specific health checks across all configured LLM providers in parallel
|
||||
/// and update their internal health flags used by weighted routing:
|
||||
/// - `strata`: probes `GET {endpoint}/health` (verifies `status == "ok"` and `loaded != false`,
|
||||
/// with fallback to `GET {endpoint}/v1/models` if `/health` returns 404).
|
||||
/// - `cpa` (CLIProxyAPI) / custom: probes `GET {endpoint}/v1/models` with Bearer token
|
||||
/// (verifies HTTP 2xx and checks whether the configured model is registered).
|
||||
pub async fn health_check_all(&self) -> Vec<ProviderHealth> {
|
||||
let mut results = Vec::with_capacity(self.providers.len());
|
||||
for (idx, provider) in self.providers.iter().enumerate() {
|
||||
let status = self.check_provider_health(provider).await;
|
||||
self.health_flags[idx].store(status.healthy, Ordering::Relaxed);
|
||||
results.push(status);
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
/// Probe a single provider's health according to its `ProviderKind`.
|
||||
pub async fn check_provider_health(&self, provider: &LlmProviderConfig) -> ProviderHealth {
|
||||
let started = Instant::now();
|
||||
let probe_res = match provider.provider {
|
||||
ProviderKind::Strata => self.probe_strata_health(provider).await,
|
||||
ProviderKind::Cpa | ProviderKind::Custom(_) => self.probe_cpa_health(provider).await,
|
||||
};
|
||||
let latency_ms = started.elapsed().as_millis() as u64;
|
||||
match probe_res {
|
||||
Ok(detail) => ProviderHealth {
|
||||
provider: provider.provider.clone(),
|
||||
endpoint: provider.endpoint.clone(),
|
||||
model: provider.model.clone(),
|
||||
weight: provider.weight,
|
||||
healthy: true,
|
||||
latency_ms,
|
||||
detail,
|
||||
},
|
||||
Err(err) => ProviderHealth {
|
||||
provider: provider.provider.clone(),
|
||||
endpoint: provider.endpoint.clone(),
|
||||
model: provider.model.clone(),
|
||||
weight: provider.weight,
|
||||
healthy: false,
|
||||
latency_ms,
|
||||
detail: err.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Strata health check:
|
||||
/// 1. `GET {endpoint}/health` (Strata native endpoint returning `{"status":"ok","loaded":true,"model":...,"images":true}`)
|
||||
/// 2. Falls back to `/v1/models` if `/health` is not exposed (404).
|
||||
async fn probe_strata_health(&self, provider: &LlmProviderConfig) -> anyhow::Result<String> {
|
||||
let url = format!("{}/health", provider.endpoint);
|
||||
let mut req = self.client.get(&url).timeout(Duration::from_secs(5));
|
||||
if !provider.api_key.is_empty() {
|
||||
req = req.bearer_auth(&provider.api_key);
|
||||
}
|
||||
let resp = req.send().await.map_err(|e| anyhow::anyhow!("Strata /health unreachable: {e}"))?;
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::NOT_FOUND {
|
||||
return self.probe_cpa_health(provider).await;
|
||||
}
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
if !status.is_success() {
|
||||
anyhow::bail!("Strata /health HTTP {status}: {}", truncate(&text, 200));
|
||||
}
|
||||
if let Ok(v) = serde_json::from_str::<Value>(&text) {
|
||||
if let Some(st) = v["status"].as_str() {
|
||||
if !st.eq_ignore_ascii_case("ok") && !st.eq_ignore_ascii_case("healthy") && !st.eq_ignore_ascii_case("ready") {
|
||||
anyhow::bail!("Strata status is '{st}': {}", truncate(&text, 200));
|
||||
}
|
||||
}
|
||||
if v["loaded"].as_bool() == Some(false) {
|
||||
anyhow::bail!("Strata engine running but model not loaded yet (loaded=false)");
|
||||
}
|
||||
let engine_model = v["model"].as_str().unwrap_or(&provider.model);
|
||||
let vision = v["images"].as_bool().map(|b| format!(", images={b}")).unwrap_or_default();
|
||||
return Ok(format!("Strata ready (model={engine_model}{vision})"));
|
||||
}
|
||||
Ok("Strata /health ok".into())
|
||||
}
|
||||
|
||||
/// CLIProxyAPI (`cpa`) health check:
|
||||
/// Queries `GET {endpoint}/v1/models` with Bearer auth and verifies the model list response.
|
||||
async fn probe_cpa_health(&self, provider: &LlmProviderConfig) -> anyhow::Result<String> {
|
||||
let url = format!("{}/v1/models", provider.endpoint);
|
||||
let mut req = self.client.get(&url).timeout(Duration::from_secs(5));
|
||||
if !provider.api_key.is_empty() {
|
||||
req = req.bearer_auth(&provider.api_key);
|
||||
}
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
anyhow::anyhow!("{} /v1/models unreachable: {e}", provider.provider.as_str())
|
||||
})?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
if !status.is_success() {
|
||||
anyhow::bail!(
|
||||
"{} /v1/models HTTP {status}: {}",
|
||||
provider.provider.as_str(),
|
||||
truncate(&text, 200)
|
||||
);
|
||||
}
|
||||
if let Ok(v) = serde_json::from_str::<Value>(&text) {
|
||||
if let Some(arr) = v["data"].as_array() {
|
||||
let ids: Vec<&str> = arr.iter().filter_map(|m| m["id"].as_str()).collect();
|
||||
if !ids.is_empty() && !ids.iter().any(|&id| id == provider.model) {
|
||||
return Ok(format!(
|
||||
"/v1/models ok ({} models online; '{}' not listed in registry, will route directly)",
|
||||
ids.len(),
|
||||
provider.model
|
||||
));
|
||||
}
|
||||
return Ok(format!("/v1/models ok ({} models, '{}' available)", ids.len(), provider.model));
|
||||
}
|
||||
}
|
||||
Ok("/v1/models ok".into())
|
||||
}
|
||||
|
||||
/// Pick the ordered list of provider indices to try for the next request:
|
||||
/// starts with the weighted round-robin winner among currently healthy providers,
|
||||
/// followed by remaining healthy providers, and finally any unhealthy providers as last-resort failover.
|
||||
fn pick_provider_order(&self) -> Vec<usize> {
|
||||
let step = self.cursor.fetch_add(1, Ordering::Relaxed);
|
||||
let mut healthy_order = Vec::with_capacity(self.providers.len());
|
||||
let mut unhealthy_order = Vec::new();
|
||||
|
||||
// Walk the weighted schedule starting from `step` so the 2:1 ratio is preserved.
|
||||
for offset in 0..self.schedule.len() {
|
||||
let idx = self.schedule[(step + offset) % self.schedule.len()];
|
||||
if self.health_flags[idx].load(Ordering::Relaxed) {
|
||||
if !healthy_order.contains(&idx) {
|
||||
healthy_order.push(idx);
|
||||
}
|
||||
} else if !unhealthy_order.contains(&idx) {
|
||||
unhealthy_order.push(idx);
|
||||
}
|
||||
}
|
||||
// Include any 0-weight backup providers not present in `schedule`.
|
||||
for idx in 0..self.providers.len() {
|
||||
if self.health_flags[idx].load(Ordering::Relaxed) {
|
||||
if !healthy_order.contains(&idx) {
|
||||
healthy_order.push(idx);
|
||||
}
|
||||
} else if !unhealthy_order.contains(&idx) {
|
||||
unhealthy_order.push(idx);
|
||||
}
|
||||
}
|
||||
healthy_order.extend(unhealthy_order);
|
||||
healthy_order
|
||||
}
|
||||
|
||||
/// Send `system` + multimodal `user` parts and parse the reply into `T`
|
||||
/// using a strict JSON schema.
|
||||
pub async fn chat_json<T: DeserializeOwned>(
|
||||
&self,
|
||||
system: &str,
|
||||
user: &[Part],
|
||||
schema_name: &str,
|
||||
schema: &Value,
|
||||
) -> anyhow::Result<T> {
|
||||
Ok(self
|
||||
.chat_json_with_meta(system, user, schema_name, schema)
|
||||
.await?
|
||||
.data)
|
||||
}
|
||||
|
||||
/// Same as `chat_json`, but also returns which provider (`strata` / `cpa`) and model served the request.
|
||||
/// Automatically fails over to the next configured provider if the chosen provider fails.
|
||||
pub async fn chat_json_with_meta<T: DeserializeOwned>(
|
||||
&self,
|
||||
system: &str,
|
||||
user: &[Part],
|
||||
schema_name: &str,
|
||||
schema: &Value,
|
||||
) -> anyhow::Result<ChatJsonOutput<T>> {
|
||||
let order = self.pick_provider_order();
|
||||
let mut last_err: Option<anyhow::Error> = None;
|
||||
|
||||
for (attempt_idx, &prov_idx) in order.iter().enumerate() {
|
||||
let provider = &self.providers[prov_idx];
|
||||
match self
|
||||
.chat_json_on_provider::<T>(provider, system, user, schema_name, schema)
|
||||
.await
|
||||
{
|
||||
Ok(data) => {
|
||||
self.health_flags[prov_idx].store(true, Ordering::Relaxed);
|
||||
return Ok(ChatJsonOutput {
|
||||
data,
|
||||
provider: provider.provider.clone(),
|
||||
model: provider.model.clone(),
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
self.health_flags[prov_idx].store(false, Ordering::Relaxed);
|
||||
if attempt_idx + 1 < order.len() {
|
||||
let next_prov = &self.providers[order[attempt_idx + 1]];
|
||||
tracing::warn!(
|
||||
failed_provider = provider.provider.as_str(),
|
||||
failed_model = %provider.model,
|
||||
failover_provider = next_prov.provider.as_str(),
|
||||
failover_model = %next_prov.model,
|
||||
error = %err,
|
||||
"LLM provider call failed, failing over to next provider"
|
||||
);
|
||||
}
|
||||
last_err = Some(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_err.unwrap_or_else(|| anyhow::anyhow!("no LLM providers available")))
|
||||
}
|
||||
|
||||
async fn chat_json_on_provider<T: DeserializeOwned>(
|
||||
&self,
|
||||
provider: &LlmProviderConfig,
|
||||
system: &str,
|
||||
user: &[Part],
|
||||
schema_name: &str,
|
||||
schema: &Value,
|
||||
) -> anyhow::Result<T> {
|
||||
let messages = vec![
|
||||
Message {
|
||||
role: "system",
|
||||
content: Value::String(system.to_string()),
|
||||
},
|
||||
Message {
|
||||
role: "user",
|
||||
content: Value::Array(user.iter().map(Part::to_json).collect()),
|
||||
},
|
||||
];
|
||||
let strict = json!({
|
||||
"type": "json_schema",
|
||||
"json_schema": { "name": schema_name, "strict": true, "schema": schema }
|
||||
});
|
||||
let raw = match self.complete(provider, &messages, Some(strict)).await {
|
||||
Ok(r) => r,
|
||||
Err(e) if should_fallback_response_format(&e, &provider.provider) => {
|
||||
tracing::warn!(
|
||||
provider = provider.provider.as_str(),
|
||||
model = %provider.model,
|
||||
"json_schema unsupported, falling back to json_object / prompt schema: {e}"
|
||||
);
|
||||
let mut msgs = messages;
|
||||
msgs[0].content = Value::String(format!(
|
||||
"{system}\n\nRespond ONLY with valid JSON matching this schema (no markdown fences, no extra prose):\n{}",
|
||||
serde_json::to_string(schema)?
|
||||
));
|
||||
match self
|
||||
.complete(provider, &msgs, Some(json!({ "type": "json_object" })))
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e2) if should_fallback_response_format(&e2, &provider.provider) => {
|
||||
self.complete(provider, &msgs, None).await?
|
||||
}
|
||||
Err(e2) => return Err(e2),
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let cleaned = strip_fences(&raw);
|
||||
serde_json::from_str(cleaned).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"LLM JSON parse error ({}/{}): {e}; raw: {}",
|
||||
provider.provider.as_str(),
|
||||
provider.model,
|
||||
truncate(&raw, 500)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
provider: &LlmProviderConfig,
|
||||
messages: &[Message],
|
||||
response_format: Option<Value>,
|
||||
) -> anyhow::Result<String> {
|
||||
let mut body = json!({
|
||||
"model": provider.model,
|
||||
"messages": messages,
|
||||
});
|
||||
if let Some(rf) = response_format {
|
||||
body["response_format"] = rf;
|
||||
}
|
||||
let started = std::time::Instant::now();
|
||||
let mut req = self
|
||||
.client
|
||||
.post(format!("{}/v1/chat/completions", provider.endpoint))
|
||||
.json(&body);
|
||||
if !provider.api_key.is_empty() {
|
||||
req = req.bearer_auth(&provider.api_key);
|
||||
}
|
||||
let resp = req.send().await?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await?;
|
||||
if !status.is_success() {
|
||||
anyhow::bail!(
|
||||
"LLM HTTP {status} ({}/{}): {}",
|
||||
provider.provider.as_str(),
|
||||
provider.model,
|
||||
truncate(&text, 800)
|
||||
);
|
||||
}
|
||||
let v: Value = serde_json::from_str(&text)?;
|
||||
let raw_content = v["choices"][0]["message"]["content"]
|
||||
.as_str()
|
||||
.ok_or_else(|| {
|
||||
anyhow::anyhow!(
|
||||
"LLM response without content ({}/{}): {}",
|
||||
provider.provider.as_str(),
|
||||
provider.model,
|
||||
truncate(&text, 500)
|
||||
)
|
||||
})?;
|
||||
let content = strip_thinking_blocks(raw_content).to_string();
|
||||
tracing::debug!(
|
||||
provider = provider.provider.as_str(),
|
||||
model = %provider.model,
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
prompt_tokens = v["usage"]["prompt_tokens"].as_u64(),
|
||||
completion_tokens = v["usage"]["completion_tokens"].as_u64(),
|
||||
"llm completion"
|
||||
);
|
||||
Ok(content)
|
||||
}
|
||||
}
|
||||
|
||||
fn should_fallback_response_format(err: &anyhow::Error, provider: &ProviderKind) -> bool {
|
||||
let msg = err.to_string().to_lowercase();
|
||||
msg.contains("response_format")
|
||||
|| msg.contains("json_schema")
|
||||
|| msg.contains("structured")
|
||||
|| (matches!(provider, ProviderKind::Strata)
|
||||
&& (msg.contains("http 400") || msg.contains("http 422") || msg.contains("unsupported")))
|
||||
}
|
||||
|
||||
/// Build a smooth interleaved weighted schedule (Smooth Weighted Round-Robin)
|
||||
/// so e.g. weights `[2, 1]` produce `[0, 0, 1]` and `[4, 2]` produce `[0, 0, 1]` (reduced by GCD).
|
||||
fn build_interleaved_schedule(providers: &[LlmProviderConfig]) -> Vec<usize> {
|
||||
let mut weights: Vec<u32> = providers.iter().map(|p| p.weight.min(100)).collect();
|
||||
let total: u32 = weights.iter().sum();
|
||||
if total == 0 {
|
||||
return (0..providers.len()).collect();
|
||||
}
|
||||
let g = weights.iter().copied().fold(0u32, gcd);
|
||||
if g > 1 {
|
||||
for w in &mut weights {
|
||||
*w /= g;
|
||||
}
|
||||
}
|
||||
let total_steps: usize = weights.iter().map(|&w| w as usize).sum();
|
||||
let total_w = total_steps as i64;
|
||||
let mut current = vec![0i64; providers.len()];
|
||||
let mut schedule = Vec::with_capacity(total_steps);
|
||||
|
||||
for _ in 0..total_steps {
|
||||
let mut best_idx = 0;
|
||||
for (i, &w) in weights.iter().enumerate() {
|
||||
if w == 0 {
|
||||
continue;
|
||||
}
|
||||
current[i] += w as i64;
|
||||
if weights[best_idx] == 0 || current[i] > current[best_idx] {
|
||||
best_idx = i;
|
||||
}
|
||||
}
|
||||
current[best_idx] -= total_w;
|
||||
schedule.push(best_idx);
|
||||
}
|
||||
schedule
|
||||
}
|
||||
|
||||
fn gcd(a: u32, b: u32) -> u32 {
|
||||
if b == 0 {
|
||||
a
|
||||
} else {
|
||||
gcd(b, a % b)
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip `<think>...</think>` blocks that local reasoning/MoE models (e.g. Qwen3.8 on Strata) may emit.
|
||||
fn strip_thinking_blocks(s: &str) -> &str {
|
||||
let mut rest = s.trim();
|
||||
while let Some(start) = rest.find("<think>") {
|
||||
if start == 0 {
|
||||
if let Some(end) = rest.find("</think>") {
|
||||
rest = rest[end + "</think>".len()..].trim();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
rest
|
||||
}
|
||||
|
||||
fn strip_fences(s: &str) -> &str {
|
||||
let t = strip_thinking_blocks(s);
|
||||
let t = t.strip_prefix("```json").or_else(|| t.strip_prefix("```")).unwrap_or(t);
|
||||
t.strip_suffix("```").unwrap_or(t).trim()
|
||||
}
|
||||
|
||||
fn truncate(s: &str, n: usize) -> String {
|
||||
if s.chars().count() <= n {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{}…", s.chars().take(n).collect::<String>())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fences_and_thinking() {
|
||||
assert_eq!(super::strip_fences("```json\n{\"a\":1}\n```"), "{\"a\":1}");
|
||||
assert_eq!(super::strip_fences("{\"a\":1}"), "{\"a\":1}");
|
||||
assert_eq!(
|
||||
super::strip_fences("<think>\nreasoning...\n</think>\n```json\n{\"a\":1}\n```"),
|
||||
"{\"a\":1}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_uppercase_and_lowercase_provider_json() {
|
||||
let raw = r#"[
|
||||
{
|
||||
"PROVIDER": "strata",
|
||||
"OPENAI_COMPATIBLE_API_ENDPOINT": "http://127.0.0.1:8080/v1/",
|
||||
"OPENAI_COMPATIBLE_API_KEY": "",
|
||||
"MODEL": "qwen3.8-flash-next",
|
||||
"WEIGHT": 2
|
||||
},
|
||||
{
|
||||
"provider": "cpa",
|
||||
"endpoint": "http://126.dahyeon.us:8317",
|
||||
"api_key": "secret-jwt",
|
||||
"model": "gpt-6.1-sol",
|
||||
"weight": 1
|
||||
}
|
||||
]"#;
|
||||
let client = LlmClient::from_json_str(raw).expect("valid json");
|
||||
assert_eq!(client.providers().len(), 2);
|
||||
assert_eq!(client.providers()[0].provider, ProviderKind::Strata);
|
||||
assert_eq!(client.providers()[0].endpoint, "http://127.0.0.1:8080");
|
||||
assert_eq!(client.providers()[0].weight, 2);
|
||||
assert_eq!(client.providers()[1].provider, ProviderKind::Cpa);
|
||||
assert_eq!(client.providers()[1].weight, 1);
|
||||
|
||||
// Weighted 2:1 schedule should pick provider 0 twice and provider 1 once every 3 calls.
|
||||
let picks: Vec<usize> = (0..6)
|
||||
.map(|_| client.pick_provider_order()[0])
|
||||
.collect();
|
||||
let count_0 = picks.iter().filter(|&&i| i == 0).count();
|
||||
let count_1 = picks.iter().filter(|&&i| i == 1).count();
|
||||
assert_eq!(count_0, 4);
|
||||
assert_eq!(count_1, 2);
|
||||
|
||||
// If provider 0 (strata) is marked unhealthy, pick_provider_order should promote provider 1 (cpa) first
|
||||
// while keeping provider 0 at the end as last-resort failover.
|
||||
client.health_flags[0].store(false, Ordering::Relaxed);
|
||||
let failover_order = client.pick_provider_order();
|
||||
assert_eq!(failover_order, vec![1, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn live_env_providers_health_and_routing() -> anyhow::Result<()> {
|
||||
let env_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.env");
|
||||
let content = std::fs::read_to_string(&env_path)?;
|
||||
let mut raw_providers = String::new();
|
||||
for line in content.lines() {
|
||||
let line = line.trim();
|
||||
if let Some(rest) = line.strip_prefix("LLM_PROVIDERS=") {
|
||||
raw_providers = rest.trim_matches('\'').trim_matches('"').to_string();
|
||||
}
|
||||
}
|
||||
assert!(!raw_providers.is_empty(), "LLM_PROVIDERS not found in .env");
|
||||
let client = LlmClient::from_json_str(&raw_providers)?;
|
||||
println!("Loaded {} providers from .env", client.providers().len());
|
||||
|
||||
let health = client.health_check_all().await;
|
||||
for h in &health {
|
||||
println!(
|
||||
"[HEALTH] provider={} endpoint={} model={} weight={} healthy={} latency={}ms detail={}",
|
||||
h.provider.as_str(),
|
||||
h.endpoint,
|
||||
h.model,
|
||||
h.weight,
|
||||
h.healthy,
|
||||
h.latency_ms,
|
||||
h.detail
|
||||
);
|
||||
}
|
||||
assert!(health.iter().all(|h| h.healthy), "Expected all providers to be healthy");
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct PingResp {
|
||||
ok: bool,
|
||||
provider_echo: String,
|
||||
}
|
||||
let schema = serde_json::json!({
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["ok", "provider_echo"],
|
||||
"properties": {
|
||||
"ok": { "type": "boolean" },
|
||||
"provider_echo": { "type": "string" }
|
||||
}
|
||||
});
|
||||
|
||||
for i in 1..=3 {
|
||||
let out: ChatJsonOutput<PingResp> = client
|
||||
.chat_json_with_meta(
|
||||
"Respond with JSON matching the schema. Set ok=true and provider_echo='pong'.",
|
||||
&[Part::text("ping")],
|
||||
"ping_check",
|
||||
&schema,
|
||||
)
|
||||
.await?;
|
||||
println!(
|
||||
"[CALL #{i}] served_by={} ({}) -> ok={}, echo={}",
|
||||
out.provider.as_str(),
|
||||
out.model,
|
||||
out.data.ok,
|
||||
out.data.provider_echo
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user