feat: initial make-space full-stack release with weighted multi-provider LLM and Dockerfile

This commit is contained in:
dahyeon committed 2026-11-04 20:49:24 +09:00
commit 37b323c9ac
55 files changed
+17087

No files matched your search

+746
View File
@@ -0,0 +1,746 @@
//! OpenAI-compatible chat client with vision input, strict JSON-schema output,
//! weighted multi-provider routing (`strata` / `cpa`), and automatic failover.
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use base64::Engine;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
/// Health probe result for a single configured LLM provider.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProviderHealth {
pub provider: ProviderKind,
pub endpoint: String,
pub model: String,
pub weight: u32,
pub healthy: bool,
pub latency_ms: u64,
pub detail: String,
}
/// Identifier for the underlying LLM inference backend.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ProviderKind {
/// Local/self-hosted Strata inference engine (Niko1221/Strata).
Strata,
/// CLIProxyAPI (`cpa`) OpenAI-compatible proxy.
#[serde(alias = "cliproxy", alias = "cli-proxy")]
Cpa,
/// Fallback for any other custom OpenAI-compatible provider name.
#[serde(untagged)]
Custom(String),
}
impl ProviderKind {
pub fn as_str(&self) -> &str {
match self {
ProviderKind::Strata => "strata",
ProviderKind::Cpa => "cpa",
ProviderKind::Custom(s) => s.as_str(),
}
}
}
impl Default for ProviderKind {
fn default() -> Self {
ProviderKind::Cpa
}
}
/// Configuration for a single weighted LLM capacity provider.
/// Accepts both uppercase env-style keys (`PROVIDER`, `OPENAI_COMPATIBLE_API_ENDPOINT`,
/// `OPENAI_COMPATIBLE_API_KEY`, `MODEL`, `WEIGHT`) and lowercase keys (`provider`,
/// `endpoint`, `api_key`, `model`, `weight`).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LlmProviderConfig {
#[serde(default, alias = "PROVIDER", alias = "provider")]
pub provider: ProviderKind,
#[serde(
alias = "OPENAI_COMPATIBLE_API_ENDPOINT",
alias = "endpoint",
alias = "ENDPOINT",
alias = "url"
)]
pub endpoint: String,
#[serde(
default,
alias = "OPENAI_COMPATIBLE_API_KEY",
alias = "api_key",
alias = "API_KEY",
alias = "key",
alias = "KEY"
)]
pub api_key: String,
#[serde(alias = "MODEL", alias = "LLM_MODEL", alias = "model")]
pub model: String,
#[serde(default = "default_weight", alias = "WEIGHT", alias = "weight")]
pub weight: u32,
}
fn default_weight() -> u32 {
1
}
impl LlmProviderConfig {
/// Display tag such as `strata/qwen3.8-flash-next` or `cpa/gpt-6.1-sol`.
pub fn display_model(&self) -> String {
format!("{}/{}", self.provider.as_str(), self.model)
}
fn normalized(mut self) -> Self {
let ep = self.endpoint.trim().trim_end_matches('/');
self.endpoint = ep.strip_suffix("/v1").unwrap_or(ep).to_string();
self.api_key = self.api_key.trim().to_string();
self.model = self.model.trim().to_string();
self
}
}
/// Successful JSON response together with the provider/model that served it.
#[derive(Debug, Clone)]
pub struct ChatJsonOutput<T> {
pub data: T,
pub provider: ProviderKind,
pub model: String,
}
impl<T> ChatJsonOutput<T> {
/// Formatted tag (`{provider}/{model}`) suitable for storing in `Layout.model`.
pub fn model_tag(&self) -> String {
format!("{}/{}", self.provider.as_str(), self.model)
}
}
#[derive(Clone)]
pub struct LlmClient {
client: reqwest::Client,
providers: Arc<Vec<LlmProviderConfig>>,
/// Latest known health flag per provider (`true` = healthy, initialized to `true`).
health_flags: Arc<Vec<AtomicBool>>,
/// Weighted schedule table of indices into `providers` (e.g. `[0, 0, 1]` for weights 2, 1).
schedule: Arc<Vec<usize>>,
cursor: Arc<AtomicUsize>,
/// Primary model name kept for backwards compatibility (`{provider}/{model}`).
pub model: String,
}
/// One part of a multimodal user message.
#[derive(Debug, Clone)]
pub enum Part {
Text(String),
/// Raw image bytes + mime type; sent as a base64 data URL.
Image { bytes: Vec<u8>, mime: String },
}
impl Part {
pub fn text(s: impl Into<String>) -> Self {
Part::Text(s.into())
}
fn to_json(&self) -> Value {
match self {
Part::Text(t) => json!({ "type": "text", "text": t }),
Part::Image { bytes, mime } => {
let b64 = base64::engine::general_purpose::STANDARD.encode(bytes);
json!({ "type": "image_url", "image_url": { "url": format!("data:{mime};base64,{b64}") } })
}
}
}
}
#[derive(Debug, Serialize, Clone)]
struct Message {
role: &'static str,
content: Value,
}
impl LlmClient {
/// Construct a single-provider client (weight = 1).
pub fn new(endpoint: impl Into<String>, key: impl Into<String>, model: impl Into<String>) -> Self {
Self::from_providers(vec![LlmProviderConfig {
provider: ProviderKind::Cpa,
endpoint: endpoint.into(),
api_key: key.into(),
model: model.into(),
weight: 1,
}])
.expect("single provider config is always non-empty")
}
/// Construct a weighted multi-provider client from a list of `LlmProviderConfig`s.
/// Providers with `weight == 0` are kept only as failover backups unless all weights are 0.
pub fn from_providers(configs: Vec<LlmProviderConfig>) -> anyhow::Result<Self> {
let providers: Vec<LlmProviderConfig> = configs
.into_iter()
.map(LlmProviderConfig::normalized)
.filter(|p| !p.endpoint.is_empty() && !p.model.is_empty())
.collect();
if providers.is_empty() {
anyhow::bail!("at least one valid LLM provider must be configured");
}
let schedule = build_interleaved_schedule(&providers);
let primary_idx = schedule.first().copied().unwrap_or(0);
let model = providers[primary_idx].display_model();
let health_flags = providers.iter().map(|_| AtomicBool::new(true)).collect();
Ok(Self {
client: reqwest::Client::builder()
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(240))
.build()
.expect("reqwest client"),
providers: Arc::new(providers),
health_flags: Arc::new(health_flags),
schedule: Arc::new(schedule),
cursor: Arc::new(AtomicUsize::new(0)),
model,
})
}
/// Parse a JSON array string of `LlmProviderConfig`s.
pub fn from_json_str(raw: &str) -> anyhow::Result<Self> {
let configs: Vec<LlmProviderConfig> = serde_json::from_str(raw.trim())
.map_err(|e| anyhow::anyhow!("invalid LLM_PROVIDERS JSON: {e}"))?;
Self::from_providers(configs)
}
pub fn providers(&self) -> &[LlmProviderConfig] {
&self.providers
}
/// Run provider-specific health checks across all configured LLM providers in parallel
/// and update their internal health flags used by weighted routing:
/// - `strata`: probes `GET {endpoint}/health` (verifies `status == "ok"` and `loaded != false`,
/// with fallback to `GET {endpoint}/v1/models` if `/health` returns 404).
/// - `cpa` (CLIProxyAPI) / custom: probes `GET {endpoint}/v1/models` with Bearer token
/// (verifies HTTP 2xx and checks whether the configured model is registered).
pub async fn health_check_all(&self) -> Vec<ProviderHealth> {
let mut results = Vec::with_capacity(self.providers.len());
for (idx, provider) in self.providers.iter().enumerate() {
let status = self.check_provider_health(provider).await;
self.health_flags[idx].store(status.healthy, Ordering::Relaxed);
results.push(status);
}
results
}
/// Probe a single provider's health according to its `ProviderKind`.
pub async fn check_provider_health(&self, provider: &LlmProviderConfig) -> ProviderHealth {
let started = Instant::now();
let probe_res = match provider.provider {
ProviderKind::Strata => self.probe_strata_health(provider).await,
ProviderKind::Cpa | ProviderKind::Custom(_) => self.probe_cpa_health(provider).await,
};
let latency_ms = started.elapsed().as_millis() as u64;
match probe_res {
Ok(detail) => ProviderHealth {
provider: provider.provider.clone(),
endpoint: provider.endpoint.clone(),
model: provider.model.clone(),
weight: provider.weight,
healthy: true,
latency_ms,
detail,
},
Err(err) => ProviderHealth {
provider: provider.provider.clone(),
endpoint: provider.endpoint.clone(),
model: provider.model.clone(),
weight: provider.weight,
healthy: false,
latency_ms,
detail: err.to_string(),
},
}
}
/// Strata health check:
/// 1. `GET {endpoint}/health` (Strata native endpoint returning `{"status":"ok","loaded":true,"model":...,"images":true}`)
/// 2. Falls back to `/v1/models` if `/health` is not exposed (404).
async fn probe_strata_health(&self, provider: &LlmProviderConfig) -> anyhow::Result<String> {
let url = format!("{}/health", provider.endpoint);
let mut req = self.client.get(&url).timeout(Duration::from_secs(5));
if !provider.api_key.is_empty() {
req = req.bearer_auth(&provider.api_key);
}
let resp = req.send().await.map_err(|e| anyhow::anyhow!("Strata /health unreachable: {e}"))?;
let status = resp.status();
if status == reqwest::StatusCode::NOT_FOUND {
return self.probe_cpa_health(provider).await;
}
let text = resp.text().await.unwrap_or_default();
if !status.is_success() {
anyhow::bail!("Strata /health HTTP {status}: {}", truncate(&text, 200));
}
if let Ok(v) = serde_json::from_str::<Value>(&text) {
if let Some(st) = v["status"].as_str() {
if !st.eq_ignore_ascii_case("ok") && !st.eq_ignore_ascii_case("healthy") && !st.eq_ignore_ascii_case("ready") {
anyhow::bail!("Strata status is '{st}': {}", truncate(&text, 200));
}
}
if v["loaded"].as_bool() == Some(false) {
anyhow::bail!("Strata engine running but model not loaded yet (loaded=false)");
}
let engine_model = v["model"].as_str().unwrap_or(&provider.model);
let vision = v["images"].as_bool().map(|b| format!(", images={b}")).unwrap_or_default();
return Ok(format!("Strata ready (model={engine_model}{vision})"));
}
Ok("Strata /health ok".into())
}
/// CLIProxyAPI (`cpa`) health check:
/// Queries `GET {endpoint}/v1/models` with Bearer auth and verifies the model list response.
async fn probe_cpa_health(&self, provider: &LlmProviderConfig) -> anyhow::Result<String> {
let url = format!("{}/v1/models", provider.endpoint);
let mut req = self.client.get(&url).timeout(Duration::from_secs(5));
if !provider.api_key.is_empty() {
req = req.bearer_auth(&provider.api_key);
}
let resp = req.send().await.map_err(|e| {
anyhow::anyhow!("{} /v1/models unreachable: {e}", provider.provider.as_str())
})?;
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
if !status.is_success() {
anyhow::bail!(
"{} /v1/models HTTP {status}: {}",
provider.provider.as_str(),
truncate(&text, 200)
);
}
if let Ok(v) = serde_json::from_str::<Value>(&text) {
if let Some(arr) = v["data"].as_array() {
let ids: Vec<&str> = arr.iter().filter_map(|m| m["id"].as_str()).collect();
if !ids.is_empty() && !ids.iter().any(|&id| id == provider.model) {
return Ok(format!(
"/v1/models ok ({} models online; '{}' not listed in registry, will route directly)",
ids.len(),
provider.model
));
}
return Ok(format!("/v1/models ok ({} models, '{}' available)", ids.len(), provider.model));
}
}
Ok("/v1/models ok".into())
}
/// Pick the ordered list of provider indices to try for the next request:
/// starts with the weighted round-robin winner among currently healthy providers,
/// followed by remaining healthy providers, and finally any unhealthy providers as last-resort failover.
fn pick_provider_order(&self) -> Vec<usize> {
let step = self.cursor.fetch_add(1, Ordering::Relaxed);
let mut healthy_order = Vec::with_capacity(self.providers.len());
let mut unhealthy_order = Vec::new();
// Walk the weighted schedule starting from `step` so the 2:1 ratio is preserved.
for offset in 0..self.schedule.len() {
let idx = self.schedule[(step + offset) % self.schedule.len()];
if self.health_flags[idx].load(Ordering::Relaxed) {
if !healthy_order.contains(&idx) {
healthy_order.push(idx);
}
} else if !unhealthy_order.contains(&idx) {
unhealthy_order.push(idx);
}
}
// Include any 0-weight backup providers not present in `schedule`.
for idx in 0..self.providers.len() {
if self.health_flags[idx].load(Ordering::Relaxed) {
if !healthy_order.contains(&idx) {
healthy_order.push(idx);
}
} else if !unhealthy_order.contains(&idx) {
unhealthy_order.push(idx);
}
}
healthy_order.extend(unhealthy_order);
healthy_order
}
/// Send `system` + multimodal `user` parts and parse the reply into `T`
/// using a strict JSON schema.
pub async fn chat_json<T: DeserializeOwned>(
&self,
system: &str,
user: &[Part],
schema_name: &str,
schema: &Value,
) -> anyhow::Result<T> {
Ok(self
.chat_json_with_meta(system, user, schema_name, schema)
.await?
.data)
}
/// Same as `chat_json`, but also returns which provider (`strata` / `cpa`) and model served the request.
/// Automatically fails over to the next configured provider if the chosen provider fails.
pub async fn chat_json_with_meta<T: DeserializeOwned>(
&self,
system: &str,
user: &[Part],
schema_name: &str,
schema: &Value,
) -> anyhow::Result<ChatJsonOutput<T>> {
let order = self.pick_provider_order();
let mut last_err: Option<anyhow::Error> = None;
for (attempt_idx, &prov_idx) in order.iter().enumerate() {
let provider = &self.providers[prov_idx];
match self
.chat_json_on_provider::<T>(provider, system, user, schema_name, schema)
.await
{
Ok(data) => {
self.health_flags[prov_idx].store(true, Ordering::Relaxed);
return Ok(ChatJsonOutput {
data,
provider: provider.provider.clone(),
model: provider.model.clone(),
});
}
Err(err) => {
self.health_flags[prov_idx].store(false, Ordering::Relaxed);
if attempt_idx + 1 < order.len() {
let next_prov = &self.providers[order[attempt_idx + 1]];
tracing::warn!(
failed_provider = provider.provider.as_str(),
failed_model = %provider.model,
failover_provider = next_prov.provider.as_str(),
failover_model = %next_prov.model,
error = %err,
"LLM provider call failed, failing over to next provider"
);
}
last_err = Some(err);
}
}
}
Err(last_err.unwrap_or_else(|| anyhow::anyhow!("no LLM providers available")))
}
async fn chat_json_on_provider<T: DeserializeOwned>(
&self,
provider: &LlmProviderConfig,
system: &str,
user: &[Part],
schema_name: &str,
schema: &Value,
) -> anyhow::Result<T> {
let messages = vec![
Message {
role: "system",
content: Value::String(system.to_string()),
},
Message {
role: "user",
content: Value::Array(user.iter().map(Part::to_json).collect()),
},
];
let strict = json!({
"type": "json_schema",
"json_schema": { "name": schema_name, "strict": true, "schema": schema }
});
let raw = match self.complete(provider, &messages, Some(strict)).await {
Ok(r) => r,
Err(e) if should_fallback_response_format(&e, &provider.provider) => {
tracing::warn!(
provider = provider.provider.as_str(),
model = %provider.model,
"json_schema unsupported, falling back to json_object / prompt schema: {e}"
);
let mut msgs = messages;
msgs[0].content = Value::String(format!(
"{system}\n\nRespond ONLY with valid JSON matching this schema (no markdown fences, no extra prose):\n{}",
serde_json::to_string(schema)?
));
match self
.complete(provider, &msgs, Some(json!({ "type": "json_object" })))
.await
{
Ok(r) => r,
Err(e2) if should_fallback_response_format(&e2, &provider.provider) => {
self.complete(provider, &msgs, None).await?
}
Err(e2) => return Err(e2),
}
}
Err(e) => return Err(e),
};
let cleaned = strip_fences(&raw);
serde_json::from_str(cleaned).map_err(|e| {
anyhow::anyhow!(
"LLM JSON parse error ({}/{}): {e}; raw: {}",
provider.provider.as_str(),
provider.model,
truncate(&raw, 500)
)
})
}
async fn complete(
&self,
provider: &LlmProviderConfig,
messages: &[Message],
response_format: Option<Value>,
) -> anyhow::Result<String> {
let mut body = json!({
"model": provider.model,
"messages": messages,
});
if let Some(rf) = response_format {
body["response_format"] = rf;
}
let started = std::time::Instant::now();
let mut req = self
.client
.post(format!("{}/v1/chat/completions", provider.endpoint))
.json(&body);
if !provider.api_key.is_empty() {
req = req.bearer_auth(&provider.api_key);
}
let resp = req.send().await?;
let status = resp.status();
let text = resp.text().await?;
if !status.is_success() {
anyhow::bail!(
"LLM HTTP {status} ({}/{}): {}",
provider.provider.as_str(),
provider.model,
truncate(&text, 800)
);
}
let v: Value = serde_json::from_str(&text)?;
let raw_content = v["choices"][0]["message"]["content"]
.as_str()
.ok_or_else(|| {
anyhow::anyhow!(
"LLM response without content ({}/{}): {}",
provider.provider.as_str(),
provider.model,
truncate(&text, 500)
)
})?;
let content = strip_thinking_blocks(raw_content).to_string();
tracing::debug!(
provider = provider.provider.as_str(),
model = %provider.model,
elapsed_ms = started.elapsed().as_millis() as u64,
prompt_tokens = v["usage"]["prompt_tokens"].as_u64(),
completion_tokens = v["usage"]["completion_tokens"].as_u64(),
"llm completion"
);
Ok(content)
}
}
fn should_fallback_response_format(err: &anyhow::Error, provider: &ProviderKind) -> bool {
let msg = err.to_string().to_lowercase();
msg.contains("response_format")
|| msg.contains("json_schema")
|| msg.contains("structured")
|| (matches!(provider, ProviderKind::Strata)
&& (msg.contains("http 400") || msg.contains("http 422") || msg.contains("unsupported")))
}
/// Build a smooth interleaved weighted schedule (Smooth Weighted Round-Robin)
/// so e.g. weights `[2, 1]` produce `[0, 0, 1]` and `[4, 2]` produce `[0, 0, 1]` (reduced by GCD).
fn build_interleaved_schedule(providers: &[LlmProviderConfig]) -> Vec<usize> {
let mut weights: Vec<u32> = providers.iter().map(|p| p.weight.min(100)).collect();
let total: u32 = weights.iter().sum();
if total == 0 {
return (0..providers.len()).collect();
}
let g = weights.iter().copied().fold(0u32, gcd);
if g > 1 {
for w in &mut weights {
*w /= g;
}
}
let total_steps: usize = weights.iter().map(|&w| w as usize).sum();
let total_w = total_steps as i64;
let mut current = vec![0i64; providers.len()];
let mut schedule = Vec::with_capacity(total_steps);
for _ in 0..total_steps {
let mut best_idx = 0;
for (i, &w) in weights.iter().enumerate() {
if w == 0 {
continue;
}
current[i] += w as i64;
if weights[best_idx] == 0 || current[i] > current[best_idx] {
best_idx = i;
}
}
current[best_idx] -= total_w;
schedule.push(best_idx);
}
schedule
}
fn gcd(a: u32, b: u32) -> u32 {
if b == 0 {
a
} else {
gcd(b, a % b)
}
}
/// Strip `<think>...</think>` blocks that local reasoning/MoE models (e.g. Qwen3.8 on Strata) may emit.
fn strip_thinking_blocks(s: &str) -> &str {
let mut rest = s.trim();
while let Some(start) = rest.find("<think>") {
if start == 0 {
if let Some(end) = rest.find("</think>") {
rest = rest[end + "</think>".len()..].trim();
continue;
}
}
break;
}
rest
}
fn strip_fences(s: &str) -> &str {
let t = strip_thinking_blocks(s);
let t = t.strip_prefix("```json").or_else(|| t.strip_prefix("```")).unwrap_or(t);
t.strip_suffix("```").unwrap_or(t).trim()
}
fn truncate(s: &str, n: usize) -> String {
if s.chars().count() <= n {
s.to_string()
} else {
format!("{}…", s.chars().take(n).collect::<String>())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fences_and_thinking() {
assert_eq!(super::strip_fences("```json\n{\"a\":1}\n```"), "{\"a\":1}");
assert_eq!(super::strip_fences("{\"a\":1}"), "{\"a\":1}");
assert_eq!(
super::strip_fences("<think>\nreasoning...\n</think>\n```json\n{\"a\":1}\n```"),
"{\"a\":1}"
);
}
#[test]
fn parse_uppercase_and_lowercase_provider_json() {
let raw = r#"[
{
"PROVIDER": "strata",
"OPENAI_COMPATIBLE_API_ENDPOINT": "http://127.0.0.1:8080/v1/",
"OPENAI_COMPATIBLE_API_KEY": "",
"MODEL": "qwen3.8-flash-next",
"WEIGHT": 2
},
{
"provider": "cpa",
"endpoint": "http://126.dahyeon.us:8317",
"api_key": "secret-jwt",
"model": "gpt-6.1-sol",
"weight": 1
}
]"#;
let client = LlmClient::from_json_str(raw).expect("valid json");
assert_eq!(client.providers().len(), 2);
assert_eq!(client.providers()[0].provider, ProviderKind::Strata);
assert_eq!(client.providers()[0].endpoint, "http://127.0.0.1:8080");
assert_eq!(client.providers()[0].weight, 2);
assert_eq!(client.providers()[1].provider, ProviderKind::Cpa);
assert_eq!(client.providers()[1].weight, 1);
// Weighted 2:1 schedule should pick provider 0 twice and provider 1 once every 3 calls.
let picks: Vec<usize> = (0..6)
.map(|_| client.pick_provider_order()[0])
.collect();
let count_0 = picks.iter().filter(|&&i| i == 0).count();
let count_1 = picks.iter().filter(|&&i| i == 1).count();
assert_eq!(count_0, 4);
assert_eq!(count_1, 2);
// If provider 0 (strata) is marked unhealthy, pick_provider_order should promote provider 1 (cpa) first
// while keeping provider 0 at the end as last-resort failover.
client.health_flags[0].store(false, Ordering::Relaxed);
let failover_order = client.pick_provider_order();
assert_eq!(failover_order, vec![1, 0]);
}
#[tokio::test]
#[ignore]
async fn live_env_providers_health_and_routing() -> anyhow::Result<()> {
let env_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.env");
let content = std::fs::read_to_string(&env_path)?;
let mut raw_providers = String::new();
for line in content.lines() {
let line = line.trim();
if let Some(rest) = line.strip_prefix("LLM_PROVIDERS=") {
raw_providers = rest.trim_matches('\'').trim_matches('"').to_string();
}
}
assert!(!raw_providers.is_empty(), "LLM_PROVIDERS not found in .env");
let client = LlmClient::from_json_str(&raw_providers)?;
println!("Loaded {} providers from .env", client.providers().len());
let health = client.health_check_all().await;
for h in &health {
println!(
"[HEALTH] provider={} endpoint={} model={} weight={} healthy={} latency={}ms detail={}",
h.provider.as_str(),
h.endpoint,
h.model,
h.weight,
h.healthy,
h.latency_ms,
h.detail
);
}
assert!(health.iter().all(|h| h.healthy), "Expected all providers to be healthy");
#[derive(Debug, Deserialize)]
struct PingResp {
ok: bool,
provider_echo: String,
}
let schema = serde_json::json!({
"type": "object",
"additionalProperties": false,
"required": ["ok", "provider_echo"],
"properties": {
"ok": { "type": "boolean" },
"provider_echo": { "type": "string" }
}
});
for i in 1..=3 {
let out: ChatJsonOutput<PingResp> = client
.chat_json_with_meta(
"Respond with JSON matching the schema. Set ok=true and provider_echo='pong'.",
&[Part::text("ping")],
"ping_check",
&schema,
)
.await?;
println!(
"[CALL #{i}] served_by={} ({}) -> ok={}, echo={}",
out.provider.as_str(),
out.model,
out.data.ok,
out.data.provider_echo
);
}
Ok(())
}
}