use anyhow::{anyhow, Context as _, Result}; use futures::{io::BufReader, stream::BoxStream, AsyncBufReadExt, AsyncReadExt, StreamExt}; use http_client::{http, AsyncBody, HttpClient, Method, Request as HttpRequest}; use serde::{Deserialize, Serialize}; use serde_json::{value::RawValue, Value}; use std::{convert::TryFrom, sync::Arc, time::Duration}; pub const LMSTUDIO_API_URL: &str = "http://localhost:1234/api/v0"; #[derive(Clone, Copy, Serialize, Deserialize, Debug, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum Role { User, Assistant, System, Tool, } impl TryFrom for Role { type Error = anyhow::Error; fn try_from(value: String) -> Result { match value.as_str() { "user" => Ok(Self::User), "assistant" => Ok(Self::Assistant), "system" => Ok(Self::System), "tool" => Ok(Self::Tool), _ => Err(anyhow!("invalid role '{value}'")), } } } impl From for String { fn from(val: Role) -> Self { match val { Role::User => "user".to_owned(), Role::Assistant => "assistant".to_owned(), Role::System => "system".to_owned(), Role::Tool => "tool".to_owned(), } } } #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] pub struct Model { pub name: String, pub display_name: Option, pub max_tokens: usize, } impl Model { pub fn new(name: &str, display_name: Option<&str>, max_tokens: Option) -> Self { Self { name: name.to_owned(), display_name: display_name.map(|s| s.to_owned()), max_tokens: max_tokens.unwrap_or(2048), } } pub fn id(&self) -> &str { &self.name } pub fn display_name(&self) -> &str { self.display_name.as_ref().unwrap_or(&self.name) } pub fn max_token_count(&self) -> usize { self.max_tokens } } #[derive(Serialize, Deserialize, Debug)] #[serde(tag = "role", rename_all = "lowercase")] pub enum ChatMessage { Assistant { #[serde(default)] content: Option, #[serde(default)] tool_calls: Option>, }, User { content: String, }, System { content: String, }, } #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "lowercase")] pub enum LmStudioToolCall { Function(LmStudioFunctionCall), } #[derive(Serialize, Deserialize, Debug)] pub struct LmStudioFunctionCall { pub name: String, pub arguments: Box, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] pub struct LmStudioFunctionTool { pub name: String, pub description: Option, pub parameters: Option, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] #[serde(tag = "type", rename_all = "lowercase")] pub enum LmStudioTool { Function { function: LmStudioFunctionTool }, } #[derive(Serialize, Debug)] pub struct ChatCompletionRequest { pub model: String, pub messages: Vec, pub stream: bool, pub max_tokens: Option, pub stop: Option>, pub temperature: Option, pub tools: Vec, } #[derive(Serialize, Deserialize, Debug)] pub struct ChatResponse { pub id: String, pub object: String, pub created: u64, pub model: String, pub choices: Vec, } #[derive(Serialize, Deserialize, Debug)] pub struct ChoiceDelta { pub index: u32, #[serde(default)] pub delta: serde_json::Value, pub finish_reason: Option, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] pub struct ToolCallChunk { pub index: usize, pub id: Option, // There is also an optional `type` field that would determine if a // function is there. Sometimes this streams in with the `function` before // it streams in the `type` pub function: Option, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] pub struct FunctionChunk { pub name: Option, pub arguments: Option, } #[derive(Serialize, Deserialize, Debug)] pub struct Usage { pub prompt_tokens: u32, pub completion_tokens: u32, pub total_tokens: u32, } #[derive(Serialize, Deserialize, Debug)] #[serde(untagged)] pub enum ResponseStreamResult { Ok(ResponseStreamEvent), Err { error: String }, } #[derive(Serialize, Deserialize, Debug)] pub struct ResponseStreamEvent { pub created: u32, pub model: String, pub choices: Vec, pub usage: Option, } #[derive(Serialize, Deserialize)] pub struct ListModelsResponse { pub data: Vec, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct ModelEntry { pub id: String, pub object: String, pub r#type: ModelType, pub publisher: String, pub arch: Option, pub compatibility_type: CompatibilityType, pub quantization: Option, pub state: ModelState, pub max_context_length: Option, pub loaded_context_length: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "lowercase")] pub enum ModelType { Llm, Embeddings, Vlm, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "kebab-case")] pub enum ModelState { Loaded, Loading, NotLoaded, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "lowercase")] pub enum CompatibilityType { Gguf, Mlx, } pub async fn complete( client: &dyn HttpClient, api_url: &str, request: ChatCompletionRequest, ) -> Result { let uri = format!("{api_url}/chat/completions"); let request_builder = HttpRequest::builder() .method(Method::POST) .uri(uri) .header("Content-Type", "application/json"); let serialized_request = serde_json::to_string(&request)?; let request = request_builder.body(AsyncBody::from(serialized_request))?; let mut response = client.send(request).await?; if response.status().is_success() { let mut body = Vec::new(); response.body_mut().read_to_end(&mut body).await?; let response_message: ChatResponse = serde_json::from_slice(&body)?; Ok(response_message) } else { let mut body = Vec::new(); response.body_mut().read_to_end(&mut body).await?; let body_str = std::str::from_utf8(&body)?; Err(anyhow!( "Failed to connect to API: {} {}", response.status(), body_str )) } } pub async fn stream_chat_completion( client: &dyn HttpClient, api_url: &str, request: ChatCompletionRequest, ) -> Result>> { let uri = format!("{api_url}/chat/completions"); let request_builder = http::Request::builder() .method(Method::POST) .uri(uri) .header("Content-Type", "application/json"); let request = request_builder.body(AsyncBody::from(serde_json::to_string(&request)?))?; let mut response = client.send(request).await?; if response.status().is_success() { let reader = BufReader::new(response.into_body()); Ok(reader .lines() .filter_map(|line| async move { match line { Ok(line) => { let line = line.strip_prefix("data: ")?; if line == "[DONE]" { None } else { let result = serde_json::from_str(&line) .context("Unable to parse chat completions response"); if let Err(ref e) = result { eprintln!("Error parsing line: {e}\nLine content: '{line}'"); } Some(result) } } Err(e) => { eprintln!("Error reading line: {e}"); Some(Err(e.into())) } } }) .boxed()) } else { let mut body = String::new(); response.body_mut().read_to_string(&mut body).await?; Err(anyhow!( "Failed to connect to LM Studio API: {} {}", response.status(), body, )) } } pub async fn get_models( client: &dyn HttpClient, api_url: &str, _: Option, ) -> Result> { let uri = format!("{api_url}/models"); let request_builder = HttpRequest::builder() .method(Method::GET) .uri(uri) .header("Accept", "application/json"); let request = request_builder.body(AsyncBody::default())?; let mut response = client.send(request).await?; let mut body = String::new(); response.body_mut().read_to_string(&mut body).await?; if response.status().is_success() { let response: ListModelsResponse = serde_json::from_str(&body).context("Unable to parse LM Studio models response")?; Ok(response.data) } else { Err(anyhow!( "Failed to connect to LM Studio API: {} {}", response.status(), body, )) } } /// Sends an empty request to LM Studio to trigger loading the model pub async fn preload_model(client: Arc, api_url: &str, model: &str) -> Result<()> { let uri = format!("{api_url}/completions"); let request = HttpRequest::builder() .method(Method::POST) .uri(uri) .header("Content-Type", "application/json") .body(AsyncBody::from(serde_json::to_string( &serde_json::json!({ "model": model, "messages": [], "stream": false, "max_tokens": 0, }), )?))?; let mut response = client.send(request).await?; if response.status().is_success() { Ok(()) } else { let mut body = String::new(); response.body_mut().read_to_string(&mut body).await?; Err(anyhow!( "Failed to connect to LM Studio API: {} {}", response.status(), body, )) } }