UI:n system prompt ja sampling-parametrit välittyvät inferenssiin asti
Frontend lähettää agentin asetukset (system_prompt, temperature, top_k, max_tokens, repeat_penalty, stop) API:lle. Hub välittää ne solmulle. Native-node ja Wasm-coder käyttävät välitettyjä arvoja hardkoodattujen sijaan.
This commit is contained in:
@@ -1,6 +1,15 @@
|
||||
use std::time::Instant;
|
||||
use std::cell::RefCell;
|
||||
|
||||
pub struct GenerateOptions {
|
||||
pub max_tokens: usize,
|
||||
pub system_prompt: Option<String>,
|
||||
pub temperature: Option<f64>,
|
||||
pub top_k: Option<u64>,
|
||||
pub repeat_penalty: Option<f64>,
|
||||
pub stop: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
pub struct LlmEngine {
|
||||
ollama_url: String,
|
||||
model: RefCell<String>,
|
||||
@@ -96,25 +105,34 @@ impl LlmEngine {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn generate(&self, prompt: &str, max_tokens: usize) -> Result<GenerateResult, String> {
|
||||
// System prompt tulee agentin konfiguraatiosta (frontend lähettää sen osana promptia).
|
||||
// Tässä ei yliajeta sitä — Ollama saa vain prompt-kentän.
|
||||
pub async fn generate(&self, prompt: &str, opts: &GenerateOptions) -> Result<GenerateResult, String> {
|
||||
let model = self.model.borrow().clone();
|
||||
|
||||
let default_stop: Vec<String> = vec![
|
||||
"<|im_end|>".into(), "\n###".into(), "\nExplanation".into(),
|
||||
"\nNote:".into(), "\nPlease note".into(), "\nThis is".into(),
|
||||
"\n```\n\n".into(), "\n// Example".into(), "\n# Example".into(),
|
||||
];
|
||||
|
||||
let mut body = serde_json::json!({
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": false,
|
||||
"options": {
|
||||
"num_predict": opts.max_tokens,
|
||||
"temperature": opts.temperature.unwrap_or(0.7),
|
||||
"top_k": opts.top_k.unwrap_or(40),
|
||||
"repeat_penalty": opts.repeat_penalty.unwrap_or(1.15),
|
||||
"stop": opts.stop.as_ref().unwrap_or(&default_stop),
|
||||
}
|
||||
});
|
||||
if let Some(ref sp) = opts.system_prompt {
|
||||
body.as_object_mut().unwrap().insert("system".to_string(), serde_json::json!(sp));
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
let resp = self.client.post(format!("{}/api/generate", self.ollama_url))
|
||||
.json(&serde_json::json!({
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": false,
|
||||
"options": {
|
||||
"num_predict": max_tokens,
|
||||
"temperature": 0.7,
|
||||
"top_k": 40,
|
||||
"repeat_penalty": 1.15,
|
||||
"stop": ["<|im_end|>", "\n###", "\nExplanation", "\nNote:", "\nPlease note", "\nThis is", "\n```\n\n", "\n// Example", "\n# Example"]
|
||||
}
|
||||
}))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Ollama generate: {}", e))?;
|
||||
|
||||
Reference in New Issue
Block a user