Compare commits
84 Commits
pre-worker
...
b074e0cb49
| Author | SHA1 | Date | |
|---|---|---|---|
| b074e0cb49 | |||
| 9307c75516 | |||
| 86191fbb6c | |||
| a6a94f7688 | |||
| 8d5c5440d2 | |||
| a12bd7ce7f | |||
| 9ac90aa540 | |||
| 32065d5818 | |||
| 321943ee3c | |||
| 1b75c89320 | |||
| 01622a960f | |||
| 4e4efda67d | |||
| f5db2eb034 | |||
| 77c8d46e7b | |||
| f14eba1b49 | |||
| 6d15298418 | |||
| cea1961183 | |||
| 21a8015ea3 | |||
| c3991193d9 | |||
| 02c6d67218 | |||
| de1cf009fa | |||
| 060f36f479 | |||
| e2ec0fa43d | |||
| 8752c0f465 | |||
| 8c95282654 | |||
| a1bc1af646 | |||
| 6b27cbbade | |||
| 4d9c51a86f | |||
| 66d1e8c4b1 | |||
| 2eeac255f6 | |||
| 6097cfc263 | |||
| 8aed9f97a2 | |||
| c0ccd76a4c | |||
| d2edb38879 | |||
| 2755794554 | |||
| dbb37b3c60 | |||
| 0e7497b627 | |||
| 6b756e2e83 | |||
| 5a52f5113c | |||
| 7b0660e46e | |||
| b35600b417 | |||
| 7693269e5d | |||
| 702c9170ad | |||
| 3feed22055 | |||
| 75310c989e | |||
| 743946a391 | |||
| 0bd5faa684 | |||
| e0c8c3586b | |||
| 3a1c5c723c | |||
| 3139d1ac65 | |||
| 49a1629646 | |||
| 13008ac693 | |||
| 30e81875db | |||
| 73bcd3143a | |||
| 216b95d15c | |||
| 34ef19472a | |||
| 54a5af96c7 | |||
| 842153a7ec | |||
| 5c25c7f9c1 | |||
| ac698a766e | |||
| f1b57a6c53 | |||
| b70cdbd24d | |||
| 01d8b597e1 | |||
| f2ca4890df | |||
| 3eb0c4d939 | |||
| d8443792a3 | |||
| ae379bdda4 | |||
| ed02e47158 | |||
| 959dc532bb | |||
| 1ef7f7c956 | |||
| e6e1f60935 | |||
| 322c98ff59 | |||
| 406e2226f0 | |||
| 9d7496157c | |||
| d332b7e910 | |||
| 8e55a15d66 | |||
| 4e3134d908 | |||
| cd45db001a | |||
| 4ad8a8793e | |||
| b2694c232e | |||
| ba58236c52 | |||
| 861f2a6902 | |||
| 11fd5b0c9e | |||
| b3646ae5d3 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -34,3 +34,6 @@ Cargo.lock
|
|||||||
*.pdb
|
*.pdb
|
||||||
|
|
||||||
# End of https://www.toptal.com/developers/gitignore/api/rust,linux
|
# End of https://www.toptal.com/developers/gitignore/api/rust,linux
|
||||||
|
|
||||||
|
# Ajonaikaiset tietokannat
|
||||||
|
*.db
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
FROM rust:slim AS builder
|
FROM rust:slim AS builder
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
pkg-config libssl-dev g++ \
|
pkg-config libssl-dev g++ libvulkan-dev \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
@@ -9,22 +9,27 @@ COPY Cargo.toml Cargo.lock ./
|
|||||||
COPY hub/Cargo.toml hub/Cargo.toml
|
COPY hub/Cargo.toml hub/Cargo.toml
|
||||||
COPY node/Cargo.toml node/Cargo.toml
|
COPY node/Cargo.toml node/Cargo.toml
|
||||||
COPY native-node/Cargo.toml native-node/Cargo.toml
|
COPY native-node/Cargo.toml native-node/Cargo.toml
|
||||||
|
COPY cli/Cargo.toml cli/Cargo.toml
|
||||||
|
|
||||||
# Tyhjät src-tiedostot riippuvuuksien esikääntämistä varten
|
# Tyhjät src-tiedostot riippuvuuksien esikääntämistä varten
|
||||||
RUN mkdir -p hub/src node/src native-node/src \
|
RUN mkdir -p hub/src node/src native-node/src cli/src \
|
||||||
&& echo "fn main(){}" > hub/src/main.rs \
|
&& echo "fn main(){}" > hub/src/main.rs \
|
||||||
&& echo "" > node/src/lib.rs \
|
&& echo "" > node/src/lib.rs \
|
||||||
&& echo "fn main(){}" > native-node/src/main.rs \
|
&& echo "fn main(){}" > native-node/src/main.rs \
|
||||||
|
&& echo "fn main(){}" > cli/src/main.rs \
|
||||||
&& cargo build --release -p native-node 2>/dev/null || true
|
&& cargo build --release -p native-node 2>/dev/null || true
|
||||||
|
|
||||||
COPY native-node/src native-node/src
|
COPY native-node/src native-node/src
|
||||||
RUN cargo build --release -p native-node
|
# Touch pakottaa rekompilauksen dummy-binaryn yli
|
||||||
|
RUN touch native-node/src/main.rs && cargo build --release -p native-node
|
||||||
|
|
||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y ca-certificates libvulkan1 && rm -rf /var/lib/apt/lists/*
|
||||||
COPY --from=builder /app/target/release/native-node /usr/local/bin/native-node
|
COPY --from=builder /app/target/release/native-node /usr/local/bin/native-node
|
||||||
|
|
||||||
ENV HUB_URL=ws://hub:3000/ws
|
ENV HUB_URL=ws://agentic-poc:3000/ws
|
||||||
|
ENV OLLAMA_URL=http://ollama:11434
|
||||||
|
ENV OLLAMA_MODEL=qwen2.5-coder:7b
|
||||||
ENV ALLOCATED_GB=4
|
ENV ALLOCATED_GB=4
|
||||||
|
|
||||||
CMD ["native-node"]
|
CMD ["native-node"]
|
||||||
|
|||||||
26
network-poc/TODO.md
Normal file
26
network-poc/TODO.md
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
# TODO — Kipinä Agentic Network
|
||||||
|
|
||||||
|
## Turvallisuus
|
||||||
|
- [ ] **Tulosten validointi** — solmu voi palauttaa haitallista koodia. Tarvitaan proof-of-work tai challenge-response -mekanismi
|
||||||
|
- [ ] **Reputaatiojärjestelmä** — solmujen luotettavuuden seuranta: onnistuneet tehtävät, vasteaika, laatu
|
||||||
|
- [ ] **Koodin sandboxaus** — generoitu koodi pitää ajaa eristetyssä ympäristössä ennen käyttäjälle näyttämistä
|
||||||
|
- [ ] **Solmun identiteetti** — rekisteröityminen ja tunnistautuminen (API-avain / token)
|
||||||
|
|
||||||
|
## Yksityisyys
|
||||||
|
- [ ] **Promptien salaus** — käyttäjän promptit menevät tuntemattomalle solmulle selkotekstinä
|
||||||
|
- [ ] **End-to-end enkryptio** — hub ei näe promptin sisältöä, vain reitittää
|
||||||
|
- [ ] **Tietosuojaseloste** — käyttäjille kerrottava miten data kulkee ja kuka sen näkee
|
||||||
|
- [ ] **Opt-in malli** — käyttäjä valitsee haluaako käyttää yhteisösolmuja vai vain omaa
|
||||||
|
|
||||||
|
## Väärinkäytön esto
|
||||||
|
- [ ] **Rate limiting per käyttäjä** — nykyinen IP-pohjainen ei riitä, tarvitaan autentikointi
|
||||||
|
- [ ] **Solmun kuormitusraja** — solmu voi asettaa max tehtävät/minuutti
|
||||||
|
- [ ] **Token-talous** — laskentaresurssien käyttö vaatii Kipinä-tokeneita (gamification jo aloitettu)
|
||||||
|
- [ ] **Abuse reporting** — mekanismi haitallisten solmujen ilmiantamiseen
|
||||||
|
|
||||||
|
## Seuraavat ominaisuudet
|
||||||
|
- [ ] Agenttien välinen keskustelu (manageri ohjaa dynaamisesti)
|
||||||
|
- [ ] Tehtävähistoria ja tulosten tallennus
|
||||||
|
- [ ] Prometheus/OpenTelemetry -metriikat
|
||||||
|
- [ ] Solmujen terveystarkistukset (ping/pong)
|
||||||
|
- [ ] Streaming-vastaukset Ollaman kautta
|
||||||
@@ -11,18 +11,14 @@ services:
|
|||||||
# Käännetään aina käynnistyksen yhteydessä varmuuden vuoksi Wasm uusimmista koodeista, ja päälle pyöräytetään Hub!
|
# Käännetään aina käynnistyksen yhteydessä varmuuden vuoksi Wasm uusimmista koodeista, ja päälle pyöräytetään Hub!
|
||||||
command: bash -c "cd node && wasm-pack build --release --target web --out-dir ../static/pkg && cd ../hub && cargo run"
|
command: bash -c "cd node && wasm-pack build --release --target web --out-dir ../static/pkg && cd ../hub && cargo run"
|
||||||
|
|
||||||
# Valinnainen natiivi-solmu — kerää oikeat laitteistotiedot (nvidia-smi-taso)
|
# Ollama — LLM-inferenssi GPU:lla (NVIDIA/AMD/Apple)
|
||||||
native-node:
|
ollama:
|
||||||
build:
|
image: ollama/ollama:latest
|
||||||
context: .
|
container_name: kipina_ollama
|
||||||
dockerfile: Dockerfile.native-node
|
ports:
|
||||||
container_name: kipina_native_node
|
- "11434:11434"
|
||||||
environment:
|
volumes:
|
||||||
- HUB_URL=ws://agentic-poc:3000/ws
|
- ollama-models:/root/.ollama
|
||||||
- ALLOCATED_GB=4
|
|
||||||
depends_on:
|
|
||||||
- agentic-poc
|
|
||||||
# GPU passthrough (valinnainen — toimii myös ilman)
|
|
||||||
deploy:
|
deploy:
|
||||||
resources:
|
resources:
|
||||||
reservations:
|
reservations:
|
||||||
@@ -32,3 +28,23 @@ services:
|
|||||||
capabilities: [gpu]
|
capabilities: [gpu]
|
||||||
profiles:
|
profiles:
|
||||||
- native
|
- native
|
||||||
|
|
||||||
|
# Natiivisolmu — yhdistää hubiin ja käyttää Ollamaa inferenssiin
|
||||||
|
native-node:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile.native-node
|
||||||
|
container_name: kipina_native_node
|
||||||
|
environment:
|
||||||
|
- HUB_URL=ws://agentic-poc:3000/ws
|
||||||
|
- OLLAMA_URL=http://ollama:11434
|
||||||
|
- OLLAMA_MODEL=qwen2.5-coder:7b
|
||||||
|
- ALLOCATED_GB=4
|
||||||
|
depends_on:
|
||||||
|
- agentic-poc
|
||||||
|
- ollama
|
||||||
|
profiles:
|
||||||
|
- native
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
ollama-models:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "hub"
|
name = "hub"
|
||||||
version = "0.2.0"
|
version = "0.2.2"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -16,3 +16,4 @@ futures = "0.3"
|
|||||||
rusqlite = { version = "0.31", features = ["bundled"] }
|
rusqlite = { version = "0.31", features = ["bundled"] }
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
base64 = "0.22"
|
base64 = "0.22"
|
||||||
|
reqwest = { version = "0.12", features = ["json"] }
|
||||||
|
|||||||
Binary file not shown.
@@ -39,6 +39,7 @@ struct AppState {
|
|||||||
ip_connections: Mutex<HashMap<IpAddr, u32>>,
|
ip_connections: Mutex<HashMap<IpAddr, u32>>,
|
||||||
node_ips: Mutex<HashMap<u64, IpAddr>>,
|
node_ips: Mutex<HashMap<u64, IpAddr>>,
|
||||||
node_tasks: Mutex<HashMap<u64, String>>, // node_id → selected_task
|
node_tasks: Mutex<HashMap<u64, String>>, // node_id → selected_task
|
||||||
|
node_types: Mutex<HashMap<u64, String>>, // node_id → "native" | "browser"
|
||||||
node_busy: Mutex<std::collections::HashSet<u64>>, // Solmut joilla on aktiivinen tehtävä
|
node_busy: Mutex<std::collections::HashSet<u64>>, // Solmut joilla on aktiivinen tehtävä
|
||||||
pending_task_ids: Mutex<std::collections::HashSet<String>>, // Hubin jakamat task_id:t (gamification-validointi)
|
pending_task_ids: Mutex<std::collections::HashSet<String>>, // Hubin jakamat task_id:t (gamification-validointi)
|
||||||
api_rate_limits: Mutex<HashMap<IpAddr, (std::time::Instant, u32)>>, // IP → (ikkuna-alku, pyyntömäärä)
|
api_rate_limits: Mutex<HashMap<IpAddr, (std::time::Instant, u32)>>, // IP → (ikkuna-alku, pyyntömäärä)
|
||||||
@@ -260,6 +261,7 @@ async fn main() {
|
|||||||
ip_connections: Mutex::new(HashMap::new()),
|
ip_connections: Mutex::new(HashMap::new()),
|
||||||
node_ips: Mutex::new(HashMap::new()),
|
node_ips: Mutex::new(HashMap::new()),
|
||||||
node_tasks: Mutex::new(HashMap::new()),
|
node_tasks: Mutex::new(HashMap::new()),
|
||||||
|
node_types: Mutex::new(HashMap::new()),
|
||||||
node_busy: Mutex::new(std::collections::HashSet::new()),
|
node_busy: Mutex::new(std::collections::HashSet::new()),
|
||||||
pending_task_ids: Mutex::new(std::collections::HashSet::new()),
|
pending_task_ids: Mutex::new(std::collections::HashSet::new()),
|
||||||
api_rate_limits: Mutex::new(HashMap::new()),
|
api_rate_limits: Mutex::new(HashMap::new()),
|
||||||
@@ -382,6 +384,9 @@ async fn main() {
|
|||||||
.route("/api/pairs", get(api_pairs))
|
.route("/api/pairs", get(api_pairs))
|
||||||
.route("/api/stats", get(api_stats))
|
.route("/api/stats", get(api_stats))
|
||||||
.route("/api/v1/chat/completions", axum::routing::post(api_chat_completions))
|
.route("/api/v1/chat/completions", axum::routing::post(api_chat_completions))
|
||||||
|
.route("/api/v1/model", axum::routing::post(api_change_model))
|
||||||
|
.route("/api/v1/hardware", get(api_hardware))
|
||||||
|
.route("/api/v1/ollama/tags", get(api_ollama_tags))
|
||||||
.route("/admin", get(admin_page))
|
.route("/admin", get(admin_page))
|
||||||
.nest_service("/", {
|
.nest_service("/", {
|
||||||
let static_dir = std::env::var("STATIC_DIR").unwrap_or_else(|_| "../static".to_string());
|
let static_dir = std::env::var("STATIC_DIR").unwrap_or_else(|_| "../static".to_string());
|
||||||
@@ -677,6 +682,7 @@ async fn handle_socket(socket: WebSocket, state: Arc<AppState>, ip: IpAddr) {
|
|||||||
state.db.insert_session(node_id, &ip.to_string(), node_type, &json);
|
state.db.insert_session(node_id, &ip.to_string(), node_type, &json);
|
||||||
}
|
}
|
||||||
state.node_tasks.lock().unwrap().insert(node_id, selected_task);
|
state.node_tasks.lock().unwrap().insert(node_id, selected_task);
|
||||||
|
state.node_types.lock().unwrap().insert(node_id, node_type.to_string());
|
||||||
|
|
||||||
if node_type == "native" {
|
if node_type == "native" {
|
||||||
let sys = json.get("system");
|
let sys = json.get("system");
|
||||||
@@ -934,6 +940,7 @@ async fn handle_socket(socket: WebSocket, state: Arc<AppState>, ip: IpAddr) {
|
|||||||
ips.remove(&node_id);
|
ips.remove(&node_id);
|
||||||
vram.remove(&node_id);
|
vram.remove(&node_id);
|
||||||
}
|
}
|
||||||
|
state.node_types.lock().unwrap().remove(&node_id);
|
||||||
tracing::info!("Solmu {} ({}) poistui verkosta.", node_id, ip);
|
tracing::info!("Solmu {} ({}) poistui verkosta.", node_id, ip);
|
||||||
broadcast_stats(&state).await;
|
broadcast_stats(&state).await;
|
||||||
sender_task.abort();
|
sender_task.abort();
|
||||||
@@ -943,6 +950,8 @@ struct ChatCompletionRequest {
|
|||||||
model: String,
|
model: String,
|
||||||
prompt: String,
|
prompt: String,
|
||||||
task_id: String,
|
task_id: String,
|
||||||
|
#[serde(default)]
|
||||||
|
max_tokens: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
@@ -952,6 +961,78 @@ struct ChatCompletionResponse {
|
|||||||
tokens_generated: u64,
|
tokens_generated: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn api_ollama_tags() -> axum::response::Response {
|
||||||
|
let ollama_url = std::env::var("OLLAMA_URL").unwrap_or_else(|_| "http://ollama:11434".to_string());
|
||||||
|
match reqwest::get(format!("{}/api/tags", ollama_url)).await {
|
||||||
|
Ok(resp) => {
|
||||||
|
if let Ok(body) = resp.json::<serde_json::Value>().await {
|
||||||
|
axum::Json(body).into_response()
|
||||||
|
} else {
|
||||||
|
axum::Json(serde_json::json!({ "models": [] })).into_response()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => axum::Json(serde_json::json!({ "models": [] })).into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn api_hardware(
|
||||||
|
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
// Etsitään natiivisolmun GPU-tiedot sessiosta
|
||||||
|
let sessions = state.db.get_sessions(50);
|
||||||
|
let native = sessions.iter().find(|s| {
|
||||||
|
s.get("node_type").and_then(|v| v.as_str()) == Some("native")
|
||||||
|
});
|
||||||
|
|
||||||
|
let (mut vram_mb, mut gpu_name, ram_mb) = if let Some(s) = native {
|
||||||
|
let gpus = s.get("gpus").and_then(|v| v.as_array());
|
||||||
|
let gpu = gpus.and_then(|g| g.first());
|
||||||
|
let vram = gpu.and_then(|g| g.get("vram_total_mb")).and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
let name = gpu.and_then(|g| g.get("name")).and_then(|v| v.as_str()).unwrap_or("").to_string();
|
||||||
|
let ram = s.get("system").and_then(|v| v.get("ram_total_mb")).and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
(vram, name, ram)
|
||||||
|
} else {
|
||||||
|
(0, String::new(), 0)
|
||||||
|
};
|
||||||
|
|
||||||
|
// Fallback: kysytään Ollamalta onko malleja ladattu (= Ollama on käynnissä)
|
||||||
|
if vram_mb == 0 {
|
||||||
|
let ollama_url = std::env::var("OLLAMA_URL").unwrap_or_else(|_| "http://ollama:11434".to_string());
|
||||||
|
if let Ok(resp) = reqwest::get(format!("{}/api/tags", ollama_url)).await {
|
||||||
|
if let Ok(body) = resp.json::<serde_json::Value>().await {
|
||||||
|
let models = body["models"].as_array().map(|a| a.len()).unwrap_or(0);
|
||||||
|
if models > 0 {
|
||||||
|
gpu_name = "Ollama (GPU/CPU)".to_string();
|
||||||
|
// Natiivisolmun RAM fallbackina
|
||||||
|
vram_mb = if ram_mb > 0 { ram_mb } else { 0 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if gpu_name.is_empty() { gpu_name = "ei natiivisolmua".to_string(); }
|
||||||
|
|
||||||
|
axum::Json(serde_json::json!({
|
||||||
|
"gpu_name": gpu_name,
|
||||||
|
"vram_mb": vram_mb,
|
||||||
|
"ram_mb": ram_mb,
|
||||||
|
})).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn api_change_model(
|
||||||
|
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
|
||||||
|
axum::Json(payload): axum::Json<serde_json::Value>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
let model = payload.get("model").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
if model.is_empty() {
|
||||||
|
return (axum::http::StatusCode::BAD_REQUEST, "model puuttuu").into_response();
|
||||||
|
}
|
||||||
|
tracing::info!("Mallin vaihto: {}", model);
|
||||||
|
let msg = serde_json::json!({ "type": "change_model", "model": model });
|
||||||
|
let _ = state.stats_tx.send(msg.to_string());
|
||||||
|
axum::Json(serde_json::json!({ "status": "ok", "model": model })).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
async fn api_chat_completions(
|
async fn api_chat_completions(
|
||||||
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
|
axum::extract::State(state): axum::extract::State<Arc<AppState>>,
|
||||||
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
ConnectInfo(addr): ConnectInfo<SocketAddr>,
|
||||||
@@ -966,16 +1047,17 @@ async fn api_chat_completions(
|
|||||||
*entry = (now, 1); // Uusi ikkuna
|
*entry = (now, 1); // Uusi ikkuna
|
||||||
} else {
|
} else {
|
||||||
entry.1 += 1;
|
entry.1 += 1;
|
||||||
if entry.1 > 10 {
|
if entry.1 > 30 {
|
||||||
return (axum::http::StatusCode::TOO_MANY_REQUESTS, "Liian monta pyyntöä — yritä minuutin kuluttua").into_response();
|
return (axum::http::StatusCode::TOO_MANY_REQUESTS, "Liian monta pyyntöä — yritä minuutin kuluttua").into_response();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Etsitään vapaa tai varattu solmu, joka vastaa pyydettyä mallia
|
// Etsitään vapaa solmu — priorisoidaan natiivisolmut (GPU) selaimen edelle
|
||||||
let (target_node_free, target_node_any, total_matching) = {
|
let (target_node_free, target_node_any, total_matching) = {
|
||||||
let tasks = state.node_tasks.lock().unwrap();
|
let tasks = state.node_tasks.lock().unwrap();
|
||||||
let busy = state.node_busy.lock().unwrap();
|
let busy = state.node_busy.lock().unwrap();
|
||||||
|
let node_types = state.node_types.lock().unwrap();
|
||||||
let matching: Vec<u64> = tasks.iter().filter(|(_, task)| {
|
let matching: Vec<u64> = tasks.iter().filter(|(_, task)| {
|
||||||
if payload.model == "qwen-coder" {
|
if payload.model == "qwen-coder" {
|
||||||
task.starts_with("qwen-coder")
|
task.starts_with("qwen-coder")
|
||||||
@@ -983,7 +1065,12 @@ async fn api_chat_completions(
|
|||||||
**task == payload.model
|
**task == payload.model
|
||||||
}
|
}
|
||||||
}).map(|(k, _)| *k).collect();
|
}).map(|(k, _)| *k).collect();
|
||||||
let free = matching.iter().find(|id| !busy.contains(id)).copied();
|
// Vapaat solmut: natiivi ensin, sitten selain
|
||||||
|
let free_native = matching.iter().find(|id| {
|
||||||
|
!busy.contains(id) && node_types.get(id).map(|t| t == "native").unwrap_or(false)
|
||||||
|
}).copied();
|
||||||
|
let free_any = matching.iter().find(|id| !busy.contains(id)).copied();
|
||||||
|
let free = free_native.or(free_any);
|
||||||
let any = matching.first().copied();
|
let any = matching.first().copied();
|
||||||
(free, any, matching.len())
|
(free, any, matching.len())
|
||||||
};
|
};
|
||||||
@@ -1059,12 +1146,15 @@ async fn api_chat_completions(
|
|||||||
state.node_busy.lock().unwrap().insert(target_node_id);
|
state.node_busy.lock().unwrap().insert(target_node_id);
|
||||||
state.pending_task_ids.lock().unwrap().insert(payload.task_id.clone());
|
state.pending_task_ids.lock().unwrap().insert(payload.task_id.clone());
|
||||||
|
|
||||||
let msg = serde_json::json!({
|
let mut msg = serde_json::json!({
|
||||||
"type": "llm_prompt",
|
"type": "llm_prompt",
|
||||||
"prompt": payload.prompt,
|
"prompt": payload.prompt,
|
||||||
"model": payload.model,
|
"model": payload.model,
|
||||||
"task_id": payload.task_id,
|
"task_id": payload.task_id,
|
||||||
});
|
});
|
||||||
|
if let Some(mt) = payload.max_tokens {
|
||||||
|
msg.as_object_mut().unwrap().insert("max_tokens".to_string(), serde_json::json!(mt));
|
||||||
|
}
|
||||||
|
|
||||||
// Odotuskanava valmiiksi (solmu palauttaa tuloksen stats_tx kautta)
|
// Odotuskanava valmiiksi (solmu palauttaa tuloksen stats_tx kautta)
|
||||||
let mut rx = state.stats_tx.subscribe();
|
let mut rx = state.stats_tx.subscribe();
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "native-node"
|
name = "native-node"
|
||||||
version = "0.1.0"
|
version = "0.2.2"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -12,10 +12,6 @@ serde_json = "1.0"
|
|||||||
sysinfo = "0.30"
|
sysinfo = "0.30"
|
||||||
nvml-wrapper = "0.10"
|
nvml-wrapper = "0.10"
|
||||||
wgpu = "24"
|
wgpu = "24"
|
||||||
candle-core = { version = "0.8", features = ["cuda"] }
|
reqwest = { version = "0.12", features = ["json"] }
|
||||||
candle-nn = "0.8"
|
|
||||||
candle-transformers = "0.8"
|
|
||||||
hf-hub = "0.4"
|
|
||||||
tokenizers = "0.19"
|
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|||||||
@@ -1,261 +1,114 @@
|
|||||||
use candle_core::{Device, Tensor, DType};
|
|
||||||
use candle_nn::VarBuilder;
|
|
||||||
use candle_transformers::models::qwen2::{Config as QwenConfig, ModelForCausalLM as QwenModel};
|
|
||||||
use hf_hub::{api::sync::Api, Repo, RepoType};
|
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
use std::cell::RefCell;
|
||||||
/// Top-k sampling with temperature and repetition penalty
|
|
||||||
fn sample_top_k(logits: &Tensor, k: usize, temperature: f64, generated_tokens: &[u32], repetition_penalty: f64, rng_state: &mut u64) -> Result<u32, String> {
|
|
||||||
let mut logits_vec: Vec<f32> = logits.to_vec1::<f32>().map_err(|e| format!("to_vec1: {}", e))?;
|
|
||||||
if logits_vec.is_empty() { return Err("Tyhjä logits".to_string()); }
|
|
||||||
|
|
||||||
// Repetition penalty: rankaisee jo generoituja tokeneita
|
|
||||||
for &token_id in generated_tokens {
|
|
||||||
if (token_id as usize) < logits_vec.len() {
|
|
||||||
let logit = &mut logits_vec[token_id as usize];
|
|
||||||
if *logit > 0.0 {
|
|
||||||
*logit /= repetition_penalty as f32;
|
|
||||||
} else {
|
|
||||||
*logit *= repetition_penalty as f32;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Temperature scaling
|
|
||||||
if temperature > 0.0 && temperature != 1.0 {
|
|
||||||
for logit in logits_vec.iter_mut() {
|
|
||||||
*logit /= temperature as f32;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Top-k: etsitään k suurinta
|
|
||||||
let mut indexed: Vec<(usize, f32)> = logits_vec.iter().enumerate().map(|(i, &v)| (i, v)).collect();
|
|
||||||
indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
|
|
||||||
indexed.truncate(k);
|
|
||||||
|
|
||||||
if k == 1 || temperature == 0.0 {
|
|
||||||
return Ok(indexed[0].0 as u32);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Softmax top-k:lle
|
|
||||||
let max_logit = indexed[0].1;
|
|
||||||
let exps: Vec<f32> = indexed.iter().map(|x| (x.1 - max_logit).exp()).collect();
|
|
||||||
let sum: f32 = exps.iter().sum();
|
|
||||||
let probs: Vec<f32> = exps.iter().map(|e| e / sum).collect();
|
|
||||||
|
|
||||||
// XorShift64 RNG
|
|
||||||
*rng_state ^= *rng_state << 13;
|
|
||||||
*rng_state ^= *rng_state >> 7;
|
|
||||||
*rng_state ^= *rng_state << 17;
|
|
||||||
let rand_val = (*rng_state % 10000) as f32 / 10000.0;
|
|
||||||
|
|
||||||
let mut cumulative = 0.0;
|
|
||||||
for (i, p) in probs.iter().enumerate() {
|
|
||||||
cumulative += p;
|
|
||||||
if rand_val < cumulative {
|
|
||||||
return Ok(indexed[i].0 as u32);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(indexed[0].0 as u32)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct LlmEngine {
|
pub struct LlmEngine {
|
||||||
tokenizer: tokenizers::Tokenizer,
|
ollama_url: String,
|
||||||
model: QwenModel,
|
model: RefCell<String>,
|
||||||
device: Device,
|
client: reqwest::Client,
|
||||||
eos_token: u32,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LlmEngine {
|
impl LlmEngine {
|
||||||
pub fn load() -> Result<Self, String> {
|
pub fn load() -> Result<Self, String> {
|
||||||
let device = Device::cuda_if_available(0).map_err(|e| format!("Device: {}", e))?;
|
let ollama_url = std::env::var("OLLAMA_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||||
let device_name = if device.is_cuda() { "CUDA" } else { "CPU" };
|
let model = std::env::var("OLLAMA_MODEL").unwrap_or_else(|_| "qwen2.5-coder:7b".to_string());
|
||||||
tracing::info!("LLM device: {}", device_name);
|
|
||||||
|
|
||||||
let dtype = if device.is_cuda() { DType::F16 } else { DType::F32 };
|
tracing::info!("Ollama backend: {} | malli: {}", ollama_url, model);
|
||||||
|
|
||||||
tracing::info!("Ladataan Qwen2.5-Coder-0.5B-Instruct...");
|
let client = reqwest::Client::builder()
|
||||||
let api = Api::new().map_err(|e| format!("HF API: {}", e))?;
|
.timeout(std::time::Duration::from_secs(600))
|
||||||
let repo = api.repo(Repo::with_revision(
|
.build()
|
||||||
"Qwen/Qwen2.5-Coder-0.5B-Instruct".to_string(),
|
.map_err(|e| format!("HTTP client: {}", e))?;
|
||||||
RepoType::Model,
|
|
||||||
"main".to_string(),
|
|
||||||
));
|
|
||||||
|
|
||||||
let tokenizer_path = repo.get("tokenizer.json").map_err(|e| format!("Tokenizer lataus: {}", e))?;
|
Ok(LlmEngine { ollama_url, model: RefCell::new(model), client })
|
||||||
let model_path = repo.get("model.safetensors").map_err(|e| format!("Malli lataus: {}", e))?;
|
|
||||||
|
|
||||||
tracing::info!("Ladataan tokenizer: {:?}", tokenizer_path);
|
|
||||||
let tokenizer = tokenizers::Tokenizer::from_file(&tokenizer_path)
|
|
||||||
.map_err(|e| format!("Tokenizer: {}", e))?;
|
|
||||||
|
|
||||||
let config = QwenConfig {
|
|
||||||
vocab_size: 151936,
|
|
||||||
hidden_size: 896,
|
|
||||||
intermediate_size: 4864,
|
|
||||||
num_hidden_layers: 24,
|
|
||||||
num_attention_heads: 14,
|
|
||||||
num_key_value_heads: 2,
|
|
||||||
max_position_embeddings: 32768,
|
|
||||||
sliding_window: 32768,
|
|
||||||
max_window_layers: 21,
|
|
||||||
tie_word_embeddings: true,
|
|
||||||
rope_theta: 1000000.0,
|
|
||||||
rms_norm_eps: 1e-6,
|
|
||||||
use_sliding_window: false,
|
|
||||||
hidden_act: candle_nn::Activation::Silu,
|
|
||||||
};
|
|
||||||
|
|
||||||
let start = Instant::now();
|
|
||||||
let vb = unsafe {
|
|
||||||
VarBuilder::from_mmaped_safetensors(&[model_path.clone()], dtype, &device)
|
|
||||||
.map_err(|e| format!("VarBuilder: {}", e))?
|
|
||||||
};
|
|
||||||
let model = QwenModel::new(&config, vb).map_err(|e| format!("Malli: {}", e))?;
|
|
||||||
tracing::info!("Malli ladattu ({:.1}s) — {}", start.elapsed().as_secs_f64(), device_name);
|
|
||||||
|
|
||||||
Ok(LlmEngine {
|
|
||||||
tokenizer,
|
|
||||||
model,
|
|
||||||
device,
|
|
||||||
eos_token: 151645,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn generate(&mut self, prompt: &str, max_tokens: usize) -> Result<GenerateResult, String> {
|
pub fn model_name(&self) -> String {
|
||||||
// Prefill: aloitetaan vastaus ```-koodiblokkilla → malli jatkaa suoraan koodilla
|
self.model.borrow().clone()
|
||||||
let formatted = format!("<|im_start|>system\nYou are a coding assistant. Respond with ONLY code. No explanations, no markdown, no comments unless asked.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n```\n", prompt);
|
}
|
||||||
|
|
||||||
let encoding = self.tokenizer.encode(formatted.as_str(), true)
|
pub fn set_model(&self, new_model: String) {
|
||||||
.map_err(|e| format!("Encode: {}", e))?;
|
*self.model.borrow_mut() = new_model;
|
||||||
let input_ids: Vec<u32> = encoding.get_ids().to_vec();
|
}
|
||||||
let input_len = input_ids.len();
|
|
||||||
|
|
||||||
// Nollataan KV-cache edellisestä promptista
|
/// Varmistaa että malli on ladattu Ollamaan (ollama pull)
|
||||||
self.model.clear_kv_cache();
|
pub async fn ensure_model(&self) -> Result<(), String> {
|
||||||
|
let model = self.model.borrow().clone();
|
||||||
|
tracing::info!("Tarkistetaan malli {}...", model);
|
||||||
|
let resp = self.client.post(format!("{}/api/pull", self.ollama_url))
|
||||||
|
.json(&serde_json::json!({ "name": model, "stream": false }))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("Ollama pull: {}", e))?;
|
||||||
|
|
||||||
// Sampling-parametrit
|
if resp.status().is_success() {
|
||||||
let temperature = 0.7;
|
tracing::info!("Malli {} valmis", model);
|
||||||
let top_k = 40;
|
Ok(())
|
||||||
let repetition_penalty = 1.15;
|
} else {
|
||||||
let mut rng_state: u64 = std::time::SystemTime::now()
|
Err(format!("Ollama pull epäonnistui: {}", resp.status()))
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
}
|
||||||
.unwrap()
|
}
|
||||||
.as_nanos() as u64;
|
|
||||||
|
pub async fn generate(&self, prompt: &str, max_tokens: usize) -> Result<GenerateResult, String> {
|
||||||
|
let system = "You are a coding assistant. Respond with ONLY code. Use proper newlines and indentation. No explanations, no markdown fences, no comments unless asked.";
|
||||||
|
let model = self.model.borrow().clone();
|
||||||
|
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
|
let resp = self.client.post(format!("{}/api/generate", self.ollama_url))
|
||||||
// Prefill
|
.json(&serde_json::json!({
|
||||||
let input = Tensor::new(input_ids.as_slice(), &self.device)
|
"model": model,
|
||||||
.and_then(|t| t.unsqueeze(0))
|
"prompt": prompt,
|
||||||
.map_err(|e| format!("Tensor: {}", e))?;
|
"system": system,
|
||||||
|
"stream": false,
|
||||||
let logits = self.model.forward(&input, 0)
|
"options": {
|
||||||
.map_err(|e| format!("Forward prefill: {}", e))?;
|
"num_predict": max_tokens,
|
||||||
|
"temperature": 0.7,
|
||||||
let logits = logits.squeeze(0).map_err(|e| format!("Squeeze: {}", e))?;
|
"top_k": 40,
|
||||||
let logits = if logits.dims().len() == 2 {
|
"repeat_penalty": 1.15,
|
||||||
let seq_len = logits.dim(0).map_err(|e| format!("Dim: {}", e))?;
|
"stop": ["<|im_end|>", "\n###", "\nExplanation", "\nNote:"]
|
||||||
if seq_len == 0 { return Err("Tyhjä tensori".to_string()); }
|
|
||||||
logits.get(seq_len - 1).map_err(|e| format!("Get: {}", e))?
|
|
||||||
} else {
|
|
||||||
logits
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut generated_text = String::new();
|
|
||||||
let mut tokens_generated: usize = 0;
|
|
||||||
let mut all_tokens: Vec<u32> = Vec::new();
|
|
||||||
|
|
||||||
let mut next_token = sample_top_k(&logits, top_k, temperature, &all_tokens, repetition_penalty, &mut rng_state)?;
|
|
||||||
|
|
||||||
if next_token != self.eos_token {
|
|
||||||
if let Ok(text) = self.tokenizer.decode(&[next_token], true) {
|
|
||||||
generated_text.push_str(&text);
|
|
||||||
}
|
|
||||||
all_tokens.push(next_token);
|
|
||||||
tokens_generated += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Autoregressive
|
|
||||||
let mut pos = input_len;
|
|
||||||
for _ in 1..max_tokens {
|
|
||||||
if next_token == self.eos_token { break; }
|
|
||||||
|
|
||||||
let input = Tensor::new(&[next_token], &self.device)
|
|
||||||
.and_then(|t| t.unsqueeze(0))
|
|
||||||
.map_err(|e| format!("Tensor: {}", e))?;
|
|
||||||
|
|
||||||
let logits = self.model.forward(&input, pos)
|
|
||||||
.map_err(|e| format!("Forward pos {}: {}", pos, e))?;
|
|
||||||
|
|
||||||
let logits = logits.squeeze(0).map_err(|e| format!("Squeeze: {}", e))?;
|
|
||||||
let logits = if logits.dims().len() == 2 {
|
|
||||||
let seq_len = logits.dim(0).map_err(|e| format!("Dim: {}", e))?;
|
|
||||||
if seq_len == 0 { break; }
|
|
||||||
logits.get(seq_len - 1).map_err(|e| format!("Get: {}", e))?
|
|
||||||
} else {
|
|
||||||
logits
|
|
||||||
};
|
|
||||||
next_token = sample_top_k(&logits, top_k, temperature, &all_tokens, repetition_penalty, &mut rng_state)?;
|
|
||||||
pos += 1;
|
|
||||||
|
|
||||||
if next_token == self.eos_token { break; }
|
|
||||||
|
|
||||||
if let Ok(text) = self.tokenizer.decode(&[next_token], true) {
|
|
||||||
generated_text.push_str(&text);
|
|
||||||
|
|
||||||
// Stop-sekvenssit: katkaistaan kun malli alkaa selittää
|
|
||||||
let lower = generated_text.to_lowercase();
|
|
||||||
if lower.contains("\n###") || lower.contains("\nexplanation") || lower.contains("\nnote:") || lower.contains("\noutput:") || lower.contains("\n```\n\n") || lower.contains("\n// example") || lower.contains("\n# example") {
|
|
||||||
for stop in &["\n###", "\nExplanation", "\nNote:", "\nOutput:", "\n```\n\n", "\n// Example", "\n// example", "\n# Example", "\n# example"] {
|
|
||||||
if let Some(pos) = generated_text.find(stop) {
|
|
||||||
generated_text.truncate(pos);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}))
|
||||||
all_tokens.push(next_token);
|
.send()
|
||||||
tokens_generated += 1;
|
.await
|
||||||
|
.map_err(|e| format!("Ollama generate: {}", e))?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("Ollama HTTP {}", resp.status()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let gen_time = start.elapsed();
|
let body: serde_json::Value = resp.json().await
|
||||||
let tokens_per_sec = if gen_time.as_secs_f64() > 0.0 {
|
.map_err(|e| format!("Ollama JSON: {}", e))?;
|
||||||
tokens_generated as f64 / gen_time.as_secs_f64()
|
|
||||||
|
let text = body["response"].as_str().unwrap_or("").to_string();
|
||||||
|
let total_duration_ns = body["total_duration"].as_u64().unwrap_or(0);
|
||||||
|
let eval_count = body["eval_count"].as_u64().unwrap_or(0) as usize;
|
||||||
|
let eval_duration_ns = body["eval_duration"].as_u64().unwrap_or(1);
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let tokens_per_sec = if eval_duration_ns > 0 {
|
||||||
|
eval_count as f64 / (eval_duration_ns as f64 / 1_000_000_000.0)
|
||||||
} else { 0.0 };
|
} else { 0.0 };
|
||||||
|
|
||||||
Ok(GenerateResult {
|
Ok(GenerateResult {
|
||||||
text: strip_markdown_wrapper(&generated_text),
|
text: strip_code_fences(&text),
|
||||||
tokens_generated,
|
tokens_generated: eval_count,
|
||||||
duration_ms: gen_time.as_millis() as f64,
|
duration_ms,
|
||||||
tokens_per_sec,
|
tokens_per_sec,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const LANG_TAGS: &[&str] = &[
|
/// Siivoa mahdolliset markdown-koodiblokki-merkit
|
||||||
"python", "py", "rust", "rs", "javascript", "js", "typescript", "ts",
|
fn strip_code_fences(text: &str) -> String {
|
||||||
"java", "kotlin", "scala", "go", "ruby", "rb", "php", "swift",
|
|
||||||
"c", "cpp", "c++", "c#", "csharp", "r", "sql", "bash", "sh", "zsh",
|
|
||||||
"html", "css", "json", "yaml", "yml", "toml", "xml", "markdown", "md",
|
|
||||||
"lua", "perl", "dart", "elixir", "haskell", "hs", "ocaml", "zig",
|
|
||||||
"plaintext", "text", "txt",
|
|
||||||
];
|
|
||||||
|
|
||||||
/// Siivoa mallin tuottama vastaus (prefill-yhteensopiva).
|
|
||||||
fn strip_markdown_wrapper(text: &str) -> String {
|
|
||||||
let mut result = text.trim().to_string();
|
let mut result = text.trim().to_string();
|
||||||
|
|
||||||
// 1. Kielitunniste — VAIN tunnettu kieli
|
// Poista aloittava ```lang
|
||||||
if let Some(nl) = result.find('\n') {
|
if result.starts_with("```") {
|
||||||
let first = result[..nl].trim().to_lowercase();
|
if let Some(nl) = result.find('\n') {
|
||||||
if LANG_TAGS.contains(&first.as_str()) {
|
|
||||||
result = result[nl + 1..].to_string();
|
result = result[nl + 1..].to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Sulkeva ``` — VAIN omalla rivillään lopussa
|
// Poista sulkeva ```
|
||||||
let trimmed = result.trim_end();
|
let trimmed = result.trim_end();
|
||||||
if trimmed.ends_with("```") {
|
if trimmed.ends_with("```") {
|
||||||
let before = &trimmed[..trimmed.len() - 3];
|
let before = &trimmed[..trimmed.len() - 3];
|
||||||
@@ -264,29 +117,7 @@ fn strip_markdown_wrapper(text: &str) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Johdantolauseet
|
result
|
||||||
let lower = result.trim().to_lowercase();
|
|
||||||
for prefix in &["sure!", "here is", "here's", "certainly!", "below is"] {
|
|
||||||
if lower.starts_with(prefix) {
|
|
||||||
if let Some(nl) = result.find('\n') { result = result[nl + 1..].to_string(); }
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Selityskommentit alusta
|
|
||||||
let mut lines: Vec<&str> = result.trim().lines().collect();
|
|
||||||
while !lines.is_empty() {
|
|
||||||
let first = lines[0].trim();
|
|
||||||
let is_preamble = first.starts_with("# ") && !first.starts_with("#!")
|
|
||||||
&& (first.to_lowercase().contains("this is")
|
|
||||||
|| first.to_lowercase().contains("simple")
|
|
||||||
|| first.to_lowercase().contains("program that")
|
|
||||||
|| first.to_lowercase().contains("here is")
|
|
||||||
|| first.to_lowercase().contains("the following")
|
|
||||||
|| first.to_lowercase().contains("below"));
|
|
||||||
if is_preamble { lines.remove(0); } else { break; }
|
|
||||||
}
|
|
||||||
lines.join("\n").trim().to_string()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct GenerateResult {
|
pub struct GenerateResult {
|
||||||
|
|||||||
@@ -285,15 +285,19 @@ async fn main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ladataan LLM-malli
|
// Ollama-backend
|
||||||
tracing::info!("Ladataan LLM-mallia...");
|
tracing::info!("Alustetaan Ollama-yhteyttä...");
|
||||||
let mut llm = match inference::LlmEngine::load() {
|
let llm = match inference::LlmEngine::load() {
|
||||||
Ok(engine) => {
|
Ok(engine) => {
|
||||||
tracing::info!("LLM valmis inferenssiin!");
|
// Varmistetaan malli (ollama pull) — odotetaan kunnes valmis
|
||||||
|
match engine.ensure_model().await {
|
||||||
|
Ok(()) => tracing::info!("Ollama valmis inferenssiin!"),
|
||||||
|
Err(e) => tracing::warn!("Mallin lataus: {} — yritetään silti", e),
|
||||||
|
}
|
||||||
Some(engine)
|
Some(engine)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("LLM-lataus epäonnistui: {} — toimitaan ilman inferenssiä", e);
|
tracing::warn!("Ollama-alustus epäonnistui: {} — toimitaan ilman inferenssiä", e);
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -324,11 +328,13 @@ async fn main() {
|
|||||||
|
|
||||||
if !prompt.is_empty() && msg_model.starts_with("qwen-coder") {
|
if !prompt.is_empty() && msg_model.starts_with("qwen-coder") {
|
||||||
|
|
||||||
if let Some(ref mut engine) = llm {
|
if let Some(ref engine) = llm {
|
||||||
busy = true;
|
busy = true;
|
||||||
tracing::info!("Generoidaan (task_id: {}): \"{}\"", task_id, prompt);
|
let max_tokens = task.get("max_tokens").and_then(|v| v.as_u64()).unwrap_or(512) as usize;
|
||||||
|
tracing::info!("Generoidaan (task_id: {}, max_tokens: {}): \"{}\"", task_id, max_tokens, &prompt[..prompt.len().min(100)]);
|
||||||
|
|
||||||
match engine.generate(prompt, 64) {
|
let model_name = engine.model_name();
|
||||||
|
match engine.generate(prompt, max_tokens).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
"Tulos: {} tokenia | {:.0}ms | {:.1} tok/s | \"{}\"",
|
"Tulos: {} tokenia | {:.0}ms | {:.1} tok/s | \"{}\"",
|
||||||
@@ -341,7 +347,7 @@ async fn main() {
|
|||||||
let done = json!({
|
let done = json!({
|
||||||
"type": "llm_done",
|
"type": "llm_done",
|
||||||
"prompt": prompt,
|
"prompt": prompt,
|
||||||
"model": "Qwen2.5-Coder-0.5B (native/GPU)",
|
"model": format!("{} (Ollama)", model_name),
|
||||||
"response": result.text,
|
"response": result.text,
|
||||||
"tokens_generated": result.tokens_generated,
|
"tokens_generated": result.tokens_generated,
|
||||||
"duration_ms": result.duration_ms,
|
"duration_ms": result.duration_ms,
|
||||||
@@ -360,7 +366,21 @@ async fn main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Ohitetaan pair_task, stats jne.
|
// Mallin vaihto lennossa
|
||||||
|
if text.contains("change_model") {
|
||||||
|
if let Ok(task) = serde_json::from_str::<serde_json::Value>(&text) {
|
||||||
|
if let Some(new_model) = task.get("model").and_then(|v| v.as_str()) {
|
||||||
|
if let Some(ref engine) = llm {
|
||||||
|
tracing::info!("Vaihdetaan malli: {}", new_model);
|
||||||
|
engine.set_model(new_model.to_string());
|
||||||
|
match engine.ensure_model().await {
|
||||||
|
Ok(()) => tracing::info!("Malli {} valmis!", new_model),
|
||||||
|
Err(e) => tracing::error!("Mallin lataus epäonnistui: {}", e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tracing::warn!("Yhteys hubiin katkesi — yritetään uudelleen 5s...");
|
tracing::warn!("Yhteys hubiin katkesi — yritetään uudelleen 5s...");
|
||||||
|
|||||||
@@ -38,17 +38,50 @@ pub fn set_gpu_load(load: u32) {
|
|||||||
console_log!("[Wasm] GPU Kuormitusraja vaihdettu -> {}%", load);
|
console_log!("[Wasm] GPU Kuormitusraja vaihdettu -> {}%", load);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Asynkroninen odotus WebAssemblylle
|
// Worker-yhteensopiva setTimeout — toimii sekä Window- että Worker-kontekstissa
|
||||||
async fn sleep_ms(ms: i32) {
|
#[wasm_bindgen]
|
||||||
|
extern "C" {
|
||||||
|
#[wasm_bindgen(js_name = setTimeout)]
|
||||||
|
fn set_timeout(closure: &js_sys::Function, ms: i32);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asynkroninen odotus WebAssemblylle (Window + Worker)
|
||||||
|
pub async fn sleep_ms(ms: i32) {
|
||||||
let promise = js_sys::Promise::new(&mut |resolve, _| {
|
let promise = js_sys::Promise::new(&mut |resolve, _| {
|
||||||
web_sys::window()
|
set_timeout(&resolve, ms);
|
||||||
.unwrap()
|
|
||||||
.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms)
|
|
||||||
.unwrap();
|
|
||||||
});
|
});
|
||||||
let _ = wasm_bindgen_futures::JsFuture::from(promise).await;
|
let _ = wasm_bindgen_futures::JsFuture::from(promise).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Worker-yhteensopiva Performance — käyttää globalThis.performance
|
||||||
|
pub fn perf_now() -> f64 {
|
||||||
|
js_sys::Reflect::get(&js_sys::global(), &"performance".into())
|
||||||
|
.ok()
|
||||||
|
.and_then(|p| js_sys::Reflect::get(&p, &"now".into()).ok())
|
||||||
|
.and_then(|f| f.dyn_into::<js_sys::Function>().ok())
|
||||||
|
.and_then(|f| {
|
||||||
|
let perf = js_sys::Reflect::get(&js_sys::global(), &"performance".into()).unwrap();
|
||||||
|
f.call0(&perf).ok()
|
||||||
|
})
|
||||||
|
.and_then(|v| v.as_f64())
|
||||||
|
.unwrap_or(0.0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Worker-yhteensopiva fetch — käyttää globalThis.fetch
|
||||||
|
pub async fn worker_fetch(url: &str) -> Result<web_sys::Response, String> {
|
||||||
|
let promise = js_sys::Reflect::get(&js_sys::global(), &"fetch".into())
|
||||||
|
.map_err(|_| "fetch ei saatavilla".to_string())?
|
||||||
|
.dyn_into::<js_sys::Function>()
|
||||||
|
.map_err(|_| "fetch ei funktio".to_string())?
|
||||||
|
.call1(&JsValue::NULL, &url.into())
|
||||||
|
.map_err(|e| format!("fetch: {:?}", e))?;
|
||||||
|
let resp = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::from(promise))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("fetch await: {:?}", e))?;
|
||||||
|
resp.dyn_into::<web_sys::Response>()
|
||||||
|
.map_err(|_| "ei Response".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
// Geneerinen tensorilaskenta — toimii millä tahansa Burn-backendillä
|
// Geneerinen tensorilaskenta — toimii millä tahansa Burn-backendillä
|
||||||
fn run_matmul<B: burn::tensor::backend::Backend>(size: usize) -> String {
|
fn run_matmul<B: burn::tensor::backend::Backend>(size: usize) -> String {
|
||||||
let device = Default::default();
|
let device = Default::default();
|
||||||
@@ -123,10 +156,9 @@ async fn run_single_tokenize(text: String, ws: Rc<RefCell<WebSocket>>) {
|
|||||||
let Some(bytes) = cached_tok else { return; };
|
let Some(bytes) = cached_tok else { return; };
|
||||||
let Ok(tokenizer) = tokenizers::Tokenizer::from_bytes(&bytes) else { return; };
|
let Ok(tokenizer) = tokenizers::Tokenizer::from_bytes(&bytes) else { return; };
|
||||||
|
|
||||||
let perf = web_sys::window().unwrap().performance().unwrap();
|
let start = perf_now();
|
||||||
let start = perf.now();
|
|
||||||
let result = tokenize_text(&tokenizer, &text);
|
let result = tokenize_text(&tokenizer, &text);
|
||||||
let duration_ms = perf.now() - start;
|
let duration_ms = perf_now() - start;
|
||||||
|
|
||||||
let token_count = result["token_count"].as_u64().unwrap_or(0);
|
let token_count = result["token_count"].as_u64().unwrap_or(0);
|
||||||
let cpt = result["chars_per_token"].as_f64().unwrap_or(0.0);
|
let cpt = result["chars_per_token"].as_f64().unwrap_or(0.0);
|
||||||
@@ -157,11 +189,10 @@ async fn run_pair_comparison(en_text: String, fi_text: String, ws: Rc<RefCell<We
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
let perf = web_sys::window().unwrap().performance().unwrap();
|
let start_time = perf_now();
|
||||||
let start_time = perf.now();
|
|
||||||
let en_result = tokenize_text(&tokenizer, &en_text);
|
let en_result = tokenize_text(&tokenizer, &en_text);
|
||||||
let fi_result = tokenize_text(&tokenizer, &fi_text);
|
let fi_result = tokenize_text(&tokenizer, &fi_text);
|
||||||
let duration_ms = perf.now() - start_time; // millisekunteja desimaalitarkkuudella
|
let duration_ms = perf_now() - start_time;
|
||||||
|
|
||||||
let en_cpt = en_result["chars_per_token"].as_f64().unwrap_or(0.0);
|
let en_cpt = en_result["chars_per_token"].as_f64().unwrap_or(0.0);
|
||||||
let fi_cpt = fi_result["chars_per_token"].as_f64().unwrap_or(0.0);
|
let fi_cpt = fi_result["chars_per_token"].as_f64().unwrap_or(0.0);
|
||||||
|
|||||||
@@ -24,10 +24,7 @@ async fn ensure_cached(key: &str, url: &str, ws: &Rc<RefCell<WebSocket>>) -> Res
|
|||||||
|
|
||||||
console_log!("[Qwen] Ladataan {}...", key);
|
console_log!("[Qwen] Ladataan {}...", key);
|
||||||
|
|
||||||
let window = web_sys::window().unwrap();
|
let resp = crate::worker_fetch(url).await?;
|
||||||
let resp_val = wasm_bindgen_futures::JsFuture::from(window.fetch_with_str(url))
|
|
||||||
.await.map_err(|e| format!("Fetch epäonnistui: {:?}", e))?;
|
|
||||||
let resp: web_sys::Response = resp_val.dyn_into().map_err(|_| "Ei Response".to_string())?;
|
|
||||||
if !resp.ok() { return Err(format!("HTTP {}", resp.status())); }
|
if !resp.ok() { return Err(format!("HTTP {}", resp.status())); }
|
||||||
|
|
||||||
let total_size: usize = resp.headers()
|
let total_size: usize = resp.headers()
|
||||||
@@ -71,7 +68,7 @@ async fn ensure_cached(key: &str, url: &str, ws: &Rc<RefCell<WebSocket>>) -> Res
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn run_qwen_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
pub async fn run_qwen_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
||||||
let perf = web_sys::window().unwrap().performance().unwrap();
|
// performance via crate::perf_now()
|
||||||
|
|
||||||
let tok_bytes = match ensure_cached("qwen05b-tokenizer.json", TOKENIZER_URL, &ws).await {
|
let tok_bytes = match ensure_cached("qwen05b-tokenizer.json", TOKENIZER_URL, &ws).await {
|
||||||
Ok(b) => b,
|
Ok(b) => b,
|
||||||
@@ -88,7 +85,7 @@ pub async fn run_qwen_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
console_log!("[Qwen] Rakennetaan mallia...");
|
console_log!("[Qwen] Rakennetaan mallia...");
|
||||||
let start_load = perf.now();
|
let start_load = crate::perf_now();
|
||||||
let device = Device::Cpu;
|
let device = Device::Cpu;
|
||||||
let dtype = DType::F32;
|
let dtype = DType::F32;
|
||||||
|
|
||||||
@@ -120,7 +117,7 @@ pub async fn run_qwen_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
|||||||
Err(e) => { console_log!("[Qwen] Mallin lataus: {}", e); return; }
|
Err(e) => { console_log!("[Qwen] Mallin lataus: {}", e); return; }
|
||||||
};
|
};
|
||||||
|
|
||||||
let load_time = perf.now() - start_load;
|
let load_time = crate::perf_now() - start_load;
|
||||||
console_log!("[Qwen] Malli ladattu ({:.0}ms). Generoidaan...", load_time);
|
console_log!("[Qwen] Malli ladattu ({:.0}ms). Generoidaan...", load_time);
|
||||||
|
|
||||||
let encoding = match tokenizer.encode(prompt.as_str(), true) {
|
let encoding = match tokenizer.encode(prompt.as_str(), true) {
|
||||||
@@ -131,7 +128,7 @@ pub async fn run_qwen_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
|||||||
let input_len = input_ids.len();
|
let input_len = input_ids.len();
|
||||||
console_log!("[Qwen] Syöte: {} tokenia", input_len);
|
console_log!("[Qwen] Syöte: {} tokenia", input_len);
|
||||||
|
|
||||||
let start_gen = perf.now();
|
let start_gen = crate::perf_now();
|
||||||
let max_new_tokens = 32;
|
let max_new_tokens = 32;
|
||||||
let mut generated_text = String::new();
|
let mut generated_text = String::new();
|
||||||
let mut tokens_generated: usize = 0;
|
let mut tokens_generated: usize = 0;
|
||||||
@@ -202,7 +199,7 @@ pub async fn run_qwen_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
|||||||
crate::sleep_ms(0).await;
|
crate::sleep_ms(0).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let gen_time = perf.now() - start_gen;
|
let gen_time = crate::perf_now() - start_gen;
|
||||||
let tokens_per_sec = if gen_time > 0.0 { (tokens_generated as f64 / gen_time) * 1000.0 } else { 0.0 };
|
let tokens_per_sec = if gen_time > 0.0 { (tokens_generated as f64 / gen_time) * 1000.0 } else { 0.0 };
|
||||||
console_log!("[Qwen] {} tokenia | {:.0}ms | {:.1} tok/s", tokens_generated, gen_time, tokens_per_sec);
|
console_log!("[Qwen] {} tokenia | {:.0}ms | {:.1} tok/s", tokens_generated, gen_time, tokens_per_sec);
|
||||||
|
|
||||||
|
|||||||
@@ -140,10 +140,7 @@ async fn ensure_cached(key: &str, url: &str, ws: &Rc<RefCell<WebSocket>>) -> Res
|
|||||||
|
|
||||||
console_log!("[Coder] Ladataan {}...", key);
|
console_log!("[Coder] Ladataan {}...", key);
|
||||||
|
|
||||||
let window = web_sys::window().unwrap();
|
let resp = crate::worker_fetch(url).await?;
|
||||||
let resp_val = wasm_bindgen_futures::JsFuture::from(window.fetch_with_str(url))
|
|
||||||
.await.map_err(|e| format!("Fetch: {:?}", e))?;
|
|
||||||
let resp: web_sys::Response = resp_val.dyn_into().map_err(|_| "Ei Response".to_string())?;
|
|
||||||
if !resp.ok() { return Err(format!("HTTP {}", resp.status())); }
|
if !resp.ok() { return Err(format!("HTTP {}", resp.status())); }
|
||||||
|
|
||||||
let total_size: usize = resp.headers()
|
let total_size: usize = resp.headers()
|
||||||
@@ -251,17 +248,16 @@ async fn get_or_build_model(use_3b: bool, ws: &Rc<RefCell<WebSocket>>) -> Result
|
|||||||
|
|
||||||
/// use_3b: false = 0.5B (nopea), true = 3B (laadukas)
|
/// use_3b: false = 0.5B (nopea), true = 3B (laadukas)
|
||||||
pub async fn run_coder_inference(prompt: String, ws: Rc<RefCell<WebSocket>>, use_3b: bool, task_id: Option<String>) {
|
pub async fn run_coder_inference(prompt: String, ws: Rc<RefCell<WebSocket>>, use_3b: bool, task_id: Option<String>) {
|
||||||
let perf = web_sys::window().unwrap().performance().unwrap();
|
|
||||||
let size_label = if use_3b { "3B" } else { "0.5B" };
|
let size_label = if use_3b { "3B" } else { "0.5B" };
|
||||||
|
|
||||||
let start_load = perf.now();
|
let start_load = crate::perf_now();
|
||||||
|
|
||||||
if let Err(e) = get_or_build_model(use_3b, &ws).await {
|
if let Err(e) = get_or_build_model(use_3b, &ws).await {
|
||||||
console_log!("[Coder] Mallin lataus: {}", e);
|
console_log!("[Coder] Mallin lataus: {}", e);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let load_time = perf.now() - start_load;
|
let load_time = crate::perf_now() - start_load;
|
||||||
if load_time > 100.0 {
|
if load_time > 100.0 {
|
||||||
console_log!("[Coder] Malli ladattu ({:.0}ms). Generoidaan...", load_time);
|
console_log!("[Coder] Malli ladattu ({:.0}ms). Generoidaan...", load_time);
|
||||||
}
|
}
|
||||||
@@ -297,7 +293,7 @@ pub async fn run_coder_inference(prompt: String, ws: Rc<RefCell<WebSocket>>, use
|
|||||||
console_log!("[Coder] Syöte: {} tokenia", input_len);
|
console_log!("[Coder] Syöte: {} tokenia", input_len);
|
||||||
|
|
||||||
let device = Device::Cpu;
|
let device = Device::Cpu;
|
||||||
let start_gen = perf.now();
|
let start_gen = crate::perf_now();
|
||||||
let eos_token = 151645u32;
|
let eos_token = 151645u32;
|
||||||
let temperature: f32 = 0.7;
|
let temperature: f32 = 0.7;
|
||||||
let top_k: usize = 40;
|
let top_k: usize = 40;
|
||||||
@@ -373,7 +369,7 @@ pub async fn run_coder_inference(prompt: String, ws: Rc<RefCell<WebSocket>>, use
|
|||||||
tokens_generated += 1;
|
tokens_generated += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
let gen_time = perf.now() - start_gen;
|
let gen_time = crate::perf_now() - start_gen;
|
||||||
|
|
||||||
// Siivotaan vastaus: poista markdown-koodiblokit ja johdantotekstit
|
// Siivotaan vastaus: poista markdown-koodiblokit ja johdantotekstit
|
||||||
let cleaned = strip_markdown_wrapper(&generated_text);
|
let cleaned = strip_markdown_wrapper(&generated_text);
|
||||||
|
|||||||
@@ -28,10 +28,7 @@ async fn ensure_cached(key: &str, url: &str, ws: &Rc<RefCell<WebSocket>>) -> Res
|
|||||||
send_progress(ws, key, 0, 0, 0);
|
send_progress(ws, key, 0, 0, 0);
|
||||||
|
|
||||||
// Fetch API:lla saadaan Content-Length ja streaming-luku
|
// Fetch API:lla saadaan Content-Length ja streaming-luku
|
||||||
let window = web_sys::window().unwrap();
|
let resp = crate::worker_fetch(url).await?;
|
||||||
let resp_val = wasm_bindgen_futures::JsFuture::from(window.fetch_with_str(url))
|
|
||||||
.await.map_err(|e| format!("Fetch epäonnistui: {:?}", e))?;
|
|
||||||
let resp: web_sys::Response = resp_val.dyn_into().map_err(|_| "Ei Response-objekti".to_string())?;
|
|
||||||
|
|
||||||
if !resp.ok() {
|
if !resp.ok() {
|
||||||
return Err(format!("HTTP {}", resp.status()));
|
return Err(format!("HTTP {}", resp.status()));
|
||||||
@@ -99,7 +96,7 @@ fn send_progress(ws: &Rc<RefCell<WebSocket>>, file: &str, pct: u32, loaded: usiz
|
|||||||
|
|
||||||
/// Lataa malli ja tokenizer, suorita inferenssi ja streamaa tokenit hubille
|
/// Lataa malli ja tokenizer, suorita inferenssi ja streamaa tokenit hubille
|
||||||
pub async fn run_smollm_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
pub async fn run_smollm_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
||||||
let perf = web_sys::window().unwrap().performance().unwrap();
|
// performance via crate::perf_now()
|
||||||
|
|
||||||
// 1. Lataa tokenizer
|
// 1. Lataa tokenizer
|
||||||
let tok_bytes = match ensure_cached("smollm-tokenizer.json", TOKENIZER_URL, &ws).await {
|
let tok_bytes = match ensure_cached("smollm-tokenizer.json", TOKENIZER_URL, &ws).await {
|
||||||
@@ -122,7 +119,7 @@ pub async fn run_smollm_inference(prompt: String, ws: Rc<RefCell<WebSocket>>) {
|
|||||||
// Burn 0.21-pre.2 cubecl-runtime ei käänny Wasmille (println! puuttuu)
|
// Burn 0.21-pre.2 cubecl-runtime ei käänny Wasmille (println! puuttuu)
|
||||||
// → NdArray kunnes Burn 0.21 stable + Wasm-tuki
|
// → NdArray kunnes Burn 0.21 stable + Wasm-tuki
|
||||||
console_log!("[SmolLM] Burn NdArray (CPU) inferenssi...");
|
console_log!("[SmolLM] Burn NdArray (CPU) inferenssi...");
|
||||||
run_burn_inference::<burn::backend::NdArray>(prompt, model_bytes, tokenizer, ws, perf.clone()).await;
|
run_burn_inference::<burn::backend::NdArray>(prompt, model_bytes, tokenizer, ws).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn run_burn_inference<B: burn::tensor::backend::Backend>(
|
async fn run_burn_inference<B: burn::tensor::backend::Backend>(
|
||||||
@@ -130,9 +127,8 @@ async fn run_burn_inference<B: burn::tensor::backend::Backend>(
|
|||||||
model_bytes: Vec<u8>,
|
model_bytes: Vec<u8>,
|
||||||
tokenizer: tokenizers::Tokenizer,
|
tokenizer: tokenizers::Tokenizer,
|
||||||
ws: Rc<RefCell<WebSocket>>,
|
ws: Rc<RefCell<WebSocket>>,
|
||||||
perf: web_sys::Performance, // Korjattu Wasm-performanssi välitettäväksi
|
|
||||||
) {
|
) {
|
||||||
let start_load = perf.now();
|
let start_load = crate::perf_now();
|
||||||
|
|
||||||
let device = Default::default();
|
let device = Default::default();
|
||||||
let config = crate::burn_smollm::config::SmolLMConfig::default();
|
let config = crate::burn_smollm::config::SmolLMConfig::default();
|
||||||
@@ -143,7 +139,7 @@ async fn run_burn_inference<B: burn::tensor::backend::Backend>(
|
|||||||
Err(e) => { console_log!("[SmolLM] Lataus epäonnistui: {}", e); return; }
|
Err(e) => { console_log!("[SmolLM] Lataus epäonnistui: {}", e); return; }
|
||||||
};
|
};
|
||||||
|
|
||||||
let load_time = perf.now() - start_load;
|
let load_time = crate::perf_now() - start_load;
|
||||||
console_log!("[SmolLM] Burn-malli ladattu ({:.0}ms). Generoidaan...", load_time);
|
console_log!("[SmolLM] Burn-malli ladattu ({:.0}ms). Generoidaan...", load_time);
|
||||||
|
|
||||||
let formatted_prompt = format!("<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", prompt);
|
let formatted_prompt = format!("<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", prompt);
|
||||||
@@ -156,7 +152,7 @@ async fn run_burn_inference<B: burn::tensor::backend::Backend>(
|
|||||||
let input_len = input_ids.len();
|
let input_len = input_ids.len();
|
||||||
console_log!("[SmolLM] Syöte: {} tokenia", input_len);
|
console_log!("[SmolLM] Syöte: {} tokenia", input_len);
|
||||||
|
|
||||||
let start_gen = perf.now();
|
let start_gen = crate::perf_now();
|
||||||
let max_new_tokens = 32;
|
let max_new_tokens = 32;
|
||||||
let mut generated_text = String::new();
|
let mut generated_text = String::new();
|
||||||
let mut tokens_generated: usize = 0;
|
let mut tokens_generated: usize = 0;
|
||||||
@@ -219,7 +215,7 @@ async fn run_burn_inference<B: burn::tensor::backend::Backend>(
|
|||||||
tokens_generated += 1;
|
tokens_generated += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
let gen_time = perf.now() - start_gen;
|
let gen_time = crate::perf_now() - start_gen;
|
||||||
let tokens_per_sec = if gen_time > 0.0 { (tokens_generated as f64 / gen_time) * 1000.0 } else { 0.0 };
|
let tokens_per_sec = if gen_time > 0.0 { (tokens_generated as f64 / gen_time) * 1000.0 } else { 0.0 };
|
||||||
|
|
||||||
let done = serde_json::json!({
|
let done = serde_json::json!({
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
38
network-poc/static/worker.js
Normal file
38
network-poc/static/worker.js
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
// Kipinä WASM Worker (ES module) — ajaa kielimallin inferenssin erillisessä säikeessä
|
||||||
|
import init, { start_agent_node, set_gpu_load, set_auto_tasks } from './pkg/node.js';
|
||||||
|
|
||||||
|
let wasmReady = false;
|
||||||
|
|
||||||
|
// Välitetään console.log -viestit pääsäikeelle jotta UI-kuuntelijat näkevät ne
|
||||||
|
const _origLog = console.log;
|
||||||
|
console.log = function(...args) {
|
||||||
|
_origLog.apply(console, args);
|
||||||
|
self.postMessage({ type: 'log', message: args.join(' ') });
|
||||||
|
};
|
||||||
|
|
||||||
|
self.onmessage = async (e) => {
|
||||||
|
const { type, data } = e.data;
|
||||||
|
|
||||||
|
if (type === 'init') {
|
||||||
|
try {
|
||||||
|
await init();
|
||||||
|
wasmReady = true;
|
||||||
|
self.postMessage({ type: 'ready' });
|
||||||
|
} catch (err) {
|
||||||
|
self.postMessage({ type: 'error', message: 'WASM init: ' + err.message });
|
||||||
|
}
|
||||||
|
} else if (type === 'start') {
|
||||||
|
if (!wasmReady) return;
|
||||||
|
const { hubUrl, hasWebGPU, deviceInfo, taskId } = data;
|
||||||
|
try {
|
||||||
|
await start_agent_node(hubUrl, hasWebGPU, deviceInfo, taskId);
|
||||||
|
self.postMessage({ type: 'started' });
|
||||||
|
} catch (err) {
|
||||||
|
self.postMessage({ type: 'error', message: 'Node: ' + err.message });
|
||||||
|
}
|
||||||
|
} else if (type === 'set_gpu_load') {
|
||||||
|
if (wasmReady) set_gpu_load(data.load);
|
||||||
|
} else if (type === 'set_auto_tasks') {
|
||||||
|
if (wasmReady) set_auto_tasks(data.enabled);
|
||||||
|
}
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user