jaakko/agentic-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Benchmark: nosta token-rajoja thinking-malleja varten

Browse Source 
qwen3.5 palauttaa ajattelun erillisessä thinking-kentässä,
content jää tyhjäksi jos tokenit loppuvat kesken.
Vaatimukset 1024→2048, speksi 2048→4096.
This commit is contained in:
jaakko
2026-04-14 08:42:32 +03:00
parent ad097ca712
commit 62c9b6e17e
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
network-poc/tests/model-benchmark.mjs
View File

@@ -79,7 +79,9 @@ async function ollamaChat(model, prompt, systemPrompt, maxTokens = 2048) {
if (!resp.ok) throw new Error(`Ollama HTTP ${resp.status}: ${await resp.text()}`); if (!resp.ok) throw new Error(`Ollama HTTP ${resp.status}: ${await resp.text()}`);
const data = await resp.json(); const data = await resp.json();
const elapsed = Date.now() - start; const elapsed = Date.now() - start;
const text = stripThinking((data.message?.content || '').trim()); // Ollama: jotkin mallit (qwen3.5) palauttavat ajattelun erillisessä thinking-kentässä
const rawContent = (data.message?.content || '').trim();
const text = stripThinking(rawContent);
const evalCount = data.eval_count || 0; const evalCount = data.eval_count || 0;
const evalDurationNs = data.eval_duration || 1; const evalDurationNs = data.eval_duration || 1;
const tokPerSec = evalCount / (evalDurationNs / 1e9); const tokPerSec = evalCount / (evalDurationNs / 1e9);
@@ -283,7 +285,7 @@ async function runPipeline(model, scenario) {
try { try {
// 1. Vaatimukset // 1. Vaatimukset
console.log(` [1/5] Vaatimukset...`); console.log(` [1/5] Vaatimukset...`);
const req = await ollamaChat(model, scenario.prompt, CLIENT_SYSTEM, 1024); const req = await ollamaChat(model, scenario.prompt, CLIENT_SYSTEM, 2048);
timings.push(req); timings.push(req);
if (!req.text || req.text.length < 50) { result.error = 'Vaatimukset liian lyhyet'; return result; } if (!req.text || req.text.length < 50) { result.error = 'Vaatimukset liian lyhyet'; return result; }
result.reqOk = true; result.reqOk = true;
@@ -291,7 +293,7 @@ async function runPipeline(model, scenario) {
// 2. JSON-speksi // 2. JSON-speksi
console.log(` [2/5] JSON-speksi...`); console.log(` [2/5] JSON-speksi...`);
const specResp = await ollamaChat(model, `${req.text}\n\nOutput a JSON spec for this project.`, SPEC_SYSTEM, 2048); const specResp = await ollamaChat(model, `${req.text}\n\nOutput a JSON spec for this project.`, SPEC_SYSTEM, 4096);
timings.push(specResp); timings.push(specResp);
const spec = extractJson(specResp.text); const spec = extractJson(specResp.text);
if (!spec || !spec.entities || spec.entities.length === 0) { result.error = 'JSON-speksi epäonnistui'; writeFileSync(`${dir}/_spec_raw.txt`, specResp.text); return result; } if (!spec || !spec.entities || spec.entities.length === 0) { result.error = 'JSON-speksi epäonnistui'; writeFileSync(`${dir}/_spec_raw.txt`, specResp.text); return result; }