jaakko/agentic-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CodeBench Taso 3: orkestrointi — pilko entiteetti kerrallaan pienille malleille

Browse Source 
Small-profiili + >1 entiteettiä → generoi entiteetti kerrallaan:
1. Author (yksi entiteetti, 8b osaa)
2. Syötä Author-koodi → generoi Post + FK
3. Yhdistä ja testaa

Large-profiili jatkaa kuten ennen (kaikki kerralla).
This commit is contained in:
jaakko
2026-04-14 15:00:40 +03:00
parent a8f731d38e
commit 0b926c2cad
1 changed files with 52 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
kipina-codebench/benchmark.mjs
View File

@@ -325,17 +325,61 @@ async function runPipeline(model, scenario) {
writeFileSync(`${dir}/_spec.json`, JSON.stringify(spec, null, 2));
// 3. LLM-koodigenerointi
console.log(` [3/5] Koodigenerointi (LLM)...`);
const fileCount = LCONF.required.length;
const goldenExample = loadGoldenExample(model);
const codePrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(spec, null, 2)}\n\nGenerate the complete project with all ${fileCount} files. Follow the reference implementation patterns exactly.`;
result.promptChars = CODE_SYSTEM.length + codePrompt.length;
result.promptTokensEst = Math.round(result.promptChars / 4);
const codeTokens = LANG === 'rust' ? 12288 : 8192;
const codeResp = await ollamaChat(model, codePrompt, CODE_SYSTEM, codeTokens);
timings.push(codeResp);
writeFileSync(`${dir}/_code_raw.txt`, codeResp.text);
const files = parseGeneratedFiles(codeResp.text);
let files;
// Orkestrointi: pilko entiteetti kerrallaan pienille malleille
if (profile === 'small' && spec.entities.length > 1 && LANG === 'python') {
console.log(` [3/5] Koodigenerointi (orkestroitu, ${spec.entities.length} entiteettiä)...`);
files = {};
let cumulativeCode = '';
for (let ei = 0; ei < spec.entities.length; ei++) {
const entity = spec.entities[ei];
const isFirst = ei === 0;
const entitySpec = {
...spec,
entities: spec.entities.slice(0, ei + 1),
relationships: (spec.relationships || []).filter(r =>
spec.entities.slice(0, ei + 1).some(e => e.name === r.from)
),
};
let entityPrompt;
if (isFirst) {
entityPrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(entitySpec, null, 2)}\n\nGenerate the complete project with all ${fileCount} files for the entity "${entity.name}". Follow the reference implementation patterns exactly.`;
} else {
entityPrompt = `${goldenExample}\n---\n\nEXISTING CODE (do not regenerate, only add to it):\n${cumulativeCode}\n\n---\n\nJSON SPECIFICATION (add entity "${entity.name}"):\n${JSON.stringify(entitySpec, null, 2)}\n\nAdd the entity "${entity.name}" to the existing code. Return ALL ${fileCount} files with === markers, including the existing entities. Follow the same patterns.`;
}
console.log(` [3/5] → ${entity.name}${isFirst ? '' : ' (+ ' + spec.entities.slice(0, ei).map(e => e.name).join(', ') + ')'}...`);
const entityResp = await ollamaChat(model, entityPrompt, CODE_SYSTEM, codeTokens);
timings.push(entityResp);
const entityFiles = parseGeneratedFiles(entityResp.text);
// Yhdistä — uudempi korvaa edellisen
for (const [fn, content] of Object.entries(entityFiles)) {
files[fn] = content;
}
cumulativeCode = Object.entries(files).map(([fn, c]) => `=== ${fn} ===\n${c}`).join('\n\n');
}
writeFileSync(`${dir}/_code_raw.txt`, cumulativeCode);
result.promptChars = CODE_SYSTEM.length + cumulativeCode.length;
result.promptTokensEst = Math.round(result.promptChars / 4);
} else {
// Normaali: kaikki kerralla
console.log(` [3/5] Koodigenerointi (LLM)...`);
const codePrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(spec, null, 2)}\n\nGenerate the complete project with all ${fileCount} files. Follow the reference implementation patterns exactly.`;
result.promptChars = CODE_SYSTEM.length + codePrompt.length;
result.promptTokensEst = Math.round(result.promptChars / 4);
const codeResp = await ollamaChat(model, codePrompt, CODE_SYSTEM, codeTokens);
timings.push(codeResp);
writeFileSync(`${dir}/_code_raw.txt`, codeResp.text);
files = parseGeneratedFiles(codeResp.text);
}
const missing = LCONF.required.filter(f => !files[f]);
if (missing.length > 0) { result.error = `Puuttuvat: ${missing.join(', ')}`; return result; }