jaakko/agentic-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CodeBench: --file-by-file generointi pienille malleille

Browse Source 
- Generoi yksi tiedosto kerrallaan: models.go → handlers.go → main.go → tests
- Edellisten tiedostojen koodi kontekstissa seuraavalle
- Max 2048 tok per tiedosto (vs 10240 kaikki kerralla)
- go.mod generoidaan aina golden examplesta (ei mallin tuotoksesta)
- Promptissa "Write ONLY the file X" + "Start with package main"
This commit is contained in:
jaakko
2026-04-14 23:31:20 +03:00
parent 178bef1277
commit a65a25c56c
1 changed files with 42 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
kipina-codebench/benchmark.mjs
View File

@@ -35,6 +35,7 @@ const RESULTS_DIR = join(__dirname, 'results');
const THINK_MODE = args.includes('--think');
const COMPACT_MODE = args.includes('--compact');
const NO_ORCHESTRATE = args.includes('--no-orchestrate');
const FILE_BY_FILE = args.includes('--file-by-file');
const SPEC_MODEL = arg('spec-model', ''); // Eri malli spec-vaiheille (1-2)
const SPEC_OLLAMA = arg('spec-ollama', ''); // Eri Ollama spec-mallille
const LANG = arg('lang', 'python'); // python | rust | go
@@ -101,6 +102,13 @@ const LANG_CONFIG = {
files: ['go.mod', 'models.go', 'handlers.go', 'main.go', 'handlers_test.go'],
required: ['go.mod', 'models.go', 'handlers.go', 'main.go', 'handlers_test.go'],
dockerImage: 'kipina-go-test',
// Tiedosto-kerrallaan generointi (--file-by-file): järjestys ja kuvaukset
fileByFile: [
{ name: 'models.go', desc: 'Go structs for all entities + Create/Update request types. Use json tags.' },
{ name: 'handlers.go', desc: 'Chi HTTP handlers as closures taking *sql.DB. Use RETURNING in INSERT/UPDATE. sql.ErrNoRows for 404.' },
{ name: 'main.go', desc: 'Chi router setup, InitDB with CREATE TABLE, main() entry point on port 3000.' },
{ name: 'handlers_test.go', desc: 'Tests using httptest.NewServer + :memory: SQLite. setupTestServer helper. CRUD tests per entity.' },
],
},
};
const LCONF = LANG_CONFIG[LANG] || LANG_CONFIG.python;
@@ -358,8 +366,37 @@ async function runPipeline(model, scenario, round = 1) {
const codeTokens = LANG === 'rust' ? 12288 : LANG === 'go' ? 10240 : 8192;
let files;
// Orkestrointi: pilko entiteetti kerrallaan pienille malleille
if (spec.entities.length > 1 && !NO_ORCHESTRATE) {
// File-by-file: generoi yksi tiedosto kerrallaan (pienille malleille)
if (FILE_BY_FILE && LCONF.fileByFile) {
const fbf = LCONF.fileByFile;
console.log(` [3/5] Koodigenerointi (file-by-file, ${fbf.length} tiedostoa)...`);
files = {};
let context = '';
for (const fileDef of fbf) {
const contextBlock = context ? `\nEXISTING CODE:\n${context}\n` : '';
const filePrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(spec, null, 2)}\n${contextBlock}\nWrite ONLY the file "${fileDef.name}": ${fileDef.desc}\nOutput raw code, no markdown fences, no explanations. Start with "package main".`;
console.log(` [3/5] → ${fileDef.name}...`);
const fileResp = await ollamaChat(model, filePrompt, CODE_SYSTEM, 2048);
timings.push(fileResp);
// Siivoa: poista markdown-aidat ja selitysteksti
let code = fileResp.text
.replace(/^```(?:go|golang)?\s*\n?/m, '').replace(/\n?```\s*$/m, '')
.replace(/^(?:Here|Sure|Below|This|The|I )[\s\S]*?(?=package\s)/m, '')
.trim();
if (code) {
files[fileDef.name] = code + '\n';
context += `=== ${fileDef.name} ===\n${code}\n\n`;
}
}
writeFileSync(`${dir}/_code_raw.txt`, context);
result.promptChars = CODE_SYSTEM.length + (context.length || 0);
result.promptTokensEst = Math.round(result.promptChars / 4);
}
// Orkestrointi: pilko entiteetti kerrallaan
else if (spec.entities.length > 1 && !NO_ORCHESTRATE) {
console.log(` [3/5] Koodigenerointi (orkestroitu, ${spec.entities.length} entiteettiä)...`);
files = {};
let cumulativeCode = '';
@@ -411,10 +448,10 @@ async function runPipeline(model, scenario, round = 1) {
const missing = LCONF.required.filter(f => !files[f]);
if (missing.length > 0) { result.error = `Puuttuvat: ${missing.join(', ')}`; return result; }
// Go: korvaa go.mod aina golden examplen versiolla (pienet mallit eivät tuota luotettavaa go.modia)
if (LANG === 'go' && files['go.mod']) {
// Go: korvaa/generoi go.mod golden examplen versiolla
if (LANG === 'go') {
const goldenMod = readFileSync(join(GOLDEN_DIR, 'todo-go', 'go.mod'), 'utf-8');
const modName = files['go.mod'].match(/^module\s+(\S+)/m)?.[1] || 'generated-api';
const modName = (files['go.mod']?.match(/^module\s+(\S+)/m)?.[1]) || spec.project_name?.replace(/[^a-z0-9-]/gi, '-') || 'generated-api';
files['go.mod'] = goldenMod.replace(/^module\s+\S+/m, `module ${modName}`);
}