diff --git a/kipina-codebench/benchmark.mjs b/kipina-codebench/benchmark.mjs index 5731c56..17db719 100644 --- a/kipina-codebench/benchmark.mjs +++ b/kipina-codebench/benchmark.mjs @@ -35,6 +35,7 @@ const RESULTS_DIR = join(__dirname, 'results'); const THINK_MODE = args.includes('--think'); const COMPACT_MODE = args.includes('--compact'); const NO_ORCHESTRATE = args.includes('--no-orchestrate'); +const FILE_BY_FILE = args.includes('--file-by-file'); const SPEC_MODEL = arg('spec-model', ''); // Eri malli spec-vaiheille (1-2) const SPEC_OLLAMA = arg('spec-ollama', ''); // Eri Ollama spec-mallille const LANG = arg('lang', 'python'); // python | rust | go @@ -101,6 +102,13 @@ const LANG_CONFIG = { files: ['go.mod', 'models.go', 'handlers.go', 'main.go', 'handlers_test.go'], required: ['go.mod', 'models.go', 'handlers.go', 'main.go', 'handlers_test.go'], dockerImage: 'kipina-go-test', + // Tiedosto-kerrallaan generointi (--file-by-file): järjestys ja kuvaukset + fileByFile: [ + { name: 'models.go', desc: 'Go structs for all entities + Create/Update request types. Use json tags.' }, + { name: 'handlers.go', desc: 'Chi HTTP handlers as closures taking *sql.DB. Use RETURNING in INSERT/UPDATE. sql.ErrNoRows for 404.' }, + { name: 'main.go', desc: 'Chi router setup, InitDB with CREATE TABLE, main() entry point on port 3000.' }, + { name: 'handlers_test.go', desc: 'Tests using httptest.NewServer + :memory: SQLite. setupTestServer helper. CRUD tests per entity.' }, + ], }, }; const LCONF = LANG_CONFIG[LANG] || LANG_CONFIG.python; @@ -358,8 +366,37 @@ async function runPipeline(model, scenario, round = 1) { const codeTokens = LANG === 'rust' ? 12288 : LANG === 'go' ? 10240 : 8192; let files; - // Orkestrointi: pilko entiteetti kerrallaan pienille malleille - if (spec.entities.length > 1 && !NO_ORCHESTRATE) { + // File-by-file: generoi yksi tiedosto kerrallaan (pienille malleille) + if (FILE_BY_FILE && LCONF.fileByFile) { + const fbf = LCONF.fileByFile; + console.log(` [3/5] Koodigenerointi (file-by-file, ${fbf.length} tiedostoa)...`); + files = {}; + let context = ''; + + for (const fileDef of fbf) { + const contextBlock = context ? `\nEXISTING CODE:\n${context}\n` : ''; + const filePrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(spec, null, 2)}\n${contextBlock}\nWrite ONLY the file "${fileDef.name}": ${fileDef.desc}\nOutput raw code, no markdown fences, no explanations. Start with "package main".`; + + console.log(` [3/5] → ${fileDef.name}...`); + const fileResp = await ollamaChat(model, filePrompt, CODE_SYSTEM, 2048); + timings.push(fileResp); + + // Siivoa: poista markdown-aidat ja selitysteksti + let code = fileResp.text + .replace(/^```(?:go|golang)?\s*\n?/m, '').replace(/\n?```\s*$/m, '') + .replace(/^(?:Here|Sure|Below|This|The|I )[\s\S]*?(?=package\s)/m, '') + .trim(); + if (code) { + files[fileDef.name] = code + '\n'; + context += `=== ${fileDef.name} ===\n${code}\n\n`; + } + } + writeFileSync(`${dir}/_code_raw.txt`, context); + result.promptChars = CODE_SYSTEM.length + (context.length || 0); + result.promptTokensEst = Math.round(result.promptChars / 4); + } + // Orkestrointi: pilko entiteetti kerrallaan + else if (spec.entities.length > 1 && !NO_ORCHESTRATE) { console.log(` [3/5] Koodigenerointi (orkestroitu, ${spec.entities.length} entiteettiä)...`); files = {}; let cumulativeCode = ''; @@ -411,10 +448,10 @@ async function runPipeline(model, scenario, round = 1) { const missing = LCONF.required.filter(f => !files[f]); if (missing.length > 0) { result.error = `Puuttuvat: ${missing.join(', ')}`; return result; } - // Go: korvaa go.mod aina golden examplen versiolla (pienet mallit eivät tuota luotettavaa go.modia) - if (LANG === 'go' && files['go.mod']) { + // Go: korvaa/generoi go.mod golden examplen versiolla + if (LANG === 'go') { const goldenMod = readFileSync(join(GOLDEN_DIR, 'todo-go', 'go.mod'), 'utf-8'); - const modName = files['go.mod'].match(/^module\s+(\S+)/m)?.[1] || 'generated-api'; + const modName = (files['go.mod']?.match(/^module\s+(\S+)/m)?.[1]) || spec.project_name?.replace(/[^a-z0-9-]/gi, '-') || 'generated-api'; files['go.mod'] = goldenMod.replace(/^module\s+\S+/m, `module ${modName}`); }