diff --git a/kipina-codebench/benchmark.mjs b/kipina-codebench/benchmark.mjs index 35e36b4..274849d 100644 --- a/kipina-codebench/benchmark.mjs +++ b/kipina-codebench/benchmark.mjs @@ -379,76 +379,90 @@ async function runPipeline(model, scenario, round = 1) { const codeTokens = isConvert ? 8192 : (LANG === 'rust' ? 12288 : LANG === 'go' ? 10240 : 8192); let files; - // File-by-file: generoi yksi tiedosto kerrallaan + välitön validointi + // File-by-file: generoi yksi tiedosto kerrallaan, build-validointi kun kaikki valmiina if (FILE_BY_FILE && LCONF.fileByFile) { const fbf = LCONF.fileByFile; - const MAX_FILE_FIX = 2; + const MAX_BUILD_FIX = 2; console.log(` [3/5] Koodigenerointi (file-by-file, ${fbf.length} tiedostoa)...`); files = {}; let context = ''; + // Vaihe 1: generoi jokainen tiedosto for (const fileDef of fbf) { const contextBlock = context ? `\nEXISTING CODE:\n${context}\n` : ''; - const basePrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(spec, null, 2)}\n${contextBlock}\nWrite ONLY the file "${fileDef.name}": ${fileDef.desc}\nOutput raw code, no markdown fences, no explanations. Start with "package main".`; + const filePrompt = `${goldenExample}\n---\n\nPROJECT REQUIREMENTS:\n${req.text}\n\nJSON SPECIFICATION:\n${JSON.stringify(spec, null, 2)}\n${contextBlock}\nWrite ONLY the file "${fileDef.name}": ${fileDef.desc}\nOutput raw code, no markdown fences, no explanations. Start with "package main".`; - let code = ''; - for (let attempt = 0; attempt <= MAX_FILE_FIX; attempt++) { - const prompt = attempt === 0 ? basePrompt - : `Fix the following Go compilation errors in "${fileDef.name}". Return ONLY the corrected file, no explanations.\n\nERRORS:\n${code.__buildErrors}\n\nCURRENT FILE:\n${code}\n\nOTHER FILES:\n${context}`; + console.log(` [3/5] → ${fileDef.name}...`); + const fileResp = await ollamaChat(model, filePrompt, CODE_SYSTEM, 2048); + timings.push(fileResp); - const label = attempt === 0 ? fileDef.name : `${fileDef.name} (fix ${attempt})`; - console.log(` [3/5] → ${label}...`); - const fileResp = await ollamaChat(model, prompt, CODE_SYSTEM, 2048); - timings.push(fileResp); + let code = fileResp.text + .replace(/^```(?:go|golang)?\s*\n?/m, '').replace(/\n?```\s*$/m, '') + .replace(/^(?:Here|Sure|Below|This|The|I )[\s\S]*?(?=package\s)/m, '') + .trim(); - code = fileResp.text - .replace(/^```(?:go|golang)?\s*\n?/m, '').replace(/\n?```\s*$/m, '') - .replace(/^(?:Here|Sure|Below|This|The|I )[\s\S]*?(?=package\s)/m, '') - .trim(); - - if (!code) break; - - const loc = code.split('\n').length; - console.log(` [3/5] ${fileResp.tokens} tok, ${loc} lines, ${fileResp.tokPerSec.toFixed(0)} tok/s`); - - // Välitön validointi: kirjoita tiedostot levylle ja aja go build - if (LANG === 'go' && !fileDef.name.endsWith('_test.go')) { - files[fileDef.name] = code + '\n'; - // Kirjoita kaikki tähänastiset tiedostot + go.mod - const goldenMod = readFileSync(join(GOLDEN_DIR, 'todo-go', 'go.mod'), 'utf-8'); - const modName = spec.project_name?.replace(/[^a-z0-9-]/gi, '-') || 'generated-api'; - writeFileSync(join(dir, 'go.mod'), goldenMod.replace(/^module\s+\S+/m, `module ${modName}`)); - for (const [fn, c] of Object.entries(files)) { - writeFileSync(join(dir, fn), c); - } - // go build tarkistus - try { - execSync( - `docker run --rm --entrypoint sh -v "${dir}:/src:ro" ${LCONF.dockerImage} -c "cp -r /src/* . && go mod tidy 2>&1 && go build ./... 2>&1"`, - { timeout: 60000, encoding: 'utf-8', stdio: 'pipe' } - ); - console.log(` [3/5] ✓ kääntyy`); - break; // OK — seuraava tiedosto - } catch (e) { - const buildErrors = (e.stdout || e.stderr || '').split('\n').filter(l => /\.go:\d+/.test(l)).slice(0, 10).join('\n'); - if (!buildErrors || attempt >= MAX_FILE_FIX) { - console.log(` [3/5] ⚠ käännösvirhe (ei korjata)`); - break; - } - console.log(` [3/5] ✗ ${buildErrors.split('\n').length} errors → fixing`); - code.__buildErrors = buildErrors; - result.fixRounds++; - } - } else { - break; // Testitiedostoa ei validoida go buildilla - } - } - - if (code && typeof code === 'string') { + if (code) { files[fileDef.name] = code + '\n'; context += `=== ${fileDef.name} ===\n${code}\n\n`; + const loc = code.split('\n').length; + console.log(` [3/5] ${fileResp.tokens} tok, ${loc} lines, ${fileResp.tokPerSec.toFixed(0)} tok/s`); } } + + // Vaihe 2: go build -validointi + per-tiedosto korjaus + if (LANG === 'go') { + for (let buildRound = 0; buildRound < MAX_BUILD_FIX; buildRound++) { + // Kirjoita kaikki tiedostot levylle + const goldenMod = readFileSync(join(GOLDEN_DIR, 'todo-go', 'go.mod'), 'utf-8'); + const modName = spec.project_name?.replace(/[^a-z0-9-]/gi, '-') || 'generated-api'; + writeFileSync(join(dir, 'go.mod'), goldenMod.replace(/^module\s+\S+/m, `module ${modName}`)); + for (const [fn, c] of Object.entries(files)) { + writeFileSync(join(dir, fn), c); + } + try { + execSync( + `docker run --rm --entrypoint sh -v "${dir}:/src:ro" ${LCONF.dockerImage} -c "cp -r /src/* . && go mod tidy 2>&1 && go build ./... 2>&1"`, + { timeout: 60000, encoding: 'utf-8', stdio: 'pipe' } + ); + console.log(` [3/5] ✓ kääntyy`); + break; + } catch (e) { + const allErrors = (e.stdout || e.stderr || '').split('\n').filter(l => /\.go:\d+/.test(l)); + if (allErrors.length === 0) { console.log(` [3/5] ⚠ build failed`); break; } + + // Ryhmittele virheet tiedostoittain + const errorsByFile = {}; + for (const line of allErrors) { + const m = line.match(/\.\/(\S+\.go):\d+/); + if (m) { (errorsByFile[m[1]] = errorsByFile[m[1]] || []).push(line); } + } + const filesToFix = Object.keys(errorsByFile).filter(f => !f.endsWith('_test.go')); + if (filesToFix.length === 0) break; + + console.log(` [3/5] ✗ ${allErrors.length} errors in ${filesToFix.join(', ')} → fixing`); + + for (const fname of filesToFix) { + const errors = errorsByFile[fname].slice(0, 10).join('\n'); + const fixPrompt = `Fix the following Go compilation errors in "${fname}". Return ONLY the corrected file, no explanations.\n\nERRORS:\n${errors}\n\nCURRENT FILE:\n${files[fname]}\n\nOTHER FILES:\n${Object.entries(files).filter(([f]) => f !== fname).map(([f, c]) => `=== ${f} ===\n${c}`).join('\n\n')}`; + console.log(` [3/5] → ${fname} (fix)...`); + const fixResp = await ollamaChat(model, fixPrompt, CODE_SYSTEM, 2048); + timings.push(fixResp); + let fixed = fixResp.text + .replace(/^```(?:go|golang)?\s*\n?/m, '').replace(/\n?```\s*$/m, '') + .replace(/^(?:Here|Sure|Below|This|The|I )[\s\S]*?(?=package\s)/m, '') + .trim(); + if (fixed) { + files[fname] = fixed + '\n'; + console.log(` [3/5] ${fixResp.tokens} tok, ${fixed.split('\n').length} lines`); + } + result.fixRounds++; + } + } + } + } + + // Päivitä context lopullisilla tiedostoilla + context = Object.entries(files).map(([fn, c]) => `=== ${fn} ===\n${c}`).join('\n\n'); writeFileSync(`${dir}/_code_raw.txt`, context); result.promptChars = CODE_SYSTEM.length + (context.length || 0); result.promptTokensEst = Math.round(result.promptChars / 4);