jaakko/agentic-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CodeBench: --convert-model Python→Go pipeline

Browse Source 
- 8b generoi Pythonia (osaa sen), 30b konvertoi Go:ksi
- convert-go.md prompti: FastAPI→Chi, SQLAlchemy→database/sql mappaukset
- Koodigenerointi käyttää Python golden+promptia kun convert-model asetettu
- Vaihe [3.5/5] konvertointia varten
This commit is contained in:
jaakko
2026-04-14 23:54:45 +03:00
parent a16c33f4fb
commit 20a1e5f015
2 changed files with 64 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
kipina-codebench/benchmark.mjs
View File

@@ -38,6 +38,8 @@ const NO_ORCHESTRATE = args.includes('--no-orchestrate');
const FILE_BY_FILE = args.includes('--file-by-file');
const SPEC_MODEL = arg('spec-model', ''); // Eri malli spec-vaiheille (1-2)
const SPEC_OLLAMA = arg('spec-ollama', ''); // Eri Ollama spec-mallille
const CONVERT_MODEL = arg('convert-model', ''); // Malli Python→Go/Rust konvertointiin
const CONVERT_OLLAMA = arg('convert-ollama', ''); // Eri Ollama konvertointimallille
const LANG = arg('lang', 'python'); // python | rust | go
const ROUNDS = parseInt(arg('rounds', '1')); // 1-10 toistoa
const MAX_FIX_ROUNDS = 2;
@@ -332,7 +334,15 @@ async function runPipeline(model, scenario, round = 1) {
error: null,
};
const timings = [];
const { system: CODE_SYSTEM, promptName, profile } = getCodePromptForModel(model);
// Konvertointi-moodissa generoidaan Python-koodia ensin, sitten konvertoidaan
const { system: CODE_SYSTEM, promptName, profile } = (() => {
if (CONVERT_MODEL) {
// Python-prompti + golden example koodigenerointiin
const pyPrompt = loadPrompt('code');
return { system: pyPrompt, promptName: 'code (→convert)', profile: PROFILES.models[model]?.profile || PROFILES.default_profile };
}
return getCodePromptForModel(model);
})();
const roundSuffix = ROUNDS > 1 ? `__r${round}` : '';
const dir = `${OUTPUT_DIR}/${model.replace(/[/:]/g, '_')}__${scenario.id}${roundSuffix}`;
mkdirSync(dir, { recursive: true });
@@ -361,9 +371,12 @@ async function runPipeline(model, scenario, round = 1) {
writeFileSync(`${dir}/_spec.json`, JSON.stringify(spec, null, 2));
// 3. LLM-koodigenerointi
const fileCount = LCONF.required.length;
const goldenExample = loadGoldenExample(model);
const codeTokens = LANG === 'rust' ? 12288 : LANG === 'go' ? 10240 : 8192;
// Konvertointi-moodissa: generoi Python ensin, golden+files Pythonista
const isConvert = !!CONVERT_MODEL;
const genConfig = isConvert ? LANG_CONFIG.python : LCONF;
const fileCount = genConfig.required.length;
const goldenExample = isConvert ? ('\n' + readFileSync(join(GOLDEN_DIR, 'todo.md'), 'utf-8').trim() + '\n') : loadGoldenExample(model);
const codeTokens = isConvert ? 8192 : (LANG === 'rust' ? 12288 : LANG === 'go' ? 10240 : 8192);
let files;
// File-by-file: generoi yksi tiedosto kerrallaan (pienille malleille)
@@ -447,6 +460,28 @@ async function runPipeline(model, scenario, round = 1) {
files = parseGeneratedFiles(codeResp.text);
}
// Konvertointi: Python→Go/Rust isommalla mallilla
if (CONVERT_MODEL && files) {
const convertUrl = CONVERT_OLLAMA || null;
const convertPromptFile = `convert-${LANG}`;
const convertSystem = existsSync(join(__dirname, 'prompts', `${convertPromptFile}.md`))
? readFileSync(join(__dirname, 'prompts', `${convertPromptFile}.md`), 'utf-8').trim()
: `Convert this Python code to ${LANG}. Return all files with === markers.`;
const pyCode = Object.entries(files).map(([fn, c]) => `=== ${fn} ===\n${c}`).join('\n\n');
console.log(` [3.5/5] Konvertointi Python→${LANG} (${CONVERT_MODEL})...`);
const convertResp = await ollamaChat(CONVERT_MODEL, pyCode, convertSystem, 12288, convertUrl);
timings.push(convertResp);
writeFileSync(`${dir}/_convert_raw.txt`, convertResp.text);
const convertedFiles = parseGeneratedFiles(convertResp.text);
if (Object.keys(convertedFiles).length > 0) {
files = convertedFiles;
const loc = Object.values(files).reduce((s, c) => s + c.split('\n').length, 0);
console.log(` [3.5/5] ${convertResp.tokens} tok, ${loc} lines, ${convertResp.tokPerSec.toFixed(0)} tok/s`);
} else {
console.log(` [3.5/5] ⚠ Konvertointi ei tuottanut tiedostoja`);
}
}
// Go: generoi go.mod golden examplen versiolla (ennen missing-tarkistusta)
if (LANG === 'go') {
const goldenMod = readFileSync(join(GOLDEN_DIR, 'todo-go', 'go.mod'), 'utf-8');