Benchmark: uv init + uv add hoitaa projektiasetuksen

LLM generoi enää 4 tiedostoa (ei pyproject.toml).
Pipeline: uv init → uv add deps → kirjoita .py → pytest.
Poistaa Poetry-yhteensopivuusongelmat kokonaan.
This commit is contained in:
2026-04-14 07:34:06 +03:00
parent 0850a139f1
commit 42ee959781

View File

@@ -147,7 +147,7 @@ const FIX_SYSTEM = 'You are a Python code fixer. Return ONLY the corrected Pytho
// === Kultainen esimerkki ===
const GOLDEN_DIR = join(__dirname, 'golden-examples', 'todo');
const GOLDEN_FILES = ['models.py', 'schemas.py', 'main.py', 'test_main.py', 'pyproject.toml'];
const GOLDEN_FILES = ['models.py', 'schemas.py', 'main.py', 'test_main.py'];
function loadGoldenExample() {
if (!existsSync(GOLDEN_DIR)) return '';
let example = '\nREFERENCE IMPLEMENTATION (todo project — follow this exact structure, style, and conventions):\n\n';
@@ -159,15 +159,16 @@ function loadGoldenExample() {
}
const GOLDEN_EXAMPLE = loadGoldenExample();
const CODE_SYSTEM = `You are a Python backend developer. Generate a complete FastAPI project with SQLAlchemy and SQLite.
const CODE_SYSTEM = `You are a Python backend developer. Generate a FastAPI project with SQLAlchemy and SQLite.
Given the project requirements, JSON specification, and a REFERENCE IMPLEMENTATION, generate these 5 files:
Given the project requirements, JSON specification, and a REFERENCE IMPLEMENTATION, generate these 4 files:
1. models.py — SQLAlchemy 2.0: DeclarativeBase, Mapped, mapped_column (NOT legacy declarative_base)
2. schemas.py — Pydantic v2: ConfigDict(from_attributes=True) (NOT class Config)
3. main.py — FastAPI CRUD endpoints for each entity
4. test_main.py — Pytest with TestClient, separate test.db, unique test data per test
5. pyproject.toml — PEP 621 [project] format (NOT [tool.poetry])
Do NOT generate pyproject.toml — it is created separately with uv.
OUTPUT FORMAT — use these exact markers to separate files:
@@ -183,16 +184,12 @@ OUTPUT FORMAT — use these exact markers to separate files:
=== test_main.py ===
<python code>
=== pyproject.toml ===
<toml content>
DOCUMENTATION — every file must have a one-line module docstring. Classes get a one-line docstring. Keep it zensical: say what it IS, not what it does. No filler.
RULES:
- Follow the REFERENCE IMPLEMENTATION patterns exactly
- SQLAlchemy 2.0: DeclarativeBase + Mapped + mapped_column (not Column())
- Python type unions: str | None (not Optional[str])
- pyproject.toml: PEP 621 [project] format, requires-python = ">=3.14"
- Tests: unique descriptive data per test, NOT generic "test_title" strings
- Absolute imports only (from models import ..., from schemas import ...)
- NO markdown fences inside file content — just raw code
@@ -307,7 +304,7 @@ async function runPipeline(model, scenario) {
timings.push(codeResp);
writeFileSync(`${dir}/_code_raw.txt`, codeResp.text);
const files = parseGeneratedFiles(codeResp.text);
const required = ['models.py', 'schemas.py', 'main.py', 'test_main.py', 'pyproject.toml'];
const required = ['models.py', 'schemas.py', 'main.py', 'test_main.py'];
const missing = required.filter(f => !files[f]);
if (missing.length > 0) { result.error = `Puuttuvat: ${missing.join(', ')}`; return result; }
@@ -338,25 +335,18 @@ async function runPipeline(model, scenario) {
result.validationIssues = issues.length;
result.fixRounds = fixRound;
// Korjaa pyproject.toml jos malli generoi Poetry-muodon
if (files['pyproject.toml'] && !files['pyproject.toml'].includes('[project]')) {
const goldenPyproject = join(GOLDEN_DIR, 'pyproject.toml');
if (existsSync(goldenPyproject)) {
const nameMatch = files['pyproject.toml'].match(/name\s*=\s*"([^"]+)"/);
const name = nameMatch ? nameMatch[1] : 'generated-app';
files['pyproject.toml'] = readFileSync(goldenPyproject, 'utf-8').replace(/name = "[^"]+"/, `name = "${name}"`);
}
}
// Kirjoita tiedostot levylle
for (const [fn, content] of Object.entries(files)) writeFileSync(`${dir}/${fn}`, content);
// 5. Pytest
// 5. Projektin alustus (uv init) + kirjoita tiedostot + pytest
console.log(` [5/5] Pytest...`);
try {
const uvPath = process.env.HOME + '/.local/bin/uv';
const uv = existsSync(uvPath) ? uvPath : 'uv';
execSync(`cd "${dir}" && ${uv} sync 2>/dev/null`, { timeout: 60000, stdio: 'pipe' });
execSync(`cd "${dir}" && ${uv} init --no-readme --python ">=3.14" 2>/dev/null && rm -f hello.py main.py`, { timeout: 30000, stdio: 'pipe' });
execSync(`cd "${dir}" && ${uv} add fastapi "uvicorn[standard]" sqlalchemy pytest httpx 2>/dev/null`, { timeout: 60000, stdio: 'pipe' });
// Kirjoita LLM:n generoimat Python-tiedostot (uv initin jälkeen)
for (const [fn, content] of Object.entries(files)) {
if (fn.endsWith('.py')) writeFileSync(`${dir}/${fn}`, content);
}
execSync(`cd "${dir}" && rm -f app.db test.db`, { stdio: 'pipe' });
const pytestOut = execSync(`cd "${dir}" && ${uv} run pytest test_main.py -v --tb=short 2>&1`, { timeout: 60000, encoding: 'utf-8' });
writeFileSync(`${dir}/_pytest.txt`, pytestOut);