From 42ee959781794e5d42b690ad5baa6f14b4374765 Mon Sep 17 00:00:00 2001 From: jaakko Date: Tue, 14 Apr 2026 07:34:06 +0300 Subject: [PATCH] Benchmark: uv init + uv add hoitaa projektiasetuksen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLM generoi enää 4 tiedostoa (ei pyproject.toml). Pipeline: uv init → uv add deps → kirjoita .py → pytest. Poistaa Poetry-yhteensopivuusongelmat kokonaan. --- network-poc/tests/model-benchmark.mjs | 38 ++++++++++----------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/network-poc/tests/model-benchmark.mjs b/network-poc/tests/model-benchmark.mjs index 8095898..7595828 100644 --- a/network-poc/tests/model-benchmark.mjs +++ b/network-poc/tests/model-benchmark.mjs @@ -147,7 +147,7 @@ const FIX_SYSTEM = 'You are a Python code fixer. Return ONLY the corrected Pytho // === Kultainen esimerkki === const GOLDEN_DIR = join(__dirname, 'golden-examples', 'todo'); -const GOLDEN_FILES = ['models.py', 'schemas.py', 'main.py', 'test_main.py', 'pyproject.toml']; +const GOLDEN_FILES = ['models.py', 'schemas.py', 'main.py', 'test_main.py']; function loadGoldenExample() { if (!existsSync(GOLDEN_DIR)) return ''; let example = '\nREFERENCE IMPLEMENTATION (todo project — follow this exact structure, style, and conventions):\n\n'; @@ -159,15 +159,16 @@ function loadGoldenExample() { } const GOLDEN_EXAMPLE = loadGoldenExample(); -const CODE_SYSTEM = `You are a Python backend developer. Generate a complete FastAPI project with SQLAlchemy and SQLite. +const CODE_SYSTEM = `You are a Python backend developer. Generate a FastAPI project with SQLAlchemy and SQLite. -Given the project requirements, JSON specification, and a REFERENCE IMPLEMENTATION, generate these 5 files: +Given the project requirements, JSON specification, and a REFERENCE IMPLEMENTATION, generate these 4 files: 1. models.py — SQLAlchemy 2.0: DeclarativeBase, Mapped, mapped_column (NOT legacy declarative_base) 2. schemas.py — Pydantic v2: ConfigDict(from_attributes=True) (NOT class Config) 3. main.py — FastAPI CRUD endpoints for each entity 4. test_main.py — Pytest with TestClient, separate test.db, unique test data per test -5. pyproject.toml — PEP 621 [project] format (NOT [tool.poetry]) + +Do NOT generate pyproject.toml — it is created separately with uv. OUTPUT FORMAT — use these exact markers to separate files: @@ -183,16 +184,12 @@ OUTPUT FORMAT — use these exact markers to separate files: === test_main.py === -=== pyproject.toml === - - DOCUMENTATION — every file must have a one-line module docstring. Classes get a one-line docstring. Keep it zensical: say what it IS, not what it does. No filler. RULES: - Follow the REFERENCE IMPLEMENTATION patterns exactly - SQLAlchemy 2.0: DeclarativeBase + Mapped + mapped_column (not Column()) - Python type unions: str | None (not Optional[str]) -- pyproject.toml: PEP 621 [project] format, requires-python = ">=3.14" - Tests: unique descriptive data per test, NOT generic "test_title" strings - Absolute imports only (from models import ..., from schemas import ...) - NO markdown fences inside file content — just raw code @@ -307,7 +304,7 @@ async function runPipeline(model, scenario) { timings.push(codeResp); writeFileSync(`${dir}/_code_raw.txt`, codeResp.text); const files = parseGeneratedFiles(codeResp.text); - const required = ['models.py', 'schemas.py', 'main.py', 'test_main.py', 'pyproject.toml']; + const required = ['models.py', 'schemas.py', 'main.py', 'test_main.py']; const missing = required.filter(f => !files[f]); if (missing.length > 0) { result.error = `Puuttuvat: ${missing.join(', ')}`; return result; } @@ -338,25 +335,18 @@ async function runPipeline(model, scenario) { result.validationIssues = issues.length; result.fixRounds = fixRound; - // Korjaa pyproject.toml jos malli generoi Poetry-muodon - if (files['pyproject.toml'] && !files['pyproject.toml'].includes('[project]')) { - const goldenPyproject = join(GOLDEN_DIR, 'pyproject.toml'); - if (existsSync(goldenPyproject)) { - const nameMatch = files['pyproject.toml'].match(/name\s*=\s*"([^"]+)"/); - const name = nameMatch ? nameMatch[1] : 'generated-app'; - files['pyproject.toml'] = readFileSync(goldenPyproject, 'utf-8').replace(/name = "[^"]+"/, `name = "${name}"`); - } - } - - // Kirjoita tiedostot levylle - for (const [fn, content] of Object.entries(files)) writeFileSync(`${dir}/${fn}`, content); - - // 5. Pytest + // 5. Projektin alustus (uv init) + kirjoita tiedostot + pytest console.log(` [5/5] Pytest...`); try { const uvPath = process.env.HOME + '/.local/bin/uv'; const uv = existsSync(uvPath) ? uvPath : 'uv'; - execSync(`cd "${dir}" && ${uv} sync 2>/dev/null`, { timeout: 60000, stdio: 'pipe' }); + execSync(`cd "${dir}" && ${uv} init --no-readme --python ">=3.14" 2>/dev/null && rm -f hello.py main.py`, { timeout: 30000, stdio: 'pipe' }); + execSync(`cd "${dir}" && ${uv} add fastapi "uvicorn[standard]" sqlalchemy pytest httpx 2>/dev/null`, { timeout: 60000, stdio: 'pipe' }); + + // Kirjoita LLM:n generoimat Python-tiedostot (uv initin jälkeen) + for (const [fn, content] of Object.entries(files)) { + if (fn.endsWith('.py')) writeFileSync(`${dir}/${fn}`, content); + } execSync(`cd "${dir}" && rm -f app.db test.db`, { stdio: 'pipe' }); const pytestOut = execSync(`cd "${dir}" && ${uv} run pytest test_main.py -v --tb=short 2>&1`, { timeout: 60000, encoding: 'utf-8' }); writeFileSync(`${dir}/_pytest.txt`, pytestOut);