Benchmark: uv init + uv add hoitaa projektiasetuksen
LLM generoi enää 4 tiedostoa (ei pyproject.toml). Pipeline: uv init → uv add deps → kirjoita .py → pytest. Poistaa Poetry-yhteensopivuusongelmat kokonaan.
This commit is contained in:
@@ -147,7 +147,7 @@ const FIX_SYSTEM = 'You are a Python code fixer. Return ONLY the corrected Pytho
|
||||
|
||||
// === Kultainen esimerkki ===
|
||||
const GOLDEN_DIR = join(__dirname, 'golden-examples', 'todo');
|
||||
const GOLDEN_FILES = ['models.py', 'schemas.py', 'main.py', 'test_main.py', 'pyproject.toml'];
|
||||
const GOLDEN_FILES = ['models.py', 'schemas.py', 'main.py', 'test_main.py'];
|
||||
function loadGoldenExample() {
|
||||
if (!existsSync(GOLDEN_DIR)) return '';
|
||||
let example = '\nREFERENCE IMPLEMENTATION (todo project — follow this exact structure, style, and conventions):\n\n';
|
||||
@@ -159,15 +159,16 @@ function loadGoldenExample() {
|
||||
}
|
||||
const GOLDEN_EXAMPLE = loadGoldenExample();
|
||||
|
||||
const CODE_SYSTEM = `You are a Python backend developer. Generate a complete FastAPI project with SQLAlchemy and SQLite.
|
||||
const CODE_SYSTEM = `You are a Python backend developer. Generate a FastAPI project with SQLAlchemy and SQLite.
|
||||
|
||||
Given the project requirements, JSON specification, and a REFERENCE IMPLEMENTATION, generate these 5 files:
|
||||
Given the project requirements, JSON specification, and a REFERENCE IMPLEMENTATION, generate these 4 files:
|
||||
|
||||
1. models.py — SQLAlchemy 2.0: DeclarativeBase, Mapped, mapped_column (NOT legacy declarative_base)
|
||||
2. schemas.py — Pydantic v2: ConfigDict(from_attributes=True) (NOT class Config)
|
||||
3. main.py — FastAPI CRUD endpoints for each entity
|
||||
4. test_main.py — Pytest with TestClient, separate test.db, unique test data per test
|
||||
5. pyproject.toml — PEP 621 [project] format (NOT [tool.poetry])
|
||||
|
||||
Do NOT generate pyproject.toml — it is created separately with uv.
|
||||
|
||||
OUTPUT FORMAT — use these exact markers to separate files:
|
||||
|
||||
@@ -183,16 +184,12 @@ OUTPUT FORMAT — use these exact markers to separate files:
|
||||
=== test_main.py ===
|
||||
<python code>
|
||||
|
||||
=== pyproject.toml ===
|
||||
<toml content>
|
||||
|
||||
DOCUMENTATION — every file must have a one-line module docstring. Classes get a one-line docstring. Keep it zensical: say what it IS, not what it does. No filler.
|
||||
|
||||
RULES:
|
||||
- Follow the REFERENCE IMPLEMENTATION patterns exactly
|
||||
- SQLAlchemy 2.0: DeclarativeBase + Mapped + mapped_column (not Column())
|
||||
- Python type unions: str | None (not Optional[str])
|
||||
- pyproject.toml: PEP 621 [project] format, requires-python = ">=3.14"
|
||||
- Tests: unique descriptive data per test, NOT generic "test_title" strings
|
||||
- Absolute imports only (from models import ..., from schemas import ...)
|
||||
- NO markdown fences inside file content — just raw code
|
||||
@@ -307,7 +304,7 @@ async function runPipeline(model, scenario) {
|
||||
timings.push(codeResp);
|
||||
writeFileSync(`${dir}/_code_raw.txt`, codeResp.text);
|
||||
const files = parseGeneratedFiles(codeResp.text);
|
||||
const required = ['models.py', 'schemas.py', 'main.py', 'test_main.py', 'pyproject.toml'];
|
||||
const required = ['models.py', 'schemas.py', 'main.py', 'test_main.py'];
|
||||
const missing = required.filter(f => !files[f]);
|
||||
if (missing.length > 0) { result.error = `Puuttuvat: ${missing.join(', ')}`; return result; }
|
||||
|
||||
@@ -338,25 +335,18 @@ async function runPipeline(model, scenario) {
|
||||
result.validationIssues = issues.length;
|
||||
result.fixRounds = fixRound;
|
||||
|
||||
// Korjaa pyproject.toml jos malli generoi Poetry-muodon
|
||||
if (files['pyproject.toml'] && !files['pyproject.toml'].includes('[project]')) {
|
||||
const goldenPyproject = join(GOLDEN_DIR, 'pyproject.toml');
|
||||
if (existsSync(goldenPyproject)) {
|
||||
const nameMatch = files['pyproject.toml'].match(/name\s*=\s*"([^"]+)"/);
|
||||
const name = nameMatch ? nameMatch[1] : 'generated-app';
|
||||
files['pyproject.toml'] = readFileSync(goldenPyproject, 'utf-8').replace(/name = "[^"]+"/, `name = "${name}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// Kirjoita tiedostot levylle
|
||||
for (const [fn, content] of Object.entries(files)) writeFileSync(`${dir}/${fn}`, content);
|
||||
|
||||
// 5. Pytest
|
||||
// 5. Projektin alustus (uv init) + kirjoita tiedostot + pytest
|
||||
console.log(` [5/5] Pytest...`);
|
||||
try {
|
||||
const uvPath = process.env.HOME + '/.local/bin/uv';
|
||||
const uv = existsSync(uvPath) ? uvPath : 'uv';
|
||||
execSync(`cd "${dir}" && ${uv} sync 2>/dev/null`, { timeout: 60000, stdio: 'pipe' });
|
||||
execSync(`cd "${dir}" && ${uv} init --no-readme --python ">=3.14" 2>/dev/null && rm -f hello.py main.py`, { timeout: 30000, stdio: 'pipe' });
|
||||
execSync(`cd "${dir}" && ${uv} add fastapi "uvicorn[standard]" sqlalchemy pytest httpx 2>/dev/null`, { timeout: 60000, stdio: 'pipe' });
|
||||
|
||||
// Kirjoita LLM:n generoimat Python-tiedostot (uv initin jälkeen)
|
||||
for (const [fn, content] of Object.entries(files)) {
|
||||
if (fn.endsWith('.py')) writeFileSync(`${dir}/${fn}`, content);
|
||||
}
|
||||
execSync(`cd "${dir}" && rm -f app.db test.db`, { stdio: 'pipe' });
|
||||
const pytestOut = execSync(`cd "${dir}" && ${uv} run pytest test_main.py -v --tb=short 2>&1`, { timeout: 60000, encoding: 'utf-8' });
|
||||
writeFileSync(`${dir}/_pytest.txt`, pytestOut);
|
||||
|
||||
Reference in New Issue
Block a user