718 lines
32 KiB
Python
718 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
"""Kipinä pipeline test loop — ajaa generoinnin, validoi, säätää prompteja ja uusii."""
|
|
|
|
import json, re, ast, sys, time, textwrap
|
|
from pathlib import Path
|
|
from urllib.request import urlopen, Request
|
|
|
|
OLLAMA = "http://localhost:11434"
|
|
MODEL = "qwen2.5-coder:7b-instruct-q4_K_M"
|
|
OUTDIR = Path(__file__).parent / "loop_runs"
|
|
MAX_RETRIES = 3
|
|
|
|
# ── Ollama-kutsu ──────────────────────────────────────────────
|
|
|
|
def llm(system: str, user: str, temperature: float = 0.2) -> str:
|
|
body = json.dumps({
|
|
"model": MODEL,
|
|
"messages": [
|
|
{"role": "system", "content": system},
|
|
{"role": "user", "content": user},
|
|
],
|
|
"stream": False,
|
|
"options": {"temperature": temperature, "num_predict": 4096, "seed": 42},
|
|
}).encode()
|
|
req = Request(f"{OLLAMA}/api/chat", data=body, headers={"Content-Type": "application/json"})
|
|
resp = json.loads(urlopen(req, timeout=120).read())
|
|
return resp["message"]["content"]
|
|
|
|
# ── Koodin erottaminen LLM-vastauksesta ──────────────────────
|
|
|
|
def extract_code(text: str) -> str:
|
|
"""Poimi koodi markdown-fencestä tai palauta koko teksti."""
|
|
# Etsi viimeinen/pisin code block
|
|
blocks = re.findall(r"```(?:python|py|toml|dockerfile|Dockerfile)?\s*\n(.*?)```", text, re.DOTALL)
|
|
if blocks:
|
|
# Palauta pisin blokki (vältä lyhyet esimerkit)
|
|
return max(blocks, key=len).strip()
|
|
# Jos ei fenceä, poista mahdolliset selitysrivit alusta/lopusta
|
|
lines = text.strip().splitlines()
|
|
code_lines = [l for l in lines if not l.startswith("Here") and not l.startswith("This") and not l.startswith("Note")]
|
|
return "\n".join(code_lines).strip()
|
|
|
|
# ── Validointi ────────────────────────────────────────────────
|
|
|
|
def validate_python(code: str, filename: str, project_files: dict) -> list[str]:
|
|
"""Tarkista syntaksi, importit, ja yleisimmät ongelmat."""
|
|
errors = []
|
|
|
|
# 1. Syntaksitarkistus
|
|
try:
|
|
tree = ast.parse(code, filename=filename)
|
|
except SyntaxError as e:
|
|
lines = code.splitlines()
|
|
bad_line = lines[e.lineno - 1] if e.lineno and e.lineno <= len(lines) else "?"
|
|
errors.append(f"SyntaxError rivi {e.lineno}: {e.msg}. Virherivin koodi: `{bad_line.strip()}`. Korjaa tämä rivi.")
|
|
return errors # ei voi jatkaa
|
|
|
|
# 2. Tarkista relatiiviset importit
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.level and node.level > 0:
|
|
errors.append(f"Relatiivinen import 'from .{node.module or ''}' — käytä absoluuttista: from {node.module} import ...")
|
|
|
|
# 3. Kerää importit
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.module:
|
|
mod = node.module.split(".")[0]
|
|
# Tarkista projektin sisäiset importit
|
|
if mod in ("models", "schemas", "main"):
|
|
source_file = f"{mod}.py"
|
|
if source_file not in project_files:
|
|
errors.append(f"Importtaa '{mod}' mutta {source_file} ei ole vielä generoitu")
|
|
else:
|
|
# Tarkista että importatut nimet löytyvät
|
|
source_code = project_files[source_file]
|
|
for alias in node.names:
|
|
name = alias.name
|
|
if name not in source_code:
|
|
errors.append(f"Importtaa '{name}' moduulista '{mod}' mutta sitä ei löydy {source_file}:stä")
|
|
|
|
# Tarkista paljaiden nimien käyttö ilman importtia
|
|
if isinstance(node, ast.Name) and node.id == "date":
|
|
has_import = any(
|
|
isinstance(n, ast.ImportFrom) and n.module == "datetime"
|
|
and any(a.name == "date" for a in n.names)
|
|
for n in ast.walk(tree)
|
|
)
|
|
if not has_import and "date" in code and "due_date: date" in code:
|
|
errors.append("Käyttää 'date'-tyyppiä mutta 'from datetime import date' puuttuu")
|
|
|
|
# 3. Tarkista ettei testi-tiedosto uudelleenmäärittele appia/modeleita
|
|
if filename == "test_main.py":
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ClassDef) and node.name in ("Todo", "Base"):
|
|
errors.append(f"test_main.py uudelleenmäärittelee '{node.name}' — pitäisi importata models.py:stä")
|
|
if isinstance(node, ast.Assign):
|
|
for target in node.targets:
|
|
if isinstance(target, ast.Name) and target.id == "app":
|
|
if isinstance(node.value, ast.Call):
|
|
func = node.value.func
|
|
if isinstance(func, ast.Name) and func.id == "FastAPI":
|
|
errors.append("test_main.py luo oman FastAPI()-instanssin — pitäisi importata main.py:stä")
|
|
|
|
# 4. Pydantic response_model ei saa olla SQLAlchemy-malli
|
|
if filename in ("test_main.py", "main.py"):
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.keyword) and node.arg == "response_model":
|
|
if isinstance(node.value, ast.Name) and node.value.id == "Todo":
|
|
errors.append(f"response_model=Todo käyttää SQLAlchemy-mallia — pitäisi olla TodoResponse (Pydantic)")
|
|
|
|
# 5. models.py: SQLite tarvitsee connect_args
|
|
if filename == "models.py":
|
|
if "sqlite" in code.lower() and "check_same_thread" not in code:
|
|
errors.append("SQLite create_engine puuttuu connect_args={'check_same_thread': False}")
|
|
# Tarkista Enum-käyttö: pitää olla python enum, ei SQLAlchemy Enum base-luokkana
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ClassDef):
|
|
for base in node.bases:
|
|
if isinstance(base, ast.Name) and base.id == "Enum" and node.name != "Base":
|
|
# Tarkista onko 'import enum' tai 'from enum import Enum'
|
|
has_enum_import = any(
|
|
(isinstance(n, ast.Import) and any(a.name == "enum" for a in n.names)) or
|
|
(isinstance(n, ast.ImportFrom) and n.module == "enum")
|
|
for n in ast.walk(tree)
|
|
)
|
|
if not has_enum_import:
|
|
errors.append(f"Luokka '{node.name}' perii Enum:in mutta 'from enum import Enum' puuttuu (SQLAlchemy Enum ei ole Python Enum)")
|
|
|
|
# 6. main.py: FastAPI route-ongelmat
|
|
if filename == "main.py":
|
|
# Tarkista query params reitissä (esim. "/search?q={query}")
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
if "?" in node.value and "{" in node.value:
|
|
errors.append(f"Reitti '{node.value}' sisältää query parametrin — FastAPI:ssa query params tulevat funktion parametreina, ei reitissä")
|
|
|
|
# Tarkista route ordering: /todos/count ja /todos/search ENNEN /todos/{{id}}
|
|
routes = []
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Call):
|
|
func = node.func
|
|
if isinstance(func, ast.Attribute) and func.attr in ("get", "post", "put", "delete", "patch"):
|
|
for arg in node.args:
|
|
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
|
routes.append(arg.value)
|
|
# Etsi {id} tai {todo_id} reitit ja tarkista tuleeko niiden jälkeen staattisia samalla prefix:llä
|
|
param_route_idx = {}
|
|
for i, r in enumerate(routes):
|
|
if re.search(r"\{[^}]+\}", r):
|
|
prefix = r.split("{")[0]
|
|
param_route_idx[prefix] = i
|
|
for i, r in enumerate(routes):
|
|
if not re.search(r"\{[^}]+\}", r):
|
|
for prefix, pidx in param_route_idx.items():
|
|
if r.startswith(prefix) and r != prefix and i > pidx:
|
|
errors.append(f"Reitti '{r}' tulee parametrisoidun reitin jälkeen — FastAPI tulkitsee sen {{id}}:ksi. Siirrä staattinen reitti ensin.")
|
|
|
|
# 7. Tarkista puuttuvat nimet: käytetäänkö Call:issa nimeä jota ei ole importattu tai määritelty
|
|
defined_names = set()
|
|
imported_names = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.ImportFrom) and node.names:
|
|
for alias in node.names:
|
|
imported_names.add(alias.asname or alias.name)
|
|
if isinstance(node, ast.Import) and node.names:
|
|
for alias in node.names:
|
|
imported_names.add(alias.asname or alias.name)
|
|
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
defined_names.add(node.name)
|
|
if isinstance(node, ast.ClassDef):
|
|
defined_names.add(node.name)
|
|
if isinstance(node, ast.Assign):
|
|
for t in node.targets:
|
|
if isinstance(t, ast.Name):
|
|
defined_names.add(t.id)
|
|
|
|
# Tunnetut builtinit ja globaalit
|
|
builtins = {"print", "len", "range", "int", "str", "float", "bool", "list", "dict", "set",
|
|
"tuple", "type", "None", "True", "False", "super", "property", "staticmethod",
|
|
"classmethod", "isinstance", "issubclass", "hasattr", "getattr", "setattr",
|
|
"Exception", "ValueError", "TypeError", "KeyError", "IndexError", "AttributeError",
|
|
"RuntimeError", "NotImplementedError", "StopIteration", "open", "enumerate", "zip",
|
|
"map", "filter", "sorted", "reversed", "any", "all", "min", "max", "sum", "abs"}
|
|
|
|
# Tarkista top-level Call-kutsut joissa käytetään tuntematonta nimeä
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Call):
|
|
func = node.func
|
|
if isinstance(func, ast.Name) and func.id not in defined_names | imported_names | builtins:
|
|
errors.append(f"Kutsutaan '{func.id}()' mutta sitä ei ole importattu eikä määritelty")
|
|
|
|
# 8. Schema/model kenttien yhteensopivuus
|
|
if filename == "schemas.py" and "models.py" in project_files:
|
|
model_code = project_files["models.py"]
|
|
# Etsi model-luokan Column-kentät
|
|
model_fields = set(re.findall(r"(\w+)\s*=\s*Column\(", model_code))
|
|
model_fields.discard("id") # id tulee automaattisesti
|
|
# Etsi schema-kentät (type-annotaatiot)
|
|
schema_fields = set()
|
|
in_response = False
|
|
for line in code.splitlines():
|
|
if "class " in line and "Response" in line:
|
|
in_response = True
|
|
elif "class " in line:
|
|
in_response = False
|
|
if in_response and ":" in line and "class " not in line and "Config" not in line and "from_attributes" not in line:
|
|
field = line.strip().split(":")[0].strip()
|
|
if field and not field.startswith("#") and not field.startswith("def"):
|
|
schema_fields.add(field)
|
|
# Response schema pitäisi sisältää kaikki model-kentät (paitsi salaiset)
|
|
SENSITIVE_FIELDS = {"password_hash", "password", "hashed_password", "secret", "token", "api_key"}
|
|
missing = model_fields - schema_fields - SENSITIVE_FIELDS
|
|
if missing and schema_fields: # vain jos löydettiin kenttiä
|
|
# Salli jos Response perii Create:sta
|
|
create_fields = set()
|
|
in_create = False
|
|
for line in code.splitlines():
|
|
if "class " in line and "Create" in line:
|
|
in_create = True
|
|
elif "class " in line:
|
|
in_create = False
|
|
if in_create and ":" in line and "class " not in line:
|
|
field = line.strip().split(":")[0].strip()
|
|
if field and not field.startswith("#") and not field.startswith("def"):
|
|
create_fields.add(field)
|
|
all_schema = schema_fields | create_fields | {"id"}
|
|
still_missing = model_fields - all_schema - SENSITIVE_FIELDS
|
|
if still_missing:
|
|
errors.append(f"Schema puuttuu kenttiä jotka ovat modelissa: {still_missing}")
|
|
|
|
return errors
|
|
|
|
|
|
def validate_toml(code: str) -> list[str]:
|
|
"""Tarkista pyproject.toml:n perusrakenne."""
|
|
errors = []
|
|
if "[project]" not in code:
|
|
errors.append("Puuttuu [project]-osio")
|
|
if "fastapi" not in code.lower():
|
|
errors.append("Puuttuu fastapi riippuvuuksista")
|
|
if "sqlalchemy" not in code.lower():
|
|
errors.append("Puuttuu sqlalchemy riippuvuuksista")
|
|
if "uvicorn" not in code.lower():
|
|
errors.append("Puuttuu uvicorn riippuvuuksista")
|
|
if "[tool.poetry]" in code or "poetry" in code.lower():
|
|
errors.append("Sisältää poetry-konfiguraation — käytä VAIN [project] (PEP 621) + uv")
|
|
if "build-backend" in code and "poetry" in code:
|
|
errors.append("build-backend käyttää poetryä — poista tai vaihda")
|
|
return errors
|
|
|
|
|
|
def validate_dockerfile(code: str) -> list[str]:
|
|
errors = []
|
|
if "poetry" in code.lower():
|
|
errors.append("Käyttää Poetryä — pitäisi käyttää uv:tä")
|
|
if "uv" not in code.lower():
|
|
errors.append("Puuttuu uv-asennus")
|
|
if "EXPOSE" not in code:
|
|
errors.append("Puuttuu EXPOSE")
|
|
# Tarkista USER-järjestys: ei saa olla ennen uv sync
|
|
lines = code.strip().splitlines()
|
|
user_line = next((i for i, l in enumerate(lines) if l.strip().startswith("USER")), None)
|
|
sync_line = next((i for i, l in enumerate(lines) if "uv sync" in l), None)
|
|
if user_line is not None and sync_line is not None and user_line < sync_line:
|
|
errors.append("USER asetettu ennen 'uv sync' — uv tarvitsee cache-kirjoitusoikeuden. Siirrä USER uv sync:n jälkeen.")
|
|
# UV_CACHE_DIR tai --no-cache
|
|
if "UV_CACHE_DIR" not in code and "--no-cache" not in code:
|
|
errors.append("Puuttuu ENV UV_CACHE_DIR=/tmp/uv-cache tai --no-cache — uv cache -virhe ei-root käyttäjällä")
|
|
return errors
|
|
|
|
# ── Promptien rakentajat ──────────────────────────────────────
|
|
|
|
SYSTEM_PROMPTS = {
|
|
"client": textwrap.dedent("""\
|
|
You are a product owner who turns vague ideas into clear, actionable software requirements.
|
|
|
|
GIVEN a short project description from the user, produce a structured brief:
|
|
1. PROJECT NAME: a short, descriptive name
|
|
2. GOAL: one sentence explaining what the software does and who it's for
|
|
3. CORE FEATURES: numbered list of 3-8 concrete features (not vague wishes)
|
|
4. DATA MODEL: list the main entities and their key fields (include field types)
|
|
5. API ENDPOINTS: list the REST endpoints (method + path + purpose)
|
|
6. CONSTRAINTS: any technical constraints (e.g. "must use SQLite", "no auth needed")
|
|
|
|
RULES:
|
|
- Be specific: "User can filter todos by status" not "todo management"
|
|
- Use plain English, no code
|
|
- Maximum 400 words total"""),
|
|
|
|
"data": textwrap.dedent("""\
|
|
You are a database architect specializing in SQLAlchemy and relational databases.
|
|
|
|
YOUR RESPONSIBILITIES:
|
|
1. Design normalized database schemas with proper column types and constraints
|
|
2. Define SQLAlchemy models with __tablename__, primary keys, indexes, and relationships
|
|
3. Set up engine, SessionLocal, and Base in the same file (models.py)
|
|
4. Use String(length) not bare String for SQLite compatibility
|
|
5. Add nullable=False for required fields, unique=True where appropriate
|
|
6. Use Column(Integer, primary_key=True, index=True) for IDs
|
|
7. SQLite: create_engine(url, connect_args={"check_same_thread": False})
|
|
|
|
ENUM HANDLING (IMPORTANT):
|
|
- For status fields, use Column(String(20)) with a default value — simpler and SQLite-compatible
|
|
- Do NOT define Python Enum classes — use plain strings instead
|
|
- Example: status = Column(String(20), default="pending")
|
|
|
|
ALWAYS INCLUDE:
|
|
- from sqlalchemy import create_engine, Column, Integer, String
|
|
- from sqlalchemy.ext.declarative import declarative_base
|
|
- from sqlalchemy.orm import sessionmaker
|
|
- DATABASE_URL, engine, SessionLocal, Base
|
|
- create_engine with connect_args={"check_same_thread": False}
|
|
|
|
Write ONLY the code, no explanations."""),
|
|
|
|
"coder": textwrap.dedent("""\
|
|
You are an expert Python developer. Write complete, production-ready code.
|
|
|
|
CRITICAL RULES:
|
|
1. Include ALL imports at the top of every file — including stdlib (from datetime import date, etc.)
|
|
2. Import from other project files: from models import Todo, SessionLocal
|
|
3. Pydantic schemas use different names than SQLAlchemy models: TodoCreate, TodoResponse (not Todo)
|
|
4. SQLAlchemy: create_engine(url, connect_args={"check_same_thread": False})
|
|
5. Pydantic v2: use model_dump() not dict(), class Config: from_attributes = True
|
|
6. All CRUD endpoints: POST (201), GET list, GET by id, PUT, DELETE (204)
|
|
|
|
NEVER:
|
|
- Leave out any import (EVERY type you use must be imported)
|
|
- Use relative imports (from .models) — ALWAYS use absolute: from models import ...
|
|
- Add explanations or comments
|
|
- Leave placeholder code or TODO comments
|
|
- Use requirements.txt or Poetry — always use pyproject.toml with [project] format (PEP 621)
|
|
|
|
Write ONLY the code, no explanations."""),
|
|
|
|
"qa": textwrap.dedent("""\
|
|
You are a QA engineer. Write pytest tests for FastAPI projects.
|
|
|
|
CRITICAL RULES FOR TESTS:
|
|
1. ALWAYS import app from main.py: from main import app
|
|
2. ALWAYS import models from models.py: from models import Base
|
|
3. NEVER redefine the app, models, or routes in the test file
|
|
4. Use in-memory SQLite for test isolation: sqlite:///:memory:
|
|
5. Override the get_db dependency to use test database
|
|
6. Use TestClient from fastapi.testclient
|
|
7. Test all CRUD: create (201), list (200), get by id (200/404), update (200), delete (204)
|
|
|
|
PATTERN:
|
|
```
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
from main import app
|
|
from models import Base
|
|
from main import get_db
|
|
|
|
engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
|
|
TestSession = sessionmaker(bind=engine)
|
|
Base.metadata.create_all(bind=engine)
|
|
|
|
def override_get_db():
|
|
db = TestSession()
|
|
try:
|
|
yield db
|
|
finally:
|
|
db.close()
|
|
|
|
app.dependency_overrides[get_db] = override_get_db
|
|
client = TestClient(app)
|
|
```
|
|
|
|
Write ONLY the code, no explanations."""),
|
|
|
|
"tester": textwrap.dedent("""\
|
|
You are a DevOps engineer. Write Dockerfiles for Python FastAPI projects.
|
|
|
|
RULES:
|
|
- Use python:3.12-slim as base
|
|
- Install uv: COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
|
|
- ENV UV_CACHE_DIR=/tmp/uv-cache (MUST set before uv sync)
|
|
- Copy pyproject.toml first, then RUN uv sync, then COPY source files
|
|
- Set USER AFTER installing dependencies (uv sync needs write access)
|
|
- NEVER use pip, poetry, or requirements.txt
|
|
- Expose port 8000
|
|
- CMD ["uv", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
|
|
|
EXACT PATTERN (follow this exactly):
|
|
```
|
|
FROM python:3.12-slim
|
|
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
|
|
ENV UV_CACHE_DIR=/tmp/uv-cache
|
|
WORKDIR /app
|
|
COPY pyproject.toml .
|
|
RUN uv sync
|
|
COPY *.py .
|
|
RUN useradd -m appuser && chown -R appuser:appuser /app /tmp/uv-cache
|
|
USER appuser
|
|
EXPOSE 8000
|
|
CMD ["uv", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
|
```
|
|
|
|
Write ONLY the Dockerfile, no explanations."""),
|
|
}
|
|
|
|
|
|
def build_user_prompt(step: str, project_files: dict, requirements: str, error_feedback: str = "") -> str:
|
|
"""Rakenna user-prompti kontekstilla ja mahdollisella virhepalautteella."""
|
|
|
|
context_files = "\n".join(
|
|
f"--- {name} ---\n{code}\n" for name, code in project_files.items()
|
|
)
|
|
|
|
error_section = ""
|
|
if error_feedback:
|
|
error_section = f"\n⚠️ PREVIOUS ATTEMPT HAD ERRORS — FIX THESE:\n{error_feedback}\n"
|
|
|
|
if step == "requirements":
|
|
return "Todo-sovellus FastAPI + SQLite, CRUD-endpointit ja testit"
|
|
|
|
elif step == "models.py":
|
|
return f"""PROJECT REQUIREMENTS:\n{requirements}\n\n{error_section}Write models.py for this project. Write ONLY the code."""
|
|
|
|
elif step == "schemas.py":
|
|
return f"""Already written files:\n{context_files}\n\nPROJECT REQUIREMENTS:\n{requirements}\n\n{error_section}Write schemas.py. Create Pydantic schemas matching the SQLAlchemy model. Include ALL imports (from datetime import date if you use date type). Write ONLY the code."""
|
|
|
|
elif step == "main.py":
|
|
return f"""Already written files:\n{context_files}\n\nPROJECT REQUIREMENTS:\n{requirements}\n\n{error_section}Write main.py with all CRUD endpoints. Import from models.py and schemas.py. Write ONLY the code."""
|
|
|
|
elif step == "pyproject.toml":
|
|
return f"""Already written files:\n{context_files}\n\n{error_section}Write pyproject.toml for this project. Include fastapi, uvicorn[standard], sqlalchemy as dependencies. Write ONLY the file."""
|
|
|
|
elif step == "test_main.py":
|
|
return f"""Already written files:\n{context_files}\n\n{error_section}Write test_main.py. IMPORT app from main.py, IMPORT Base from models.py. NEVER redefine models or app. Use in-memory SQLite. Write ONLY the code."""
|
|
|
|
elif step == "Dockerfile":
|
|
return f"""Project files: {', '.join(project_files.keys())}\n\n{error_section}Write a Dockerfile. Use uv (NOT pip, NOT poetry). Write ONLY the Dockerfile."""
|
|
|
|
return ""
|
|
|
|
|
|
# ── Pipeline ──────────────────────────────────────────────────
|
|
|
|
STEPS = [
|
|
("requirements", "client", None),
|
|
("models.py", "data", "python"),
|
|
("schemas.py", "coder", "python"),
|
|
("main.py", "coder", "python"),
|
|
("pyproject.toml","coder", "toml"),
|
|
("test_main.py", "qa", "python"),
|
|
("Dockerfile", "tester", "dockerfile"),
|
|
]
|
|
|
|
|
|
def run_pipeline(run_id: str) -> dict:
|
|
run_dir = OUTDIR / run_id
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
project_files = {}
|
|
requirements = ""
|
|
results = {"run_id": run_id, "steps": [], "model": MODEL}
|
|
|
|
for step_name, agent, file_type in STEPS:
|
|
print(f"\n{'='*60}")
|
|
print(f" Step: {step_name} (agent: {agent})")
|
|
print(f"{'='*60}")
|
|
|
|
system = SYSTEM_PROMPTS[agent]
|
|
error_feedback = ""
|
|
final_code = ""
|
|
step_result = {"step": step_name, "agent": agent, "attempts": []}
|
|
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
|
print(f" Yritys {attempt}/{MAX_RETRIES}...", end=" ", flush=True)
|
|
|
|
user_prompt = build_user_prompt(step_name, project_files, requirements, error_feedback)
|
|
|
|
t0 = time.time()
|
|
raw = llm(system, user_prompt)
|
|
elapsed = time.time() - t0
|
|
print(f"({elapsed:.1f}s)")
|
|
|
|
if step_name == "requirements":
|
|
final_code = raw.strip()
|
|
errors = []
|
|
else:
|
|
final_code = extract_code(raw)
|
|
if file_type == "python":
|
|
errors = validate_python(final_code, step_name, project_files)
|
|
elif file_type == "toml":
|
|
errors = validate_toml(final_code)
|
|
elif file_type == "dockerfile":
|
|
errors = validate_dockerfile(final_code)
|
|
else:
|
|
errors = []
|
|
|
|
# Deduplikoi virheet
|
|
errors = list(dict.fromkeys(errors))
|
|
|
|
attempt_data = {
|
|
"attempt": attempt,
|
|
"elapsed": round(elapsed, 1),
|
|
"errors": errors,
|
|
"code_length": len(final_code),
|
|
}
|
|
step_result["attempts"].append(attempt_data)
|
|
|
|
if errors:
|
|
print(f" ❌ {len(errors)} virhettä:")
|
|
for e in errors:
|
|
print(f" - {e}")
|
|
error_feedback = "\n".join(f"- {e}" for e in errors)
|
|
# Lisää koko generoitu koodi virhefeedbackiin jotta malli näkee kontekstin
|
|
error_feedback += f"\n\nYOUR PREVIOUS OUTPUT (fix the errors above):\n```\n{final_code}\n```"
|
|
else:
|
|
print(f" ✅ OK")
|
|
break
|
|
|
|
# Tallenna tulos
|
|
if step_name == "requirements":
|
|
requirements = final_code
|
|
else:
|
|
project_files[step_name] = final_code
|
|
|
|
step_result["final_errors"] = errors
|
|
step_result["passed"] = len(errors) == 0
|
|
results["steps"].append(step_result)
|
|
|
|
# Kirjoita tiedosto
|
|
(run_dir / step_name.replace("/", "_")).write_text(
|
|
final_code if step_name != "requirements" else final_code,
|
|
encoding="utf-8"
|
|
)
|
|
|
|
# Yhteenveto
|
|
passed = sum(1 for s in results["steps"] if s["passed"])
|
|
total = len(results["steps"])
|
|
results["summary"] = f"{passed}/{total} passed"
|
|
|
|
# Tallenna raportti
|
|
(run_dir / "report.json").write_text(json.dumps(results, indent=2, ensure_ascii=False))
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f" TULOS: {passed}/{total} stepiä OK")
|
|
print(f" Output: {run_dir}")
|
|
print(f"{'='*60}")
|
|
|
|
for s in results["steps"]:
|
|
status = "✅" if s["passed"] else "❌"
|
|
retries = len(s["attempts"])
|
|
retry_info = f" ({retries} yritystä)" if retries > 1 else ""
|
|
errs = f" — {s['final_errors']}" if not s["passed"] else ""
|
|
print(f" {status} {s['step']}{retry_info}{errs}")
|
|
|
|
return results
|
|
|
|
|
|
# ── Docker-testaus ────────────────────────────────────────────
|
|
|
|
import subprocess
|
|
|
|
def docker_test(run_dir: Path) -> dict:
|
|
"""Rakenna ja testaa generoitu projekti Dockerilla."""
|
|
results = {"build": False, "pytest": False, "api": False, "errors": []}
|
|
|
|
# Luo docker-compose.yml
|
|
compose = textwrap.dedent("""\
|
|
services:
|
|
app:
|
|
build: .
|
|
ports:
|
|
- "18765:8000"
|
|
healthcheck:
|
|
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/docs')"]
|
|
interval: 3s
|
|
timeout: 5s
|
|
retries: 5
|
|
""")
|
|
(run_dir / "docker-compose.yml").write_text(compose)
|
|
|
|
# Varmista ettei vanha kontti roiku
|
|
subprocess.run(["docker", "compose", "down", "--remove-orphans", "-t", "2"],
|
|
cwd=run_dir, capture_output=True, timeout=15)
|
|
|
|
# 1. Build
|
|
print(" 🔨 Docker build...", end=" ", flush=True)
|
|
r = subprocess.run(["docker", "compose", "build", "--no-cache"],
|
|
cwd=run_dir, capture_output=True, text=True, timeout=120)
|
|
if r.returncode != 0:
|
|
results["errors"].append(f"Build failed:\n{r.stderr[-500:]}")
|
|
print("❌")
|
|
return results
|
|
results["build"] = True
|
|
print("✅")
|
|
|
|
# 2. Pytest kontissa
|
|
print(" 🧪 pytest...", end=" ", flush=True)
|
|
r = subprocess.run(
|
|
["docker", "compose", "run", "--rm", "--no-deps", "app",
|
|
"uv", "run", "pytest", "test_main.py", "-v", "--tb=short"],
|
|
cwd=run_dir, capture_output=True, text=True, timeout=90)
|
|
pytest_output = r.stdout + r.stderr
|
|
if r.returncode == 0:
|
|
results["pytest"] = True
|
|
print("✅")
|
|
else:
|
|
results["errors"].append(f"pytest failed (exit {r.returncode}):\n{pytest_output[-800:]}")
|
|
print("❌")
|
|
|
|
# 3. API smoke test
|
|
print(" 🌐 API smoke test...", end=" ", flush=True)
|
|
# Käynnistä kontti taustalle
|
|
subprocess.run(["docker", "compose", "up", "-d"],
|
|
cwd=run_dir, capture_output=True, timeout=30)
|
|
|
|
# Odota terveyden tarkistusta (max 15s)
|
|
api_ok = False
|
|
for i in range(15):
|
|
time.sleep(1)
|
|
try:
|
|
from urllib.request import urlopen
|
|
resp = urlopen("http://localhost:18765/docs", timeout=3)
|
|
if resp.status == 200:
|
|
api_ok = True
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if api_ok:
|
|
try:
|
|
import urllib.request
|
|
# POST
|
|
req = urllib.request.Request(
|
|
"http://localhost:18765/todos/",
|
|
data=json.dumps({"title": "Testi", "description": "Docker-testi"}).encode(),
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST"
|
|
)
|
|
# Kokeile myös /tasks/ koska malli saattaa käyttää eri nimeä
|
|
try:
|
|
resp = urlopen(req, timeout=5)
|
|
post_ok = resp.status == 201
|
|
except Exception:
|
|
req2 = urllib.request.Request(
|
|
"http://localhost:18765/tasks/",
|
|
data=json.dumps({"title": "Testi", "description": "Docker-testi"}).encode(),
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST"
|
|
)
|
|
try:
|
|
resp = urlopen(req2, timeout=5)
|
|
post_ok = resp.status == 201
|
|
except Exception as e:
|
|
post_ok = False
|
|
results["errors"].append(f"POST /todos/ ja /tasks/ molemmat failasivat: {e}")
|
|
|
|
if post_ok:
|
|
results["api"] = True
|
|
print("✅")
|
|
else:
|
|
print("❌ POST failed")
|
|
except Exception as e:
|
|
results["errors"].append(f"API test error: {e}")
|
|
print(f"❌ {e}")
|
|
else:
|
|
results["errors"].append("Kontti ei käynnistynyt 15s:ssa")
|
|
print("❌ timeout")
|
|
|
|
# Cleanup
|
|
subprocess.run(["docker", "compose", "down", "-t", "2"],
|
|
cwd=run_dir, capture_output=True, timeout=15)
|
|
|
|
return results
|
|
|
|
|
|
# ── Main ──────────────────────────────────────────────────────
|
|
|
|
if __name__ == "__main__":
|
|
run_id = f"run_{int(time.time())}"
|
|
skip_docker = False
|
|
args = sys.argv[1:]
|
|
if args and args[0] == "--no-docker":
|
|
skip_docker = True
|
|
args = args[1:]
|
|
if args:
|
|
run_id = args[0]
|
|
|
|
results = run_pipeline(run_id)
|
|
|
|
if not skip_docker and all(s["passed"] for s in results["steps"]):
|
|
run_dir = OUTDIR / run_id
|
|
print(f"\n{'='*60}")
|
|
print(f" Docker end-to-end test")
|
|
print(f"{'='*60}")
|
|
|
|
# Lisää pytest riippuvuus pyproject.toml:iin jos puuttuu
|
|
toml_path = run_dir / "pyproject.toml"
|
|
toml_content = toml_path.read_text()
|
|
if "pytest" not in toml_content:
|
|
toml_content = toml_content.replace(
|
|
'"sqlalchemy"',
|
|
'"sqlalchemy",\n "pytest",\n "httpx"'
|
|
)
|
|
toml_path.write_text(toml_content)
|
|
|
|
docker_results = docker_test(run_dir)
|
|
results["docker"] = docker_results
|
|
|
|
print(f"\n Docker: build={'✅' if docker_results['build'] else '❌'} "
|
|
f"pytest={'✅' if docker_results['pytest'] else '❌'} "
|
|
f"api={'✅' if docker_results['api'] else '❌'}")
|
|
if docker_results["errors"]:
|
|
for e in docker_results["errors"]:
|
|
print(f" ⚠️ {e[:200]}")
|
|
|
|
# Päivitä raportti
|
|
(run_dir / "report.json").write_text(json.dumps(results, indent=2, ensure_ascii=False))
|
|
elif not all(s["passed"] for s in results["steps"]):
|
|
print("\n ⏭️ Docker-testi ohitettu — staattinen validointi ei mennyt läpi")
|