diff --git a/kipina-codebench/benchmark.mjs b/kipina-codebench/benchmark.mjs index cff00d0..d8e2085 100644 --- a/kipina-codebench/benchmark.mjs +++ b/kipina-codebench/benchmark.mjs @@ -308,7 +308,7 @@ function starsForScore(score) { } // === Pipeline: yhdelle mallille ja skenaariolle === -async function runPipeline(model, scenario) { +async function runPipeline(model, scenario, round = 1) { const result = { model, scenario: scenario.id, reqOk: false, specOk: false, specEntities: 0, @@ -321,7 +321,8 @@ async function runPipeline(model, scenario) { }; const timings = []; const { system: CODE_SYSTEM, promptName, profile } = getCodePromptForModel(model); - const dir = `${OUTPUT_DIR}/${model.replace(/[/:]/g, '_')}__${scenario.id}`; + const roundSuffix = ROUNDS > 1 ? `__r${round}` : ''; + const dir = `${OUTPUT_DIR}/${model.replace(/[/:]/g, '_')}__${scenario.id}${roundSuffix}`; mkdirSync(dir, { recursive: true }); try { @@ -651,7 +652,7 @@ async function main() { for (const scenario of scenarios) { const roundLabel = ROUNDS > 1 ? ` [${round}/${ROUNDS}]` : ''; console.log(`\n━━━ ${model} × ${scenario.id}${roundLabel} ━━━`); - const r = await runPipeline(model, scenario); + const r = await runPipeline(model, scenario, round); if (ROUNDS > 1) r.round = round; results.push(r); diff --git a/kipina-codebench/golden-examples/todo-go.md b/kipina-codebench/golden-examples/todo-go.md index c2d3249..fbd2b4f 100644 --- a/kipina-codebench/golden-examples/todo-go.md +++ b/kipina-codebench/golden-examples/todo-go.md @@ -302,16 +302,13 @@ import ( "net/http/httptest" "strings" "testing" - _ "modernc.org/sqlite" ) func setupTestServer(t *testing.T) (*httptest.Server, *sql.DB) { t.Helper() db, err := sql.Open("sqlite", ":memory:") - if err != nil { - t.Fatal(err) - } + if err != nil { t.Fatal(err) } InitDB(db) return httptest.NewServer(NewRouter(db)), db } @@ -320,147 +317,57 @@ func TestCreateTodo(t *testing.T) { ts, db := setupTestServer(t) defer ts.Close() defer db.Close() - resp, err := http.Post(ts.URL+"/todos", "application/json", strings.NewReader(`{"title":"Buy groceries","priority":2}`)) - if err != nil { - t.Fatal(err) - } + if err != nil { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - t.Fatalf("expected 201, got %d", resp.StatusCode) - } + if resp.StatusCode != http.StatusCreated { t.Fatalf("expected 201, got %d", resp.StatusCode) } var body map[string]interface{} json.NewDecoder(resp.Body).Decode(&body) - if body["title"] != "Buy groceries" { - t.Fatalf("expected title 'Buy groceries', got %v", body["title"]) - } - if body["id"] == nil { - t.Fatal("expected id to be present") - } -} - -func TestListTodos(t *testing.T) { - ts, db := setupTestServer(t) - defer ts.Close() - defer db.Close() - - http.Post(ts.URL+"/todos", "application/json", - strings.NewReader(`{"title":"Listable task"}`)) - - resp, err := http.Get(ts.URL + "/todos") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } - var body []map[string]interface{} - json.NewDecoder(resp.Body).Decode(&body) - if len(body) < 1 { - t.Fatal("expected at least 1 todo") - } + if body["title"] != "Buy groceries" { t.Fatalf("expected 'Buy groceries', got %v", body["title"]) } + if body["id"] == nil { t.Fatal("expected id") } } func TestGetTodoByID(t *testing.T) { ts, db := setupTestServer(t) defer ts.Close() defer db.Close() - resp, _ := http.Post(ts.URL+"/todos", "application/json", strings.NewReader(`{"title":"Fetchable task"}`)) var created map[string]interface{} json.NewDecoder(resp.Body).Decode(&created) resp.Body.Close() - id := created["id"].(float64) - resp, err := http.Get(ts.URL + "/todos/" + fmt.Sprintf("%.0f", id)) - if err != nil { - t.Fatal(err) - } + resp, _ = http.Get(ts.URL + "/todos/" + fmt.Sprintf("%.0f", id)) defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } - var body map[string]interface{} - json.NewDecoder(resp.Body).Decode(&body) - if body["id"] != id { - t.Fatalf("expected id %.0f, got %v", id, body["id"]) - } + if resp.StatusCode != http.StatusOK { t.Fatalf("expected 200, got %d", resp.StatusCode) } } func TestGetTodoNotFound(t *testing.T) { ts, db := setupTestServer(t) defer ts.Close() defer db.Close() - - resp, err := http.Get(ts.URL + "/todos/99999") - if err != nil { - t.Fatal(err) - } + resp, _ := http.Get(ts.URL + "/todos/99999") defer resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("expected 404, got %d", resp.StatusCode) - } -} - -func TestUpdateTodo(t *testing.T) { - ts, db := setupTestServer(t) - defer ts.Close() - defer db.Close() - - resp, _ := http.Post(ts.URL+"/todos", "application/json", - strings.NewReader(`{"title":"Old title"}`)) - var created map[string]interface{} - json.NewDecoder(resp.Body).Decode(&created) - resp.Body.Close() - - id := created["id"].(float64) - req, _ := http.NewRequest(http.MethodPut, ts.URL+"/todos/"+fmt.Sprintf("%.0f", id), - strings.NewReader(`{"title":"New title"}`)) - req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected 200, got %d", resp.StatusCode) - } - var body map[string]interface{} - json.NewDecoder(resp.Body).Decode(&body) - if body["title"] != "New title" { - t.Fatalf("expected 'New title', got %v", body["title"]) - } + if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404, got %d", resp.StatusCode) } } func TestDeleteTodo(t *testing.T) { ts, db := setupTestServer(t) defer ts.Close() defer db.Close() - resp, _ := http.Post(ts.URL+"/todos", "application/json", strings.NewReader(`{"title":"Deletable task"}`)) var created map[string]interface{} json.NewDecoder(resp.Body).Decode(&created) resp.Body.Close() - id := created["id"].(float64) req, _ := http.NewRequest(http.MethodDelete, ts.URL+"/todos/"+fmt.Sprintf("%.0f", id), nil) - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatal(err) - } + resp, _ = http.DefaultClient.Do(req) defer resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - t.Fatalf("expected 204, got %d", resp.StatusCode) - } - + if resp.StatusCode != http.StatusNoContent { t.Fatalf("expected 204, got %d", resp.StatusCode) } resp, _ = http.Get(ts.URL + "/todos/" + fmt.Sprintf("%.0f", id)) defer resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("expected 404 after delete, got %d", resp.StatusCode) - } + if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404 after delete, got %d", resp.StatusCode) } } ```