diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index c0b59a9469d..42f289091a9 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -15,59 +15,69 @@ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_000_empty_reducers", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-000-empty-reducers-golden", + "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-000-empty-reducers-golden", + "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084853617Z", - "finished_at": "2026-01-15T08:44:05.084853617Z" + "started_at": "2026-01-16T00:55:32.994654009Z", + "finished_at": "2026-01-16T00:57:50.266999019Z" }, "t_001_basic_tables": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_001_basic_tables", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "use spacetimedb::table;\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = products)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = notes)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-001-basic-tables-golden", + "llm_db": "basics-t-001-basic-tables-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_001_basic_tables/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-001-basic-tables-golden", + "llm_db": "basics-t-001-basic-tables-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084882788Z", - "finished_at": "2026-01-15T08:44:05.084882788Z" + "started_at": "2026-01-16T00:57:55.725635804Z", + "finished_at": "2026-01-16T00:59:36.074188986Z" }, "t_002_scheduled_table": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", @@ -77,432 +87,778 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: &TickTimer) {\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-002-scheduled-table-golden", + "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", "scorer_details": { "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n\u001b[1m\u001b[32m Updating\u001b[0m crates.io index\n\u001b[1m\u001b[32m Locking\u001b[0m 72 packages to latest compatible versions\n\u001b[1m\u001b[36m Adding\u001b[0m generic-array v0.14.7 \u001b[1m\u001b[33m(available: v0.14.9)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-macro v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-sys v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-lib v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-primitives v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-sats v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[32m Compiling\u001b[0m proc-macro2 v1.0.105\n\u001b[1m\u001b[32m Compiling\u001b[0m unicode-ident v1.0.22\n\u001b[1m\u001b[32m Compiling\u001b[0m quote v1.0.43\n\u001b[1m\u001b[32m Compiling\u001b[0m version_check v0.9.5\n\u001b[1m\u001b[32m Compiling\u001b[0m typenum v1.19.0\n\u001b[1m\u001b[32m Compiling\u001b[0m autocfg v1.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m generic-array v0.14.7\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_core v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m num-traits v0.2.19\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m cfg-if v1.0.4\n\u001b[1m\u001b[32m Compiling\u001b[0m syn v2.0.114\n\u001b[1m\u001b[32m Compiling\u001b[0m shlex v1.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m either v1.15.0\n\u001b[1m\u001b[32m Compiling\u001b[0m serde v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m find-msvc-tools v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m zerocopy v0.8.33\n\u001b[1m\u001b[32m Compiling\u001b[0m itertools v0.12.1\n\u001b[1m\u001b[32m Compiling\u001b[0m cc v1.2.52\n\u001b[1m\u001b[32m Compiling\u001b[0m crypto-common v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m block-buffer v0.10.4\n\u001b[1m\u001b[32m Compiling\u001b[0m bitflags v2.10.0\n\u001b[1m\u001b[32m Compiling\u001b[0m nohash-hasher v0.2.0\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m anyhow v1.0.100\n\u001b[1m\u001b[32m Compiling\u001b[0m digest v0.10.7\n\u001b[1m\u001b[32m Compiling\u001b[0m blake3 v1.8.3\n\u001b[1m\u001b[32m Compiling\u001b[0m approx v0.3.2\n\u001b[1m\u001b[32m Compiling\u001b[0m getrandom v0.2.17\n\u001b[1m\u001b[32m Compiling\u001b[0m bytes v1.11.0\n\u001b[1m\u001b[32m Compiling\u001b[0m zmij v1.0.14\n\u001b[1m\u001b[32m Compiling\u001b[0m humantime v2.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.4.1\n\u001b[1m\u001b[32m Compiling\u001b[0m keccak v0.1.5\n\u001b[1m\u001b[32m Compiling\u001b[0m convert_case v0.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m enum-as-inner v0.6.1\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror-impl v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayvec v0.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-primitives v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ppv-lite86 v0.2.21\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-macro v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m derive_more v0.99.20\n\u001b[1m\u001b[32m Compiling\u001b[0m sha3 v0.10.8\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_core v0.6.4\n\u001b[1m\u001b[32m Compiling\u001b[0m decorum v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ethnum v1.5.2\n\u001b[1m\u001b[32m Compiling\u001b[0m chrono v0.4.43\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_json v1.0.149\n\u001b[1m\u001b[32m Compiling\u001b[0m constant_time_eq v0.4.2\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayref v0.3.9\n\u001b[1m\u001b[32m Compiling\u001b[0m second-stack v0.3.5\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-lib v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m bytemuck v1.24.0\n\u001b[1m\u001b[32m Compiling\u001b[0m itoa v1.0.17\n\u001b[1m\u001b[32m Compiling\u001b[0m smallvec v1.15.1\n\u001b[1m\u001b[32m Compiling\u001b[0m hex v0.4.3\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_chacha v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-sats v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m memchr v2.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m log v0.4.29\n\u001b[1m\u001b[32m Compiling\u001b[0m rand v0.8.5\n\u001b[1m\u001b[32m Compiling\u001b[0m http v1.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-sys v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m scoped-tls v1.0.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetime-module v0.1.0 (/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `public`, `private`, `name`, `index`, `scheduled`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:4:28\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m4\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:22:11\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer(scheduled)]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0422]\u001b[0m\u001b[0m\u001b[1m: cannot find struct, variant or union type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:36\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0412]\u001b[0m\u001b[0m\u001b[1m: cannot find type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:43\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: &TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:14:15\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m14\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if ctx.db.tick_timer().count() == 0 {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:16\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:17:39\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m17\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m scheduled_at: ScheduleAt::repeat_micros(50_000),\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1mSome errors have detailed explanations: E0412, E0422, E0599.\u001b[0m\n\u001b[0m\u001b[1mFor more information about an error, try `rustc --explain E0412`.\u001b[0m\n\u001b[1m\u001b[31merror\u001b[0m\u001b[1m:\u001b[0m could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084873891Z", - "finished_at": "2026-01-15T08:44:05.084873891Z" + "started_at": "2026-01-16T00:57:40.526882193Z", + "finished_at": "2026-01-16T00:59:39.076539651Z" }, "t_003_struct_in_table": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_003_struct_in_table", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-003-struct-in-table-golden", + "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-003-struct-in-table-golden", + "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084870202Z", - "finished_at": "2026-01-15T08:44:05.084870202Z" + "started_at": "2026-01-16T00:57:55.081651973Z", + "finished_at": "2026-01-16T00:59:39.595370748Z" }, "t_004_insert": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_004_insert", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 2, + "passed_tests": 2, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "data_parity_insert_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT id, name, age, active FROM users WHERE id=1", + "reducer": "insert_user", + "server": "http://127.0.0.1:39633" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084895209Z", - "finished_at": "2026-01-15T08:44:05.084895209Z" + "started_at": "2026-01-16T00:57:58.000448174Z", + "finished_at": "2026-01-16T00:59:37.992407740Z" }, "t_005_update": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_005_update", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let users = ctx.db.users();\n if users.id().find(id).is_some() {\n users.id().update(User { id, name, age, active });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "data_parity_update_user": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1, + "Alice2", + 31, + false + ], + "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-005-update-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM users WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:39633" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084862637Z", - "finished_at": "2026-01-15T08:44:05.084862637Z" + "started_at": "2026-01-16T00:55:40.398470793Z", + "finished_at": "2026-01-16T00:57:57.999596161Z" }, "t_006_delete": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_006_delete", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.users().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-006-delete-golden", + "llm_db": "basics-t-006-delete-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "seed_users_row": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-006-delete-golden", + "llm_db": "basics-t-006-delete-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084830908Z", - "finished_at": "2026-01-15T08:44:05.084830908Z" + "started_at": "2026-01-16T00:55:36.775033097Z", + "finished_at": "2026-01-16T00:57:55.081598062Z" }, "t_007_crud": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_007_crud", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 4, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.users();\n\n let mut alice = users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n alice.name = \"Alice2\".to_string();\n alice.age = 31;\n alice.active = false;\n users.id().update(alice);\n\n users.id().delete(&2);\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM users" + } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM users WHERE id=1", + "reducer": "crud", + "server": "http://127.0.0.1:39633" + } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084835177Z", - "finished_at": "2026-01-15T08:44:05.084835177Z" + "started_at": "2026-01-16T00:55:35.376875489Z", + "finished_at": "2026-01-16T00:57:57.388761441Z" }, "t_008_index_lookup": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_008_index_lookup", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.users().id().find(id) {\n ctx.db.results().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "index_lookup_projection_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1 + ], + "golden_db": "basics-t-008-index-lookup-golden", + "golden_out": "id | name ----+--------- 1 | \"Alice\"", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "llm_out": "id | name ----+--------- 1 | \"Alice\"", + "query": "SELECT id, name FROM results WHERE id=1", + "reducer": "lookup_user_name", + "server": "http://127.0.0.1:39633" + } + }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084850033Z", - "finished_at": "2026-01-15T08:44:05.084850033Z" + "started_at": "2026-01-16T00:55:40.201797723Z", + "finished_at": "2026-01-16T00:57:58.991030144Z" }, "t_009_init": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_009_init", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 4, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "init_total_two": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM users" + } + }, + "init_seed_alice": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2 AND name='Bob' AND age=22 AND active=false" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084857594Z", - "finished_at": "2026-01-15T08:44:05.084857594Z" + "started_at": "2026-01-16T00:55:40.398058582Z", + "finished_at": "2026-01-16T00:57:59.118749309Z" }, "t_010_connect": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_010_connect", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-010-connect-golden", + "llm_db": "basics-t-010-connect-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_010_connect/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_010_connect/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-010-connect-golden", + "llm_db": "basics-t-010-connect-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084888101Z", - "finished_at": "2026-01-15T08:44:05.084888101Z" + "started_at": "2026-01-16T00:57:57.999646665Z", + "finished_at": "2026-01-16T00:59:40.062984353Z" }, "t_011_helper_function": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_011_helper_function", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.results().insert(ResultRow { id, sum });\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "helper_func_sum_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "id | sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "id | sum ----+----- 1 | 5", + "query": "SELECT id, sum FROM results WHERE id=1", + "reducer": "compute_sum", + "server": "http://127.0.0.1:39633" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1 AND sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084879130Z", - "finished_at": "2026-01-15T08:44:05.084879130Z" + "started_at": "2026-01-16T00:57:50.267037300Z", + "finished_at": "2026-01-16T00:59:13.310786115Z" }, "t_012_spacetime_product_type": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_012_spacetime_product_type", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = results)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "product_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:39633/v1/database/c20053a41b3acf73960cda9a96c51b51aa6f343d021f0109b2f2da5e9537d18d/sql)\n", + "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084824707Z", - "finished_at": "2026-01-15T08:44:05.084824707Z" + "started_at": "2026-01-16T00:53:05.557696880Z", + "finished_at": "2026-01-16T00:55:24.481587160Z" }, "t_013_spacetime_sum_type": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_013_spacetime_sum_type", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", + "query": "SELECT id, value FROM results WHERE id=1", + "reducer": "set_circle", + "server": "http://127.0.0.1:39633" + } + }, + "sum_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084827283Z", - "finished_at": "2026-01-15T08:44:05.084827283Z" + "started_at": "2026-01-16T00:55:29.605929261Z", + "finished_at": "2026-01-16T00:57:40.526817081Z" }, "t_014_elementary_columns": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_014_elementary_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitives WHERE id=1" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "elementary_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT id, count, total, price, ratio, active, name FROM primitives WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:39633" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084777176Z", - "finished_at": "2026-01-15T08:44:05.084777176Z" + "started_at": "2026-01-16T00:53:05.557387542Z", + "finished_at": "2026-01-16T00:55:29.605867355Z" }, "t_015_product_type_columns": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_015_product_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profiles().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:39633/v1/database/c2009d57e3266caa27aacc56f0c67f35ff7831a1e8e050f94ea2e7168e4d2d29/sql)\n", + "phase": "sql_golden" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profiles WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084822119Z", - "finished_at": "2026-01-15T08:44:05.084822119Z" + "started_at": "2026-01-16T00:53:05.558011008Z", + "finished_at": "2026-01-16T00:55:32.994601591Z" }, "t_016_sum_type_columns": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_016_sum_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\nstruct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawings();\n let ids: Vec = drawings.iter().map(|d| d.id).collect();\n for id in ids {\n drawings.id().delete(&id);\n }\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "sum_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM drawings WHERE id=1" + } + }, + "sum_type_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-016-sum-type-columns-golden", + "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", + "query": "SELECT id, a, b FROM drawings WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:39633" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084866461Z", - "finished_at": "2026-01-15T08:44:05.084866461Z" + "started_at": "2026-01-16T00:55:24.481644858Z", + "finished_at": "2026-01-16T00:57:55.725583576Z" }, "t_017_scheduled_columns": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", @@ -510,144 +866,280 @@ "lang": "rust", "golden_published": false, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 2, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-017-scheduled-columns-golden", + "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n\u001b[1m\u001b[32m Updating\u001b[0m crates.io index\n\u001b[1m\u001b[32m Locking\u001b[0m 72 packages to latest compatible versions\n\u001b[1m\u001b[36m Adding\u001b[0m generic-array v0.14.7 \u001b[1m\u001b[33m(available: v0.14.9)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-macro v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-sys v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-lib v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-primitives v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-sats v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[32m Compiling\u001b[0m proc-macro2 v1.0.105\n\u001b[1m\u001b[32m Compiling\u001b[0m unicode-ident v1.0.22\n\u001b[1m\u001b[32m Compiling\u001b[0m quote v1.0.43\n\u001b[1m\u001b[32m Compiling\u001b[0m typenum v1.19.0\n\u001b[1m\u001b[32m Compiling\u001b[0m version_check v0.9.5\n\u001b[1m\u001b[32m Compiling\u001b[0m autocfg v1.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m generic-array v0.14.7\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_core v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m num-traits v0.2.19\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m cfg-if v1.0.4\n\u001b[1m\u001b[32m Compiling\u001b[0m syn v2.0.114\n\u001b[1m\u001b[32m Compiling\u001b[0m zerocopy v0.8.33\n\u001b[1m\u001b[32m Compiling\u001b[0m either v1.15.0\n\u001b[1m\u001b[32m Compiling\u001b[0m shlex v1.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m find-msvc-tools v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m serde v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m itertools v0.12.1\n\u001b[1m\u001b[32m Compiling\u001b[0m cc v1.2.52\n\u001b[1m\u001b[32m Compiling\u001b[0m block-buffer v0.10.4\n\u001b[1m\u001b[32m Compiling\u001b[0m crypto-common v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m anyhow v1.0.100\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m bitflags v2.10.0\n\u001b[1m\u001b[32m Compiling\u001b[0m nohash-hasher v0.2.0\n\u001b[1m\u001b[32m Compiling\u001b[0m digest v0.10.7\n\u001b[1m\u001b[32m Compiling\u001b[0m blake3 v1.8.3\n\u001b[1m\u001b[32m Compiling\u001b[0m approx v0.3.2\n\u001b[1m\u001b[32m Compiling\u001b[0m getrandom v0.2.17\n\u001b[1m\u001b[32m Compiling\u001b[0m zmij v1.0.14\n\u001b[1m\u001b[32m Compiling\u001b[0m humantime v2.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayvec v0.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m bytes v1.11.0\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.4.1\n\u001b[1m\u001b[32m Compiling\u001b[0m convert_case v0.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m keccak v0.1.5\n\u001b[1m\u001b[32m Compiling\u001b[0m sha3 v0.10.8\n\u001b[1m\u001b[32m Compiling\u001b[0m enum-as-inner v0.6.1\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror-impl v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m ppv-lite86 v0.2.21\n\u001b[1m\u001b[32m Compiling\u001b[0m derive_more v0.99.20\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-primitives v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-macro v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_core v0.6.4\n\u001b[1m\u001b[32m Compiling\u001b[0m decorum v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ethnum v1.5.2\n\u001b[1m\u001b[32m Compiling\u001b[0m chrono v0.4.43\n\u001b[1m\u001b[32m Compiling\u001b[0m itoa v1.0.17\n\u001b[1m\u001b[32m Compiling\u001b[0m smallvec v1.15.1\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayref v0.3.9\n\u001b[1m\u001b[32m Compiling\u001b[0m constant_time_eq v0.4.2\n\u001b[1m\u001b[32m Compiling\u001b[0m hex v0.4.3\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-lib v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m second-stack v0.3.5\n\u001b[1m\u001b[32m Compiling\u001b[0m bytemuck v1.24.0\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_json v1.0.149\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_chacha v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-sats v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m log v0.4.29\n\u001b[1m\u001b[32m Compiling\u001b[0m memchr v2.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m rand v0.8.5\n\u001b[1m\u001b[32m Compiling\u001b[0m http v1.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-sys v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m scoped-tls v1.0.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetime-module v0.1.0 (/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `public`, `private`, `name`, `index`, `scheduled`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:4:28\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m4\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0422]\u001b[0m\u001b[0m\u001b[1m: cannot find struct, variant or union type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:36\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0412]\u001b[0m\u001b[0m\u001b[1m: cannot find type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:42\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:14:15\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m14\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if ctx.db.tick_timer().count() == 0 {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:16\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:17:39\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m17\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m scheduled_at: ScheduleAt::repeat_micros(50_000),\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0277]\u001b[0m\u001b[0m\u001b[1m: invalid reducer signature\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:8\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m----------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound introduced by this call\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mthis reducer signature is not valid\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mhelp\u001b[0m\u001b[0m: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: reducer signatures must match the following pattern:\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: where each `Ti` type implements `SpacetimeType`.\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;10mnote\u001b[0m\u001b[0m: required by a bound in `register_reducer`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0m/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m746\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10m^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10mrequired by this bound in `register_reducer`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0277]\u001b[0m\u001b[0m\u001b[1m: invalid reducer signature\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:8\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m----------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound introduced by this call\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mthis reducer signature is not valid\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mhelp\u001b[0m\u001b[0m: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: reducer signatures must match the following pattern:\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: where each `Ti` type implements `SpacetimeType`.\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;10mnote\u001b[0m\u001b[0m: required by a bound in `invoke_reducer`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0m/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m44\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn invoke_reducer<'a, A: Args<'a>>(\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--------------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound in this function\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m45\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m reducer: impl Reducer<'a, A>,\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10m^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10mrequired by this bound in `invoke_reducer`\u001b[0m\n\n\u001b[0m\u001b[1mSome errors have detailed explanations: E0277, E0412, E0422, E0599.\u001b[0m\n\u001b[0m\u001b[1mFor more information about an error, try `rustc --explain E0277`.\u001b[0m\n\u001b[1m\u001b[31merror\u001b[0m\u001b[1m:\u001b[0m could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084809104Z", - "finished_at": "2026-01-15T08:44:05.084809104Z" + "started_at": "2026-01-16T00:53:05.556796888Z", + "finished_at": "2026-01-16T00:55:36.775006738Z" }, "t_018_constraints": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_018_constraints", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = accounts, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let accounts = ctx.db.accounts();\n let ids: Vec = accounts.iter().map(|a| a.id).collect();\n for id in ids {\n accounts.id().delete(&id);\n }\n accounts.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n accounts.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "constraints_row_parity_after_seed": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:39633/v1/database/c2001c7a5adcf4f79a3117c2c3671a576d84bde5f0e9152c5c660a77c5074614/sql)\n", + "phase": "sql_golden" + } + }, + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM accounts WHERE id=2" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084805389Z", - "finished_at": "2026-01-15T08:44:05.084805389Z" + "started_at": "2026-01-16T00:53:05.556487089Z", + "finished_at": "2026-01-16T00:55:35.376822731Z" }, "t_019_many_to_many": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 5, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = groups)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = memberships,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let mids: Vec = ctx.db.memberships().iter().map(|m| m.id).collect();\n for id in mids {\n ctx.db.memberships().id().delete(&id);\n }\n let uids: Vec = ctx.db.users().iter().map(|u| u.user_id).collect();\n for uid in uids {\n ctx.db.users().user_id().delete(&uid);\n }\n let gids: Vec = ctx.db.groups().iter().map(|g| g.group_id).collect();\n for gid in gids {\n ctx.db.groups().group_id().delete(&gid);\n }\n\n ctx.db.users().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.users().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=10" + } + }, + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=20" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=2 AND group_id=20" + } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM memberships" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084816252Z", - "finished_at": "2026-01-15T08:44:05.084816252Z" + "started_at": "2026-01-16T00:53:05.555877349Z", + "finished_at": "2026-01-16T00:55:39.920644423Z" }, "t_020_ecs": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_020_ecs", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 5, + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table, UniqueColumn};\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = positions)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entities().try_insert(Entity { id: 1 });\n let _ = ctx.db.entities().try_insert(Entity { id: 2 });\n\n let p1 = Position {\n entity_id: 1,\n x: 0,\n y: 0,\n };\n if ctx.db.positions().entity_id().find(1).is_some() {\n let _ = ctx.db.positions().entity_id().update(p1);\n } else {\n let _ = ctx.db.positions().insert(p1);\n }\n\n let v1 = Velocity {\n entity_id: 1,\n vx: 1,\n vy: 0,\n };\n if ctx.db.velocities().entity_id().find(1).is_some() {\n let _ = ctx.db.velocities().entity_id().update(v1);\n } else {\n let _ = ctx.db.velocities().insert(v1);\n }\n\n let p2 = Position {\n entity_id: 2,\n x: 10,\n y: 0,\n };\n if ctx.db.positions().entity_id().find(2).is_some() {\n let _ = ctx.db.positions().entity_id().update(p2);\n } else {\n let _ = ctx.db.positions().insert(p2);\n }\n\n let v2 = Velocity {\n entity_id: 2,\n vx: -2,\n vy: 3,\n };\n if ctx.db.velocities().entity_id().find(2).is_some() {\n let _ = ctx.db.velocities().entity_id().update(v2);\n } else {\n let _ = ctx.db.velocities().insert(v2);\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.positions().iter() {\n if let Some(v) = ctx.db.velocities().entity_id().find(p.entity_id) {\n let np = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n if ctx\n .db\n .next_positions()\n .entity_id()\n .find(p.entity_id)\n .is_some()\n {\n let _ = ctx.db.next_positions().entity_id().update(np);\n } else {\n let _ = ctx.db.next_positions().insert(np);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "ecs_seed_positions_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM positions" + } + }, + "ecs_next_pos_entity1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE entity_id=1 AND x=1 AND y=0" + } + }, + "ecs_step_next_positions_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM next_positions" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "ecs_next_pos_entity2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE entity_id=2 AND x=8 AND y=3" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084812996Z", - "finished_at": "2026-01-15T08:44:05.084812996Z" + "started_at": "2026-01-16T00:53:05.556208899Z", + "finished_at": "2026-01-16T00:55:40.201729445Z" }, "t_021_multi_column_index": { "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a", "task": "t_021_multi_column_index", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 4, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = logs, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let logs = ctx.db.logs();\n\n let existing_ids: Vec = logs.iter().map(|r| r.id).collect();\n for id in existing_ids {\n logs.id().delete(&id);\n }\n\n logs.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n logs.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n logs.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=2" + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM logs" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:39633", + "tables_diff": null, + "tables_equal": true + } + }, + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=1" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:44:05.084891997Z", - "finished_at": "2026-01-15T08:44:05.084891997Z" + "started_at": "2026-01-16T00:53:05.557086109Z", + "finished_at": "2026-01-16T00:55:40.398017385Z" } } } @@ -848,378 +1340,640 @@ "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_000_empty_reducers", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-000-empty-reducers-golden", + "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-000-empty-reducers-golden", + "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350758519Z", - "finished_at": "2026-01-15T08:48:04.350758519Z" + "started_at": "2026-01-16T01:08:18.856841039Z", + "finished_at": "2026-01-16T01:08:44.106092534Z" }, "t_001_basic_tables": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_001_basic_tables", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n}\n\n[SpacetimeDB.Table(Name = \"users\", Public = true)]\npublic partial struct Users\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n}\n\n[SpacetimeDB.Table(Name = \"products\", Public = true)]\npublic partial struct Products\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n}\n\n[SpacetimeDB.Table(Name = \"notes\", Public = true)]\npublic partial struct Notes\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-001-basic-tables-golden", + "llm_db": "basics-t-001-basic-tables-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_001_basic_tables/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-001-basic-tables-golden", + "llm_db": "basics-t-001-basic-tables-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350802382Z", - "finished_at": "2026-01-15T08:48:04.350802382Z" + "started_at": "2026-01-16T01:10:46.550782288Z", + "finished_at": "2026-01-16T01:11:21.093863299Z" }, "t_002_scheduled_table": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_002_scheduled_table", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-002-scheduled-table-golden", + "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-002-scheduled-table-golden", + "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350787405Z", - "finished_at": "2026-01-15T08:48:04.350787405Z" + "started_at": "2026-01-16T01:09:55.247537588Z", + "finished_at": "2026-01-16T01:10:36.710734521Z" }, "t_003_struct_in_table": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_003_struct_in_table", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-003-struct-in-table-golden", + "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-003-struct-in-table-golden", + "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350798386Z", - "finished_at": "2026-01-15T08:48:04.350798386Z" + "started_at": "2026-01-16T01:10:36.710775076Z", + "finished_at": "2026-01-16T01:11:05.707794534Z" }, "t_004_insert": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_004_insert", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 2, + "passed_tests": 2, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "data_parity_insert_user": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1, + "Alice", + 30, + true + ], + "golden_db": "basics-t-004-insert-golden", + "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", + "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1", + "reducer": "InsertUser", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350810397Z", - "finished_at": "2026-01-15T08:48:04.350810397Z" + "started_at": "2026-01-16T01:11:21.093906981Z", + "finished_at": "2026-01-16T01:11:54.905124723Z" }, "t_005_update": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_005_update", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.users.Id.Find(id);\n if (existing != null)\n {\n var u = existing.Value;\n u.Name = name;\n u.Age = age;\n u.Active = active;\n ctx.Db.users.Id.Update(u);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } + }, + "data_parity_update_user": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], + "golden_db": "basics-t-005-update-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-005-update-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1", + "reducer": "UpdateUser", + "server": "http://127.0.0.1:37321" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350783269Z", - "finished_at": "2026-01-15T08:48:04.350783269Z" + "started_at": "2026-01-16T01:09:35.751798751Z", + "finished_at": "2026-01-16T01:10:09.499012619Z" }, "t_006_delete": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_006_delete", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-006-delete-golden", + "llm_db": "basics-t-006-delete-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1" + } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-006-delete-golden", + "llm_db": "basics-t-006-delete-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350772180Z", - "finished_at": "2026-01-15T08:48:04.350772180Z" + "started_at": "2026-01-16T01:08:44.106133480Z", + "finished_at": "2026-01-16T01:09:18.793710872Z" }, "t_007_crud": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_007_crud", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n var user1 = ctx.Db.users.Id.Find(1);\n if (user1 is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.users.Id.Update(u1);\n }\n\n ctx.Db.users.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2" + } + }, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM users" + } + }, + "crud_row_id1_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "basics-t-007-crud-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1", + "reducer": "Crud", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350768636Z", - "finished_at": "2026-01-15T08:48:04.350768636Z" + "started_at": "2026-01-16T01:08:24.082507036Z", + "finished_at": "2026-01-16T01:09:02.249009409Z" }, "t_008_index_lookup": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_008_index_lookup", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "index_lookup_projection_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1 + ], + "golden_db": "basics-t-008-index-lookup-golden", + "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", + "query": "SELECT Id, Name FROM results WHERE Id=1", + "reducer": "LookupUserName", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350775111Z", - "finished_at": "2026-01-15T08:48:04.350775111Z" + "started_at": "2026-01-16T01:09:02.249052590Z", + "finished_at": "2026-01-16T01:09:35.751755801Z" }, "t_009_init": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_009_init", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "init_seed_alice": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + } + }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "init_total_two": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM users" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350777872Z", - "finished_at": "2026-01-15T08:48:04.350777872Z" + "started_at": "2026-01-16T01:09:18.793757880Z", + "finished_at": "2026-01-16T01:09:55.247472877Z" }, "t_010_connect": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_010_connect", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "passed_tests": 1, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"events\")]\n public partial struct Event\n {\n [PrimaryKey, AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event\n {\n Kind = \"connected\"\n });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event\n {\n Kind = \"disconnected\"\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-010-connect-golden", + "llm_db": "basics-t-010-connect-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_010_connect/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_010_connect/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "basics-t-010-connect-golden", + "llm_db": "basics-t-010-connect-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350806555Z", - "finished_at": "2026-01-15T08:48:04.350806555Z" + "started_at": "2026-01-16T01:11:05.707824440Z", + "finished_at": "2026-01-16T01:11:41.874533069Z" }, "t_011_helper_function": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_011_helper_function", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n var sum = Add(a, b);\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Sum = sum\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", + "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "helper_func_sum_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "Id | Sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "Id | Sum ----+----- 1 | 5", + "query": "SELECT Id, Sum FROM results WHERE Id=1", + "reducer": "ComputeSum", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350794264Z", - "finished_at": "2026-01-15T08:48:04.350794264Z" + "started_at": "2026-01-16T01:10:09.499058365Z", + "finished_at": "2026-01-16T01:10:46.550746821Z" }, "t_012_spacetime_product_type": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_012_spacetime_product_type", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\n[Type]\npublic partial struct Score\n{\n public int Left;\n public int Right;\n}\n\npublic static partial class Module\n{\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "product_type_row_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", + "query": "SELECT Id, Value FROM results WHERE Id=1", + "reducer": "SetScore", + "server": "http://127.0.0.1:37321" + } + }, + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350747625Z", - "finished_at": "2026-01-15T08:48:04.350747625Z" + "started_at": "2026-01-16T01:06:05.088532998Z", + "finished_at": "2026-01-16T01:06:46.603812945Z" }, "t_013_spacetime_sum_type": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", @@ -1227,86 +1981,142 @@ "lang": "csharp", "golden_published": false, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 3, "passed_tests": 0, - "llm_output": null, + "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>;\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/StdbModule.csproj]\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350765485Z", - "finished_at": "2026-01-15T08:48:04.350765485Z" + "started_at": "2026-01-16T01:07:27.375289930Z", + "finished_at": "2026-01-16T01:08:18.856814059Z" }, "t_014_elementary_columns": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_014_elementary_columns", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var row = new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n };\n\n var existing = ctx.Db.primitives.Id.Find(1);\n if (existing is null)\n {\n ctx.Db.primitives.Insert(row);\n }\n else\n {\n ctx.Db.primitives.Id.Update(row);\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "elementary_columns_row_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [], + "golden_db": "schema-t-014-elementary-columns-golden", + "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", + "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350744275Z", - "finished_at": "2026-01-15T08:48:04.350744275Z" + "started_at": "2026-01-16T01:05:40.135748267Z", + "finished_at": "2026-01-16T01:07:03.394849892Z" }, "t_015_product_type_columns": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_015_product_type_columns", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Address\n{\n public string Street;\n public int Zip;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Position\n{\n public int X;\n public int Y;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var home = new Address { Street = \"1 Main\", Zip = 11111 };\n var work = new Address { Street = \"2 Broad\", Zip = 22222 };\n var pos = new Position { X = 7, Y = 9 };\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = home,\n Work = work,\n Pos = pos\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350753908Z", - "finished_at": "2026-01-15T08:48:04.350753908Z" + "started_at": "2026-01-16T01:06:46.603867126Z", + "finished_at": "2026-01-16T01:07:27.375238274Z" }, "t_016_sum_type_columns": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", @@ -1314,173 +2124,329 @@ "lang": "csharp", "golden_published": false, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 3, "passed_tests": 0, - "llm_output": null, + "llm_output": "using SpacetimeDB;\nusing Shape = SpacetimeDB.TaggedEnum<(Module.Circle, Module.Rectangle)>;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Circle { Radius = 10 };\n var b = new Rectangle { Width = 4, Height = 6 };\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'drawings' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,188): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,206): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,227): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,252): error CS0246: The type or namespace name 'ARW' could not be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,58): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,73): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,92): error CS0426: The type name 'BSATN<,>' does not exist in the type 'TaggedEnum<(Circle, Rectangle)>' [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,99): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,114): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,134): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,149): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,167): error CS8657: Static member 'Module.Drawing.BSATN.BSATN(Module.Circle, Rectangle, Circle.BSATN, Rectangle.BSATN, ARW)' cannot be marked 'readonly'. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,188): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,206): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,227): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,252): error CS0246: The type or namespace name 'BRW' could not be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,58): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,73): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,92): error CS0426: The type name 'BSATN<,>' does not exist in the type 'TaggedEnum<(Circle, Rectangle)>' [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,99): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,114): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,134): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,149): error CS0704: Cannot do non-virtual member lookup in 'Module' because it is a type parameter [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,167): error CS8657: Static member 'Module.Drawing.BSATN.BSATN(Module.Circle, Rectangle, Circle.BSATN, Rectangle.BSATN, BRW)' cannot be marked 'readonly'. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,167): error CS0542: 'BSATN': member names cannot be the same as their enclosing type [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,167): error CS0542: 'BSATN': member names cannot be the same as their enclosing type [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(27,258): error CS1736: Default parameter value for '' must be a compile-time constant [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/Module.Drawing.cs(28,258): error CS1736: Default parameter value for '' must be a compile-time constant [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350750774Z", - "finished_at": "2026-01-15T08:48:04.350750774Z" + "started_at": "2026-01-16T01:07:03.394897852Z", + "finished_at": "2026-01-16T01:08:24.082465708Z" }, "t_017_scheduled_columns": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_017_scheduled_columns", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 2, + "passed_tests": 2, + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer tick)\n {\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-017-scheduled-columns-golden", + "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "golden_db": "schema-t-017-scheduled-columns-golden", + "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "scheduled_seeded_one_row": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350734182Z", - "finished_at": "2026-01-15T08:48:04.350734182Z" + "started_at": "2026-01-16T01:04:58.696136087Z", + "finished_at": "2026-01-16T01:05:40.135697722Z" }, "t_018_constraints": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_018_constraints", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 3, + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2" + } + }, + "constraints_row_parity_after_seed": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-018-constraints-golden", + "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:37321" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350737033Z", - "finished_at": "2026-01-15T08:48:04.350737033Z" + "started_at": "2026-01-16T01:04:39.589075714Z", + "finished_at": "2026-01-16T01:05:26.703647120Z" }, "t_019_many_to_many": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_019_many_to_many", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 5, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM memberships" + } + }, + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20" + } + }, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350730826Z", - "finished_at": "2026-01-15T08:48:04.350730826Z" + "started_at": "2026-01-16T01:03:27.648698788Z", + "finished_at": "2026-01-16T01:04:39.589013436Z" }, "t_020_ecs": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_020_ecs", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 5, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n UpsertEntity(ctx, 1);\n UpsertPosition(ctx, 1, 0, 0);\n UpsertVelocity(ctx, 1, 1, 0);\n\n UpsertEntity(ctx, 2);\n UpsertPosition(ctx, 2, 10, 0);\n UpsertVelocity(ctx, 2, -2, 3);\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n var vel = ctx.Db.velocities.EntityId.Find(pos.EntityId);\n if (!vel.HasValue)\n {\n continue;\n }\n\n var nx = pos.X + vel.Value.VX;\n var ny = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.next_positions.EntityId.Find(pos.EntityId);\n if (existing.HasValue)\n {\n var np = existing.Value;\n np.X = nx;\n np.Y = ny;\n ctx.Db.next_positions.EntityId.Update(np);\n }\n else\n {\n ctx.Db.next_positions.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n\n private static void UpsertEntity(ReducerContext ctx, int id)\n {\n var e = ctx.Db.entities.Id.Find(id);\n if (!e.HasValue)\n {\n ctx.Db.entities.Insert(new Entity { Id = id });\n }\n }\n\n private static void UpsertPosition(ReducerContext ctx, int entityId, int x, int y)\n {\n var p = ctx.Db.positions.EntityId.Find(entityId);\n if (p.HasValue)\n {\n var row = p.Value;\n row.X = x;\n row.Y = y;\n ctx.Db.positions.EntityId.Update(row);\n }\n else\n {\n ctx.Db.positions.Insert(new Position\n {\n EntityId = entityId,\n X = x,\n Y = y\n });\n }\n }\n\n private static void UpsertVelocity(ReducerContext ctx, int entityId, int vx, int vy)\n {\n var v = ctx.Db.velocities.EntityId.Find(entityId);\n if (v.HasValue)\n {\n var row = v.Value;\n row.VX = vx;\n row.VY = vy;\n ctx.Db.velocities.EntityId.Update(row);\n }\n else\n {\n ctx.Db.velocities.Insert(new Velocity\n {\n EntityId = entityId,\n VX = vx,\n VY = vy\n });\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "ecs_seed_positions_count": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM positions" + } + }, + "ecs_next_pos_entity1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0" + } + }, + "ecs_next_pos_entity2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "ecs_step_next_positions_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM next_positions" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350721102Z", - "finished_at": "2026-01-15T08:48:04.350721102Z" + "started_at": "2026-01-16T01:03:27.652097672Z", + "finished_at": "2026-01-16T01:04:58.695969194Z" }, "t_021_multi_column_index": { "hash": "57aa5b6bb986daddf9d3a9f4992cdb0c6bab78257186c00ab1284c1b40281b46", "task": "t_021_multi_column_index", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 4, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [\"UserId\", \"Day\"])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 429 Too Many Requests: {\n \"error\": {\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors.\",\n \"type\": \"insufficient_quota\",\n \"param\": null,\n \"code\": \"insufficient_quota\"\n }\n}", - "phase": "build_or_publish" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37321", + "tables_diff": null, + "tables_equal": true + } + }, + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1" + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM logs" } } }, "vendor": "openai", - "started_at": "2026-01-15T08:48:04.350740315Z", - "finished_at": "2026-01-15T08:48:04.350740315Z" + "started_at": "2026-01-16T01:05:26.703696953Z", + "finished_at": "2026-01-16T01:06:05.088463308Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index fc0c66a436f..3f8c0800dbc 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,6 +1,6 @@ { "version": 1, - "generated_at": "2026-01-15T08:48:04.366Z", + "generated_at": "2026-01-16T01:11:54.962Z", "by_language": { "csharp": { "modes": { @@ -11,28 +11,28 @@ "categories": { "basics": { "tasks": 12, - "total_tests": 12, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 27, + "passed_tests": 27, + "pass_pct": 100.0, + "task_pass_equiv": 12.0, + "task_pass_pct": 100.0 }, "schema": { "tasks": 10, - "total_tests": 10, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 34, + "passed_tests": 28, + "pass_pct": 82.35294, + "task_pass_equiv": 8.0, + "task_pass_pct": 80.0 } }, "totals": { "tasks": 22, - "total_tests": 22, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 61, + "passed_tests": 55, + "pass_pct": 90.16393, + "task_pass_equiv": 20.0, + "task_pass_pct": 90.909096 } } } @@ -48,28 +48,28 @@ "categories": { "basics": { "tasks": 12, - "total_tests": 12, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 27, + "passed_tests": 26, + "pass_pct": 96.296295, + "task_pass_equiv": 11.0, + "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, - "total_tests": 10, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 34, + "passed_tests": 29, + "pass_pct": 85.29412, + "task_pass_equiv": 8.0, + "task_pass_pct": 80.0 } }, "totals": { "tasks": 22, - "total_tests": 22, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 61, + "passed_tests": 55, + "pass_pct": 90.16393, + "task_pass_equiv": 19.0, + "task_pass_pct": 86.36364 } } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt index d0b31a0deaf..a20841aa2fb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt @@ -10,7 +10,7 @@ TABLES - active: bool - results - - Struct: Result + - Struct: ResultRow - Fields: - id: i32 (primary key) - name: String diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/rust.txt index 9bdd66364d2..f3d011a8251 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/rust.txt @@ -2,7 +2,7 @@ TABLE - results - - Struct: Result + - Struct: ResultRow - Fields: - id: i32 (primary key) - sum: i32 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_012_spacetime_product_type/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_012_spacetime_product_type/tasks/rust.txt index 330e8532475..ce12c115ded 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_012_spacetime_product_type/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_012_spacetime_product_type/tasks/rust.txt @@ -8,7 +8,7 @@ TYPES TABLE - results - - Struct: Result + - Struct: ResultRow - Fields: - id: i32 (primary key) - value: Score diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/tasks/rust.txt index eb16d58a882..68ccae99f50 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/tasks/rust.txt @@ -12,7 +12,7 @@ TYPES TABLE - results - - Struct: Result + - Struct: ResultRow - Fields: - id: i32 (primary key) - value: Shape