Traces

Every run: the agent's full trajectory, its submissions over the 20-hour budget, and the final score.

Kolmogorov Audio Compression

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.0779	$45.19	77.6M	262k	17h 49m	334	16
2	Claude Fable 5.1	0.0754	$52.35	88.6M	331k	12h 35m	339	10
3	Claude Fable 5.1	0.0711	$71.87	125.0M	385k	17h 43m	414	18
4	Claude Fable 5.1	0.0695	$54.46	93.0M	327k	16h 46m	347	11
5	Claude Fable 5.1	0.0678	$47.98	81.1M	304k	16h 17m	333	10
6	GPT-5.6	0.0416	$183.22	165.9M	262k	15h 47m	471	12
7	GPT-5.6	0.0408	$43.32	47.2M	136k	19h 34m	249	10
8	GLM-5.3	0.0369	$28.71	91.1M	330k	19h 38m	343	2
9	GLM-5.3	0.0349	$48.94	118.7M	352k	19h 6m	384	2
10	Kimi K3	0.0313	$38.45	102.2M	362k	20h 1m	397	10
11	GLM-5.3	0.0298	$35.18	91.1M	311k	19h 41m	366	13
12	DeepSeek V4 Flash Vision Exp	0.0268	$2.15	112.2M	397k	7h 43m	389	3
13	Kimi K3	0.0266	$26.15	60.2M	321k	19h 29m	303	13
14	GLM-5.3	0.0264	$29.01	75.1M	335k	17h 41m	311	6
15	Kimi K3	0.0251	$50.26	120.4M	460k	19h 45m	410	7
16	Kimi K3	0.0250	$34.53	93.2M	354k	19h 44m	395	8
17	GLM-5.3	0.0243	$29.39	96.9M	323k	19h 43m	356	1
18	Gemini 3.7 Flash	0.0240	$13.44	120.0M	296k	7h 42m	465	7
19	Gemini 3.7 Flash	0.0232	$21.26	112.5M	588k	9h 45m	340	20
20	DeepSeek V4 Flash Vision Exp	0.0228	$5.13	304.7M	583k	13h 30m	712	7
21	GPT-5.6	0.0224	$49.42	77.2M	169k	19h 53m	422	11
22	Grok 4.6	0.0221	$127.10	131.5M	617k	19h 58m	617	30
23	Grok 4.6	0.0192	$168.00	174.0M	498k	19h 58m	768	68
24	Gemini 3.7 Flash	0.0189	$26.85	83.8M	686k	9h 43m	267	16
25	Grok 4.6	0.0171	$114.81	118.3M	558k	19h 40m	599	47
26	Qwen3.8-Max	0.0171	$21.55	69.7M	606k	19h 26m	446	15
27	GPT-5.6	0.0163	$47.54	55.3M	185k	19h 43m	291	10
28	GPT-5.6	0.0159	$41.60	61.4M	164k	18h 50m	368	11
29	Qwen3.8-Max	0.0155	$18.80	60.7M	524k	19h 36m	400	4
30	Muse Spark 1.2	0.0126	$49.13	229.2M	341k	8h 52m	575	6
31	Grok 4.6	0.0126	$162.44	156.4M	577k	19h 37m	634	61
32	Kimi K3	0.0120	$62.95	172.7M	576k	20h 0m	473	15
33	Qwen3.8-Max	0.0118	$16.99	53.1M	556k	19h 0m	415	2
34	Gemini 3.7 Flash	0.0110	$6.63	19.2M	229k	5h 4m	166	5
35	Grok 4.6	0.0106	$206.03	201.8M	768k	19h 59m	845	63
36	Muse Spark 1.2	0.0104	$27.73	109.9M	451k	7h 13m	358	10
37	Qwen3.8-Max	0.0098	$20.15	54.4M	994k	19h 8m	321	6
38	Muse Spark 1.2	0.0097	$69.69	161.5M	457k	8h 41m	441	5
39	Gemini 3.7 Flash	0.0097	$3.47	13.3M	97k	5h 22m	153	4
40	DeepSeek V4 Flash Vision Exp	0.0094	$6.17	373.9M	627k	11h 53m	798	9
41	Qwen3.8-Max	0.0089	$23.54	77.9M	588k	18h 20m	403	2
42	Muse Spark 1.2	0.0087	$13.25	53.2M	190k	6h 44m	295	3
43	DeepSeek V4 Flash Vision Exp	0.0081	$3.17	180.3M	434k	7h 52m	554	2
44	Inkling	0.0073	$7.83	8.2M	21k	1h 33m	141	2
45	Muse Spark 1.2	0.0065	$21.18	65.3M	269k	5h 22m	313	3
46	Inkling	0.0061	$5.43	5.5M	41k	1h 2m	147	2
47	Inkling	0.0061	$13.67	14.2M	40k	2h 24m	225	2
48	DeepSeek V4 Flash Vision Exp	0.0056	$3.93	222.7M	553k	14h 38m	530	7
49	Inkling	0.0039	$4.62	4.8M	24k	1h 6m	101	1
50	Inkling	0.0030	$5.70	5.9M	31k	2h 11m	145	1

Astronomy Toolkit

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.8953	$64.06	109.1M	391k	4h 49m	305	17
2	Claude Fable 5.1	0.8935	$57.22	97.5M	350k	6h 25m	298	24
3	Claude Fable 5.1	0.8000	$36.76	59.7M	285k	3h 53m	220	11
4	Claude Fable 5.1	0.7999	$32.01	51.0M	267k	3h 57m	202	12
5	Claude Fable 5.1	0.7955	$42.10	68.8M	317k	4h 11m	232	12
6	Gemini 3.7 Flash	0.7294	$8.39	49.1M	408k	2h 12m	218	1
7	GLM-5.3	0.6985	$72.52	246.9M	707k	19h 16m	599	6
8	Kimi K3	0.6250	$60.98	152.4M	478k	16h 31m	414	12
9	Grok 4.6	0.5299	$15.57	15.6M	323k	1h 52m	106	4
10	GPT-5.6	0.4849	$43.38	52.6M	186k	3h 16m	263	5
11	GLM-5.3	0.4028	$57.26	195.8M	552k	12h 7m	426	8
12	GLM-5.3	0.4017	$72.89	226.5M	514k	17h 29m	476	6
13	Gemini 3.7 Flash	0.4014	$3.09	23.0M	225k	38m	153	3
14	Grok 4.6	0.3993	$48.11	46.2M	395k	3h 31m	208	6
15	Kimi K3	0.3983	$90.41	238.0M	505k	20h 1m	587	20
16	Kimi K3	0.3959	$20.08	46.3M	279k	6h 2m	205	5
17	GLM-5.3	0.3930	$77.97	220.3M	559k	15h 22m	481	2
18	Kimi K3	0.3804	$70.49	167.8M	452k	19h 35m	468	8
19	Kimi K3	0.2995	$53.30	137.6M	418k	11h 54m	430	7
20	GLM-5.3	0.2893	$64.65	196.5M	549k	19h 1m	419	8
21	Grok 4.6	0.2000	$34.68	34.5M	354k	2h 42m	175	8
22	Grok 4.6	0.1971	$39.66	35.3M	400k	3h 43m	165	11
23	GPT-5.6	0.1823	$39.28	48.9M	195k	2h 7m	255	4
24	Muse Spark 1.2	0.1013	$11.07	59.7M	393k	2h 7m	259	1
25	Muse Spark 1.2	0.1000	$12.16	66.0M	402k	2h 34m	231	4
26	Muse Spark 1.2	0.0563	$26.33	90.8M	427k	5h 8m	297	2
27	Qwen3.8-Max	0.0000	$42.82	145.6M	925k	17h 28m	524	2
28	Inkling	0.0000	$4.59	4.7M	24k	22m	105	2
29	DeepSeek V4 Flash Vision Exp	0.0000	$4.02	226.3M	587k	18h 16m	503	5
30	Inkling	0.0000	$5.98	6.1M	33k	29m	144	1
31	DeepSeek V4 Flash Vision Exp	0.0000	$4.14	229.6M	637k	20h 0m	483	1
32	GPT-5.6	0.0000	$29.74	37.3M	170k	2h 5m	211	8
33	Qwen3.8-Max	0.0000	$48.24	157.1M	1.3M	19h 42m	518	2
34	Gemini 3.7 Flash	0.0000	$5.44	38.0M	299k	1h 47m	197	3
35	Inkling	0.0000	$2.97	2.9M	45k	43m	109	2
36	Muse Spark 1.2	0.0000	$53.10	223.4M	907k	9h 16m	535	3
37	DeepSeek V4 Flash Vision Exp	0.0000	$3.11	165.3M	552k	9h 15m	409	6
38	Muse Spark 1.2	0.0000	$11.05	58.0M	328k	3h 11m	241	2
39	Qwen3.8-Max	0.0000	$42.25	133.5M	1.3M	19h 46m	412	2
40	GPT-5.6	0.0000	$23.28	33.7M	166k	2h 33m	207	5
41	DeepSeek V4 Flash Vision Exp	0.0000	$3.44	189.2M	548k	10h 29m	453	4
42	GPT-5.6	0.0000	$47.11	54.0M	192k	2h 34m	262	4
43	Inkling	0.0000	$6.49	6.6M	47k	1h 53m	142	1
44	Qwen3.8-Max	0.0000	$56.35	191.3M	1.2M	19h 55m	589	1
45	Grok 4.6	0.0000	$23.09	23.9M	317k	4h 4m	152	3
46	Gemini 3.7 Flash	0.0000	$3.91	24.5M	212k	1h 28m	155	2
47	Gemini 3.7 Flash	0.0000	$3.82	23.6M	220k	46m	162	2
48	Inkling	0.0000	$6.06	6.2M	36k	1h 0m	118	2
49	Qwen3.8-Max	0.0000	$28.32	86.1M	1.0M	14h 43m	386	3
50	DeepSeek V4 Flash Vision Exp	0.0000	$3.67	205.7M	547k	18h 16m	510	3

Cranelift Codegen Optimization

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.3484	$166.03	303.2M	604k	16h 29m	825	13
2	Claude Fable 5.1	0.3356	$233.87	425.4M	876k	16h 37m	1021	18
3	Claude Fable 5.1	0.2852	$199.91	365.0M	723k	15h 17m	974	15
4	Claude Fable 5.1	0.2009	$170.71	311.4M	625k	17h 11m	813	17
5	Gemini 3.7 Flash	0.0802	$35.86	231.6M	242k	6h 15m	574	20
6	Kimi K3	0.0603	$148.59	441.7M	786k	16h 22m	1028	15
7	Kimi K3	0.0478	$126.89	362.2M	801k	17h 53m	936	9
8	DeepSeek V4 Flash Vision Exp	0.0237	$8.27	504.6M	723k	9h 18m	1260	11
9	Gemini 3.7 Flash	0.0189	$20.57	157.5M	204k	2h 49m	432	17
10	GPT-5.6	0.0176	$225.59	218.6M	216k	6h 56m	506	6
11	GLM-5.3	0.0162	$93.99	346.3M	474k	11h 28m	766	2
12	Kimi K3	0.0132	$145.36	415.9M	1.0M	20h 3m	1079	7
13	GLM-5.3	0.0129	$109.37	311.0M	484k	17h 22m	770	9
14	Qwen3.8-Max	0.0117	$41.08	108.0M	2.2M	18h 38m	484	6
15	Gemini 3.7 Flash	0.0110	$23.00	164.8M	188k	4h 11m	495	19
16	DeepSeek V4 Flash Vision Exp	0.0107	$5.01	303.8M	439k	7h 50m	723	8
17	GLM-5.3	0.0102	$60.68	223.7M	392k	7h 14m	615	4
18	DeepSeek V4 Flash Vision Exp	0.0030	$5.08	311.7M	419k	8h 3m	781	6
19	Grok 4.6	0.0000	$113.60	103.4M	918k	9h 35m	426	6
20	Inkling	0.0000	$8.38	8.8M	12k	55m	94	1
21	GPT-5.6	0.0000	$297.21	276.6M	238k	7h 17m	549	5
22	Muse Spark 1.2	0.0000	$33.79	127.0M	205k	10h 23m	356	5
23	Muse Spark 1.2	0.0000	$18.72	115.3M	194k	3h 23m	350	4
24	Qwen3.8-Max	0.0000	$34.88	81.5M	2.3M	20h 0m	413	1
25	Kimi K3	0.0000	$136.80	389.4M	1.1M	20h 0m	1006	19
26	Gemini 3.7 Flash	0.0000	$30.61	210.1M	211k	5h 15m	593	21
27	GPT-5.6	0.0000	$234.61	222.5M	240k	7h 38m	461	5
28	Grok 4.6	0.0000	$240.25	234.7M	1.3M	16h 47m	895	24
29	Inkling	0.0000	$13.75	14.4M	22k	1h 40m	135	2
30	Kimi K3	0.0000	$136.72	403.0M	752k	19h 8m	1147	11
31	Inkling	0.0000	$11.79	12.4M	15k	1h 2m	121	1
32	Muse Spark 1.2	0.0000	$87.58	303.3M	711k	10h 13m	562	5
33	GPT-5.6	0.0000	$212.19	205.4M	208k	5h 52m	461	5
34	Inkling	0.0000	$8.24	8.6M	21k	30m	104	2
35	DeepSeek V4 Flash Vision Exp	0.0000	$3.68	220.8M	344k	8h 44m	670	8
36	Inkling	0.0000	$13.82	14.5M	21k	59m	137	2
37	Grok 4.6	0.0000	$138.74	131.9M	1.2M	8h 47m	587	6
38	Claude Fable 5.1	0.0000	$232.11	427.5M	765k	16h 0m	1023	35
39	Grok 4.6	0.0000	$138.58	127.2M	927k	7h 29m	488	9
40	Muse Spark 1.2	0.0000	$76.77	223.4M	655k	10h 10m	450	7
41	Muse Spark 1.2	0.0000	$49.18	242.6M	532k	8h 28m	495	9
42	Qwen3.8-Max	0.0000	$43.78	119.5M	2.2M	18h 46m	496	2
43	Qwen3.8-Max	0.0000	$50.74	159.1M	1.7M	16h 16m	624	4
44	GLM-5.3	0.0000	$110.08	353.7M	515k	18h 43m	761	18
45	Gemini 3.7 Flash	0.0000	$47.46	341.2M	378k	8h 44m	828	22
46	Grok 4.6	0.0000	$108.30	103.9M	740k	6h 37m	447	6
47	DeepSeek V4 Flash Vision Exp	0.0000	$4.87	296.6M	422k	6h 43m	730	3
48	GLM-5.3	0.0000	$69.79	258.7M	392k	7h 49m	703	7
49	GPT-5.6	0.0000	$355.12	344.3M	254k	10h 32m	627	3
50	Qwen3.8-Max	0.0000	$42.54	115.4M	2.2M	20h 0m	491	1

Dart Style in Haskell

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9491	$569.86	1069.9M	1.4M	17h 10m	2285	28
2	Claude Fable 5.1	0.9284	$552.06	1031.1M	1.5M	18h 56m	2091	46
3	Claude Fable 5.1	0.9275	$456.14	848.4M	1.3M	18h 57m	2026	39
4	Claude Fable 5.1	0.8919	$415.43	774.8M	1.2M	19h 19m	1796	47
5	Claude Fable 5.1	0.8712	$313.43	577.0M	1.0M	18h 15m	1283	12
6	GPT-5.6	0.7949	$998.14	972.7M	983k	19h 18m	2146	16
7	GPT-5.6	0.6779	$433.00	423.6M	391k	6h 1m	844	4
8	Gemini 3.7 Flash	0.6517	$199.28	2123.1M	2.6M	19h 55m	4917	62
9	GPT-5.6	0.6344	$89.80	91.5M	201k	1h 55m	362	5
10	Muse Spark 1.2	0.6241	$2.29	11.0M	77k	1h 6m	128	3
11	GPT-5.6	0.6068	$48.47	52.9M	204k	1h 35m	274	3
12	Grok 4.6	0.5864	$513.86	512.7M	2.7M	20h 0m	2054	59
13	Kimi K3	0.5650	$209.81	607.6M	1.3M	20h 1m	1454	19
14	Kimi K3	0.5395	$190.44	541.2M	1.2M	19h 50m	1336	3
15	Gemini 3.7 Flash	0.5376	$121.24	1110.9M	1.5M	19h 27m	2524	37
16	GLM-5.3	0.5299	$165.74	599.8M	1.1M	19h 6m	1398	51
17	Kimi K3	0.4874	$196.59	547.1M	1.3M	19h 46m	1241	8
18	Qwen3.8-Max	0.4828	$83.97	303.2M	1.2M	16h 38m	893	2
19	GLM-5.3	0.4696	$158.77	595.9M	645k	19h 33m	1177	12
20	GLM-5.3	0.4653	$144.84	516.1M	611k	19h 39m	1066	22
21	GLM-5.3	0.4595	$150.71	560.0M	706k	19h 32m	1263	35
22	GPT-5.6	0.4547	$71.79	84.5M	237k	2h 25m	406	7
23	Gemini 3.7 Flash	0.4365	$75.56	629.3M	934k	19h 55m	1331	3
24	Kimi K3	0.4305	$123.27	338.1M	738k	19h 52m	670	1
25	Gemini 3.7 Flash	0.4242	$33.92	324.0M	891k	3h 25m	780	8
26	DeepSeek V4 Flash Vision Exp	0.4014	$18.12	1122.6M	1.6M	19h 51m	2602	23
27	GLM-5.3	0.3788	$98.82	359.7M	600k	20h 0m	738	1
28	Qwen3.8-Max	0.3212	$156.83	579.3M	1.7M	20h 1m	1506	1
29	Gemini 3.7 Flash	0.3137	$25.24	282.5M	578k	1h 28m	590	2
30	Grok 4.6	0.3015	$389.00	386.7M	2.2M	19h 53m	1645	42
31	Qwen3.8-Max	0.2726	$83.47	304.6M	1.0M	17h 28m	843	2
32	Qwen3.8-Max	0.2674	$50.86	177.6M	941k	18h 25m	662	12
33	DeepSeek V4 Flash Vision Exp	0.2342	$7.03	436.2M	602k	8h 13m	914	3
34	DeepSeek V4 Flash Vision Exp	0.2234	$21.23	1304.8M	2.0M	15h 38m	2836	7
35	Grok 4.6	0.1785	$547.31	552.2M	2.4M	19h 52m	2177	74
36	DeepSeek V4 Flash Vision Exp	0.1638	$15.31	956.5M	1.3M	19h 4m	2023	3
37	Muse Spark 1.2	0.1043	$2.53	9.5M	123k	1h 21m	101	1
38	Muse Spark 1.2	0.1040	$2.18	10.0M	104k	1h 4m	113	2
39	Muse Spark 1.2	0.1040	$3.41	16.0M	109k	59m	110	2
40	Qwen3.8-Max	0.0853	$104.77	373.0M	1.7M	19h 28m	957	1
41	DeepSeek V4 Flash Vision Exp	0.0766	$15.69	953.5M	1.5M	16h 48m	2199	5
42	Muse Spark 1.2	0.0377	$5.55	30.5M	146k	1h 16m	174	2
43	Inkling	0.0000	$2.95	3.1M	10k	11m	87	2
44	Inkling	0.0000	$20.49	21.4M	45k	32m	241	1
45	Grok 4.6	0.0000	$485.05	493.3M	2.2M	19h 52m	1992	109
46	Grok 4.6	0.0000	$402.09	394.7M	2.1M	17h 23m	1524	78
47	Inkling	0.0000	$19.88	20.7M	49k	1h 2m	297	1
48	Inkling	0.0000	$7.80	8.1M	20k	1h 3m	140	1
49	Inkling	0.0000	$4.62	4.8M	11k	1h 10m	101	2
50	Kimi K3	0.0000	$86.57	238.4M	589k	20h 1m	593	1

FFmpeg libswscale Optimization

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.8080	$283.04	487.0M	1.6M	13h 9m	1070	46
2	Claude Fable 5.1	0.5509	$202.06	343.4M	1.3M	9h 13m	819	28
3	Claude Fable 5.1	0.5106	$148.93	254.0M	906k	6h 11m	641	18
4	GLM-5.3	0.1752	$226.40	818.4M	2.0M	19h 33m	1884	18
5	GLM-5.3	0.1410	$148.35	524.2M	1.7M	16h 26m	1303	8
6	GLM-5.3	0.1369	$139.76	505.8M	1.2M	11h 14m	1097	9
7	Claude Fable 5.1	0.1368	$219.11	385.2M	1.1M	10h 27m	835	22
8	Kimi K3	0.1260	$127.79	342.6M	1.2M	13h 59m	827	13
9	Kimi K3	0.1252	$211.22	606.6M	1.4M	19h 43m	1382	3
10	GLM-5.3	0.1231	$175.55	619.5M	1.7M	16h 47m	1509	8
11	Kimi K3	0.0799	$127.97	356.2M	1.1M	15h 4m	914	11
12	GLM-5.3	0.0783	$177.95	625.3M	1.7M	17h 1m	1420	8
13	DeepSeek V4 Flash Vision Exp	0.0668	$8.52	493.3M	1.0M	6h 44m	1095	8
14	Kimi K3	0.0639	$138.03	382.3M	1.1M	17h 32m	902	2
15	DeepSeek V4 Flash Vision Exp	0.0636	$9.11	528.7M	1.1M	6h 10m	1172	5
16	DeepSeek V4 Flash Vision Exp	0.0602	$13.77	800.3M	1.6M	9h 14m	1831	19
17	DeepSeek V4 Flash Vision Exp	0.0483	$9.39	554.1M	1.0M	5h 54m	1197	16
18	Qwen3.8-Max	0.0373	$86.44	286.2M	2.3M	17h 43m	706	6
19	Grok 4.6	0.0360	$419.25	404.2M	2.4M	19h 56m	1517	114
20	Qwen3.8-Max	0.0181	$64.05	193.9M	2.4M	18h 48m	555	1
21	Gemini 3.7 Flash	0.0125	$20.83	184.0M	617k	2h 35m	524	2
22	Qwen3.8-Max	0.0053	$46.03	124.6M	2.3M	19h 58m	424	1
23	Gemini 3.7 Flash	0.0032	$19.94	167.8M	795k	2h 4m	496	7
24	Grok 4.6	0.0000	$1121.42	1126.0M	2.4M	18h 11m	3693	434
25	Muse Spark 1.2	0.0000	$15.42	90.0M	327k	41m	286	2
26	Grok 4.6	0.0000	$68.40	59.6M	603k	3h 50m	229	4
27	Muse Spark 1.2	0.0000	$16.97	85.6M	384k	1h 38m	227	4
28	Claude Fable 5.1	0.0000	$217.24	363.8M	1.5M	10h 22m	893	27
29	Qwen3.8-Max	0.0000	$54.69	158.2M	2.4M	18h 58m	475	1
30	Grok 4.6	0.0000	$132.56	129.2M	778k	5h 28m	562	27
31	Inkling	0.0000	$11.24	11.7M	27k	1h 20m	165	2
32	Inkling	0.0000	$15.45	16.1M	35k	22m	188	2
33	Qwen3.8-Max	0.0000	$30.12	66.3M	2.1M	19h 55m	298	1
34	Inkling	0.0000	$39.66	41.4M	77k	41m	354	2
35	Gemini 3.7 Flash	0.0000	$25.27	228.2M	1.1M	2h 14m	655	5
36	GPT-5.6	0.0000	$66.82	75.1M	179k	1h 59m	318	8
37	Inkling	0.0000	$15.51	16.2M	36k	32m	190	1
38	Muse Spark 1.2	0.0000	$11.90	64.7M	340k	1h 14m	223	2
39	GPT-5.6	0.0000	$78.43	84.8M	190k	2h 4m	348	5
40	Grok 4.6	0.0000	$380.75	368.6M	1.8M	11h 52m	1370	67
41	Muse Spark 1.2	0.0000	$35.70	203.8M	978k	3h 29m	468	8
42	GPT-5.6	0.0000	$123.31	124.9M	233k	2h 12m	422	3
43	Kimi K3	0.0000	$171.14	500.1M	1.1M	19h 41m	1175	47
44	GPT-5.6	0.0000	$76.67	86.4M	186k	3h 5m	369	10
45	DeepSeek V4 Flash Vision Exp	0.0000	$12.34	718.0M	1.4M	8h 32m	1678	16
46	GPT-5.6	0.0000	$117.96	120.8M	233k	3h 42m	405	8
47	Inkling	0.0000	$11.17	11.6M	27k	16m	167	1
48	Gemini 3.7 Flash	0.0000	$16.99	131.7M	937k	2h 45m	448	6
49	Muse Spark 1.2	0.0000	$27.98	159.1M	568k	1h 40m	345	2
50	Gemini 3.7 Flash	0.0000	$44.74	251.9M	922k	8h 1m	758	6

Crash-Proof Flash Filesystem

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9970	$49.34	77.1M	443k	2h 57m	208	6
2	Claude Fable 5.1	0.9927	$43.72	70.9M	340k	3h 17m	229	4
3	Claude Fable 5.1	0.9927	$61.85	100.4M	479k	3h 5m	243	10
4	Claude Fable 5.1	0.8898	$81.53	137.5M	526k	5h 0m	326	10
5	Grok 4.6	0.0610	$858.44	859.7M	2.4M	19h 46m	2990	501
6	Kimi K3	0.0431	$197.20	550.4M	1.5M	20h 1m	1197	1
7	Grok 4.6	0.0241	$445.27	424.9M	2.8M	19h 59m	1572	200
8	GPT-5.6	0.0225	$66.23	68.5M	180k	1h 47m	274	8
9	GPT-5.6	0.0225	$81.19	79.4M	199k	1h 55m	291	7
10	GPT-5.6	0.0224	$73.40	72.5M	171k	1h 41m	273	8
11	Gemini 3.7 Flash	0.0217	$93.80	813.5M	2.3M	19h 56m	2141	1
12	Claude Fable 5.1	0.0173	$56.44	90.9M	453k	4h 18m	228	8
13	GPT-5.6	0.0173	$159.04	153.2M	259k	4h 31m	440	16
14	GPT-5.6	0.0173	$92.72	87.0M	212k	1h 59m	268	7
15	GLM-5.3	0.0173	$171.41	627.5M	1.2M	19h 32m	1385	14
16	GLM-5.3	0.0173	$236.98	874.4M	1.4M	19h 51m	1908	31
17	Grok 4.6	0.0117	$262.00	255.9M	1.7M	13h 11m	1039	20
18	Kimi K3	0.0112	$202.76	569.3M	1.5M	19h 50m	1237	2
19	Gemini 3.7 Flash	0.0112	$89.95	756.9M	1.7M	20h 0m	1907	1
20	GLM-5.3	0.0111	$148.21	526.6M	1.0M	18h 49m	1175	16
21	DeepSeek V4 Flash Vision Exp	0.0110	$29.12	1761.2M	2.9M	19h 19m	3911	24
22	DeepSeek V4 Flash Vision Exp	0.0053	$22.61	1343.5M	2.5M	19h 20m	2867	26
23	GLM-5.3	0.0049	$108.31	386.0M	1.3M	16h 54m	851	5
24	DeepSeek V4 Flash Vision Exp	0.0049	$20.65	1205.0M	2.5M	19h 56m	2498	19
25	Muse Spark 1.2	0.0049	$36.55	207.9M	989k	3h 16m	456	11
26	Kimi K3	0.0049	$158.40	411.0M	1.7M	20h 0m	1010	1
27	DeepSeek V4 Flash Vision Exp	0.0049	$17.94	1097.8M	1.6M	18h 58m	2348	14
28	Gemini 3.7 Flash	0.0049	$62.15	472.1M	1.5M	20h 0m	1139	1
29	DeepSeek V4 Flash Vision Exp	0.0049	$20.78	1251.0M	2.1M	19h 4m	2804	15
30	Kimi K3	0.0049	$190.49	508.0M	1.5M	19h 54m	1111	9
31	Muse Spark 1.2	0.0047	$53.94	336.5M	554k	4h 31m	660	7
32	Grok 4.6	0.0046	$267.41	252.9M	1.7M	10h 33m	927	132
33	GLM-5.3	0.0039	$154.89	566.6M	1.0M	19h 45m	1350	18
34	Muse Spark 1.2	0.0038	$13.60	58.8M	346k	1h 50m	189	3
35	Muse Spark 1.2	0.0033	$35.75	222.8M	379k	2h 39m	489	7
36	Grok 4.6	0.0031	$181.59	175.3M	1.4M	8h 20m	693	24
37	Gemini 3.7 Flash	0.0024	$124.70	1220.5M	2.9M	20h 0m	2795	1
38	Qwen3.8-Max	0.0013	$12.18	16.0M	1.3M	19h 33m	130	1
39	Gemini 3.7 Flash	0.0010	$64.73	520.0M	1.4M	20h 0m	1356	1
40	Kimi K3	0.0008	$195.88	529.5M	1.7M	20h 0m	1266	2
41	Qwen3.8-Max	0.0001	$13.81	19.3M	1.5M	19h 35m	164	3
42	Qwen3.8-Max	0.0000	$14.39	19.8M	1.5M	20h 1m	160	1
43	Inkling	0.0000	$14.16	14.7M	37k	1h 30m	184	2
44	Qwen3.8-Max	0.0000	$9.18	10.7M	1.0M	19h 50m	100	1
45	Inkling	0.0000	$9.30	9.7M	22k	11m	123	2
46	Inkling	0.0000	$12.08	12.6M	23k	14m	161	2
47	Qwen3.8-Max	0.0000	$13.72	18.7M	1.5M	18h 42m	146	3
48	Muse Spark 1.2	0.0000	$18.50	97.7M	207k	1h 10m	269	4
49	Inkling	0.0000	$16.45	17.2M	21k	16m	173	2
50	Inkling	0.0000	$13.17	13.7M	28k	1h 27m	186	2

FrogsGame Post-Training

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9740	$14.41	21.9M	144k	17h 0m	153	5
2	Claude Fable 5.1	0.8920	$12.71	19.5M	122k	16h 12m	146	4
3	GLM-5.3	0.8380	$21.95	47.4M	206k	19h 31m	233	2
4	Claude Fable 5.1	0.7800	$16.40	25.5M	151k	15h 15m	165	3
5	GLM-5.3	0.7220	$12.91	44.6M	176k	19h 17m	247	2
6	Claude Fable 5.1	0.6180	$15.24	23.4M	145k	17h 43m	162	6
7	Kimi K3	0.4620	$12.73	24.9M	159k	19h 34m	172	1
8	Claude Fable 5.1	0.4080	$13.04	19.3M	141k	15h 46m	135	4
9	GLM-5.3	0.3120	$16.40	57.5M	195k	19h 21m	314	2
10	Kimi K3	0.3000	$16.00	38.1M	240k	20h 0m	187	1
11	Grok 4.6	0.2460	$18.57	19.8M	283k	19h 59m	152	1
12	Grok 4.6	0.2420	$9.68	12.9M	262k	19h 22m	132	2
13	Qwen3.8-Max	0.2160	$11.97	38.7M	277k	19h 34m	297	3
14	Kimi K3	0.1900	$13.63	32.0M	177k	20h 0m	198	1
15	GLM-5.3	0.1640	$20.82	41.0M	222k	19h 4m	214	3
16	GLM-5.3	0.1580	$16.59	58.5M	233k	19h 38m	265	1
17	Grok 4.6	0.1280	$47.67	45.1M	287k	18h 35m	259	5
18	DeepSeek V4 Flash Vision Exp	0.1100	$1.27	65.0M	240k	20h 4m	306	1
19	Kimi K3	0.1040	$11.13	20.9M	147k	19h 47m	174	1
20	Qwen3.8-Max	0.0980	$10.69	32.2M	339k	19h 46m	272	1
21	GPT-5.6	0.0900	$19.46	28.7M	100k	17h 8m	206	2
22	Qwen3.8-Max	0.0880	$5.85	16.5M	229k	19h 7m	195	1
23	DeepSeek V4 Flash Vision Exp	0.0860	$1.47	79.6M	227k	19h 17m	390	2
24	Kimi K3	0.0660	$8.10	17.2M	123k	19h 18m	153	1
25	Grok 4.6	0.0640	$49.02	43.5M	397k	19h 5m	221	4
26	Gemini 3.7 Flash	0.0620	$5.03	14.3M	76k	5h 27m	128	1
27	GPT-5.6	0.0540	$20.87	32.1M	115k	12h 56m	241	3
28	Gemini 3.7 Flash	0.0420	$30.72	72.9M	130k	16h 34m	301	1
29	GPT-5.6	0.0380	$22.11	34.6M	103k	18h 7m	223	2
30	GPT-5.6	0.0360	$61.31	64.5M	145k	15h 2m	277	2
31	Qwen3.8-Max	0.0360	$35.74	120.9M	650k	19h 55m	546	2
32	Grok 4.6	0.0340	$48.29	46.2M	363k	16h 43m	237	5
33	Gemini 3.7 Flash	0.0320	$45.33	138.6M	173k	19h 58m	396	1
34	Qwen3.8-Max	0.0300	$11.98	37.6M	292k	18h 44m	323	4
35	Muse Spark 1.2	0.0300	$18.36	45.0M	166k	4h 11m	176	5
36	Gemini 3.7 Flash	0.0220	$3.80	24.6M	101k	2h 41m	142	2
37	Muse Spark 1.2	0.0220	$21.74	48.9M	286k	5h 11m	239	11
38	Gemini 3.7 Flash	0.0200	$37.07	96.2M	140k	17h 26m	321	1
39	DeepSeek V4 Flash Vision Exp	0.0160	$1.33	68.9M	247k	19h 56m	345	2
40	DeepSeek V4 Flash Vision Exp	0.0160	$1.13	55.9M	221k	19h 37m	264	2
41	GPT-5.6	0.0120	$91.18	51.7M	86k	17h 41m	228	4
42	Muse Spark 1.2	0.0100	$18.40	105.2M	433k	4h 43m	251	5
43	Inkling	0.0080	$2.60	2.6M	22k	28m	84	2
44	Muse Spark 1.2	0.0060	$13.73	65.6M	312k	5h 25m	183	9
45	Inkling	0.0000	$8.37	8.7M	31k	38m	169	2
46	DeepSeek V4 Flash Vision Exp	0.0000	$1.61	87.5M	249k	20h 2m	395	1
47	Inkling	0.0000	$3.54	3.7M	16k	24m	97	2
48	Inkling	0.0000	$1.63	1.6M	18k	24m	71	2
49	Muse Spark 1.2	0.0000	$196.24	1245.9M	587k	4h 6m	2389	8
50	Inkling	0.0000	$2.52	2.6M	17k	28m	103	2

Stepper Music Sequencer GBA

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9662	$443.40	798.0M	1.8M	18h 14m	1802	27
2	Claude Fable 5.1	0.9569	$440.73	799.7M	1.7M	15h 10m	1825	14
3	Claude Fable 5.1	0.9473	$365.73	654.8M	1.6M	15h 4m	1521	20
4	Claude Fable 5.1	0.9459	$457.39	831.4M	1.7M	14h 37m	1849	32
5	Claude Fable 5.1	0.9053	$549.23	1015.6M	1.7M	16h 2m	2234	18
6	GPT-5.6	0.7701	$944.54	934.7M	1.1M	15h 1m	2411	21
7	GPT-5.6	0.7406	$409.56	413.9M	542k	8h 34m	1251	15
8	GPT-5.6	0.7219	$253.31	245.2M	326k	3h 19m	679	6
9	GLM-5.3	0.6108	$356.80	1301.6M	3.0M	19h 1m	2671	19
10	GPT-5.6	0.6048	$566.59	550.5M	752k	14h 19m	1513	11
11	Grok 4.6	0.5671	$305.84	301.9M	2.2M	11h 0m	1227	45
12	Kimi K3	0.5098	$230.68	643.2M	1.9M	20h 0m	1545	3
13	GPT-5.6	0.4981	$490.17	481.7M	531k	5h 24m	1274	32
14	GLM-5.3	0.4715	$281.50	1008.4M	2.3M	16h 6m	2264	16
15	GLM-5.3	0.4046	$276.19	1010.3M	1.8M	15h 31m	2070	21
16	Kimi K3	0.3706	$227.60	629.2M	1.9M	19h 47m	1504	2
17	GLM-5.3	0.3427	$275.26	1018.0M	1.7M	17h 39m	2090	39
18	Grok 4.6	0.3420	$868.26	872.5M	3.4M	16h 30m	3114	373
19	Kimi K3	0.3170	$227.44	622.4M	1.9M	19h 52m	1439	1
20	Grok 4.6	0.2607	$223.96	223.8M	1.5M	7h 20m	882	18
21	Grok 4.6	0.2416	$205.23	195.4M	1.9M	7h 34m	792	28
22	Gemini 3.7 Flash	0.2415	$49.29	499.9M	1.0M	4h 48m	1302	2
23	Grok 4.6	0.2197	$129.50	120.8M	2.0M	7h 52m	540	19
24	Gemini 3.7 Flash	0.1895	$40.63	436.9M	982k	3h 1m	1134	2
25	Gemini 3.7 Flash	0.1724	$88.83	943.5M	2.1M	6h 27m	2449	1
26	Gemini 3.7 Flash	0.1698	$29.14	318.3M	646k	2h 19m	856	2
27	GLM-5.3	0.1440	$63.57	232.5M	498k	3h 14m	487	6
28	DeepSeek V4 Flash Vision Exp	0.1348	$19.48	1192.0M	1.8M	11h 51m	2670	7
29	Kimi K3	0.1336	$196.38	508.8M	1.8M	19h 51m	1209	4
30	Gemini 3.7 Flash	0.1224	$55.17	592.3M	1.3M	5h 23m	1507	5
31	DeepSeek V4 Flash Vision Exp	0.1138	$28.55	1772.4M	2.4M	15h 1m	3833	23
32	Kimi K3	0.0909	$184.74	495.2M	1.5M	19h 58m	1154	2
33	DeepSeek V4 Flash Vision Exp	0.0534	$18.46	1143.5M	1.6M	13h 42m	2700	16
34	DeepSeek V4 Flash Vision Exp	0.0419	$18.93	1164.8M	1.7M	11h 44m	2764	29
35	DeepSeek V4 Flash Vision Exp	0.0319	$17.22	1033.6M	1.8M	11h 35m	2306	13
36	Qwen3.8-Max	0.0240	$114.04	338.0M	2.8M	19h 40m	884	6
37	Qwen3.8-Max	0.0224	$119.80	401.3M	2.9M	20h 1m	934	—
38	Qwen3.8-Max	0.0183	$113.87	373.6M	3.0M	19h 42m	990	—
39	Muse Spark 1.2	0.0175	$37.47	155.3M	1.3M	5h 29m	460	7
40	Qwen3.8-Max	0.0159	$129.76	393.1M	2.8M	19h 53m	1102	—
41	Qwen3.8-Max	0.0002	$139.67	464.7M	2.7M	19h 56m	1203	—
42	Inkling	0.0000	$3.82	3.9M	21k	15m	103	2
43	Inkling	0.0000	$6.62	6.9M	28k	22m	168	1
44	Inkling	0.0000	$8.48	8.8M	35k	24m	171	2
45	Muse Spark 1.2	0.0000	$44.79	140.1M	936k	4h 21m	543	4
46	Inkling	0.0000	$10.64	11.1M	32k	19m	193	1
47	Muse Spark 1.2	0.0000†	$4.68	18.3M	123k	58m	156	2
48	Muse Spark 1.2	0.0000†	$53.19	95.9M	776k	4h 50m	351	2
49	Inkling	0.0000	$5.72	5.9M	31k	22m	140	2
50	Muse Spark 1.2	0.0000†	$10.59	43.3M	223k	2h 38m	259	3

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Git to Zig

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.5599	$663.30	1199.7M	2.6M	18h 57m	2674	17
2	Claude Fable 5.1	0.5146	$566.08	1019.0M	2.3M	19h 11m	2360	13
3	Claude Fable 5.1	0.5049	$432.88	762.8M	2.1M	18h 58m	1732	13
4	Claude Fable 5.1	0.4886	$363.29	631.2M	2.0M	18h 58m	1562	11
5	Claude Fable 5.1	0.4844	$390.64	686.7M	2.0M	19h 13m	1611	26
6	GPT-5.6	0.3440	$1238.08	1225.0M	944k	19h 8m	2627	21
7	GPT-5.6	0.2804	$1082.72	1080.1M	850k	19h 36m	2314	42
8	GPT-5.6	0.2801	$1175.97	1171.4M	884k	19h 37m	2602	29
9	GPT-5.6	0.2369	$955.56	945.7M	722k	19h 4m	1982	13
10	GPT-5.6	0.1874	$770.31	763.5M	629k	19h 0m	1668	6
11	Gemini 3.7 Flash	0.1808	$157.01	1517.8M	2.4M	19h 43m	3256	1
12	Kimi K3	0.1768	$251.00	752.4M	1.2M	20h 2m	1619	1
13	Kimi K3	0.1733	$200.20	602.7M	965k	19h 43m	1483	13
14	Grok 4.6	0.1693	$652.70	657.8M	2.2M	19h 59m	2502	122
15	Grok 4.6	0.1618	$614.44	610.4M	2.6M	18h 0m	2324	73
16	Kimi K3	0.1616	$254.18	761.8M	1.1M	19h 47m	1668	9
17	Kimi K3	0.1574	$259.84	779.9M	1.1M	19h 53m	1765	2
18	GLM-5.3	0.1556	$225.36	842.0M	964k	19h 45m	2012	17
19	GLM-5.3	0.1531	$195.54	692.6M	689k	19h 54m	1480	2
20	Grok 4.6	0.1516	$806.68	819.7M	2.2M	19h 59m	2981	186
21	Inkling	0.1510	$17.77	18.6M	32k	1h 29m	255	1
22	Gemini 3.7 Flash	0.1495	$79.59	838.0M	1.2M	20h 0m	1855	2
23	Kimi K3	0.1473	$222.64	660.4M	1.1M	19h 44m	1491	1
24	Gemini 3.7 Flash	0.1433	$101.57	1069.8M	1.5M	19h 58m	2372	1
25	GLM-5.3	0.1390	$212.77	708.2M	852k	19h 49m	1640	49
26	GLM-5.3	0.1378	$248.79	932.5M	906k	19h 55m	2219	21
27	Grok 4.6	0.1352	$482.88	483.0M	1.6M	11h 22m	1767	84
28	DeepSeek V4 Flash Vision Exp	0.1304	$28.18	1759.3M	2.2M	19h 24m	3723	15
29	Grok 4.6	0.1269	$408.61	405.7M	1.8M	19h 55m	1558	48
30	Gemini 3.7 Flash	0.1237	$71.55	746.3M	1.3M	19h 57m	1690	4
31	GLM-5.3	0.1219	$188.86	712.0M	621k	19h 43m	1318	16
32	DeepSeek V4 Flash Vision Exp	0.1213	$28.06	1756.1M	2.2M	19h 43m	3793	11
33	Qwen3.8-Max	0.1212	$184.24	696.5M	1.3M	19h 33m	1805	12
34	DeepSeek V4 Flash Vision Exp	0.1193	$26.16	1630.6M	2.1M	19h 37m	3584	8
35	DeepSeek V4 Flash Vision Exp	0.1079	$21.83	1364.0M	1.7M	17h 17m	2977	8
36	Qwen3.8-Max	0.1038	$196.74	742.9M	1.4M	19h 38m	1831	26
37	Qwen3.8-Max	0.1023	$180.44	685.7M	1.2M	19h 14m	1776	2
38	DeepSeek V4 Flash Vision Exp	0.0988	$20.50	1272.8M	1.7M	19h 17m	2732	11
39	Qwen3.8-Max	0.0987	$178.20	662.0M	1.7M	19h 50m	1710	6
40	Qwen3.8-Max	0.0927	$190.64	725.2M	1.3M	19h 52m	1620	35
41	Muse Spark 1.2	0.0185	$42.33	250.7M	911k	3h 1m	578	5
42	Inkling	0.0093	$17.52	18.3M	28k	1h 29m	208	3
43	Inkling	0.0019	$16.36	17.1M	30k	17m	280	1
44	Muse Spark 1.2	0.0000†	$2.45	9.3M	123k	59m	86	2
45	Inkling	0.0000	$3.15	3.3M	13k	5m	86	2
46	Muse Spark 1.2	0.0000†	$5.53	26.0M	135k	1h 56m	175	4
47	Muse Spark 1.2	0.0000†	$59.99	358.3M	658k	4h 22m	779	1
48	Gemini 3.7 Flash	0.0000†	$76.41	819.6M	1.5M	19h 43m	1734	1
49	Inkling	0.0000†	$5.57	5.8M	17k	1h 55m	118	2
50	Muse Spark 1.2	0.0000†	$2.14	7.7M	83k	1h 10m	93	3

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Granite Mamba2 Inference Optimization

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GLM-5.3	0.8927	$85.79	297.0M	636k	6h 7m	634	7
2	GLM-5.3	0.8719	$51.39	174.8M	618k	4h 8m	382	8
3	DeepSeek V4 Flash Vision Exp	0.8708	$2.68	149.8M	376k	16h 23m	473	5
4	Claude Fable 5.1	0.8458	$138.31	228.0M	1000k	9h 1m	580	15
5	GLM-5.3	0.8401	$45.93	158.7M	655k	5h 28m	343	6
6	Claude Fable 5.1	0.8393	$189.96	320.0M	1.2M	8h 30m	718	28
7	Claude Fable 5.1	0.8171	$168.52	281.0M	1.2M	8h 8m	667	16
8	Muse Spark 1.2	0.8168	$5.16	25.8M	238k	1h 23m	169	3
9	Qwen3.8-Max	0.8086	$63.86	200.9M	2.1M	18h 31m	572	2
10	Claude Fable 5.1	0.8047	$204.26	349.5M	1.2M	8h 27m	784	31
11	Inkling	0.7968	$5.41	5.6M	24k	1h 10m	98	3
12	Inkling	0.7910	$2.64	2.7M	12k	47m	61	2
13	DeepSeek V4 Flash Vision Exp	0.7876	$3.53	195.8M	540k	19h 32m	519	8
14	Qwen3.8-Max	0.7876	$54.77	177.9M	1.6M	16h 1m	520	5
15	Kimi K3	0.7872	$122.78	337.3M	986k	19h 10m	771	36
16	Claude Fable 5.1	0.7867	$157.78	258.9M	1.2M	7h 40m	582	11
17	Gemini 3.7 Flash	0.7839	$8.97	84.4M	442k	56m	281	3
18	Kimi K3	0.7805	$103.11	286.2M	907k	14h 49m	651	10
19	Inkling	0.7670	$5.68	5.8M	36k	1h 22m	101	2
20	GPT-5.6	0.7649	$74.74	74.2M	207k	2h 36m	270	5
21	DeepSeek V4 Flash Vision Exp	0.7578	$8.78	501.2M	1.2M	13h 5m	1157	3
22	Kimi K3	0.7571	$91.87	248.8M	620k	20h 0m	531	50
23	GLM-5.3	0.7528	$38.94	126.7M	486k	4h 14m	351	5
24	Kimi K3	0.7477	$74.48	205.5M	639k	8h 3m	456	20
25	DeepSeek V4 Flash Vision Exp	0.7467	$8.27	472.9M	1.1M	11h 58m	1140	23
26	GLM-5.3	0.7402	$85.05	310.6M	673k	18h 16m	645	8
27	Grok 4.6	0.7305	$58.74	52.8M	626k	4h 20m	206	13
28	Grok 4.6	0.7245	$46.67	39.7M	692k	3h 57m	169	10
29	Qwen3.8-Max	0.7188	$63.39	207.4M	1.7M	17h 14m	557	9
30	Muse Spark 1.2	0.7107	$36.53	207.7M	800k	2h 20m	527	1
31	Qwen3.8-Max	0.7023	$29.64	70.4M	1.9M	19h 40m	330	1
32	Qwen3.8-Max	0.6658	$53.16	162.1M	1.8M	18h 29m	495	2
33	Grok 4.6	0.6586	$55.70	47.5M	700k	4h 6m	194	10
34	Grok 4.6	0.6473	$57.88	51.1M	725k	4h 13m	210	5
35	GPT-5.6	0.6303	$111.93	116.6M	261k	11h 2m	425	3
36	Kimi K3	0.6169	$112.73	307.9M	980k	13h 16m	708	31
37	GPT-5.6	0.6093	$23.53	32.7M	162k	1h 28m	207	3
38	DeepSeek V4 Flash Vision Exp	0.4680	$3.00	159.1M	520k	6h 30m	442	5
39	GPT-5.6	0.4658	$119.15	122.5M	308k	3h 17m	463	6
40	Gemini 3.7 Flash	0.3912	$15.89	144.8M	798k	2h 17m	479	5
41	GPT-5.6	0.3383	$180.46	175.3M	266k	4h 7m	474	10
42	Gemini 3.7 Flash	0.3149	$10.74	97.9M	514k	1h 12m	278	2
43	Grok 4.6	0.2777	$27.64	23.0M	551k	3h 18m	127	2
44	Gemini 3.7 Flash	0.2425	$12.51	121.4M	559k	2h 10m	326	2
45	Gemini 3.7 Flash	0.2101	$6.70	49.8M	379k	47m	212	3
46	Inkling	0.0000	$7.50	7.7M	51k	1h 46m	135	1
47	Muse Spark 1.2	0.0000	$19.40	92.0M	1.0M	4h 3m	307	3
48	Muse Spark 1.2	0.0000	$9.93	53.1M	354k	2h 11m	258	6
49	Muse Spark 1.2	0.0000	$8.05	39.0M	223k	1h 17m	201	4
50	Inkling	0.0000	$4.66	4.8M	18k	1h 13m	91	2

Higgs Uncertainty Inference

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9930	$40.59	65.5M	322k	8h 40m	241	9
2	Claude Fable 5.1	0.9895	$33.56	52.1M	309k	8h 2m	221	8
3	Qwen3.8-Max	0.9778	$23.74	79.2M	556k	12h 13m	428	14
4	DeepSeek V4 Flash Vision Exp	0.9597	$1.51	70.8M	347k	3h 23m	283	4
5	Kimi K3	0.9593	$29.55	62.1M	341k	12h 49m	285	15
6	Grok 4.6	0.9143	$37.23	34.8M	407k	3h 22m	158	10
7	Muse Spark 1.2	0.9041	$8.87	38.4M	436k	2h 57m	150	1
8	Grok 4.6	0.8991	$42.77	39.2M	529k	3h 38m	183	9
9	Grok 4.6	0.8827	$39.92	37.7M	462k	3h 38m	181	8
10	Inkling	0.8760	$6.24	6.3M	65k	37m	140	1
11	Grok 4.6	0.8188	$274.70	264.2M	1.9M	19h 38m	1078	67
12	GPT-5.6	0.7863	$99.90	97.4M	228k	4h 26m	327	20
13	Qwen3.8-Max	0.7685	$16.66	51.1M	571k	13h 28m	342	5
14	DeepSeek V4 Flash Vision Exp	0.7308	$2.17	110.7M	423k	3h 32m	374	9
15	Muse Spark 1.2	0.7187	$4.40	15.5M	216k	1h 29m	110	2
16	Muse Spark 1.2	0.7132	$3.96	18.1M	225k	49m	126	4
17	Qwen3.8-Max	0.4289	$16.21	49.4M	555k	8h 58m	285	8
18	Kimi K3	0.4200	$32.18	58.0M	281k	15h 50m	280	8
19	Qwen3.8-Max	0.3449	$15.61	47.3M	543k	7h 37m	278	8
20	Claude Fable 5.1	0.3345	$30.45	45.8M	311k	4h 14m	174	15
21	Qwen3.8-Max	0.3078	$18.56	57.9M	578k	8h 28m	322	6
22	Grok 4.6	0.2104	$116.88	112.5M	824k	9h 43m	470	52
23	GPT-5.6	0.2084	$52.26	56.6M	152k	2h 42m	244	5
24	Claude Fable 5.1	0.2017	$30.51	47.2M	284k	6h 48m	194	9
25	Gemini 3.7 Flash	0.1681	$1.72	9.3M	135k	1h 3m	102	1
26	GLM-5.3	0.1632	$24.11	84.1M	350k	8h 48m	310	8
27	Muse Spark 1.2	0.1494	$13.10	65.0M	597k	2h 52m	267	1
28	Gemini 3.7 Flash	0.1295	$1.98	10.2M	141k	50m	105	3
29	Gemini 3.7 Flash	0.1075	$1.62	9.2M	146k	1h 7m	102	1
30	GLM-5.3	0.1005	$24.46	71.1M	367k	17h 31m	283	7
31	Kimi K3	0.0960	$32.35	77.4M	380k	13h 56m	311	15
32	Gemini 3.7 Flash	0.0798	$2.53	10.3M	161k	1h 26m	110	1
33	Claude Fable 5.1	0.0778	$29.88	44.4M	316k	2h 48m	183	13
34	GPT-5.6	0.0777	$44.33	50.5M	161k	1h 56m	229	6
35	DeepSeek V4 Flash Vision Exp	0.0718	$2.02	102.8M	397k	15h 29m	346	4
36	DeepSeek V4 Flash Vision Exp	0.0700	$1.50	70.3M	352k	6h 13m	290	6
37	Gemini 3.7 Flash	0.0647	$3.79	20.7M	194k	1h 41m	139	2
38	Kimi K3	0.0626	$34.08	68.6M	372k	14h 20m	275	12
39	DeepSeek V4 Flash Vision Exp	0.0559	$1.32	62.9M	305k	6h 32m	278	4
40	GLM-5.3	0.0517	$9.91	30.6M	263k	2h 10m	167	5
41	Muse Spark 1.2	0.0498	$2.47	8.0M	164k	1h 0m	83	2
42	GLM-5.3	0.0375	$14.94	49.5M	308k	3h 28m	223	2
43	Kimi K3	0.0289	$90.08	230.0M	659k	20h 0m	555	1
44	GLM-5.3	0.0251	$17.68	60.0M	356k	3h 29m	246	3
45	GPT-5.6	0.0080	$56.44	60.4M	164k	6h 2m	259	12
46	Inkling	0.0078	$3.62	3.6M	51k	2h 38m	120	1
47	GPT-5.6	0.0049	$70.63	65.2M	124k	2h 29m	194	6
48	Inkling	0.0004	$3.37	3.3M	54k	1h 37m	104	1
49	Inkling	0.0004	$4.08	4.1M	54k	1h 12m	112	2
50	Inkling	0.0000	$2.47	2.5M	36k	1h 12m	106	1

SGLang Inference System Optimization

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GPT-5.6	0.1092	$291.23	284.1M	251k	5h 12m	616	12
2	GPT-5.6	0.0675	$443.00	437.3M	281k	6h 23m	877	14
3	GLM-5.3	0.0534	$68.77	241.2M	448k	15h 36m	650	8
4	Muse Spark 1.2	0.0532	$132.47	498.4M	1.4M	17h 13m	1444	3
5	Grok 4.6	0.0322	$488.14	454.2M	2.3M	19h 33m	1703	83
6	Qwen3.8-Max	0.0214	$45.80	153.5M	1.1M	15h 59m	715	7
7	GLM-5.3	0.0211	$67.37	244.9M	383k	14h 17m	698	20
8	Gemini 3.7 Flash	0.0169	$19.15	195.9M	289k	3h 33m	505	4
9	GPT-5.6	0.0168	$283.36	279.0M	247k	6h 47m	655	21
10	Kimi K3	0.0103	$110.14	311.1M	507k	20h 1m	740	11
11	Muse Spark 1.2	0.0099	$100.58	403.5M	1.7M	11h 46m	1193	8
12	Muse Spark 1.2	0.0081	$15.03	90.2M	152k	1h 36m	211	7
13	Qwen3.8-Max	0.0081	$33.42	99.7M	1.3M	15h 1m	574	8
14	Muse Spark 1.2	0.0047	$67.55	167.8M	657k	12h 12m	541	6
15	Grok 4.6	0.0025	$162.38	158.0M	937k	10h 27m	649	14
16	Grok 4.6	0.0012	$313.02	296.5M	1.9M	18h 41m	1120	19
17	Grok 4.6	0.0007	$96.00	91.3M	481k	3h 53m	329	3
18	Qwen3.8-Max	0.0004	$47.32	149.7M	1.5M	17h 59m	651	3
19	GLM-5.3	0.0004	$46.58	170.2M	369k	10h 37m	591	7
20	GLM-5.3	0.0000	$50.83	180.8M	386k	12h 52m	576	4
21	Claude Fable 5.1	0.0000	$219.62	392.8M	963k	7h 58m	844	9
22	Muse Spark 1.2	0.0000	$32.81	65.0M	127k	4h 50m	276	3
23	Claude Fable 5.1	0.0000	$193.70	342.2M	936k	9h 59m	820	11
24	DeepSeek V4 Flash Vision Exp	0.0000	$2.44	138.4M	299k	9h 52m	531	4
25	DeepSeek V4 Flash Vision Exp	0.0000	$5.46	332.4M	492k	10h 53m	782	15
26	DeepSeek V4 Flash Vision Exp	0.0000	$3.59	206.9M	443k	9h 31m	593	16
27	Kimi K3	0.0000	$185.83	517.0M	1.5M	20h 0m	1188	13
28	GPT-5.6	0.0000	$109.76	114.5M	139k	5h 57m	401	7
29	Inkling	0.0000	$4.87	5.1M	12k	1h 40m	92	3
30	Claude Fable 5.1	0.0000	$177.26	316.2M	794k	11h 11m	833	9
31	Kimi K3	0.0000	$246.45	710.4M	1.8M	20h 0m	1736	14
32	GLM-5.3	0.0000	$68.85	206.5M	390k	14h 23m	607	17
33	Inkling	0.0000	$7.60	8.0M	10k	1h 46m	120	2
34	Gemini 3.7 Flash	0.0000	$9.47	100.2M	149k	1h 43m	316	4
35	GPT-5.6	0.0000	$182.18	175.0M	189k	6h 4m	485	8
36	Inkling	0.0000	$5.18	5.4M	12k	1h 28m	91	2
37	DeepSeek V4 Flash Vision Exp	0.0000	$3.41	198.5M	360k	8h 1m	562	4
38	Grok 4.6	0.0000	$419.76	405.9M	2.2M	19h 4m	1586	60
39	Claude Fable 5.1	0.0000	$182.53	329.2M	746k	9h 31m	841	21
40	Inkling	0.0000	$6.42	6.7M	16k	1h 12m	100	1
41	DeepSeek V4 Flash Vision Exp	0.0000	$3.54	209.2M	364k	8h 32m	632	9
42	Qwen3.8-Max	0.0000	$42.20	143.0M	952k	17h 51m	604	1
43	Kimi K3	0.0000	$49.64	141.5M	328k	6h 19m	529	7
44	Inkling	0.0000	$14.76	15.4M	36k	34m	194	2
45	Gemini 3.7 Flash	0.0000	$9.02	85.7M	143k	1h 52m	306	1
46	Qwen3.8-Max	0.0000	$47.25	152.8M	1.4M	17h 10m	642	3
47	Gemini 3.7 Flash	0.0000	$12.58	90.8M	182k	4h 36m	317	1
48	Kimi K3	0.0000	$124.72	355.5M	911k	15h 41m	948	24
49	Claude Fable 5.1	0.0000	$226.14	406.3M	953k	11h 14m	902	15
50	Gemini 3.7 Flash	0.0000	$6.58	51.9M	91k	1h 41m	296	4

Lean 4 Kernel Type Checker in Pascal

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	1.0000	$34.17	49.9M	378k	2h 37m	164	6
2	Claude Fable 5.1	1.0000	$45.30	70.2M	418k	4h 0m	200	9
3	Claude Fable 5.1	1.0000	$34.22	50.9M	360k	1h 59m	159	8
4	Claude Fable 5.1	1.0000	$43.08	67.7M	379k	4h 3m	206	11
5	Claude Fable 5.1	1.0000	$50.91	83.1M	385k	3h 9m	232	8
6	GPT-5.6	0.9807	$221.16	222.7M	429k	6h 13m	691	19
7	GPT-5.6	0.9791	$78.01	86.8M	272k	4h 32m	395	2
8	GPT-5.6	0.9757	$207.31	199.1M	381k	7h 23m	620	10
9	Gemini 3.7 Flash	0.9448	$95.08	678.0M	1.4M	19h 49m	1660	1
10	Gemini 3.7 Flash	0.9361	$54.83	453.2M	1.3M	12h 43m	1224	1
11	GPT-5.6	0.9339	$340.15	337.0M	382k	8h 35m	789	11
12	Kimi K3	0.9198	$192.68	537.9M	1.4M	20h 0m	1218	1
13	GLM-5.3	0.8650	$139.71	481.8M	637k	18h 8m	859	6
14	GLM-5.3	0.8372	$111.12	401.3M	867k	19h 42m	904	10
15	Kimi K3	0.8313	$155.97	405.9M	1.5M	19h 47m	935	3
16	GPT-5.6	0.7874	$397.16	404.0M	396k	8h 54m	1134	7
17	Kimi K3	0.5265	$147.65	387.7M	1.4M	20h 2m	894	1
18	Kimi K3	0.4229	$171.33	477.1M	1.3M	19h 55m	1130	12
19	GLM-5.3	0.4065	$133.87	385.6M	660k	19h 35m	677	3
20	Grok 4.6	0.3824	$105.13	105.0M	1.0M	6h 34m	490	2
21	DeepSeek V4 Flash Vision Exp	0.3599	$12.71	725.7M	1.6M	10h 38m	1614	4
22	DeepSeek V4 Flash Vision Exp	0.3164	$9.48	551.4M	1.2M	7h 58m	1228	8
23	Kimi K3	0.2644	$151.43	423.8M	1.1M	20h 1m	928	1
24	DeepSeek V4 Flash Vision Exp	0.2513	$13.44	768.2M	1.8M	13h 43m	1760	7
25	Gemini 3.7 Flash	0.1248	$74.81	557.5M	1.1M	14h 22m	1217	1
26	GLM-5.3	0.1056	$134.85	474.9M	964k	19h 7m	1105	3
27	DeepSeek V4 Flash Vision Exp	0.0143	$12.56	761.2M	1.2M	12h 33m	1627	35
28	Gemini 3.7 Flash	0.0087	$48.94	384.1M	939k	13h 20m	1118	1
29	Muse Spark 1.2	0.0000	$19.14	108.5M	529k	2h 29m	348	1
30	Qwen3.8-Max	0.0000	$14.14	28.2M	1.1M	19h 59m	169	1
31	Inkling	0.0000	$5.56	5.7M	37k	1h 16m	115	2
32	Inkling	0.0000	$5.46	5.6M	34k	55m	107	2
33	Muse Spark 1.2	0.0000	$21.09	53.6M	699k	3h 48m	205	2
34	Muse Spark 1.2	0.0000	$29.46	162.8M	830k	6h 1m	404	3
35	Qwen3.8-Max	0.0000	$8.31	10.6M	914k	19h 59m	109	2
36	Inkling	0.0000	$5.70	5.9M	22k	14m	80	2
37	Grok 4.6	0.0000	$114.14	110.5M	1.1M	7h 47m	492	4
38	Muse Spark 1.2	0.0000	$27.30	88.9M	859k	4h 46m	278	1
39	Grok 4.6	0.0000	$407.54	406.7M	2.0M	19h 47m	1527	137
40	Grok 4.6	0.0000	$250.10	240.1M	1.8M	13h 13m	970	43
41	Inkling	0.0000	$13.81	14.4M	43k	1h 26m	175	1
42	DeepSeek V4 Flash Vision Exp	0.0000	$11.64	694.6M	1.3M	11h 22m	1413	33
43	Grok 4.6	0.0000	$233.84	217.8M	2.5M	19h 53m	929	30
44	Muse Spark 1.2	0.0000	$9.51	48.0M	382k	2h 35m	213	2
45	Gemini 3.7 Flash	0.0000	$101.50	861.0M	1.8M	19h 58m	2092	1
46	Qwen3.8-Max	0.0000	$13.92	25.5M	1.2M	19h 56m	167	2
47	Qwen3.8-Max	0.0000	$19.68	45.8M	1.3M	20h 0m	211	1
48	GLM-5.3	0.0000	$113.59	400.7M	1.0M	18h 31m	962	5
49	Qwen3.8-Max	0.0000	$10.35	13.6M	1.1M	19h 57m	115	1
50	Inkling	0.0000	$6.56	6.8M	25k	12m	109	6

libexpat Optimization

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.5674	$350.25	628.4M	1.5M	16h 3m	1456	82
2	Claude Fable 5.1	0.5395	$313.80	554.2M	1.5M	11h 26m	1289	34
3	Claude Fable 5.1	0.4885	$375.68	676.2M	1.5M	17h 3m	1505	50
4	Claude Fable 5.1	0.4469	$289.07	503.7M	1.5M	13h 41m	1156	50
5	GPT-5.6	0.3847	$146.13	140.6M	311k	2h 25m	439	7
6	GPT-5.6	0.3225	$67.06	75.3M	223k	1h 44m	326	9
7	GPT-5.6	0.3025	$18.35	26.1M	127k	53m	144	4
8	Claude Fable 5.1	0.2880	$179.04	308.9M	1.0M	10h 29m	755	12
9	GPT-5.6	0.2868	$49.11	53.9M	193k	1h 32m	238	5
10	GPT-5.6	0.2751	$110.05	104.7M	243k	2h 24m	350	7
11	Gemini 3.7 Flash	0.2399	$41.35	394.7M	960k	4h 30m	1009	6
12	GLM-5.3	0.2372	$181.79	673.8M	1.1M	8h 14m	1501	3
13	GLM-5.3	0.2303	$183.58	670.1M	1.1M	9h 50m	1460	18
14	GLM-5.3	0.1823	$327.58	1209.3M	1.8M	18h 23m	2635	46
15	Kimi K3	0.1778	$238.01	679.3M	1.8M	19h 52m	1588	83
16	GLM-5.3	0.1583	$236.28	865.5M	1.6M	18h 53m	1991	34
17	Gemini 3.7 Flash	0.1572	$41.30	358.7M	785k	6h 1m	899	5
18	Gemini 3.7 Flash	0.1555	$24.69	216.5M	877k	2h 56m	624	2
19	Kimi K3	0.1513	$169.56	484.2M	1.2M	19h 34m	1155	16
20	Grok 4.6	0.1479	$150.76	148.7M	804k	4h 11m	580	26
21	Grok 4.6	0.1393	$108.10	108.1M	736k	4h 5m	435	11
22	Kimi K3	0.1288	$173.14	509.0M	1.1M	13h 23m	1149	28
23	Grok 4.6	0.1022	$79.42	75.8M	516k	2h 56m	273	7
24	DeepSeek V4 Flash Vision Exp	0.0891	$17.04	1034.3M	1.6M	6h 42m	2273	6
25	DeepSeek V4 Flash Vision Exp	0.0545	$16.46	992.0M	1.7M	6h 43m	2222	11
26	Muse Spark 1.2	0.0000	$40.00	150.5M	651k	3h 23m	388	3
27	Muse Spark 1.2	0.0000	$41.66	132.7M	956k	3h 49m	473	3
28	Inkling	0.0000	$23.47	24.5M	46k	39m	359	2
29	Muse Spark 1.2	0.0000	$16.54	87.9M	378k	1h 23m	309	3
30	Qwen3.8-Max	0.0000	$21.15	36.8M	1.9M	20h 0m	260	—
31	Inkling	0.0000	$3.70	3.7M	39k	18m	85	2
32	Qwen3.8-Max	0.0000	$16.28	22.3M	1.7M	19h 46m	202	—
33	Gemini 3.7 Flash	0.0000	$33.92	328.7M	960k	2h 58m	810	20
34	DeepSeek V4 Flash Vision Exp	0.0000	$13.51	827.8M	1.3M	5h 43m	1720	13
35	DeepSeek V4 Flash Vision Exp	0.0000	$21.93	1323.6M	2.2M	8h 50m	3095	7
36	Gemini 3.7 Flash	0.0000	$29.56	325.1M	599k	2h 10m	830	2
37	DeepSeek V4 Flash Vision Exp	0.0000	$33.36	1965.0M	3.8M	19h 31m	4102	48
38	Kimi K3	0.0000	$214.44	584.8M	1.8M	20h 0m	1459	1
39	Inkling	0.0000	$18.36	19.0M	75k	47m	251	4
40	Qwen3.8-Max	0.0000	$19.23	33.9M	1.7M	19h 53m	233	1
41	Grok 4.6	0.0000	$93.77	99.5M	677k	3h 26m	435	10
42	Muse Spark 1.2	0.0000	$39.83	172.4M	582k	3h 18m	559	6
43	Inkling	0.0000	$3.58	3.6M	31k	16m	74	2
44	Qwen3.8-Max	0.0000	$25.12	46.8M	2.2M	19h 55m	301	—
45	Muse Spark 1.2	0.0000	$32.92	157.9M	806k	3h 52m	344	3
46	Grok 4.6	0.0000	$71.78	70.3M	399k	1h 58m	277	5
47	Qwen3.8-Max	0.0000	$13.06	14.1M	1.6M	19h 35m	156	—
48	Inkling	0.0000	$3.11	3.2M	24k	13m	82	3
49	Kimi K3	0.0000	$256.75	739.1M	1.4M	20h 0m	1576	1
50	GLM-5.3	0.0000	$216.67	774.3M	1.6M	16h 7m	1754	19

Lua Native Compiler

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	1.0000	$44.19	69.8M	381k	3h 24m	215	8
2	Claude Fable 5.1	1.0000	$51.03	82.5M	402k	3h 56m	245	12
3	Claude Fable 5.1	1.0000	$55.84	88.8M	470k	3h 14m	235	6
4	GLM-5.3	0.9976	$201.34	735.2M	1.4M	11h 37m	1660	3
5	GLM-5.3	0.9953	$142.13	511.6M	1.2M	9h 22m	1178	2
6	DeepSeek V4 Flash Vision Exp	0.9788	$23.06	1383.4M	2.4M	13h 19m	2911	21
7	GLM-5.3	0.9599	$274.76	1000.1M	1.8M	19h 32m	2273	5
8	Kimi K3	0.9575	$183.51	524.3M	1.3M	19h 59m	1203	20
9	GLM-5.3	0.9527	$236.30	883.7M	1.1M	17h 20m	1835	7
10	Kimi K3	0.8750	$185.22	530.7M	1.3M	20h 0m	1207	1
11	Claude Fable 5.1	0.8463	$43.78	69.0M	381k	3h 39m	231	1
12	Gemini 3.7 Flash	0.6998	$21.69	183.4M	580k	2h 58m	400	1
13	Claude Fable 5.1	0.6667	$57.38	92.1M	466k	4h 40m	248	7
14	Gemini 3.7 Flash	0.6541	$62.39	519.3M	1.2M	7h 8m	1260	1
15	Gemini 3.7 Flash	0.6517	$26.08	255.3M	824k	1h 58m	670	1
16	Gemini 3.7 Flash	0.6392	$29.13	278.2M	822k	3h 5m	751	1
17	Grok 4.6	0.6384	$176.45	167.1M	1.5M	7h 26m	653	8
18	Kimi K3	0.5063	$220.93	645.6M	1.2M	20h 2m	1461	1
19	Gemini 3.7 Flash	0.3270	$79.34	743.6M	1.7M	8h 42m	1815	1
20	Kimi K3	0.2862	$159.88	431.8M	1.4M	19h 11m	1009	1
21	DeepSeek V4 Flash Vision Exp	0.2005	$21.27	1250.2M	2.4M	15h 16m	2676	17
22	Inkling	0.0000	$27.74	28.9M	72k	42m	300	1
23	Inkling	0.0000	$16.92	17.5M	67k	44m	221	1
24	Qwen3.8-Max	0.0000	$64.51	206.8M	2.0M	19h 8m	639	2
25	DeepSeek V4 Flash Vision Exp	0.0000	$11.39	676.4M	1.3M	7h 45m	1507	15
26	Grok 4.6	0.0000	$110.36	107.0M	986k	5h 11m	447	8
27	Inkling	0.0000	$48.62	50.6M	132k	1h 1m	367	2
28	GPT-5.6	0.0000	$35.07	44.0M	172k	2h 41m	211	10
29	Muse Spark 1.2	0.0000	$44.95	118.1M	616k	6h 24m	354	2
30	Kimi K3	0.0000	$228.55	670.7M	1.4M	19h 8m	1408	3
31	Inkling	0.0000	$14.81	15.3M	75k	53m	223	1
32	GPT-5.6	0.0000†	$7.45	8.9M	65k	25m	65	3
33	GLM-5.3	0.0000	$251.10	909.6M	1.3M	19h 55m	1918	6
34	Qwen3.8-Max	0.0000	$121.10	426.7M	2.2M	19h 59m	984	1
35	Grok 4.6	0.0000	$271.55	264.4M	2.0M	9h 49m	1061	31
36	Muse Spark 1.2	0.0000	$10.77	39.0M	388k	1h 41m	164	4
37	DeepSeek V4 Flash Vision Exp	0.0000	$22.43	1385.4M	2.0M	11h 31m	2860	2
38	Qwen3.8-Max	0.0000	$135.25	480.8M	2.3M	19h 7m	1081	2
39	Qwen3.8-Max	0.0000	$60.45	189.1M	2.1M	19h 48m	599	3
40	Qwen3.8-Max	0.0000	$31.00	87.6M	1.4M	13h 21m	385	6
41	Inkling	0.0000	$7.91	8.2M	23k	26m	165	2
42	Grok 4.6	0.0000	$676.54	663.0M	4.1M	19h 59m	2429	277
43	GPT-5.6	0.0000	$127.72	124.4M	217k	2h 5m	382	24
44	Muse Spark 1.2	0.0000	$4.46	22.6M	149k	52m	145	2
45	DeepSeek V4 Flash Vision Exp	0.0000	$18.34	1059.5M	2.2M	17h 1m	2383	9
46	GPT-5.6	0.0000	$24.84	37.1M	163k	1h 9m	219	13
47	Grok 4.6	0.0000	$250.76	240.6M	2.1M	11h 13m	930	16
48	Muse Spark 1.2	0.0000	$22.61	124.4M	368k	1h 49m	323	2
49	Muse Spark 1.2	0.0000	$12.00	53.2M	328k	1h 52m	189	2
50	GPT-5.6	0.0000	$20.50	39.9M	168k	1h 9m	231	7

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Machine-Learned Interatomic Potential

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GPT-5.6	0.9925	$108.64	109.2M	151k	11h 16m	348	8
2	GPT-5.6	0.9918	$109.29	114.5M	152k	15h 47m	404	16
3	GLM-5.3	0.9901	$21.35	50.8M	160k	19h 5m	353	1
4	Qwen3.8-Max	0.9731	$4.10	11.4M	180k	17h 54m	185	3
5	GLM-5.3	0.9615	$15.62	37.8M	156k	19h 29m	294	2
6	Claude Fable 5.1	0.9586	$14.22	21.7M	140k	16h 34m	179	11
7	GPT-5.6	0.9437	$26.20	41.3M	108k	14h 19m	237	3
8	Claude Fable 5.1	0.9369	$12.11	18.4M	119k	17h 11m	175	15
9	Kimi K3	0.9282	$14.81	21.6M	194k	19h 20m	175	3
10	Qwen3.8-Max	0.9220	$5.13	15.3M	180k	19h 14m	209	2
11	GLM-5.3	0.9184	$31.31	62.0M	243k	18h 46m	307	2
12	Kimi K3	0.9177	$11.72	20.5M	161k	18h 47m	165	3
13	DeepSeek V4 Flash Vision Exp	0.9173	$3.12	179.4M	413k	18h 25m	558	6
14	Claude Fable 5.1	0.9113	$11.26	16.5M	124k	17h 50m	150	5
15	Grok 4.6	0.8964	$27.94	18.3M	258k	19h 42m	147	2
16	Kimi K3	0.8916	$6.54	9.0M	95k	18h 28m	126	2
17	Muse Spark 1.2	0.8888	$4.08	20.1M	191k	2h 11m	165	12
18	Claude Fable 5.1	0.8820	$17.05	26.5M	157k	16h 32m	179	7
19	Grok 4.6	0.8799	$71.42	58.5M	227k	19h 56m	307	9
20	Muse Spark 1.2	0.8642	$31.30	153.8M	371k	2h 47m	464	12
21	Gemini 3.7 Flash	0.8422	$11.45	30.2M	159k	8h 5m	208	3
22	Grok 4.6	0.8342	$80.88	55.9M	251k	19h 37m	265	10
23	Claude Fable 5.1	0.8229	$11.99	18.2M	119k	18h 9m	171	13
24	Muse Spark 1.2	0.8223	$3.45	15.9M	178k	49m	127	3
25	GLM-5.3	0.8163	$20.32	55.4M	207k	19h 30m	350	2
26	Qwen3.8-Max	0.8093	$6.94	20.8M	251k	18h 48m	231	2
27	Kimi K3	0.8033	$15.11	21.0M	176k	19h 11m	168	3
28	GLM-5.3	0.7858	$25.28	49.5M	200k	18h 31m	285	2
29	Muse Spark 1.2	0.7533	$3.65	17.8M	164k	1h 53m	126	3
30	Kimi K3	0.7378	$9.88	13.2M	119k	19h 28m	143	2
31	Gemini 3.7 Flash	0.7285	$10.95	29.5M	154k	8h 16m	213	7
32	Grok 4.6	0.7267	$28.23	20.6M	259k	19h 44m	159	3
33	Muse Spark 1.2	0.7264	$3.55	17.0M	170k	2h 17m	144	3
34	DeepSeek V4 Flash Vision Exp	0.6906	$1.80	93.9M	332k	18h 0m	378	6
35	Grok 4.6	0.5686	$61.59	46.3M	291k	18h 29m	250	4
36	Gemini 3.7 Flash	0.5113	$8.21	27.8M	178k	5h 29m	184	2
37	DeepSeek V4 Flash Vision Exp	0.4712	$1.05	54.4M	192k	19h 29m	342	2
38	Gemini 3.7 Flash	0.4203	$5.25	21.8M	116k	4h 45m	178	1
39	DeepSeek V4 Flash Vision Exp	0.2438	$1.59	84.5M	269k	19h 1m	389	5
40	Qwen3.8-Max	0.2071	$6.23	16.9M	295k	18h 26m	218	2
41	Inkling	0.0572	$3.46	3.5M	36k	2h 37m	136	1
42	DeepSeek V4 Flash Vision Exp	0.0562	$2.23	120.5M	374k	18h 41m	436	3
43	Inkling	0.0431	$4.94	5.0M	59k	45m	120	1
44	Inkling	0.0393	$6.84	7.0M	43k	1h 26m	181	1
45	Inkling	0.0179	$5.35	5.4M	56k	1h 32m	126	2
46	Inkling	0.0126	$4.98	5.1M	42k	1h 33m	148	2
47	GPT-5.6	0.0076	$48.99	56.5M	122k	14h 29m	293	5
48	GPT-5.6	0.0010	$36.55	49.8M	109k	7h 20m	272	3
49	Qwen3.8-Max	0.0010	$4.41	12.0M	206k	19h 18m	179	1
50	Gemini 3.7 Flash	0.0010	$6.01	20.4M	174k	4h 35m	154	2

Medium-Range Weather Forecast

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.6214	$15.11	24.2M	125k	16h 58m	204	11
2	Claude Fable 5.1	0.6178	$12.69	20.4M	104k	18h 1m	175	12
3	Claude Fable 5.1	0.6011	$16.81	27.4M	129k	16h 5m	208	16
4	Claude Fable 5.1	0.5960	$16.06	25.6M	135k	16h 36m	193	12
5	GPT-5.6	0.5936	$28.82	47.8M	113k	15h 13m	294	4
6	GLM-5.3	0.5873	$22.61	71.9M	228k	18h 20m	314	2
7	Grok 4.6	0.5804	$120.40	104.0M	444k	19h 13m	464	37
8	Gemini 3.7 Flash	0.5747	$22.40	49.9M	145k	13h 34m	256	7
9	Gemini 3.7 Flash	0.5566	$16.56	54.7M	160k	9h 58m	285	2
10	Grok 4.6	0.5522	$208.02	184.0M	911k	19h 55m	826	53
11	Grok 4.6	0.5495	$238.57	230.6M	626k	20h 0m	861	62
12	Gemini 3.7 Flash	0.5365	$13.11	37.7M	147k	6h 21m	241	3
13	Gemini 3.7 Flash	0.5356	$19.36	48.1M	172k	8h 5m	252	4
14	Gemini 3.7 Flash	0.5332	$4.99	21.4M	136k	4h 20m	171	1
15	GLM-5.3	0.4818	$19.75	47.5M	201k	18h 1m	273	5
16	GLM-5.3	0.4745	$28.03	96.3M	327k	19h 20m	363	7
17	GLM-5.3	0.4723	$21.03	47.8M	206k	18h 36m	293	2
18	GLM-5.3	0.4679	$26.30	77.9M	252k	19h 29m	364	2
19	DeepSeek V4 Flash Vision Exp	0.4639	$1.55	83.0M	253k	19h 5m	403	5
20	Grok 4.6	0.4530	$43.07	35.9M	245k	19h 35m	225	1
21	Grok 4.6	0.4154	$92.48	66.7M	263k	19h 27m	303	6
22	Qwen3.8-Max	0.4005	$8.94	29.1M	230k	19h 22m	281	1
23	Qwen3.8-Max	0.3989	$14.91	48.7M	387k	19h 25m	357	1
24	Qwen3.8-Max	0.3606	$13.12	44.7M	268k	19h 18m	385	3
25	Muse Spark 1.2	0.3456	$4.38	18.4M	145k	37m	104	2
26	Qwen3.8-Max	0.3410	$4.91	14.6M	172k	18h 25m	189	4
27	Qwen3.8-Max	0.3360	$8.01	25.7M	213k	18h 56m	239	8
28	Muse Spark 1.2	0.3227	$38.37	235.4M	385k	2h 26m	592	5
29	Muse Spark 1.2	0.3018	$11.62	45.2M	202k	1h 9m	181	7
30	Muse Spark 1.2	0.2944	$2.12	8.7M	99k	50m	86	1
31	Muse Spark 1.2	0.2758	$11.17	51.5M	378k	2h 40m	211	6
32	Inkling	0.2568	$3.54	3.6M	37k	1h 43m	97	1
33	Inkling	0.2237	$3.91	3.9M	47k	1h 43m	120	1
34	Inkling	0.1542	$5.24	5.4M	39k	1h 50m	132	2
35	Inkling	0.1482	$3.49	3.6M	19k	1h 32m	102	1
36	GPT-5.6	0.0000	$122.35	137.1M	209k	17h 41m	576	7
37	Kimi K3	0.0000	$20.25	49.9M	257k	19h 31m	296	8
38	Kimi K3	0.0000	$16.03	42.3M	172k	19h 35m	282	1
39	Kimi K3	0.0000	$13.29	32.3M	193k	19h 22m	245	2
40	Inkling	0.0000	$6.20	6.2M	72k	38m	132	1
41	DeepSeek V4 Flash Vision Exp	0.0000	$1.66	90.1M	258k	19h 29m	392	3
42	DeepSeek V4 Flash Vision Exp	0.0000	$1.99	110.4M	289k	18h 13m	487	4
43	DeepSeek V4 Flash Vision Exp	0.0000	$1.55	84.2M	243k	18h 42m	398	5
44	GPT-5.6	0.0000	$63.62	70.4M	130k	17h 56m	290	3
45	Kimi K3	0.0000	$11.04	28.1M	135k	19h 31m	261	6
46	GPT-5.6	0.0000	$72.57	94.1M	189k	15h 53m	478	14
47	DeepSeek V4 Flash Vision Exp	0.0000	$1.68	93.8M	242k	19h 27m	452	2
48	Claude Fable 5.1	0.0000	$14.60	23.4M	121k	14h 28m	186	8
49	GPT-5.6	0.0000	$88.96	100.2M	138k	14h 52m	416	8
50	Kimi K3	0.0000	$19.09	48.5M	196k	18h 52m	326	6

MEG Speech Decoding

task page →43 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.3166	$19.44	30.2M	179k	16h 28m	214	8
2	Claude Fable 5.1	0.3123	$26.35	42.8M	204k	16h 51m	240	12
3	GPT-5.6	0.3119	$41.21	43.5M	133k	5h 23m	204	2
4	Claude Fable 5.1	0.3116	$13.99	21.5M	133k	17h 11m	192	11
5	Claude Fable 5.1	0.3057	$16.57	26.2M	144k	13h 23m	202	11
6	Grok 4.6	0.3041	$128.46	122.4M	984k	15h 5m	545	23
7	GPT-5.6	0.3039	$14.10	19.5M	85k	2h 25m	137	2
8	GPT-5.6	0.2311	$153.75	128.1M	180k	10h 45m	298	17
9	Grok 4.6	0.2265	$217.69	204.9M	1.2M	19h 25m	878	23
10	GLM-5.3	0.2244	$22.68	51.0M	188k	18h 6m	315	2
11	Claude Fable 5.1	0.2155	$25.98	41.7M	212k	14h 56m	237	10
12	Grok 4.6	0.2107	$132.86	129.3M	708k	19h 25m	571	33
13	GLM-5.3	0.2036	$34.96	74.2M	244k	14h 55m	372	2
14	GPT-5.6	0.1902	$64.35	67.7M	163k	7h 2m	279	4
15	GPT-5.6	0.1862	$104.80	98.3M	165k	7h 53m	318	8
16	Grok 4.6	0.1844	$95.16	83.3M	284k	18h 58m	362	11
17	DeepSeek V4 Flash Vision Exp	0.1749	$1.32	68.7M	240k	18h 36m	359	6
18	Muse Spark 1.2	0.1659	$17.10	36.3M	144k	3h 7m	130	6
19	GLM-5.3	0.1492	$27.34	90.5M	318k	15h 42m	371	3
20	Qwen3.8-Max	0.1460	$13.18	43.8M	302k	19h 37m	333	7
21	Grok 4.6	0.1245	$71.38	67.2M	365k	9h 53m	293	10
22	Muse Spark 1.2	0.1175	$9.52	47.7M	272k	3h 44m	183	8
23	Gemini 3.7 Flash	0.1130	$4.87	33.1M	132k	3h 51m	226	1
24	Gemini 3.7 Flash	0.1121	$12.12	51.3M	122k	7h 24m	312	4
25	Kimi K3	0.1102	$14.90	32.2M	173k	18h 22m	254	3
26	GLM-5.3	0.1032	$33.80	77.7M	262k	19h 7m	353	2
27	Muse Spark 1.2	0.1011	$11.96	61.9M	415k	4h 37m	241	3
28	Qwen3.8-Max	0.0985	$20.47	70.2M	412k	18h 53m	478	2
29	Gemini 3.7 Flash	0.0948	$9.93	38.7M	111k	7h 46m	266	5
30	Gemini 3.7 Flash	0.0854	$10.11	35.8M	110k	7h 53m	303	1
31	DeepSeek V4 Flash Vision Exp	0.0826	$1.73	93.5M	273k	12h 25m	457	2
32	GLM-5.3	0.0000	$49.90	134.3M	340k	19h 4m	469	21
33	Qwen3.8-Max	0.0000	$13.84	45.7M	331k	19h 21m	336	2
34	Inkling	0.0000	$5.29	5.5M	26k	1h 28m	151	1
35	Qwen3.8-Max	0.0000	$27.96	99.7M	412k	19h 32m	527	4
36	Muse Spark 1.2	0.0000	$13.11	77.0M	195k	5h 0m	202	3
37	Inkling	0.0000	$5.01	5.2M	28k	1h 29m	128	1
38	Gemini 3.7 Flash	0.0000	$12.20	48.7M	173k	7h 50m	285	2
39	Muse Spark 1.2	0.0000	$16.36	73.7M	275k	2h 42m	244	7
40	Inkling	0.0000	$6.23	6.3M	53k	2h 6m	139	1
41	Qwen3.8-Max	0.0000	$8.55	26.2M	281k	19h 56m	242	1
42	Inkling	0.0000	$5.31	5.5M	23k	2h 3m	134	2
43	Inkling	0.0000	$5.42	5.6M	24k	1h 30m	116	1

Wan 2.1 on MAX/Mojo

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9331	$62.26	106.8M	367k	8h 14m	284	8
2	Claude Fable 5.1	0.9314	$52.87	90.3M	319k	5h 49m	274	9
3	Kimi K3	0.9307	$34.42	90.2M	306k	9h 26m	306	5
4	GPT-5.6	0.9241	$103.11	107.4M	168k	5h 28m	369	8
5	Claude Fable 5.1	0.9205	$68.80	118.8M	390k	9h 16m	323	6
6	Kimi K3	0.9196	$29.20	75.3M	345k	8h 0m	286	8
7	Claude Fable 5.1	0.9192	$42.87	72.2M	281k	5h 13m	226	5
8	Claude Fable 5.1	0.9186	$55.53	95.2M	329k	8h 32m	279	10
9	GLM-5.3	0.9144	$51.16	175.7M	465k	14h 14m	461	2
10	GLM-5.3	0.9094	$42.10	140.4M	401k	16h 39m	404	2
11	Kimi K3	0.9090	$15.06	37.9M	188k	7h 2m	204	7
12	GLM-5.3	0.9087	$43.47	155.3M	374k	16h 47m	478	2
13	GPT-5.6	0.9087	$34.04	44.6M	108k	4h 5m	199	9
14	Grok 4.6	0.9085	$149.23	138.7M	1.3M	12h 2m	586	2
15	GPT-5.6	0.9062	$69.52	78.1M	124k	3h 50m	295	3
16	Grok 4.6	0.9047	$29.23	30.9M	270k	3h 12m	158	1
17	GPT-5.6	0.9014	$57.11	66.4M	129k	4h 8m	267	5
18	Grok 4.6	0.9010	$76.51	66.4M	639k	9h 42m	273	2
19	Kimi K3	0.8899	$31.51	84.6M	291k	12h 22m	348	3
20	GPT-5.6	0.8681	$66.06	72.8M	105k	4h 8m	267	4
21	Qwen3.8-Max	0.4407	$34.51	98.7M	1.3M	19h 41m	423	2
22	Qwen3.8-Max	0.4404	$41.19	95.2M	1.6M	20h 1m	427	1
23	DeepSeek V4 Flash Vision Exp	0.4396	$4.92	297.3M	464k	19h 11m	681	11
24	DeepSeek V4 Flash Vision Exp	0.4384	$3.53	204.1M	419k	17h 55m	562	3
25	DeepSeek V4 Flash Vision Exp	0.3550	$2.99	168.7M	386k	19h 32m	453	3
26	Qwen3.8-Max	0.2981	$45.25	145.8M	997k	19h 53m	526	1
27	Kimi K3	0.2838	$85.29	208.6M	954k	19h 56m	558	3
28	DeepSeek V4 Flash Vision Exp	0.2720	$3.61	208.7M	436k	14h 28m	562	11
29	Grok 4.6	0.2716	$127.41	108.5M	1.7M	19h 30m	464	4
30	DeepSeek V4 Flash Vision Exp	0.2506	$3.27	186.8M	413k	17h 5m	493	5
31	Grok 4.6	0.2296	$214.90	198.0M	1.7M	19h 41m	759	13
32	GLM-5.3	0.0945	$60.08	134.7M	351k	20h 1m	432	1
33	Qwen3.8-Max	0.0664	$30.34	85.4M	1.2M	19h 11m	360	2
34	Inkling	0.0429	$17.12	17.9M	30k	2h 11m	192	1
35	Inkling	0.0404	$13.85	14.5M	23k	1h 11m	142	1
36	Gemini 3.7 Flash	0.0349	$23.63	178.6M	776k	6h 26m	569	1
37	Gemini 3.7 Flash	0.0347	$27.48	166.6M	453k	6h 47m	438	1
38	Gemini 3.7 Flash	0.0331	$42.63	248.9M	797k	11h 49m	716	1
39	Muse Spark 1.2	0.0325	$10.93	49.7M	405k	2h 44m	182	3
40	Gemini 3.7 Flash	0.0317	$28.92	149.4M	462k	8h 34m	380	1
41	Gemini 3.7 Flash	0.0310	$36.97	183.1M	560k	8h 49m	483	1
42	Inkling	0.0305	$12.93	13.5M	35k	14m	141	1
43	Muse Spark 1.2	0.0252	$38.37	181.5M	771k	3h 21m	481	13
44	GLM-5.3	0.0239	$77.39	173.5M	511k	19h 46m	464	11
45	Qwen3.8-Max	0.0000	$50.91	123.3M	1.9M	19h 56m	467	1
46	Muse Spark 1.2	0.0000	$29.55	111.8M	325k	3h 53m	331	2
47	Muse Spark 1.2	0.0000†	$16.08	77.6M	385k	3h 9m	254	2
48	Inkling	0.0000	$16.05	16.8M	29k	1h 23m	176	2
49	Muse Spark 1.2	0.0000	$10.80	49.7M	240k	3h 10m	226	3
50	Inkling	0.0000	$11.47	11.9M	30k	1h 18m	145	2

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

MS/MS De Novo Generation

task page →45 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GPT-5.6	0.1316	$55.65	69.9M	170k	12h 29m	332	2
2	GPT-5.6	0.1312	$87.71	101.7M	199k	15h 19m	453	6
3	Kimi K3	0.1101	$21.35	46.7M	229k	17h 35m	282	6
4	Kimi K3	0.1097	$26.90	62.8M	258k	19h 1m	325	7
5	Kimi K3	0.1091	$18.21	39.5M	209k	19h 3m	239	5
6	Gemini 3.7 Flash	0.1064	$8.88	40.2M	172k	5h 7m	265	2
7	GLM-5.3	0.1050	$34.13	75.8M	305k	18h 42m	336	2
8	GLM-5.3	0.1036	$36.94	94.3M	365k	18h 38m	359	3
9	Grok 4.6	0.1023	$289.71	280.9M	1.3M	20h 0m	1109	91
10	GLM-5.3	0.0941	$44.44	143.9M	417k	17h 20m	450	8
11	Qwen3.8-Max	0.0879	$12.07	38.5M	344k	19h 44m	290	3
12	Gemini 3.7 Flash	0.0865	$5.25	20.8M	119k	3h 41m	172	1
13	Gemini 3.7 Flash	0.0846	$10.26	48.6M	211k	7h 0m	264	3
14	Grok 4.6	0.0709	$86.33	77.3M	437k	10h 44m	317	6
15	Qwen3.8-Max	0.0652	$23.10	77.1M	550k	18h 46m	436	2
16	Muse Spark 1.2	0.0557	$16.81	90.3M	275k	5h 28m	342	4
17	Muse Spark 1.2	0.0507	$33.64	201.0M	350k	2h 20m	488	3
18	Inkling	0.0000	$4.78	4.9M	42k	32m	136	1
19	GLM-5.3	0.0000	$28.55	71.5M	295k	19h 12m	332	2
20	GLM-5.3	0.0000	$43.34	125.4M	382k	18h 56m	463	2
21	GPT-5.6	0.0000	$29.44	43.7M	172k	11h 11m	291	4
22	Inkling	0.0000	$9.60	9.7M	91k	1h 37m	187	1
23	Qwen3.8-Max	0.0000	$18.80	59.9M	558k	19h 4m	366	1
24	DeepSeek V4 Flash Vision Exp	0.0000	$1.89	103.3M	298k	19h 46m	435	3
25	DeepSeek V4 Flash Vision Exp	0.0000	$2.64	142.6M	438k	12h 9m	474	5
26	Inkling	0.0000	$4.22	4.3M	40k	1h 20m	125	2
27	Muse Spark 1.2	0.0000	$8.11	42.2M	266k	2h 10m	169	2
28	Qwen3.8-Max	0.0000	$22.77	75.2M	580k	19h 13m	492	3
29	Grok 4.6	0.0000	$51.24	39.8M	426k	15h 47m	243	5
30	Kimi K3	0.0000	$18.66	38.3M	209k	20h 0m	254	4
31	DeepSeek V4 Flash Vision Exp	0.0000	$2.54	140.9M	373k	17h 33m	473	5
32	Qwen3.8-Max	0.0000	$31.11	106.6M	635k	17h 29m	500	3
33	Inkling	0.0000	$9.21	9.4M	67k	2h 19m	235	1
34	GPT-5.6	0.0000	$115.28	137.3M	233k	11h 7m	603	10
35	Kimi K3	0.0000	$31.98	77.8M	334k	19h 25m	339	4
36	DeepSeek V4 Flash Vision Exp	0.0000	$2.76	151.1M	447k	15h 16m	503	6
37	GPT-5.6	0.0000	$33.43	41.9M	177k	7h 37m	257	4
38	Gemini 3.7 Flash	0.0000	$6.93	28.3M	150k	3h 32m	203	4
39	Grok 4.6	0.0000	$192.75	185.2M	1.2M	14h 10m	766	64
40	DeepSeek V4 Flash Vision Exp	0.0000	$2.75	153.9M	409k	18h 4m	510	2
41	Grok 4.6	0.0000	$86.81	76.6M	468k	16h 53m	336	9
42	Gemini 3.7 Flash	0.0000	$15.58	62.6M	304k	13h 22m	285	2
43	Muse Spark 1.2	0.0000	$7.49	39.5M	207k	4h 45m	218	2
44	Muse Spark 1.2	0.0000	$9.64	33.9M	298k	3h 46m	194	5
45	Inkling	0.0000	$6.19	6.3M	42k	29m	120	1

Multi-GPU Efficient Finetuning

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Inkling	0.5238	$11.78	12.1M	59k	1h 35m	166	1
2	Muse Spark 1.2	0.3810	$64.94	126.7M	662k	16h 40m	386	27
3	DeepSeek V4 Flash Vision Exp	0.3333	$0.82	36.7M	207k	19h 58m	238	1
4	Kimi K3	0.3333	$19.15	18.0M	131k	19h 9m	167	6
5	Muse Spark 1.2	0.3095	$11.95	41.8M	358k	12h 44m	199	24
6	Kimi K3	0.3095	$16.56	15.3M	148k	19h 10m	134	3
7	GLM-5.3	0.2857	$12.88	26.8M	144k	19h 0m	234	2
8	Claude Fable 5.1	0.2857	$5.65	7.6M	75k	17h 4m	98	2
9	Qwen3.8-Max	0.2857	$6.42	17.1M	316k	18h 44m	213	5
10	Claude Fable 5.1	0.2857	$5.38	7.9M	60k	16h 16m	118	3
11	GLM-5.3	0.2857	$10.42	18.4M	127k	19h 10m	175	6
12	Muse Spark 1.2	0.2619	$355.31	2070.9M	1.9M	15h 46m	5920	22
13	Claude Fable 5.1	0.2619	$6.12	8.6M	75k	18h 44m	112	4
14	Qwen3.8-Max	0.2619	$6.15	16.6M	296k	17h 38m	220	10
15	GPT-5.6	0.2381	$19.33	23.5M	110k	15h 47m	167	3
16	Claude Fable 5.1	0.2143	$5.26	7.1M	71k	18h 19m	92	3
17	Gemini 3.7 Flash	0.2143	$10.24	29.6M	72k	9h 49m	181	31
18	Muse Spark 1.2	0.1905	$5.52	20.2M	214k	2h 15m	135	3
19	Kimi K3	0.1905	$16.13	11.8M	128k	18h 47m	116	1
20	Gemini 3.7 Flash	0.1667	$11.85	31.1M	102k	19h 53m	202	10
21	Claude Fable 5.1	0.1667	$6.92	9.6M	88k	17h 1m	101	4
22	DeepSeek V4 Flash Vision Exp	0.1429	$0.92	41.7M	226k	19h 50m	251	2
23	GLM-5.3	0.0714	$17.61	33.2M	221k	18h 56m	211	2
24	Muse Spark 1.2	0.0476	$35.64	132.6M	581k	10h 53m	451	13
25	Kimi K3	0.0476	$25.13	35.1M	233k	19h 37m	236	1
26	Inkling	0.0476	$5.35	5.5M	24k	1h 26m	132	2
27	GPT-5.6	0.0238	$19.65	25.0M	71k	12h 53m	133	3
28	Inkling	0.0238	$14.22	14.8M	45k	2h 19m	219	1
29	DeepSeek V4 Flash Vision Exp	0.0238	$0.66	29.1M	168k	19h 41m	226	2
30	Qwen3.8-Max	0.0000	$4.65	11.0M	287k	19h 49m	183	5
31	Gemini 3.7 Flash	0.0000	$22.18	47.8M	107k	18h 31m	299	2
32	GPT-5.6	0.0000	$16.45	25.9M	78k	16h 50m	180	2
33	GLM-5.3	0.0000	$20.26	34.9M	208k	19h 31m	211	4
34	Inkling	0.0000	$6.66	6.8M	46k	2h 1m	135	1
35	GPT-5.6	0.0000	$24.13	34.7M	95k	16h 10m	181	2
36	GPT-5.6	0.0000	$30.41	39.1M	90k	14h 8m	199	3
37	Grok 4.6	0.0000	$47.03	50.4M	373k	19h 22m	261	13
38	Qwen3.8-Max	0.0000	$3.67	9.9M	171k	19h 13m	153	3
39	GLM-5.3	0.0000	$17.67	32.5M	199k	19h 29m	215	2
40	DeepSeek V4 Flash Vision Exp	0.0000	$0.76	37.0M	151k	19h 59m	275	1
41	Qwen3.8-Max	0.0000	$7.44	21.5M	284k	19h 29m	253	2
42	Grok 4.6	0.0000	$18.53	24.3M	327k	17h 14m	194	7
43	DeepSeek V4 Flash Vision Exp	0.0000	$1.23	60.8M	248k	19h 49m	323	1
44	Grok 4.6	0.0000	$32.88	28.8M	249k	18h 12m	196	5
45	Gemini 3.7 Flash	0.0000	$8.30	27.1M	102k	9h 38m	188	10
46	Grok 4.6	0.0000	$23.95	18.0M	271k	19h 15m	145	4
47	Inkling	0.0000	$4.16	4.3M	15k	34m	107	3
48	Kimi K3	0.0000	$10.94	8.7M	101k	18h 40m	121	1
49	Gemini 3.7 Flash	0.0000	$30.31	73.0M	132k	18h 49m	310	13
50	Grok 4.6	0.0000	$20.07	18.4M	217k	18h 24m	156	8

Notebook Compression

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.6002	$67.85	117.2M	379k	14h 26m	401	17
2	Claude Fable 5.1	0.5736	$77.68	134.5M	428k	13h 49m	450	16
3	Claude Fable 5.1	0.5457	$68.00	118.2M	365k	13h 15m	419	21
4	Claude Fable 5.1	0.5279	$64.70	113.1M	334k	17h 24m	419	17
5	Claude Fable 5.1	0.4824	$61.77	107.1M	338k	15h 49m	418	16
6	GLM-5.3	0.2770	$25.86	86.5M	327k	19h 49m	370	3
7	GLM-5.3	0.2744	$34.96	116.4M	391k	19h 2m	435	2
8	Kimi K3	0.2647	$37.66	96.4M	363k	19h 31m	409	17
9	GPT-5.6	0.2623	$123.00	167.4M	279k	11h 19m	628	8
10	GPT-5.6	0.2599	$99.01	115.0M	257k	15h 3m	528	10
11	GPT-5.6	0.2438	$58.48	76.1M	190k	17h 55m	412	14
12	Kimi K3	0.2338	$43.76	111.9M	405k	17h 35m	403	12
13	GPT-5.6	0.2326	$88.19	110.8M	211k	11h 35m	518	6
14	Grok 4.6	0.2286	$85.31	84.1M	498k	7h 48m	331	14
15	Kimi K3	0.2242	$36.27	87.9M	412k	11h 52m	382	12
16	Qwen3.8-Max	0.2190	$26.32	83.3M	818k	19h 14m	454	2
17	Qwen3.8-Max	0.2153	$10.82	27.5M	610k	19h 5m	314	2
18	Kimi K3	0.2151	$24.45	60.7M	274k	11h 45m	302	10
19	Grok 4.6	0.2146	$36.98	34.6M	412k	13h 20m	220	5
20	DeepSeek V4 Flash Vision Exp	0.2139	$1.97	101.8M	387k	13h 26m	420	1
21	Qwen3.8-Max	0.2125	$16.15	47.4M	646k	17h 46m	362	2
22	Grok 4.6	0.2098	$96.38	88.3M	636k	11h 39m	377	12
23	GLM-5.3	0.2089	$39.28	89.8M	362k	20h 0m	369	1
24	Grok 4.6	0.2060	$71.54	75.2M	388k	6h 34m	381	27
25	GLM-5.3	0.2058	$59.65	209.0M	565k	15h 11m	548	8
26	DeepSeek V4 Flash Vision Exp	0.2028	$2.58	144.1M	374k	19h 46m	508	2
27	Grok 4.6	0.2019	$87.92	83.6M	470k	8h 37m	361	10
28	Qwen3.8-Max	0.2005	$15.43	43.7M	693k	18h 44m	414	1
29	Kimi K3	0.1961	$13.59	21.1M	194k	9h 53m	179	9
30	DeepSeek V4 Flash Vision Exp	0.1948	$3.01	169.4M	449k	11h 0m	551	4
31	DeepSeek V4 Flash Vision Exp	0.1939	$2.93	164.0M	435k	14h 31m	526	4
32	Qwen3.8-Max	0.1831	$20.18	58.6M	851k	19h 8m	443	2
33	DeepSeek V4 Flash Vision Exp	0.1818	$2.40	126.7M	415k	10h 56m	421	6
34	Gemini 3.7 Flash	0.1775	$2.95	13.6M	108k	3h 17m	129	2
35	Gemini 3.7 Flash	0.1714	$6.92	27.7M	160k	6h 28m	236	5
36	Muse Spark 1.2	0.1646	$89.68	227.7M	1.2M	20h 2m	728	7
37	GPT-5.6	0.1417	$64.27	80.6M	214k	10h 2m	389	3
38	Muse Spark 1.2	0.1353	$32.24	170.7M	487k	10h 37m	551	5
39	GLM-5.3	0.1261	$28.93	86.9M	409k	13h 16m	328	2
40	Muse Spark 1.2	0.0816	$32.40	192.3M	534k	7h 46m	570	4
41	Gemini 3.7 Flash	0.0417	$2.99	16.0M	112k	3h 8m	165	6
42	Gemini 3.7 Flash	0.0412	$3.05	19.8M	150k	3h 10m	194	10
43	Gemini 3.7 Flash	0.0403	$4.56	17.9M	128k	4h 20m	145	8
44	Muse Spark 1.2	0.0387	$183.05	621.0M	1.3M	19h 7m	1497	7
45	Muse Spark 1.2	0.0356	$37.95	196.7M	1.2M	19h 2m	669	3
46	Inkling	0.0214	$5.30	5.4M	31k	3h 11m	184	2
47	Inkling	0.0209	$6.76	6.9M	44k	2h 24m	193	2
48	Inkling	0.0205	$8.12	8.4M	30k	3h 20m	185	1
49	Inkling	0.0203	$5.44	5.6M	27k	1h 39m	121	3
50	Inkling	0.0179	$5.86	6.0M	36k	2h 37m	166	2

Flight-Sim Renderer in OpenGL

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.4716	$208.91	376.5M	866k	17h 50m	1017	10
2	Claude Fable 5.1	0.3532	$168.41	314.0M	539k	12h 53m	608	4
3	Claude Fable 5.1	0.3481	$106.70	193.6M	429k	9h 8m	454	13
4	Claude Fable 5.1	0.3402	$125.75	229.9M	473k	10h 25m	482	6
5	GPT-5.6	0.2171	$270.86	274.0M	362k	12h 30m	806	4
6	Claude Fable 5.1	0.1985	$113.40	205.2M	463k	5h 28m	430	3
7	GPT-5.6	0.1552	$336.67	340.6M	347k	11h 28m	906	14
8	GPT-5.6	0.1222	$253.61	257.1M	305k	8h 28m	760	3
9	GPT-5.6	0.1119	$309.74	312.3M	342k	9h 22m	857	12
10	GPT-5.6	0.1032	$420.32	406.2M	354k	11h 35m	880	13
11	Grok 4.6	0.0526	$317.62	274.4M	1.8M	19h 55m	1078	110
12	Grok 4.6	0.0382	$286.38	260.8M	1.4M	19h 57m	984	103
13	Grok 4.6	0.0273	$488.84	437.7M	2.0M	19h 56m	1536	307
14	Grok 4.6	0.0266	$454.46	403.4M	2.0M	19h 58m	1435	75
15	Kimi K3	0.0249	$126.07	350.4M	1.1M	19h 56m	936	7
16	Kimi K3	0.0214	$168.60	476.4M	1.3M	20h 1m	1079	5
17	GLM-5.3	0.0166	$91.30	316.9M	852k	19h 35m	891	7
18	Grok 4.6	0.0147	$328.36	283.8M	1.8M	19h 57m	1084	57
19	Gemini 3.7 Flash	0.0116	$11.28	68.7M	237k	4h 7m	304	2
20	Kimi K3	0.0104	$167.00	460.9M	1.4M	20h 1m	1039	1
21	Kimi K3	0.0079	$184.18	516.4M	1.3M	19h 37m	1094	2
22	DeepSeek V4 Flash Vision Exp	0.0069	$10.96	663.9M	1.1M	19h 2m	1527	2
23	DeepSeek V4 Flash Vision Exp	0.0065	$10.60	633.4M	1.1M	15h 56m	1516	8
24	Gemini 3.7 Flash	0.0064	$35.75	195.3M	360k	8h 28m	555	1
25	GLM-5.3	0.0054	$111.06	362.5M	755k	19h 29m	893	10
26	Gemini 3.7 Flash	0.0054	$6.52	51.3M	224k	2h 40m	274	2
27	GLM-5.3	0.0038	$97.66	297.0M	573k	19h 4m	633	5
28	DeepSeek V4 Flash Vision Exp	0.0028	$12.40	756.6M	1.2M	19h 8m	1756	39
29	DeepSeek V4 Flash Vision Exp	0.0027	$6.77	415.2M	638k	18h 28m	837	7
30	Gemini 3.7 Flash	0.0020	$17.05	105.9M	300k	5h 32m	381	1
31	Qwen3.8-Max	0.0018	$183.02	558.1M	1.7M	19h 56m	1375	1
32	Qwen3.8-Max	0.0016	$153.43	448.8M	1.7M	19h 59m	1202	1
33	Qwen3.8-Max	0.0010	$149.46	419.0M	1.2M	19h 25m	993	2
34	Gemini 3.7 Flash	0.0009	$11.28	77.4M	230k	5h 1m	369	3
35	GLM-5.3	0.0006	$94.18	300.9M	621k	18h 57m	592	10
36	Qwen3.8-Max	0.0001	$169.64	519.3M	1.6M	20h 0m	1239	1
37	Muse Spark 1.2	0.0000	$7.77	40.8M	287k	2h 9m	182	5
38	Inkling	0.0000	$6.72	7.0M	25k	22m	138	2
39	Qwen3.8-Max	0.0000	$189.91	559.5M	1.9M	19h 51m	1464	3
40	Kimi K3	0.0000	$182.81	498.0M	1.4M	20h 1m	1063	2
41	Muse Spark 1.2	0.0000	$8.07	38.9M	412k	3h 7m	202	4
42	DeepSeek V4 Flash Vision Exp	0.0000	$11.01	666.7M	1.1M	19h 11m	1467	11
43	Muse Spark 1.2	0.0000	$6.30	24.2M	135k	1h 37m	159	4
44	GLM-5.3	0.0000	$103.09	375.1M	554k	19h 46m	790	12
45	Inkling	0.0000	$6.59	6.8M	31k	2h 10m	146	2
46	Muse Spark 1.2	0.0000†	$2.64	12.4M	42k	2h 25m	90	2
47	Inkling	0.0000	$6.57	6.8M	31k	35m	156	1
48	Inkling	0.0000	$4.24	4.4M	17k	41m	109	1
49	Inkling	0.0000	$8.87	9.2M	43k	1h 49m	170	2
50	Muse Spark 1.2	0.0000†	$1.72	8.1M	39k	1h 17m	88	1

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Optimizer Design

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GPT-5.6	0.3487	$65.53	80.8M	133k	10h 22m	433	4
2	GPT-5.6	0.3194	$196.66	203.2M	202k	11h 49m	678	23
3	Kimi K3	0.3057	$15.06	37.2M	181k	19h 11m	275	4
4	GPT-5.6	0.2846	$55.29	72.7M	133k	15h 42m	423	5
5	GPT-5.6	0.2809	$71.65	93.2M	177k	14h 22m	523	21
6	Grok 4.6	0.2557	$99.14	98.0M	624k	19h 20m	424	18
7	GLM-5.3	0.2349	$19.98	35.0M	224k	17h 41m	192	3
8	Claude Fable 5.1	0.2272	$15.45	22.4M	175k	11h 13m	164	5
9	GPT-5.6	0.2234	$31.29	52.9M	100k	13h 17m	373	2
10	GLM-5.3	0.2192	$17.37	36.9M	195k	19h 6m	241	2
11	GLM-5.3	0.2082	$16.48	26.8M	198k	19h 23m	184	3
12	Grok 4.6	0.1957	$53.90	66.7M	503k	19h 24m	395	24
13	Claude Fable 5.1	0.1750	$17.15	26.3M	165k	12h 46m	182	6
14	Grok 4.6	0.1570	$59.53	57.5M	295k	19h 26m	258	11
15	Kimi K3	0.1475	$41.51	100.0M	508k	20h 1m	354	4
16	Claude Fable 5.1	0.1459	$14.90	22.4M	152k	15h 9m	171	4
17	Kimi K3	0.1370	$16.05	38.0M	228k	19h 25m	222	4
18	Grok 4.6	0.1368	$54.49	49.3M	462k	19h 16m	240	10
19	Claude Fable 5.1	0.1343	$11.08	15.3M	141k	13h 28m	134	3
20	Gemini 3.7 Flash	0.1194	$22.13	241.0M	134k	10h 36m	834	2
21	DeepSeek V4 Flash Vision Exp	0.1123	$1.12	52.3M	255k	18h 23m	256	2
22	Kimi K3	0.0990	$22.58	49.0M	297k	17h 53m	241	6
23	GLM-5.3	0.0984	$17.01	55.5M	261k	19h 4m	277	5
24	GLM-5.3	0.0965	$13.47	23.0M	159k	18h 59m	180	2
25	Grok 4.6	0.0839	$80.45	75.7M	333k	19h 42m	279	3
26	Qwen3.8-Max	0.0827	$7.68	17.4M	516k	19h 49m	226	3
27	DeepSeek V4 Flash Vision Exp	0.0770	$1.22	63.5M	222k	19h 1m	334	2
28	Qwen3.8-Max	0.0741	$7.61	18.8M	448k	19h 35m	241	3
29	Qwen3.8-Max	0.0655	$6.87	15.9M	443k	19h 59m	203	3
30	Gemini 3.7 Flash	0.0537	$10.37	85.1M	122k	13h 22m	435	2
31	Gemini 3.7 Flash	0.0405	$25.34	217.9M	182k	10h 29m	744	2
32	DeepSeek V4 Flash Vision Exp	0.0348	$1.44	75.8M	242k	19h 19m	349	2
33	Claude Fable 5.1	0.0226	$15.74	21.7M	202k	14h 5m	141	4
34	Qwen3.8-Max	0.0143	$7.32	18.3M	421k	19h 31m	238	1
35	Muse Spark 1.2	0.0000	$39.25	85.0M	351k	13h 51m	275	7
36	Muse Spark 1.2	0.0000	$46.70	116.8M	475k	9h 50m	364	8
37	DeepSeek V4 Flash Vision Exp	0.0000	$1.51	79.4M	262k	19h 14m	406	2
38	Muse Spark 1.2	0.0000	$55.59	163.2M	672k	19h 37m	567	1
39	Gemini 3.7 Flash	0.0000	$28.88	259.5M	185k	18h 35m	908	1
40	Gemini 3.7 Flash	0.0000	$12.41	47.0M	144k	13h 48m	297	2
41	DeepSeek V4 Flash Vision Exp	0.0000	$1.33	65.8M	275k	19h 51m	313	1
42	Muse Spark 1.2	0.0000	$27.04	48.5M	345k	6h 22m	256	3
43	Muse Spark 1.2	0.0000	$16.21	19.8M	148k	19h 57m	188	1
44	Inkling	0.0000	$1.08	1.1M	14k	1h 4m	58	1
45	Kimi K3	0.0000	$19.05	42.1M	297k	14h 45m	219	3
46	Inkling	0.0000	$1.53	1.5M	16k	1h 13m	73	2
47	Qwen3.8-Max	0.0000	$12.42	33.0M	636k	19h 59m	303	1
48	Inkling	0.0000	$3.84	3.9M	25k	5h 18m	124	1
49	Inkling	0.0000	$2.54	2.6M	28k	2h 1m	88	1
50	Inkling	0.0000	$1.63	1.6M	27k	1h 46m	73	1

PostgreSQL 18 on SQLite

task page →49 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GPT-5.6	0.5275	$209.42	198.9M	277k	4h 5m	589	12
2	Claude Fable 5.1	0.4011	$588.09	1056.3M	2.5M	17h 46m	2515	13
3	Claude Fable 5.1	0.3901	$721.01	1319.3M	2.5M	17h 44m	2872	14
4	GPT-5.6	0.3901	$654.39	640.1M	687k	19h 58m	1472	28
5	Claude Fable 5.1	0.3516	$384.99	680.6M	1.8M	18h 59m	1688	9
6	Claude Fable 5.1	0.3352	$475.86	850.2M	2.1M	18h 4m	1894	7
7	Gemini 3.7 Flash	0.2582	$124.97	1333.8M	2.5M	11h 51m	3014	37
8	Gemini 3.7 Flash	0.2418	$144.25	1592.6M	2.6M	19h 46m	3450	39
9	Claude Fable 5.1	0.2308	$392.90	697.7M	1.8M	19h 16m	1679	4
10	Gemini 3.7 Flash	0.2253	$193.89	2112.9M	3.0M	19h 55m	4680	16
11	Gemini 3.7 Flash	0.1758	$159.99	1689.8M	2.8M	19h 49m	3751	13
12	Grok 4.6	0.1593	$509.89	496.5M	1.6M	19h 56m	1696	70
13	GLM-5.3	0.1593	$191.86	711.5M	862k	19h 53m	1746	17
14	Grok 4.6	0.1538	$461.37	465.7M	1.9M	17h 59m	1842	49
15	DeepSeek V4 Flash Vision Exp	0.1484	$22.50	1379.3M	2.0M	19h 53m	3146	1
16	GLM-5.3	0.1484	$202.20	744.7M	1.0M	19h 42m	1789	7
17	GLM-5.3	0.1374	$201.89	706.7M	843k	19h 32m	1585	9
18	GLM-5.3	0.1374	$155.77	570.5M	772k	19h 52m	1384	2
19	Qwen3.8-Max	0.1374	$186.86	700.0M	1.6M	19h 30m	1822	2
20	Qwen3.8-Max	0.1319	$178.70	667.7M	1.7M	19h 47m	1549	4
21	DeepSeek V4 Flash Vision Exp	0.1264	$22.61	1383.3M	2.0M	19h 51m	3272	3
22	DeepSeek V4 Flash Vision Exp	0.1044	$19.66	1214.6M	1.7M	19h 25m	2688	9
23	Grok 4.6	0.0989	$257.56	255.7M	1.5M	8h 44m	952	17
24	GLM-5.3	0.0989	$219.76	822.5M	966k	19h 50m	1903	11
25	Qwen3.8-Max	0.0769	$170.86	638.4M	1.5M	19h 53m	1587	2
26	Grok 4.6	0.0714	$398.74	383.4M	2.7M	19h 51m	1457	34
27	Kimi K3	0.0714	$227.56	673.8M	1.2M	20h 0m	1528	1
28	DeepSeek V4 Flash Vision Exp	0.0659	$23.23	1428.3M	2.0M	19h 44m	3348	1
29	Qwen3.8-Max	0.0659	$174.14	653.5M	1.5M	19h 26m	1536	1
30	Kimi K3	0.0659	$265.37	772.2M	1.4M	20h 1m	1427	1
31	Kimi K3	0.0604	$238.20	702.8M	1.4M	20h 1m	1513	1
32	Qwen3.8-Max	0.0549	$158.21	583.9M	1.7M	19h 23m	1405	3
33	Kimi K3	0.0495	$197.52	561.9M	1.3M	20h 0m	1168	1
34	Grok 4.6	0.0495	$148.44	141.0M	985k	6h 17m	559	10
35	Gemini 3.7 Flash	0.0495	$17.96	182.1M	620k	1h 34m	402	2
36	DeepSeek V4 Flash Vision Exp	0.0165	$8.18	489.5M	856k	20h 0m	1194	2
37	Muse Spark 1.2	0.0055	$37.82	217.7M	976k	3h 48m	496	10
38	Muse Spark 1.2	0.0055	$40.72	146.5M	561k	3h 14m	420	4
39	GPT-5.6	0.0055†	$293.05	285.8M	311k	4h 20m	671	6
40	Muse Spark 1.2	0.0000	$32.40	176.5M	473k	2h 53m	436	3
41	Inkling	0.0000	$22.34	23.3M	56k	1h 53m	340	3
42	Muse Spark 1.2	0.0000	$14.08	83.6M	240k	1h 8m	314	2
43	GPT-5.6	0.0000	$137.59	141.5M	243k	3h 27m	480	2
44	GPT-5.6	0.0000	$142.29	142.1M	259k	4h 34m	462	11
45	Inkling	0.0000	$19.06	19.8M	53k	1h 41m	302	2
46	Inkling	0.0000	$19.04	19.9M	46k	23m	252	1
47	Inkling	0.0000	$16.56	17.2M	48k	32m	214	2
48	Inkling	0.0000	$16.62	17.3M	42k	34m	287	1
49	Muse Spark 1.2	0.0000	$33.19	162.4M	603k	2h 4m	458	1

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Quantum ESPRESSO pw.x in Rust

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9431	$100.68	158.8M	876k	5h 35m	408	11
2	Claude Fable 5.1	0.9350	$63.80	99.1M	584k	3h 4m	218	8
3	Claude Fable 5.1	0.9268	$63.95	99.7M	578k	3h 23m	222	5
4	Claude Fable 5.1	0.9187	$72.59	116.9M	580k	4h 7m	245	5
5	Claude Fable 5.1	0.9106	$77.90	124.4M	643k	3h 55m	256	4
6	GPT-5.6	0.5203	$304.21	296.3M	260k	4h 50m	606	9
7	GPT-5.6	0.4634	$57.42	64.9M	189k	2h 47m	279	13
8	GPT-5.6	0.4553	$120.99	116.9M	214k	5h 5m	404	8
9	GPT-5.6	0.4065	$104.97	107.5M	204k	3h 16m	391	10
10	Grok 4.6	0.3984	$196.95	181.0M	1.6M	13h 51m	681	14
11	Grok 4.6	0.3008	$282.67	273.1M	1.6M	13h 49m	976	40
12	Grok 4.6	0.2683	$1211.08	1191.6M	2.2M	19h 59m	3559	243
13	Kimi K3	0.2602	$75.00	147.3M	1.2M	19h 51m	428	1
14	GLM-5.3	0.1707	$145.94	507.1M	1.3M	19h 30m	1082	13
15	Gemini 3.7 Flash	0.1545	$46.71	314.3M	747k	10h 28m	740	2
16	GLM-5.3	0.1382	$132.96	476.8M	1.3M	19h 42m	984	12
17	GLM-5.3	0.1301	$108.12	382.4M	1.3M	19h 32m	838	2
18	Grok 4.6	0.1057	$123.16	111.5M	1.7M	10h 28m	429	13
19	Grok 4.6	0.0976	$96.14	88.3M	1.1M	7h 19m	360	12
20	Gemini 3.7 Flash	0.0976	$71.79	423.8M	777k	17h 50m	948	3
21	Gemini 3.7 Flash	0.0650	$36.49	288.8M	712k	6h 10m	725	2
22	GLM-5.3	0.0650	$123.53	403.7M	1.6M	19h 19m	957	11
23	Kimi K3	0.0488	$160.13	415.5M	1.7M	20h 0m	999	1
24	Muse Spark 1.2	0.0244	$48.27	287.1M	1.0M	3h 0m	598	1
25	Kimi K3	0.0244	$100.42	211.3M	1.8M	20h 0m	566	1
26	Kimi K3	0.0081	$131.87	355.0M	1.3M	19h 56m	785	6
27	Qwen3.8-Max	0.0000	$48.53	146.5M	1.8M	19h 49m	431	1
28	DeepSeek V4 Flash Vision Exp	0.0000	$7.62	422.1M	1.1M	12h 5m	912	6
29	Gemini 3.7 Flash	0.0000	$51.00	393.0M	804k	11h 3m	1002	4
30	Kimi K3	0.0000	$72.76	158.2M	1.2M	20h 1m	418	1
31	Muse Spark 1.2	0.0000	$22.08	58.0M	685k	1h 30m	194	3
32	Qwen3.8-Max	0.0000	$54.35	159.6M	2.3M	19h 59m	413	1
33	Qwen3.8-Max	0.0000	$33.37	86.6M	1.8M	19h 52m	301	2
34	Gemini 3.7 Flash	0.0000	$24.96	238.1M	690k	3h 15m	539	2
35	Inkling	0.0000	$18.68	19.4M	66k	28m	186	2
36	DeepSeek V4 Flash Vision Exp	0.0000	$6.14	336.4M	959k	16h 29m	853	10
37	Inkling	0.0000	$22.94	24.0M	46k	17m	274	1
38	DeepSeek V4 Flash Vision Exp	0.0000	$4.17	227.6M	660k	6h 35m	447	3
39	GPT-5.6	0.0000	$210.32	203.5M	250k	6h 57m	577	6
40	Inkling	0.0000	$8.19	8.5M	35k	16m	142	1
41	Muse Spark 1.2	0.0000	$8.00	39.5M	393k	1h 21m	166	2
42	Inkling	0.0000	$9.25	9.5M	49k	17m	146	2
43	DeepSeek V4 Flash Vision Exp	0.0000	$6.64	359.4M	1.1M	12h 22m	882	5
44	GLM-5.3	0.0000	$103.68	348.9M	1.2M	19h 23m	795	14
45	DeepSeek V4 Flash Vision Exp	0.0000	$6.97	389.0M	1.0M	14h 29m	887	5
46	Muse Spark 1.2	0.0000	$5.45	30.5M	131k	29m	160	3
47	Qwen3.8-Max	0.0000	$35.43	95.7M	1.8M	20h 1m	338	1
48	Inkling	0.0000	$9.32	9.6M	44k	15m	138	2
49	Muse Spark 1.2	0.0000	$10.30	54.9M	392k	1h 20m	248	4
50	Qwen3.8-Max	0.0000	$11.62	13.4M	1.3M	19h 53m	112	1

Qubit Routing

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.8857	$37.59	60.9M	295k	9h 18m	239	6
2	Claude Fable 5.1	0.8752	$40.65	67.2M	290k	7h 39m	274	9
3	Claude Fable 5.1	0.8680	$40.94	67.5M	295k	10h 23m	279	9
4	Claude Fable 5.1	0.8599	$41.23	67.7M	303k	9h 29m	256	9
5	Claude Fable 5.1	0.8541	$59.53	101.1M	368k	11h 5m	353	7
6	GPT-5.6	0.7946	$58.77	77.9M	203k	6h 45m	423	8
7	GPT-5.6	0.7781	$75.11	86.9M	202k	10h 41m	410	11
8	Gemini 3.7 Flash	0.7591	$2.33	9.2M	139k	1h 43m	99	2
9	GLM-5.3	0.7535	$23.46	64.7M	277k	19h 3m	267	6
10	Kimi K3	0.7526	$40.44	107.3M	401k	19h 37m	387	17
11	Grok 4.6	0.7350	$123.13	116.9M	502k	9h 53m	414	28
12	GLM-5.3	0.7343	$45.48	90.2M	303k	16h 27m	365	4
13	GPT-5.6	0.7314	$114.07	131.6M	228k	8h 56m	569	14
14	GPT-5.6	0.7190	$154.11	169.8M	283k	6h 40m	688	13
15	Kimi K3	0.7151	$55.15	146.5M	497k	20h 0m	484	14
16	GPT-5.6	0.7125	$69.33	92.1M	185k	8h 7m	475	5
17	Kimi K3	0.6987	$79.07	224.2M	641k	20h 1m	556	12
18	GLM-5.3	0.6907	$44.40	135.6M	383k	15h 8m	392	12
19	Grok 4.6	0.6883	$68.03	60.3M	544k	3h 28m	249	6
20	Qwen3.8-Max	0.6875	$16.94	52.6M	563k	18h 44m	340	2
21	Grok 4.6	0.6689	$82.90	75.9M	477k	3h 6m	283	17
22	Gemini 3.7 Flash	0.6672	$3.04	15.3M	126k	1h 37m	140	1
23	Grok 4.6	0.6619	$46.47	48.9M	429k	4h 33m	274	7
24	Gemini 3.7 Flash	0.6446	$3.44	16.9M	214k	2h 28m	151	2
25	GLM-5.3	0.6370	$18.59	55.0M	231k	19h 13m	287	3
26	Kimi K3	0.6270	$30.79	76.7M	394k	12h 20m	318	20
27	Kimi K3	0.6254	$97.66	258.3M	819k	19h 58m	576	17
28	DeepSeek V4 Flash Vision Exp	0.6244	$2.53	133.2M	457k	10h 50m	402	7
29	Muse Spark 1.2	0.6114	$9.23	47.6M	407k	3h 16m	262	4
30	Gemini 3.7 Flash	0.6074	$16.26	72.9M	429k	6h 43m	280	5
31	GLM-5.3	0.6071	$32.20	106.0M	372k	12h 23m	375	5
32	Muse Spark 1.2	0.5947	$9.36	45.3M	339k	3h 54m	244	2
33	Grok 4.6	0.5924	$84.01	82.0M	496k	4h 20m	336	10
34	Muse Spark 1.2	0.5914	$9.21	41.8M	337k	3h 53m	230	2
35	Gemini 3.7 Flash	0.5912	$5.63	44.5M	188k	5h 26m	270	1
36	Muse Spark 1.2	0.5787	$10.64	52.8M	389k	2h 30m	272	8
37	Qwen3.8-Max	0.5728	$25.90	85.8M	648k	18h 2m	442	6
38	Muse Spark 1.2	0.5545	$7.70	42.7M	226k	2h 46m	257	5
39	DeepSeek V4 Flash Vision Exp	0.5485	$2.27	120.4M	400k	9h 24m	405	3
40	DeepSeek V4 Flash Vision Exp	0.4883	$3.26	177.3M	519k	9h 8m	426	7
41	DeepSeek V4 Flash Vision Exp	0.4768	$2.33	127.1M	362k	12h 24m	407	4
42	Qwen3.8-Max	0.4459	$25.19	78.5M	819k	19h 40m	374	2
43	DeepSeek V4 Flash Vision Exp	0.4348	$2.35	127.0M	382k	9h 0m	395	7
44	Inkling	0.4268	$4.27	4.3M	39k	1h 28m	101	2
45	Qwen3.8-Max	0.4074	$19.97	63.7M	596k	17h 56m	390	2
46	Qwen3.8-Max	0.2765	$43.05	138.5M	1.3M	20h 1m	512	1
47	Inkling	0.2712	$3.61	3.7M	31k	1h 49m	90	2
48	Inkling	0.2062	$7.66	7.9M	44k	1h 46m	158	2
49	Inkling	0.1413	$4.71	4.8M	39k	39m	98	2
50	Inkling	0.0057	$3.81	3.8M	60k	1h 38m	95	1

Reconnaissance Blind Chess Recovery

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9536	$56.70	98.3M	311k	9h 23m	314	7
2	Claude Fable 5.1	0.9536	$46.08	78.9M	273k	10h 59m	276	9
3	Claude Fable 5.1	0.9429	$31.38	51.1M	239k	11h 53m	233	6
4	GPT-5.6	0.8893	$28.42	45.1M	149k	6h 33m	291	7
5	GPT-5.6	0.8607	$35.99	57.3M	138k	6h 13m	336	13
6	Claude Fable 5.1	0.8357	$41.84	71.9M	245k	9h 32m	252	11
7	GPT-5.6	0.8071	$20.62	32.5M	99k	10h 50m	196	10
8	GPT-5.6	0.8000	$20.53	30.5M	123k	2h 52m	197	3
9	Claude Fable 5.1	0.7929	$38.22	64.0M	257k	7h 37m	245	4
10	Kimi K3	0.7571	$29.23	77.1M	277k	14h 20m	314	2
11	GPT-5.6	0.7357	$15.51	21.1M	106k	11h 38m	168	9
12	Grok 4.6	0.6036	$87.29	70.2M	604k	8h 33m	259	9
13	Grok 4.6	0.5857	$104.51	99.3M	819k	14h 17m	453	24
14	GLM-5.3	0.5357	$48.03	138.0M	410k	14h 33m	388	9
15	Kimi K3	0.5071	$31.51	82.4M	325k	14h 46m	311	4
16	Kimi K3	0.4286	$27.43	64.3M	286k	14h 50m	256	11
17	Gemini 3.7 Flash	0.4250	$12.09	51.0M	293k	4h 11m	266	3
18	GLM-5.3	0.4214	$29.52	79.9M	302k	14h 26m	305	6
19	Grok 4.6	0.4071	$107.37	99.7M	852k	13h 36m	443	17
20	Muse Spark 1.2	0.4036	$5.41	24.7M	331k	4h 44m	150	2
21	Muse Spark 1.2	0.4036	$6.73	30.6M	391k	4h 23m	132	4
22	Muse Spark 1.2	0.3750	$11.18	57.5M	431k	6h 54m	254	6
23	GLM-5.3	0.3607	$36.05	73.5M	339k	13h 30m	259	4
24	Muse Spark 1.2	0.3429	$9.01	40.4M	451k	7h 47m	197	4
25	Kimi K3	0.3143	$28.91	69.9M	341k	12h 50m	253	2
26	Gemini 3.7 Flash	0.3000	$23.11	214.3M	578k	5h 22m	524	1
27	GLM-5.3	0.2893	$25.26	83.8M	284k	14h 38m	339	4
28	Grok 4.6	0.2786	$78.34	65.1M	979k	10h 29m	301	8
29	GLM-5.3	0.2571	$27.02	69.8M	315k	14h 43m	253	6
30	Gemini 3.7 Flash	0.2393	$20.54	111.4M	562k	4h 43m	327	1
31	Muse Spark 1.2	0.2071	$9.69	39.5M	748k	4h 22m	138	7
32	Inkling	0.2000	$1.23	1.2M	15k	1h 37m	46	1
33	DeepSeek V4 Flash Vision Exp	0.1786	$2.02	107.0M	337k	14h 34m	328	2
34	Kimi K3	0.1429	$43.61	116.1M	397k	14h 40m	395	2
35	Qwen3.8-Max	0.1357	$20.20	59.6M	794k	13h 51m	345	3
36	DeepSeek V4 Flash Vision Exp	0.1214	$2.99	164.7M	448k	8h 44m	423	5
37	Grok 4.6	0.0929	$172.71	159.1M	1.1M	14h 32m	609	14
38	Inkling	0.0786	$1.37	1.4M	14k	43m	62	1
39	Qwen3.8-Max	0.0750	$16.70	49.8M	638k	13h 27m	335	3
40	Qwen3.8-Max	0.0536	$12.92	35.1M	623k	13h 47m	238	3
41	Gemini 3.7 Flash	0.0536	$30.51	186.7M	1.3M	5h 45m	513	2
42	Qwen3.8-Max	0.0536	$22.79	70.5M	780k	13h 7m	415	3
43	Gemini 3.7 Flash	0.0429	$30.88	192.0M	568k	8h 45m	506	2
44	Inkling	0.0393	$5.27	5.4M	45k	1h 49m	127	2
45	DeepSeek V4 Flash Vision Exp	0.0357	$2.71	145.6M	448k	14h 17m	377	7
46	DeepSeek V4 Flash Vision Exp	0.0143	$2.49	130.6M	440k	11h 4m	400	10
47	Inkling	0.0143	$2.10	2.1M	21k	28m	83	1
48	Qwen3.8-Max	0.0071	$26.05	82.8M	805k	13h 22m	436	3
49	DeepSeek V4 Flash Vision Exp	0.0036	$2.74	145.9M	463k	9h 28m	372	6
50	Inkling	0.0000	$3.91	4.0M	36k	2h 12m	98	2

Fitness-Recap Video in Remotion

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	GPT-5.6	0.2152	$950.51	938.4M	1.0M	19h 3m	2343	45
2	Claude Fable 5.1	0.2055	$157.69	293.5M	491k	6h 51m	665	8
3	Claude Fable 5.1	0.2018	$157.38	293.1M	476k	6h 7m	644	7
4	GPT-5.6	0.1930	$656.93	636.3M	650k	19h 12m	1478	11
5	Claude Fable 5.1	0.1927	$120.12	223.7M	414k	5h 14m	587	7
6	GPT-5.6	0.1913	$298.28	288.7M	261k	7h 23m	651	12
7	GPT-5.6	0.1895	$283.99	275.5M	274k	5h 43m	687	6
8	GLM-5.3	0.1894	$78.50	212.7M	442k	18h 20m	530	6
9	GLM-5.3	0.1828	$113.14	407.8M	483k	19h 2m	804	9
10	Kimi K3	0.1767	$191.75	571.0M	982k	19h 53m	1356	3
11	Claude Fable 5.1	0.1730	$560.60	1048.2M	1.5M	18h 2m	2274	26
12	Kimi K3	0.1723	$171.83	504.1M	1.1M	19h 44m	1234	8
13	Kimi K3	0.1716	$266.42	802.2M	1.3M	20h 1m	1618	1
14	DeepSeek V4 Flash Vision Exp	0.1621	$10.73	658.6M	994k	19h 13m	1651	18
15	Qwen3.8-Max	0.1474	$77.77	271.1M	994k	13h 7m	800	4
16	GLM-5.3	0.1426	$118.44	380.0M	784k	18h 55m	986	18
17	GPT-5.6	0.1374	$190.12	189.9M	230k	5h 21m	589	2
18	GLM-5.3	0.1053	$110.52	388.9M	908k	15h 36m	961	9
19	GLM-5.3	0.0837	$90.93	307.2M	485k	16h 10m	636	8
20	DeepSeek V4 Flash Vision Exp	0.0738	$6.53	405.2M	565k	19h 26m	994	10
21	Qwen3.8-Max	0.0733	$163.48	458.4M	1.7M	19h 54m	1249	1
22	Claude Fable 5.1	0.0694	$335.29	627.1M	960k	15h 45m	1456	19
23	Kimi K3	0.0657	$243.01	727.7M	1.1M	20h 1m	1633	1
24	Kimi K3	0.0536	$202.63	587.9M	1.0M	19h 57m	1479	12
25	DeepSeek V4 Flash Vision Exp	0.0483	$6.64	411.0M	525k	16h 0m	983	21
26	Grok 4.6	0.0338	$1762.94	1204.0M	2.8M	20h 0m	4175	289
27	DeepSeek V4 Flash Vision Exp	0.0324	$12.46	767.8M	1.1M	18h 4m	1789	24
28	Grok 4.6	0.0279	$646.10	557.5M	1.5M	19h 48m	2043	149
29	Grok 4.6	0.0245	$555.19	490.3M	1.5M	19h 56m	2020	90
30	Grok 4.6	0.0242	$1149.83	792.5M	1.8M	19h 58m	2836	250
31	Grok 4.6	0.0216	$746.94	531.5M	1.4M	19h 55m	1941	163
32	Gemini 3.7 Flash	0.0129	$13.97	114.7M	211k	4h 23m	382	2
33	Qwen3.8-Max	0.0127	$196.38	482.4M	1.8M	19h 51m	1048	1
34	DeepSeek V4 Flash Vision Exp	0.0121	$4.01	242.4M	395k	12h 19m	777	16
35	Gemini 3.7 Flash	0.0091	$2.94	21.2M	86k	1h 20m	169	2
36	Gemini 3.7 Flash	0.0089	$10.44	67.7M	164k	2h 48m	328	4
37	Gemini 3.7 Flash	0.0080	$7.68	61.7M	148k	2h 7m	300	1
38	Muse Spark 1.2	0.0079	$12.46	62.2M	180k	3h 28m	270	3
39	Qwen3.8-Max	0.0062	$52.83	157.6M	675k	9h 28m	578	4
40	Muse Spark 1.2	0.0060	$91.70	184.1M	1.3M	8h 42m	509	3
41	Gemini 3.7 Flash	0.0060	$4.67	30.1M	112k	2h 7m	222	1
42	Muse Spark 1.2	0.0032	$14.71	29.2M	224k	3h 30m	183	2
43	Muse Spark 1.2	0.0031	$37.85	75.3M	303k	4h 52m	285	4
44	Inkling	0.0006	$14.05	14.6M	38k	1h 0m	183	1
45	Inkling	0.0001	$7.96	8.3M	26k	31m	141	2
46	Inkling	0.0000	$13.72	14.2M	49k	59m	205	1
47	Inkling	0.0000	$10.65	11.1M	29k	1h 18m	172	2
48	Inkling	0.0000	$11.15	11.6M	44k	40m	138	1
49	Qwen3.8-Max	0.0000†	$11.91	31.2M	238k	4h 2m	292	2
50	Muse Spark 1.2	0.0000†	$12.66	38.1M	209k	4h 41m	190	1

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Snooker Prediction

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.5126	$87.17	154.8M	404k	6h 52m	408	12
2	Claude Fable 5.1	0.4984	$96.24	173.0M	403k	7h 34m	486	21
3	GPT-5.6	0.4857	$119.18	111.7M	242k	2h 5m	299	11
4	GPT-5.6	0.4839	$170.16	165.0M	275k	4h 12m	432	9
5	Claude Fable 5.1	0.4822	$86.06	152.3M	409k	8h 44m	402	23
6	Claude Fable 5.1	0.4767	$88.31	156.7M	413k	3h 39m	397	10
7	GPT-5.6	0.4428	$157.87	150.8M	208k	2h 34m	354	9
8	GPT-5.6	0.4295	$132.32	126.4M	240k	4h 1m	369	6
9	Grok 4.6	0.3789	$466.11	391.2M	2.0M	19h 43m	1488	73
10	Claude Fable 5.1	0.3738	$86.02	153.2M	388k	5h 28m	424	9
11	Grok 4.6	0.3048	$106.12	86.0M	725k	5h 58m	378	5
12	GPT-5.6	0.2890	$179.27	154.6M	213k	9h 23m	357	5
13	GLM-5.3	0.1976	$71.12	251.5M	765k	18h 56m	654	15
14	Grok 4.6	0.1779	$108.45	87.9M	851k	6h 23m	344	15
15	Grok 4.6	0.1175	$125.27	104.6M	768k	11h 27m	446	9
16	GLM-5.3	0.1013	$57.78	201.7M	793k	14h 16m	531	13
17	Qwen3.8-Max	0.0882	$56.82	135.8M	1.2M	18h 54m	506	1
18	GLM-5.3	0.0590	$53.37	163.6M	430k	17h 18m	373	6
19	Inkling	0.0335	$2.61	2.6M	26k	1h 16m	72	2
20	GLM-5.3	0.0311	$89.97	246.3M	519k	19h 15m	495	13
21	Grok 4.6	0.0252	$77.16	60.3M	816k	4h 29m	271	13
22	Inkling	0.0203	$2.91	2.9M	46k	1h 53m	87	2
23	Kimi K3	0.0000	$126.53	346.0M	738k	19h 31m	778	8
24	Qwen3.8-Max	0.0000	$80.65	199.9M	2.1M	18h 39m	673	1
25	Gemini 3.7 Flash	0.0000	$7.82	66.5M	374k	1h 5m	279	1
26	Gemini 3.7 Flash	0.0000	$8.33	59.1M	343k	2h 40m	282	3
27	Inkling	0.0000	$3.24	3.2M	40k	1h 45m	103	1
28	Gemini 3.7 Flash	0.0000	$5.61	40.2M	288k	59m	224	2
29	Muse Spark 1.2	0.0000	$26.92	100.5M	603k	1h 48m	377	5
30	DeepSeek V4 Flash Vision Exp	0.0000	$4.83	277.6M	638k	5h 11m	644	13
31	Muse Spark 1.2	0.0000	$12.08	49.7M	275k	1h 6m	250	8
32	DeepSeek V4 Flash Vision Exp	0.0000	$3.23	179.6M	479k	19h 58m	466	1
33	DeepSeek V4 Flash Vision Exp	0.0000	$3.55	189.8M	629k	6h 0m	485	3
34	Qwen3.8-Max	0.0000	$115.97	289.9M	2.1M	19h 35m	811	1
35	Qwen3.8-Max	0.0000	$102.79	307.3M	1.7M	19h 10m	794	2
36	Gemini 3.7 Flash	0.0000	$6.64	48.6M	323k	1h 55m	247	1
37	Kimi K3	0.0000	$140.38	372.9M	1.4M	19h 51m	925	26
38	Inkling	0.0000	$2.84	2.7M	69k	45m	82	2
39	Muse Spark 1.2	0.0000	$16.11	78.5M	517k	1h 19m	309	4
40	Qwen3.8-Max	0.0000	$61.40	180.5M	1.6M	20h 1m	551	1
41	Inkling	0.0000	$1.80	1.8M	26k	14m	79	2
42	Muse Spark 1.2	0.0000	$13.70	75.9M	319k	1h 35m	305	3
43	Kimi K3	0.0000	$122.14	346.5M	798k	19h 47m	793	8
44	Gemini 3.7 Flash	0.0000	$7.40	56.9M	413k	1h 21m	266	2
45	DeepSeek V4 Flash Vision Exp	0.0000	$4.28	248.4M	549k	14h 16m	593	8
46	Kimi K3	0.0000	$117.31	316.7M	1.0M	19h 47m	806	6
47	Kimi K3	0.0000	$105.21	273.3M	660k	20h 0m	548	3
48	GLM-5.3	0.0000	$94.15	290.7M	465k	18h 52m	627	11
49	DeepSeek V4 Flash Vision Exp	0.0000	$5.35	314.7M	642k	14h 37m	709	3
50	Muse Spark 1.2	0.0000	$16.58	87.1M	508k	2h 14m	321	6

SPICE Circuit Simulator in Rust

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.8586	$166.41	276.3M	1.2M	9h 57m	632	17
2	Claude Fable 5.1	0.7677	$93.05	145.4M	837k	3h 54m	417	4
3	Claude Fable 5.1	0.7071	$96.79	153.4M	827k	4h 2m	413	8
4	Claude Fable 5.1	0.6970	$69.08	107.8M	623k	2h 37m	227	2
5	Claude Fable 5.1	0.6667	$96.05	148.5M	898k	4h 16m	406	6
6	Gemini 3.7 Flash	0.6263	$99.31	1037.7M	2.0M	6h 16m	2577	4
7	Grok 4.6	0.5657	$717.25	714.4M	2.7M	18h 22m	2483	266
8	GLM-5.3	0.5354	$171.96	618.6M	1.7M	18h 15m	1441	4
9	Gemini 3.7 Flash	0.5253	$121.42	1230.5M	2.0M	12h 5m	2949	3
10	Grok 4.6	0.5253	$835.65	836.5M	3.1M	20h 0m	3128	205
11	GLM-5.3	0.4949	$235.96	867.9M	1.6M	17h 11m	1905	18
12	Grok 4.6	0.4848	$586.10	582.0M	3.0M	17h 34m	2322	89
13	Gemini 3.7 Flash	0.4747	$76.85	790.2M	1.6M	4h 46m	2017	2
14	Grok 4.6	0.4646	$1191.20	1187.2M	2.5M	19h 58m	3879	96
15	Grok 4.6	0.4545	$165.93	154.1M	860k	5h 37m	563	37
16	Gemini 3.7 Flash	0.4444	$80.51	766.1M	1.3M	6h 23m	1793	7
17	GPT-5.6	0.4444	$366.07	350.2M	455k	6h 25m	797	22
18	GLM-5.3	0.4444	$158.88	578.0M	1.3M	10h 53m	1155	4
19	GLM-5.3	0.4141	$190.24	693.2M	1.5M	19h 41m	1705	30
20	GPT-5.6	0.4141	$307.32	296.1M	386k	4h 30m	783	15
21	GPT-5.6	0.3939	$538.95	528.2M	617k	6h 30m	1280	21
22	Gemini 3.7 Flash	0.3737	$126.28	1246.3M	2.0M	12h 6m	2921	2
23	Qwen3.8-Max	0.3737	$145.98	524.1M	2.1M	19h 26m	1338	1
24	Qwen3.8-Max	0.3434	$127.04	463.9M	1.6M	19h 13m	987	4
25	GPT-5.6	0.3333	$187.27	186.7M	262k	2h 15m	515	8
26	GLM-5.3	0.3131	$152.26	545.3M	1.3M	16h 57m	1164	23
27	Qwen3.8-Max	0.3030	$96.60	342.7M	1.6M	18h 43m	760	2
28	Qwen3.8-Max	0.3030	$110.14	386.5M	2.0M	19h 56m	947	1
29	Qwen3.8-Max	0.2727	$126.58	455.7M	1.9M	19h 26m	951	7
30	GPT-5.6	0.2626	$226.84	221.2M	327k	3h 7m	592	13
31	Muse Spark 1.2	0.1313	$32.05	183.6M	731k	2h 8m	465	1
32	Muse Spark 1.2	0.0707	$10.81	53.3M	351k	1h 35m	223	2
33	Muse Spark 1.2	0.0707	$32.82	175.7M	815k	3h 21m	461	1
34	Muse Spark 1.2	0.0505	$8.77	48.8M	242k	1h 16m	196	2
35	Muse Spark 1.2	0.0404	$5.34	27.9M	188k	26m	143	2
36	DeepSeek V4 Flash Vision Exp	0.0000	$11.65	704.6M	1.2M	10h 40m	1549	15
37	Kimi K3	0.0000	$179.57	474.0M	1.9M	19h 42m	1215	9
38	Kimi K3	0.0000	$215.15	588.2M	1.9M	19h 56m	1405	3
39	DeepSeek V4 Flash Vision Exp	0.0000	$19.65	1192.1M	2.0M	18h 3m	2398	10
40	Kimi K3	0.0000	$199.08	565.4M	1.4M	19h 58m	1271	1
41	Kimi K3	0.0000	$204.94	570.3M	1.7M	19h 34m	1399	3
42	Kimi K3	0.0000	$193.03	529.1M	1.8M	19h 58m	1263	1
43	DeepSeek V4 Flash Vision Exp	0.0000	$20.48	1199.6M	2.4M	11h 32m	2563	9
44	Inkling	0.0000	$4.42	4.6M	23k	45m	83	1
45	DeepSeek V4 Flash Vision Exp	0.0000	$21.51	1287.0M	2.2M	19h 10m	2904	13
46	Inkling	0.0000	$19.30	20.1M	40k	1h 20m	170	2
47	Inkling	0.0000	$1.04	1.1M	10k	27m	52	2
48	DeepSeek V4 Flash Vision Exp	0.0000	$19.68	1182.5M	2.0M	19h 8m	2617	23
49	Inkling	0.0000	$5.07	5.3M	18k	1h 13m	110	2
50	Inkling	0.0000	$5.21	5.3M	32k	15m	124	2

Synthetic Music Diarization

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.5457	$12.11	18.5M	118k	17h 49m	156	8
2	Claude Fable 5.1	0.5346	$11.41	17.0M	120k	17h 44m	158	7
3	GPT-5.6	0.5247	$44.19	53.3M	154k	7h 34m	267	3
4	Claude Fable 5.1	0.5104	$17.60	28.7M	134k	17h 45m	230	11
5	GPT-5.6	0.5065	$80.65	78.5M	178k	15h 57m	339	11
6	GPT-5.6	0.4952	$59.62	69.7M	147k	15h 37m	359	3
7	Claude Fable 5.1	0.4905	$10.29	15.1M	114k	17h 40m	140	7
8	Claude Fable 5.1	0.4889	$10.86	15.6M	125k	15h 34m	138	6
9	GLM-5.3	0.4816	$35.34	125.4M	361k	17h 30m	462	11
10	GLM-5.3	0.4802	$31.62	79.0M	303k	18h 9m	361	1
11	Gemini 3.7 Flash	0.4776	$32.86	129.1M	265k	12h 34m	438	2
12	GPT-5.6	0.4737	$78.78	76.5M	133k	14h 47m	286	4
13	GPT-5.6	0.4727	$68.59	82.8M	153k	14h 29m	425	7
14	Gemini 3.7 Flash	0.4664	$11.27	42.4M	155k	7h 29m	254	2
15	Grok 4.6	0.4661	$133.75	133.0M	655k	19h 52m	578	42
16	Qwen3.8-Max	0.4605	$29.62	103.3M	529k	18h 43m	509	11
17	GLM-5.3	0.4602	$19.77	41.7M	196k	19h 9m	253	4
18	Qwen3.8-Max	0.4425	$26.66	93.3M	455k	18h 13m	478	8
19	Gemini 3.7 Flash	0.4422	$14.36	133.2M	250k	5h 49m	389	2
20	DeepSeek V4 Flash Vision Exp	0.4412	$1.72	93.1M	282k	19h 4m	430	2
21	Grok 4.6	0.4323	$188.19	192.5M	1.1M	19h 45m	871	56
22	Kimi K3	0.4318	$13.36	33.2M	179k	18h 52m	244	5
23	Qwen3.8-Max	0.4301	$34.00	119.2M	572k	19h 17m	508	10
24	GLM-5.3	0.4285	$28.03	77.4M	258k	18h 22m	366	9
25	GLM-5.3	0.4265	$27.27	78.5M	240k	19h 37m	374	6
26	Gemini 3.7 Flash	0.4264	$42.51	256.4M	394k	13h 10m	681	9
27	DeepSeek V4 Flash Vision Exp	0.4247	$2.96	173.2M	352k	17h 55m	617	4
28	Kimi K3	0.4157	$22.81	60.7M	202k	19h 10m	347	7
29	Grok 4.6	0.4120	$228.84	232.3M	993k	19h 44m	918	63
30	DeepSeek V4 Flash Vision Exp	0.4065	$1.73	95.5M	262k	17h 34m	448	5
31	Qwen3.8-Max	0.4043	$33.94	117.1M	655k	18h 57m	501	5
32	Kimi K3	0.4034	$13.34	31.5M	191k	18h 19m	247	4
33	Gemini 3.7 Flash	0.4000	$15.67	83.4M	201k	4h 25m	322	1
34	Grok 4.6	0.3985	$299.53	304.5M	1.6M	19h 27m	1281	54
35	Qwen3.8-Max	0.3951	$17.02	56.1M	424k	19h 10m	391	4
36	Kimi K3	0.3852	$16.48	39.9M	237k	19h 5m	234	2
37	Muse Spark 1.2	0.3829	$88.55	356.6M	541k	7h 12m	754	3
38	DeepSeek V4 Flash Vision Exp	0.3803	$1.85	99.9M	306k	18h 37m	455	3
39	Muse Spark 1.2	0.3711	$10.87	42.0M	256k	3h 30m	227	4
40	Kimi K3	0.3635	$16.00	39.1M	227k	18h 59m	234	6
41	Muse Spark 1.2	0.3611	$12.50	32.3M	167k	4h 4m	213	5
42	Muse Spark 1.2	0.3544	$13.53	50.0M	335k	4h 26m	248	4
43	Muse Spark 1.2	0.3434	$13.40	47.8M	217k	5h 21m	241	5
44	DeepSeek V4 Flash Vision Exp	0.2979	$2.02	115.0M	266k	19h 32m	508	4
45	Inkling	0.2089	$6.22	6.4M	44k	35m	131	2
46	Inkling	0.0477	$6.40	6.6M	38k	28m	123	1
47	Inkling	0.0074	$12.08	12.5M	45k	59m	160	1
48	Inkling	0.0053	$5.67	5.8M	48k	1h 31m	99	2
49	Inkling	0.0012	$9.51	9.9M	36k	38m	170	1
50	Grok 4.6	0.0000	$501.54	509.4M	1.9M	18h 17m	1937	96

Vision-only TORCS Racing Bot

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.3604	$65.32	119.8M	246k	14h 32m	399	16
2	Claude Fable 5.1	0.3524	$77.13	147.0M	311k	14h 51m	472	5
3	Claude Fable 5.1	0.2715	$77.56	137.7M	370k	10h 32m	426	7
4	Claude Fable 5.1	0.2491	$104.18	191.5M	350k	15h 56m	562	8
5	Claude Fable 5.1	0.1812	$69.87	126.5M	304k	15h 13m	418	9
6	GPT-5.6	0.1207	$165.68	160.5M	163k	9h 38m	416	7
7	Grok 4.6	0.1206	$114.81	93.3M	882k	19h 41m	426	10
8	Qwen3.8-Max	0.1078	$31.56	67.3M	1.0M	19h 45m	456	2
9	DeepSeek V4 Flash Vision Exp	0.0671	$3.79	218.1M	451k	16h 2m	591	4
10	Kimi K3	0.0612	$51.38	138.7M	388k	19h 16m	456	9
11	Inkling	0.0559	$112.11	117.1M	218k	1h 53m	811	1
12	GLM-5.3	0.0490	$52.08	118.9M	330k	18h 59m	354	2
13	DeepSeek V4 Flash Vision Exp	0.0421	$4.04	235.7M	470k	19h 34m	605	5
14	GLM-5.3	0.0406	$79.08	158.6M	318k	18h 36m	406	8
15	Qwen3.8-Max	0.0385	$33.83	72.9M	764k	18h 52m	422	2
16	Kimi K3	0.0382	$61.72	163.7M	377k	19h 53m	534	3
17	GLM-5.3	0.0363	$101.89	234.7M	397k	19h 55m	487	1
18	Grok 4.6	0.0335	$207.50	166.7M	1.1M	18h 40m	692	23
19	GLM-5.3	0.0278	$99.55	248.0M	367k	18h 54m	563	14
20	Gemini 3.7 Flash	0.0235	$5.06	24.8M	145k	2h 40m	197	1
21	Qwen3.8-Max	0.0216	$32.92	71.7M	721k	19h 16m	425	2
22	Kimi K3	0.0211	$44.44	114.5M	382k	20h 0m	386	1
23	Inkling	0.0186	$4.32	4.4M	39k	1h 1m	122	1
24	GLM-5.3	0.0180	$78.52	153.1M	353k	18h 42m	396	9
25	DeepSeek V4 Flash Vision Exp	0.0178	$3.85	221.0M	459k	11h 59m	555	7
26	Kimi K3	0.0143	$59.95	154.4M	389k	20h 1m	507	2
27	Muse Spark 1.2	0.0135	$59.82	110.7M	431k	11h 54m	331	6
28	Inkling	0.0129	$1.44	1.4M	20k	2h 0m	84	1
29	Muse Spark 1.2	0.0128	$52.14	70.4M	490k	14h 49m	289	4
30	Gemini 3.7 Flash	0.0127	$18.66	92.0M	417k	5h 34m	318	2
31	Grok 4.6	0.0122	$159.04	128.5M	975k	18h 46m	569	20
32	Gemini 3.7 Flash	0.0121	$11.51	63.9M	262k	4h 49m	310	2
33	Gemini 3.7 Flash	0.0118	$6.81	26.5M	177k	3h 38m	183	2
34	DeepSeek V4 Flash Vision Exp	0.0114	$4.73	279.3M	524k	19h 31m	681	4
35	Qwen3.8-Max	0.0061	$53.75	151.0M	965k	20h 0m	695	2
36	Muse Spark 1.2	0.0051	$32.90	57.9M	354k	9h 41m	372	3
37	Grok 4.6	0.0043	$218.28	183.9M	1.3M	14h 37m	752	15
38	Gemini 3.7 Flash	0.0041	$7.39	32.0M	185k	3h 17m	213	1
39	Inkling	0.0031	$5.44	5.4M	67k	1h 58m	120	2
40	Inkling	0.0025	$2.42	2.4M	30k	2h 1m	89	2
41	DeepSeek V4 Flash Vision Exp	0.0000†	$2.30	127.9M	331k	19h 43m	454	3
42	GPT-5.6	0.0000†	$58.14	63.0M	111k	13h 6m	288	4
43	GPT-5.6	0.0000†	$25.75	38.4M	104k	16h 16m	218	4
44	Grok 4.6	0.0000†	$57.50	52.5M	468k	7h 59m	252	3
45	Qwen3.8-Max	0.0000†	$18.02	51.8M	431k	20h 1m	433	2
46	GPT-5.6	0.0000†	$51.99	57.9M	124k	4h 17m	277	6
47	GPT-5.6	0.0000†	$79.43	80.8M	113k	10h 50m	285	4
48	Kimi K3	0.0000†	$56.92	159.4M	376k	20h 0m	521	1
49	Muse Spark 1.2	0.0000†	$35.10	65.9M	265k	9h 20m	229	3
50	Muse Spark 1.2	0.0000†	$27.43	124.8M	316k	10h 1m	385	4

† score adjusted by manual review; the trace's Judge tab carries the measured score and the rationale.

Verilog Simulator in Swift

task page →50 runs · indexed 2026-09-02

#	Model	Reward	Cost	In tok	Out tok	Time	Steps	Submits
1	Claude Fable 5.1	0.9332	$262.30	478.6M	947k	9h 23m	1148	17
2	Claude Fable 5.1	0.9315	$216.99	391.4M	878k	7h 55m	966	6
3	Claude Fable 5.1	0.9195	$211.66	382.5M	840k	9h 40m	989	17
4	Claude Fable 5.1	0.9161	$266.36	483.7M	1.0M	10h 14m	1167	34
5	Claude Fable 5.1	0.9092	$168.96	308.4M	606k	6h 47m	586	10
6	GPT-5.6	0.8853	$427.16	420.2M	394k	14h 37m	921	9
7	GPT-5.6	0.8596	$271.89	260.9M	315k	4h 56m	650	5
8	GPT-5.6	0.8493	$335.11	326.9M	349k	4h 23m	746	9
9	Kimi K3	0.8373	$248.18	704.4M	1.6M	20h 0m	1636	11
10	GPT-5.6	0.8339	$184.09	180.1M	273k	4h 24m	480	6
11	Gemini 3.7 Flash	0.8305	$103.65	1018.6M	2.0M	19h 58m	2418	1
12	Gemini 3.7 Flash	0.8288	$74.39	745.3M	2.1M	17h 44m	1828	1
13	Grok 4.6	0.8253	$223.65	211.2M	1.9M	12h 53m	884	31
14	GLM-5.3	0.8236	$236.70	879.4M	1.3M	20h 1m	1924	11
15	GPT-5.6	0.8236	$181.11	176.3M	277k	4h 52m	476	3
16	Gemini 3.7 Flash	0.8202	$75.29	725.4M	1.4M	19h 4m	1651	2
17	GLM-5.3	0.8151	$195.25	722.0M	1.1M	19h 28m	1706	10
18	Grok 4.6	0.8116	$347.28	325.2M	2.2M	19h 55m	1222	56
19	GLM-5.3	0.7757	$229.93	817.6M	1.1M	19h 47m	1800	19
20	GLM-5.3	0.7723	$247.89	875.5M	1.2M	19h 55m	1804	17
21	Kimi K3	0.7551	$141.93	387.6M	1.1M	20h 1m	1009	1
22	Qwen3.8-Max	0.7551	$162.17	597.6M	1.8M	19h 37m	1491	1
23	GLM-5.3	0.7551	$285.37	1061.6M	1.4M	19h 46m	2367	23
24	Grok 4.6	0.7517	$518.10	508.9M	2.5M	19h 53m	1842	50
25	Kimi K3	0.7175	$256.44	747.8M	1.5M	20h 1m	1669	2
26	Grok 4.6	0.6901	$168.65	160.9M	1.6M	9h 35m	691	25
27	Kimi K3	0.6849	$265.41	774.0M	1.6M	19h 54m	1682	2
28	Kimi K3	0.6250	$235.66	685.5M	1.4M	19h 49m	1521	1
29	Gemini 3.7 Flash	0.5976	$41.43	380.1M	776k	20h 1m	944	4
30	DeepSeek V4 Flash Vision Exp	0.5582	$23.11	1453.0M	1.8M	19h 40m	3118	3
31	Grok 4.6	0.5188	$107.03	99.0M	943k	6h 25m	402	19
32	Qwen3.8-Max	0.4384	$128.73	471.1M	1.6M	19h 41m	965	2
33	DeepSeek V4 Flash Vision Exp	0.4212	$16.08	998.7M	1.4M	20h 0m	2128	4
34	DeepSeek V4 Flash Vision Exp	0.3784	$22.06	1380.9M	1.8M	19h 50m	2999	12
35	Gemini 3.7 Flash	0.3545	$53.22	180.3M	689k	19h 59m	574	1
36	Qwen3.8-Max	0.3527	$147.05	546.0M	1.5M	19h 49m	1109	2
37	DeepSeek V4 Flash Vision Exp	0.3253	$27.96	1735.1M	2.4M	17h 46m	3798	22
38	DeepSeek V4 Flash Vision Exp	0.3065	$23.64	1484.7M	1.9M	18h 25m	3157	30
39	Qwen3.8-Max	0.2894	$161.74	608.8M	1.4M	19h 56m	1406	2
40	Qwen3.8-Max	0.2414	$54.19	188.0M	1.1M	20h 0m	597	1
41	Muse Spark 1.2	0.0445	$10.85	58.8M	321k	1h 31m	229	2
42	Muse Spark 1.2	0.0325	$16.81	84.3M	312k	1h 17m	329	2
43	Muse Spark 1.2	0.0120	$23.14	130.5M	679k	1h 19m	395	3
44	Muse Spark 1.2	0.0051	$4.27	21.5M	175k	51m	138	1
45	Inkling	0.0017	$19.93	20.8M	47k	1h 36m	283	1
46	Inkling	0.0000	$3.27	3.4M	20k	1h 3m	98	2
47	Inkling	0.0000	$13.36	13.9M	43k	1h 28m	227	1
48	Muse Spark 1.2	0.0000	$10.69	54.0M	468k	1h 50m	208	1
49	Inkling	0.0000	$2.17	2.2M	13k	1h 11m	80	1
50	Inkling	0.0000	$9.64	10.0M	34k	1h 28m	169	2