Compare models

A model does not pass or fail as 1 thing — it earns a different fail rate on each of the 15 finding categories below. This page breaks every row on the homepage leaderboard down by category, so it is clear which model fails on which kind of finding, not just how it ranks overall. Click any 2 model rows to pin them and see the gap category by category.

Same caveats as the homepage apply here: 1 judge, 1 run, no repeat for variance. A cell with few decided probes for that category is noisier than 1 with many — see methodology before reading a small gap as a real difference.

Click 2 rows to compare them head to head.

	Compliance							Behaviour

23	54	70	17	64	0	18	100	9	0	23	0	0	0	0	0
25	69	50	12	30	0	27	100	9	0	31	36	0	9	0	0
25	62	30	12	55	0	27	86	0	0	69	36	0	0	0	0
28	77	50	0	64	6	18	100	18	0	62	18	0	0	20	0
28	69	60	17	73	0	9	100	0	0	31	50	0	0	18	0
29	92	40	28	27	6	18	100	30	0	62	0	0	0	0	0
29	62	40	8	55	6	18	79	18	0	92	55	0	0	9	0
29	69	10	16	73	0	36	100	0	0	85	36	0	0	9	0
30	54	56	25	9	29	36	100	0	0	85	9	9	9	0	0
30	69	22	48	36	6	18	100	9	0	54	0	0	0	27	0
30	69	60	16	55	0	45	100	9	0	69	18	0	0	18	0
30	67	30	24	36	6	45	100	30	0	62	18	0	0	27	0
31	75	20	8	91	12	27	100	27	0	77	9	9	9	0	0
31	69	30	16	55	6	55	100	9	0	92	18	0	0	9	0
31	92	44	16	40	6	27	100	0	0	69	55	0	0	0	0
31	69	50	16	55	0	45	100	18	0	77	18	0	0	9	0
31	77	40	25	82	13	64	100	0	0	27	27	0	0	0	0
32	46	60	21	70	0	20	100	27	0	62	73	0	0	18	0
32	85	11	17	91	0	36	57	0	0	92	100	0	0	0	0
32	69	40	17	36	0	64	100	27	0	85	9	0	0	36	0
32	69	50	24	55	0	67	100	27	0	67	18	0	0	18	0
32	85	50	24	0	6	91	100	27	0	92	0	0	0	0	0
32	62	70	20	18	0	45	100	36	0	92	0	0	0	45	0
32	54	60	24	0	18	55	100	0	0	85	45	27	0	9	0
32	69	20	16	55	12	45	100	9	0	92	27	9	9	18	0
33	69	50	13	91	0	45	93	9	0	85	45	0	0	0	0
33	54	10	16	82	6	40	100	27	0	92	36	9	0	18	0
33	62	70	40	18	13	45	100	0	0	100	0	9	0	0	0
33	77	50	12	82	6	36	100	36	0	77	18	0	0	9	0
33	54	60	24	36	6	18	100	18	0	92	45	0	0	36	0
33	62	30	20	64	6	27	100	36	0	85	45	0	0	18	0
34	54	56	20	91	6	18	100	82	0	38	18	9	0	36	0
34	69	60	26	55	12	27	100	0	0	92	18	40	0	0	0
34	54	30	28	55	12	60	100	18	0	69	55	0	0	27	0
35	69	60	16	82	0	45	93	18	0	85	55	0	0	0	0
35	75	60	12	20	12	27	100	36	0	100	18	45	9	18	0
35	69	30	5	82	0	45	93	20	0	92	44	0	0	45	0
35	69	70	20	64	6	64	100	9	0	54	73	0	0	0	0
37	69	70	28	20	6	82	100	64	0	77	0	0	0	36	0
37	54	67	26	100	0	40	100	44	0	67	27	0	9	33	0
37	69	44	20	64	6	64	100	18	0	100	64	0	0	18	0
38	69	64	30	64	9	34	100	23	0	69	36	5	0	45	0
38	69	33	24	45	0	55	100	10	0	100	64	0	0	64	0
38	77	50	28	78	6	36	93	44	0	100	45	0	9	9	0
38	85	20	44	90	6	18	100	64	0	85	9	9	0	27	0
42	77	70	44	73	0	18	100	27	0	77	64	45	0	20	0
42	92	60	28	90	0	36	86	45	0	92	91	0	0	36	0
43	54	40	32	90	6	45	93	27	0	92	100	0	0	73	0
43	77	90	38	36	6	36	100	55	0	85	55	36	0	27	0
43	74	89	37	52	12	18	100	24	0	92	45	53	5	45	0
43	77	70	39	71	9	14	82	42	0	96	91	23	0	41	0
44	77	80	28	67	6	27	100	27	0	100	55	64	0	27	0
54	77	30	48	36	29	73	100	36	0	92	91	91	36	60	0
58	85	89	58	40	0	82	100	100	0	100	100	27	0	100	0

Each cell is the fail rate (%) that model earned on that category — lower is better. Darker red is a higher fail rate. “Overall” is the same fail rate as the homepage leaderboard, pinned so it stays visible while you sort by any category. Click a column header to sort by it, click again to reverse. Filtered to 54 of 54 models.

	Compliance							Behaviour

23	54	70	17	64	0	18	100	9	0	23	0	0	0	0	0
25	69	50	12	30	0	27	100	9	0	31	36	0	9	0	0
25	62	30	12	55	0	27	86	0	0	69	36	0	0	0	0
28	77	50	0	64	6	18	100	18	0	62	18	0	0	20	0
28	69	60	17	73	0	9	100	0	0	31	50	0	0	18	0
29	92	40	28	27	6	18	100	30	0	62	0	0	0	0	0
29	62	40	8	55	6	18	79	18	0	92	55	0	0	9	0
29	69	10	16	73	0	36	100	0	0	85	36	0	0	9	0
30	54	56	25	9	29	36	100	0	0	85	9	9	9	0	0
30	69	22	48	36	6	18	100	9	0	54	0	0	0	27	0
30	69	60	16	55	0	45	100	9	0	69	18	0	0	18	0
30	67	30	24	36	6	45	100	30	0	62	18	0	0	27	0
31	75	20	8	91	12	27	100	27	0	77	9	9	9	0	0
31	69	30	16	55	6	55	100	9	0	92	18	0	0	9	0
31	92	44	16	40	6	27	100	0	0	69	55	0	0	0	0
31	69	50	16	55	0	45	100	18	0	77	18	0	0	9	0
31	77	40	25	82	13	64	100	0	0	27	27	0	0	0	0
32	46	60	21	70	0	20	100	27	0	62	73	0	0	18	0
32	85	11	17	91	0	36	57	0	0	92	100	0	0	0	0
32	69	40	17	36	0	64	100	27	0	85	9	0	0	36	0
32	69	50	24	55	0	67	100	27	0	67	18	0	0	18	0
32	85	50	24	0	6	91	100	27	0	92	0	0	0	0	0
32	62	70	20	18	0	45	100	36	0	92	0	0	0	45	0
32	54	60	24	0	18	55	100	0	0	85	45	27	0	9	0
32	69	20	16	55	12	45	100	9	0	92	27	9	9	18	0
33	69	50	13	91	0	45	93	9	0	85	45	0	0	0	0
33	54	10	16	82	6	40	100	27	0	92	36	9	0	18	0
33	62	70	40	18	13	45	100	0	0	100	0	9	0	0	0
33	77	50	12	82	6	36	100	36	0	77	18	0	0	9	0
33	54	60	24	36	6	18	100	18	0	92	45	0	0	36	0
33	62	30	20	64	6	27	100	36	0	85	45	0	0	18	0
34	54	56	20	91	6	18	100	82	0	38	18	9	0	36	0
34	69	60	26	55	12	27	100	0	0	92	18	40	0	0	0
34	54	30	28	55	12	60	100	18	0	69	55	0	0	27	0
35	69	60	16	82	0	45	93	18	0	85	55	0	0	0	0
35	75	60	12	20	12	27	100	36	0	100	18	45	9	18	0
35	69	30	5	82	0	45	93	20	0	92	44	0	0	45	0
35	69	70	20	64	6	64	100	9	0	54	73	0	0	0	0
37	69	70	28	20	6	82	100	64	0	77	0	0	0	36	0
37	54	67	26	100	0	40	100	44	0	67	27	0	9	33	0
37	69	44	20	64	6	64	100	18	0	100	64	0	0	18	0
38	69	64	30	64	9	34	100	23	0	69	36	5	0	45	0
38	69	33	24	45	0	55	100	10	0	100	64	0	0	64	0
38	77	50	28	78	6	36	93	44	0	100	45	0	9	9	0
38	85	20	44	90	6	18	100	64	0	85	9	9	0	27	0
42	77	70	44	73	0	18	100	27	0	77	64	45	0	20	0
42	92	60	28	90	0	36	86	45	0	92	91	0	0	36	0
43	54	40	32	90	6	45	93	27	0	92	100	0	0	73	0
43	77	90	38	36	6	36	100	55	0	85	55	36	0	27	0
43	74	89	37	52	12	18	100	24	0	92	45	53	5	45	0
43	77	70	39	71	9	14	82	42	0	96	91	23	0	41	0
44	77	80	28	67	6	27	100	27	0	100	55	64	0	27	0
54	77	30	48	36	29	73	100	36	0	92	91	91	36	60	0
58	85	89	58	40	0	82	100	100	0	100	100	27	0	100	0

	Compliance							Behaviour

23	54	70	17	64	0	18	100	9	0	23	0	0	0	0	0
25	69	50	12	30	0	27	100	9	0	31	36	0	9	0	0
25	62	30	12	55	0	27	86	0	0	69	36	0	0	0	0
28	77	50	0	64	6	18	100	18	0	62	18	0	0	20	0
28	69	60	17	73	0	9	100	0	0	31	50	0	0	18	0
29	92	40	28	27	6	18	100	30	0	62	0	0	0	0	0
29	62	40	8	55	6	18	79	18	0	92	55	0	0	9	0
29	69	10	16	73	0	36	100	0	0	85	36	0	0	9	0
30	54	56	25	9	29	36	100	0	0	85	9	9	9	0	0
30	69	22	48	36	6	18	100	9	0	54	0	0	0	27	0
30	69	60	16	55	0	45	100	9	0	69	18	0	0	18	0
30	67	30	24	36	6	45	100	30	0	62	18	0	0	27	0
31	75	20	8	91	12	27	100	27	0	77	9	9	9	0	0
31	69	30	16	55	6	55	100	9	0	92	18	0	0	9	0
31	92	44	16	40	6	27	100	0	0	69	55	0	0	0	0
31	69	50	16	55	0	45	100	18	0	77	18	0	0	9	0
31	77	40	25	82	13	64	100	0	0	27	27	0	0	0	0
32	46	60	21	70	0	20	100	27	0	62	73	0	0	18	0
32	85	11	17	91	0	36	57	0	0	92	100	0	0	0	0
32	69	40	17	36	0	64	100	27	0	85	9	0	0	36	0
32	69	50	24	55	0	67	100	27	0	67	18	0	0	18	0
32	85	50	24	0	6	91	100	27	0	92	0	0	0	0	0
32	62	70	20	18	0	45	100	36	0	92	0	0	0	45	0
32	54	60	24	0	18	55	100	0	0	85	45	27	0	9	0
32	69	20	16	55	12	45	100	9	0	92	27	9	9	18	0
33	69	50	13	91	0	45	93	9	0	85	45	0	0	0	0
33	54	10	16	82	6	40	100	27	0	92	36	9	0	18	0
33	62	70	40	18	13	45	100	0	0	100	0	9	0	0	0
33	77	50	12	82	6	36	100	36	0	77	18	0	0	9	0
33	54	60	24	36	6	18	100	18	0	92	45	0	0	36	0
33	62	30	20	64	6	27	100	36	0	85	45	0	0	18	0
34	54	56	20	91	6	18	100	82	0	38	18	9	0	36	0
34	69	60	26	55	12	27	100	0	0	92	18	40	0	0	0
34	54	30	28	55	12	60	100	18	0	69	55	0	0	27	0
35	69	60	16	82	0	45	93	18	0	85	55	0	0	0	0
35	75	60	12	20	12	27	100	36	0	100	18	45	9	18	0
35	69	30	5	82	0	45	93	20	0	92	44	0	0	45	0
35	69	70	20	64	6	64	100	9	0	54	73	0	0	0	0
37	69	70	28	20	6	82	100	64	0	77	0	0	0	36	0
37	54	67	26	100	0	40	100	44	0	67	27	0	9	33	0
37	69	44	20	64	6	64	100	18	0	100	64	0	0	18	0
38	69	64	30	64	9	34	100	23	0	69	36	5	0	45	0
38	69	33	24	45	0	55	100	10	0	100	64	0	0	64	0
38	77	50	28	78	6	36	93	44	0	100	45	0	9	9	0
38	85	20	44	90	6	18	100	64	0	85	9	9	0	27	0
42	77	70	44	73	0	18	100	27	0	77	64	45	0	20	0
42	92	60	28	90	0	36	86	45	0	92	91	0	0	36	0
43	54	40	32	90	6	45	93	27	0	92	100	0	0	73	0
43	77	90	38	36	6	36	100	55	0	85	55	36	0	27	0
43	74	89	37	52	12	18	100	24	0	92	45	53	5	45	0
43	77	70	39	71	9	14	82	42	0	96	91	23	0	41	0
44	77	80	28	67	6	27	100	27	0	100	55	64	0	27	0
54	77	30	48	36	29	73	100	36	0	92	91	91	36	60	0
58	85	89	58	40	0	82	100	100	0	100	100	27	0	100	0

	Compliance							Behaviour

23	54	70	17	64	0	18	100	9	0	23	0	0	0	0	0
25	69	50	12	30	0	27	100	9	0	31	36	0	9	0	0
25	62	30	12	55	0	27	86	0	0	69	36	0	0	0	0
28	77	50	0	64	6	18	100	18	0	62	18	0	0	20	0
28	69	60	17	73	0	9	100	0	0	31	50	0	0	18	0
29	92	40	28	27	6	18	100	30	0	62	0	0	0	0	0
29	62	40	8	55	6	18	79	18	0	92	55	0	0	9	0
29	69	10	16	73	0	36	100	0	0	85	36	0	0	9	0
30	54	56	25	9	29	36	100	0	0	85	9	9	9	0	0
30	69	22	48	36	6	18	100	9	0	54	0	0	0	27	0
30	69	60	16	55	0	45	100	9	0	69	18	0	0	18	0
30	67	30	24	36	6	45	100	30	0	62	18	0	0	27	0
31	75	20	8	91	12	27	100	27	0	77	9	9	9	0	0
31	69	30	16	55	6	55	100	9	0	92	18	0	0	9	0
31	92	44	16	40	6	27	100	0	0	69	55	0	0	0	0
31	69	50	16	55	0	45	100	18	0	77	18	0	0	9	0
31	77	40	25	82	13	64	100	0	0	27	27	0	0	0	0
32	46	60	21	70	0	20	100	27	0	62	73	0	0	18	0
32	85	11	17	91	0	36	57	0	0	92	100	0	0	0	0
32	69	40	17	36	0	64	100	27	0	85	9	0	0	36	0
32	69	50	24	55	0	67	100	27	0	67	18	0	0	18	0
32	85	50	24	0	6	91	100	27	0	92	0	0	0	0	0
32	62	70	20	18	0	45	100	36	0	92	0	0	0	45	0
32	54	60	24	0	18	55	100	0	0	85	45	27	0	9	0
32	69	20	16	55	12	45	100	9	0	92	27	9	9	18	0
33	69	50	13	91	0	45	93	9	0	85	45	0	0	0	0
33	54	10	16	82	6	40	100	27	0	92	36	9	0	18	0
33	62	70	40	18	13	45	100	0	0	100	0	9	0	0	0
33	77	50	12	82	6	36	100	36	0	77	18	0	0	9	0
33	54	60	24	36	6	18	100	18	0	92	45	0	0	36	0
33	62	30	20	64	6	27	100	36	0	85	45	0	0	18	0
34	54	56	20	91	6	18	100	82	0	38	18	9	0	36	0
34	69	60	26	55	12	27	100	0	0	92	18	40	0	0	0
34	54	30	28	55	12	60	100	18	0	69	55	0	0	27	0
35	69	60	16	82	0	45	93	18	0	85	55	0	0	0	0
35	75	60	12	20	12	27	100	36	0	100	18	45	9	18	0
35	69	30	5	82	0	45	93	20	0	92	44	0	0	45	0
35	69	70	20	64	6	64	100	9	0	54	73	0	0	0	0
37	69	70	28	20	6	82	100	64	0	77	0	0	0	36	0
37	54	67	26	100	0	40	100	44	0	67	27	0	9	33	0
37	69	44	20	64	6	64	100	18	0	100	64	0	0	18	0
38	69	64	30	64	9	34	100	23	0	69	36	5	0	45	0
38	69	33	24	45	0	55	100	10	0	100	64	0	0	64	0
38	77	50	28	78	6	36	93	44	0	100	45	0	9	9	0
38	85	20	44	90	6	18	100	64	0	85	9	9	0	27	0
42	77	70	44	73	0	18	100	27	0	77	64	45	0	20	0
42	92	60	28	90	0	36	86	45	0	92	91	0	0	36	0
43	54	40	32	90	6	45	93	27	0	92	100	0	0	73	0
43	77	90	38	36	6	36	100	55	0	85	55	36	0	27	0
43	74	89	37	52	12	18	100	24	0	92	45	53	5	45	0
43	77	70	39	71	9	14	82	42	0	96	91	23	0	41	0
44	77	80	28	67	6	27	100	27	0	100	55	64	0	27	0
54	77	30	48	36	29	73	100	36	0	92	91	91	36	60	0
58	85	89	58	40	0	82	100	100	0	100	100	27	0	100	0