{ "schema": "csoai.rating-the-raters/0.1", "result_id": "RTR-001", "axis": "RTR-A1", "axis_name": "Human-Reference Rule Match", "programme": "Rating the Raters", "programme_status": "This is result 001. One rating organisation has been measured on one axis. This is not a survey and no cross-rater comparison exists.", "subject": "ARC Prize -- ARC-AGI-2 public evaluation human baseline", "measured_on": "2026-08-26", "license": "CC-BY-4.0", "publisher": "Council of AI (CSOAI Ltd, UK Companies House 16939677)", "verdict": "MISMATCHED", "verdict_meaning": "ARC's published human figure is computed under a different attempt budget than the one ARC's scoring rule applies to AI systems, and ARC does not publish a rule-matched human figure alongside it.", "fairness_statement": "ARC never claimed its 66% figure was a pass@2 score. ARC's own wording is 'average human performance', and ARC states its solvability claim -- at least 2 people in no more than 2 attempts -- separately and correctly. Nothing ARC published is false. This measurement exists because the rule-matched comparable is absent upstream, so the field routinely sets AI pass@2 scores against a human number measured under a looser rule. It is also worth saying plainly that this audit was possible ONLY because ARC publishes its participant-level rows under MIT, publishes its scoring rule verbatim, publishes gold-label corrections in a changelog rather than editing silently, and self-reports its own contamination problem. Most benchmark operators publish none of that, and cannot be audited at all. ARC's transparency is the reason it can be measured, and it should not be penalised in reputation for being the one organisation that made the check possible.", "criterion": { "id": "RTR-A1", "name": "Human-Reference Rule Match", "question": "Is the benchmark's published human-performance figure computed under the same scoring rule the benchmark applies to machines?", "applies_to": "Any benchmark operator that publishes BOTH a headline human performance figure AND machine scores on the same benchmark under a stated scoring rule.", "procedure": [ "1. Read the operator's stated machine scoring rule. Extract the attempt budget k (trials permitted per test item) and the grading function.", "2. Read the operator's published human figure and the computation the operator states for it.", "3. If the operator publishes participant-level rows, recompute the human figure under the machine rule -- same k, same grading, same aggregation -- to obtain H_matched.", "4. Report H_published, H_matched, and gap = H_published - H_matched in percentage points." ], "verdicts": { "MATCHED": "H_published is computed at the same k and grading as machine scores. Gap is 0 by construction.", "MISMATCHED": "H_published is computed at a different k or grading, and the operator publishes no rule-matched human figure. Gap reported in pp.", "NOT_APPLICABLE": "The operator publishes no headline human figure.", "UNMEASURABLE_FROM_PUBLIC_ROWS": "The operator publishes a human figure but not the rows needed to recompute it. This is a limit on what CSOAI can check, not a finding against the operator." }, "constraints": [ "Deterministic arithmetic only. No model judges any output.", "Grading is the operator's own grading function, unmodified.", "n >= 30 on the aggregation unit or the result is not published.", "Every published number recomputable from published rows." ], "how_to_dispute": "The criterion is stated so it can be argued with. An operator disputing this result should name which step it rejects: the extracted budget k, the aggregation, the stop-at-first-correct assumption, or the claim that no rule-matched figure is published. Send to nicholas@csoai.org; disputes are published verbatim alongside the result." }, "finding": { "arc_machine_scoring_rule": "For each test input, the test-taker is allowed 2 trials. This holds for all test-takers, either humans or AI.", "arc_machine_rule_source": "github.com/arcprize/ARC-AGI-2 readme", "human_figure_published_by_arc_pct": 66.0, "human_figure_basis": "unlimited submissions", "human_figure_recomputed_pct": 66.07, "human_figure_reconciles": true, "human_figure_rule_matched_pct": 55.27, "human_figure_rule_matched_ci95": [ 50.97, 59.51 ], "gap_pp": 10.8, "attempts_using_more_than_2_submissions": 263, "attempts_using_more_than_2_submissions_pct": 20.9, "solved_attempts_needing_more_than_2_submissions": 126, "solved_attempts_needing_more_than_2_submissions_pct": 16.3 }, "what_is_not_measured": { "note": "One rating organisation has been measured, on one axis, on one benchmark. Everything below is UNMEASURED. They are named only to state that they have NOT been measured -- no finding about any of them is expressed or implied, and their appearance here is not a ranking, a shortlist, or a queue.", "unmeasured": [ { "organisation": "LMArena", "status": "unmeasured" }, { "organisation": "Vals AI", "status": "unmeasured" }, { "organisation": "Artificial Analysis", "status": "unmeasured" } ], "also_not_measured_about_arc": [ "ARC-AGI-2 semi-private and private evaluation results. These are not third-party recomputable by design, for a defensible anti-contamination reason. CSOAI cites them, never restates them as CSOAI-measured.", "ARC-AGI-1 and ARC-AGI-3 on this axis.", "The 5 public-eval tasks the published rows do not cover.", "Whether ARC's grading of model outputs is correct. That is a separate measurement and has not been done." ] }, "limits": [ "Scoped to the 115 of 120 public-eval tasks the published rows cover. Not a statement about the other 5.", "Human rows are public-eval; the leaderboard's AI scores are semi-private. The two populations are different task sets, so the human and AI figures are not directly comparable even after the rule is matched. This result narrows one of the two mismatches, not both.", "The pass@k derivation rests on the stop-at-first-correct assumption documented in assumption_check.", "Human participants were not incentivised, timed, or selected the way an evaluated model is prompted. This measures the published figures' rule-consistency, not human ability." ], "source": { "dataset": "arcprize/arc_agi_2_human_testing", "file": "test_pair_attempts.csv", "dataset_license": "MIT", "dataset_sha256": "d13f1a05580285a04f6bba4dafefdc4fd81a45014c134f6625d1ab625d7deea5", "rows_in_file": 4681, "rows_used_public_eval": 1260, "arc_agi_2_repo_commit": "f3283f727488ad98fe575ea6a5ac981e4a188e49", "why_pinned": "ARC corrected public-eval gold labels 20+ times between 2025-03-24 and 2025-04-17, published in a changelog rather than applied silently. An unpinned ARC number is not meaningful.", "recompute_with": "scripts/rating_the_raters_001_arc.py" }, "n": { "tasks": 115, "test_pairs": 161, "attempts": 1260, "distinct_sessions": 392, "floor": 30, "floor_met": true }, "calibration_claim": { "arc_claim": "Every ARC-AGI-2 evaluation task was solved by at least 2 people in no more than 2 attempts.", "pairs_tested": 161, "pairs_reproduced": 161, "verdict": "REPRODUCES", "minimum_distinct_solvers_on_any_pair": 2, "note": "Reproduces on every (task, test_index) pair the published rows cover. The margin is exact rather than comfortable: at least one pair was solved by exactly 2 distinct sessions within 2 submissions, which is the claim's floor. That is a property of the claim, not a defect in it -- ARC calibrated to a threshold and the threshold is met." }, "aggregation_search": { "arc_published_figure_pct": 66.0, "arc_published_wording": "Average human performance on these tasks in our test sample was 66%.", "candidates_under_unlimited_submissions": { "attempt_level_micro": 61.35, "macro_over_test_pairs": 65.69, "macro_over_tasks": 66.07, "macro_over_tasks_all_test_pairs": 65.03, "task_level_micro": 60.64 }, "unique_reconciliation": "macro_over_tasks", "note": "Exactly one of the five candidate aggregations reproduces ARC's published 66%: the macro-average over tasks under unlimited submissions, at 66.07%. ARC's figure is correct and correctly computed. No discrepancy was found in it." }, "assumption_check": { "assumption": "A participant stopped submitting once correct, so a solved attempt's correct submission is its last submission.", "why_it_matters": "ARC publishes submission counts, not the index of the correct submission. Without this assumption an attempt recorded as '5 submissions, 1 correct' cannot be placed inside or outside a 2-submission budget.", "solved_attempts": 773, "solved_attempts_with_exactly_one_correct": 772, "solved_attempts_with_more_than_one_correct": 1, "submission_count_distribution_over_solved_attempts": [ 472, 175, 70, 30, 16, 6, 3, 1 ], "distribution_monotonically_decreasing": true, "sensitivity_pass_at_2_if_anomalous_row_counted_as_solved_pct": 55.34, "how_arc_could_settle_this": "Publish the 1-based index of the correct submission as a column. One column removes the assumption entirely." }, "coverage": { "public_eval_tasks_in_benchmark": 120, "public_eval_tasks_covered_by_published_rows": 115, "coverage_pct": 95.8, "tasks_not_covered": 5, "status_of_uncovered_tasks": "UNMEASURED", "note": "A limit on what public data can settle, not a defect in ARC. ARC's own dataset card states the released rows are not comprehensive. Every figure here is scoped to the 115 covered tasks and none of them should be read as covering the other 5." }, "results_by_rule": { "unlimited": { "macro_over_tasks_pct": 66.07, "macro_ci95_bootstrap_tasks": [ 61.91, 70.03 ], "macro_ci95_method": "percentile bootstrap over the 115 tasks, 10000 resamples, seed 20260826", "micro_over_attempts_pct": 61.35, "micro_numerator": 773, "micro_denominator": 1260, "micro_ci95_wilson": [ 58.63, 64.0 ], "micro_ci95_method": "Wilson score interval; applies exactly, the attempt-level rate is a single binomial proportion. Attempts are clustered within sessions and tasks, so nominal coverage is optimistic.", "macro_ci95_wilson_naive_DO_NOT_USE": [ 57.04, 74.09 ], "macro_ci95_wilson_naive_note": "Published only to show what the estate's default method yields when misapplied to a mean of proportions. It treats each task as one Bernoulli trial and throws away the session data behind each task's rate. It is NOT the interval for the headline." }, "pass_at_2": { "macro_over_tasks_pct": 55.27, "macro_ci95_bootstrap_tasks": [ 50.97, 59.51 ], "macro_ci95_method": "percentile bootstrap over the 115 tasks, 10000 resamples, seed 20260826", "micro_over_attempts_pct": 51.35, "micro_numerator": 647, "micro_denominator": 1260, "micro_ci95_wilson": [ 48.59, 54.1 ], "micro_ci95_method": "Wilson score interval; applies exactly, the attempt-level rate is a single binomial proportion. Attempts are clustered within sessions and tasks, so nominal coverage is optimistic.", "macro_ci95_wilson_naive_DO_NOT_USE": [ 46.54, 64.4 ], "macro_ci95_wilson_naive_note": "Published only to show what the estate's default method yields when misapplied to a mean of proportions. It treats each task as one Bernoulli trial and throws away the session data behind each task's rate. It is NOT the interval for the headline." }, "pass_at_1": { "macro_over_tasks_pct": 40.44, "macro_ci95_bootstrap_tasks": [ 36.12, 44.77 ], "macro_ci95_method": "percentile bootstrap over the 115 tasks, 10000 resamples, seed 20260826", "micro_over_attempts_pct": 37.46, "micro_numerator": 472, "micro_denominator": 1260, "micro_ci95_wilson": [ 34.83, 40.17 ], "micro_ci95_method": "Wilson score interval; applies exactly, the attempt-level rate is a single binomial proportion. Attempts are clustered within sessions and tasks, so nominal coverage is optimistic.", "macro_ci95_wilson_naive_DO_NOT_USE": [ 32.32, 50.01 ], "macro_ci95_wilson_naive_note": "Published only to show what the estate's default method yields when misapplied to a mean of proportions. It treats each task as one Bernoulli trial and throws away the session data behind each task's rate. It is NOT the interval for the headline." } }, "task_rows": [ { "task_id": "0934a4d8", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 44.44, "rate_within_2_pct": 22.22, "rate_within_1_pct": 11.11 }, { "task_id": "135a2760", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 6, "solved_within_1": 4, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 66.67, "rate_within_1_pct": 44.44 }, { "task_id": "136b0064", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 55.56, "rate_within_1_pct": 33.33 }, { "task_id": "13e47133", "attempts": 13, "distinct_sessions": 13, "test_pairs": 2, "solved_unlimited": 7, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 53.85, "rate_within_2_pct": 38.46, "rate_within_1_pct": 23.08 }, { "task_id": "142ca369", "attempts": 10, "distinct_sessions": 5, "test_pairs": 2, "solved_unlimited": 7, "solved_within_2": 7, "solved_within_1": 5, "rate_unlimited_pct": 70.0, "rate_within_2_pct": 70.0, "rate_within_1_pct": 50.0 }, { "task_id": "16b78196", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 55.56, "rate_within_1_pct": 33.33 }, { "task_id": "16de56c4", "attempts": 14, "distinct_sessions": 14, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 8, "solved_within_1": 3, "rate_unlimited_pct": 64.29, "rate_within_2_pct": 57.14, "rate_within_1_pct": 21.43 }, { "task_id": "1818057f", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 6, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 66.67 }, { "task_id": "195c6913", "attempts": 23, "distinct_sessions": 18, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 39.13, "rate_within_2_pct": 26.09, "rate_within_1_pct": 21.74 }, { "task_id": "1ae2feb7", "attempts": 24, "distinct_sessions": 8, "test_pairs": 3, "solved_unlimited": 10, "solved_within_2": 8, "solved_within_1": 4, "rate_unlimited_pct": 41.67, "rate_within_2_pct": 33.33, "rate_within_1_pct": 16.67 }, { "task_id": "20270e3b", "attempts": 17, "distinct_sessions": 17, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 8, "solved_within_1": 6, "rate_unlimited_pct": 52.94, "rate_within_2_pct": 47.06, "rate_within_1_pct": 35.29 }, { "task_id": "20a9e565", "attempts": 17, "distinct_sessions": 14, "test_pairs": 2, "solved_unlimited": 11, "solved_within_2": 7, "solved_within_1": 2, "rate_unlimited_pct": 64.71, "rate_within_2_pct": 41.18, "rate_within_1_pct": 11.76 }, { "task_id": "21897d95", "attempts": 13, "distinct_sessions": 9, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 4, "solved_within_1": 1, "rate_unlimited_pct": 46.15, "rate_within_2_pct": 30.77, "rate_within_1_pct": 7.69 }, { "task_id": "221dfab4", "attempts": 15, "distinct_sessions": 11, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 5, "solved_within_1": 2, "rate_unlimited_pct": 40.0, "rate_within_2_pct": 33.33, "rate_within_1_pct": 13.33 }, { "task_id": "247ef758", "attempts": 10, "distinct_sessions": 5, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 60.0, "rate_within_2_pct": 60.0, "rate_within_1_pct": 50.0 }, { "task_id": "269e22fb", "attempts": 22, "distinct_sessions": 19, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 27.27, "rate_within_2_pct": 27.27, "rate_within_1_pct": 22.73 }, { "task_id": "271d71e2", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 37.5, "rate_within_2_pct": 25.0, "rate_within_1_pct": 12.5 }, { "task_id": "28a6681f", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 80.0, "rate_within_2_pct": 70.0, "rate_within_1_pct": 60.0 }, { "task_id": "291dc1e1", "attempts": 6, "distinct_sessions": 6, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 50.0, "rate_within_2_pct": 33.33, "rate_within_1_pct": 16.67 }, { "task_id": "2b83f449", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 2, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 22.22, "rate_within_2_pct": 22.22, "rate_within_1_pct": 11.11 }, { "task_id": "2ba387bc", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 8, "solved_within_1": 8, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 88.89, "rate_within_1_pct": 88.89 }, { "task_id": "2c181942", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 55.56, "rate_within_1_pct": 33.33 }, { "task_id": "2d0172a1", "attempts": 15, "distinct_sessions": 12, "test_pairs": 2, "solved_unlimited": 7, "solved_within_2": 5, "solved_within_1": 2, "rate_unlimited_pct": 46.67, "rate_within_2_pct": 33.33, "rate_within_1_pct": 13.33 }, { "task_id": "31f7f899", "attempts": 3, "distinct_sessions": 3, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 0, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 66.67, "rate_within_1_pct": 0.0 }, { "task_id": "332f06d7", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 77.78, "rate_within_1_pct": 66.67 }, { "task_id": "35ab12c3", "attempts": 12, "distinct_sessions": 12, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 3, "solved_within_1": 2, "rate_unlimited_pct": 33.33, "rate_within_2_pct": 25.0, "rate_within_1_pct": 16.67 }, { "task_id": "36a08778", "attempts": 6, "distinct_sessions": 3, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 83.33 }, { "task_id": "38007db0", "attempts": 6, "distinct_sessions": 3, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 4, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 66.67 }, { "task_id": "3a25b0d8", "attempts": 15, "distinct_sessions": 9, "test_pairs": 2, "solved_unlimited": 12, "solved_within_2": 12, "solved_within_1": 11, "rate_unlimited_pct": 80.0, "rate_within_2_pct": 80.0, "rate_within_1_pct": 73.33 }, { "task_id": "3dc255db", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 3, "solved_within_1": 3, "rate_unlimited_pct": 87.5, "rate_within_2_pct": 37.5, "rate_within_1_pct": 37.5 }, { "task_id": "3e6067c3", "attempts": 13, "distinct_sessions": 9, "test_pairs": 2, "solved_unlimited": 13, "solved_within_2": 13, "solved_within_1": 12, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 92.31 }, { "task_id": "409aa875", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 3, "solved_within_1": 2, "rate_unlimited_pct": 50.0, "rate_within_2_pct": 37.5, "rate_within_1_pct": 25.0 }, { "task_id": "446ef5d2", "attempts": 13, "distinct_sessions": 13, "test_pairs": 2, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 4, "rate_unlimited_pct": 61.54, "rate_within_2_pct": 53.85, "rate_within_1_pct": 30.77 }, { "task_id": "45a5af55", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 77.78, "rate_within_1_pct": 66.67 }, { "task_id": "4c3d4a41", "attempts": 10, "distinct_sessions": 6, "test_pairs": 2, "solved_unlimited": 7, "solved_within_2": 4, "solved_within_1": 2, "rate_unlimited_pct": 70.0, "rate_within_2_pct": 40.0, "rate_within_1_pct": 20.0 }, { "task_id": "4c416de3", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 6, "solved_within_2": 5, "solved_within_1": 4, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 62.5, "rate_within_1_pct": 50.0 }, { "task_id": "4c7dc4dd", "attempts": 15, "distinct_sessions": 10, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 60.0, "rate_within_2_pct": 46.67, "rate_within_1_pct": 40.0 }, { "task_id": "4e34c42c", "attempts": 20, "distinct_sessions": 17, "test_pairs": 2, "solved_unlimited": 7, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 35.0, "rate_within_2_pct": 30.0, "rate_within_1_pct": 25.0 }, { "task_id": "53fb4810", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 77.78, "rate_within_1_pct": 66.67 }, { "task_id": "5545f144", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 2, "solved_within_2": 2, "solved_within_1": 2, "rate_unlimited_pct": 20.0, "rate_within_2_pct": 20.0, "rate_within_1_pct": 20.0 }, { "task_id": "581f7754", "attempts": 10, "distinct_sessions": 5, "test_pairs": 2, "solved_unlimited": 7, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 70.0, "rate_within_2_pct": 60.0, "rate_within_1_pct": 50.0 }, { "task_id": "58490d8a", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 8, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 88.89 }, { "task_id": "58f5dbd5", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 8, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 88.89 }, { "task_id": "5961cc34", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 3, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 75.0, "rate_within_1_pct": 37.5 }, { "task_id": "5dbc8537", "attempts": 12, "distinct_sessions": 10, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 8, "solved_within_1": 6, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 66.67, "rate_within_1_pct": 50.0 }, { "task_id": "62593bfd", "attempts": 22, "distinct_sessions": 20, "test_pairs": 2, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 7, "rate_unlimited_pct": 36.36, "rate_within_2_pct": 31.82, "rate_within_1_pct": 31.82 }, { "task_id": "64efde09", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 8, "solved_within_1": 6, "rate_unlimited_pct": 80.0, "rate_within_2_pct": 80.0, "rate_within_1_pct": 60.0 }, { "task_id": "65b59efc", "attempts": 13, "distinct_sessions": 12, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 46.15, "rate_within_2_pct": 46.15, "rate_within_1_pct": 38.46 }, { "task_id": "67e490f4", "attempts": 4, "distinct_sessions": 4, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 4, "solved_within_1": 4, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 100.0 }, { "task_id": "6e453dd6", "attempts": 4, "distinct_sessions": 4, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 50.0, "rate_within_1_pct": 25.0 }, { "task_id": "6e4f6532", "attempts": 13, "distinct_sessions": 13, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 4, "rate_unlimited_pct": 46.15, "rate_within_2_pct": 46.15, "rate_within_1_pct": 30.77 }, { "task_id": "6ffbe589", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 2, "solved_within_2": 2, "solved_within_1": 0, "rate_unlimited_pct": 22.22, "rate_within_2_pct": 22.22, "rate_within_1_pct": 0.0 }, { "task_id": "71e489b6", "attempts": 13, "distinct_sessions": 13, "test_pairs": 2, "solved_unlimited": 11, "solved_within_2": 9, "solved_within_1": 7, "rate_unlimited_pct": 84.62, "rate_within_2_pct": 69.23, "rate_within_1_pct": 53.85 }, { "task_id": "7491f3cf", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 55.56, "rate_within_1_pct": 33.33 }, { "task_id": "7666fa5d", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 6, "solved_within_1": 6, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 66.67, "rate_within_1_pct": 66.67 }, { "task_id": "78332cb0", "attempts": 12, "distinct_sessions": 12, "test_pairs": 2, "solved_unlimited": 11, "solved_within_2": 9, "solved_within_1": 7, "rate_unlimited_pct": 91.67, "rate_within_2_pct": 75.0, "rate_within_1_pct": 58.33 }, { "task_id": "7b0280bc", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 5, "solved_within_1": 4, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 55.56, "rate_within_1_pct": 44.44 }, { "task_id": "7b3084d4", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 2, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 22.22, "rate_within_2_pct": 22.22, "rate_within_1_pct": 11.11 }, { "task_id": "7b5033c1", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 77.78, "rate_within_1_pct": 66.67 }, { "task_id": "7b80bb43", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 4, "solved_within_1": 2, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 44.44, "rate_within_1_pct": 22.22 }, { "task_id": "7c66cb00", "attempts": 11, "distinct_sessions": 11, "test_pairs": 1, "solved_unlimited": 6, "solved_within_2": 3, "solved_within_1": 2, "rate_unlimited_pct": 54.55, "rate_within_2_pct": 27.27, "rate_within_1_pct": 18.18 }, { "task_id": "7ed72f31", "attempts": 12, "distinct_sessions": 12, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 58.33, "rate_within_1_pct": 50.0 }, { "task_id": "800d221b", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 5, "solved_within_1": 2, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 55.56, "rate_within_1_pct": 22.22 }, { "task_id": "80a900e0", "attempts": 3, "distinct_sessions": 3, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 66.67, "rate_within_1_pct": 33.33 }, { "task_id": "8698868d", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 6, "rate_unlimited_pct": 90.0, "rate_within_2_pct": 90.0, "rate_within_1_pct": 60.0 }, { "task_id": "88bcf3b4", "attempts": 17, "distinct_sessions": 15, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 4, "solved_within_1": 4, "rate_unlimited_pct": 35.29, "rate_within_2_pct": 23.53, "rate_within_1_pct": 23.53 }, { "task_id": "88e364bc", "attempts": 15, "distinct_sessions": 11, "test_pairs": 2, "solved_unlimited": 10, "solved_within_2": 10, "solved_within_1": 6, "rate_unlimited_pct": 66.67, "rate_within_2_pct": 66.67, "rate_within_1_pct": 40.0 }, { "task_id": "89565ca0", "attempts": 3, "distinct_sessions": 3, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 3, "solved_within_1": 3, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 100.0 }, { "task_id": "898e7135", "attempts": 3, "distinct_sessions": 3, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 3, "solved_within_1": 3, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 100.0 }, { "task_id": "8b7bacbf", "attempts": 14, "distinct_sessions": 10, "test_pairs": 2, "solved_unlimited": 10, "solved_within_2": 10, "solved_within_1": 8, "rate_unlimited_pct": 71.43, "rate_within_2_pct": 71.43, "rate_within_1_pct": 57.14 }, { "task_id": "8b9c3697", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 55.56, "rate_within_1_pct": 33.33 }, { "task_id": "8e5c0c38", "attempts": 23, "distinct_sessions": 19, "test_pairs": 2, "solved_unlimited": 11, "solved_within_2": 8, "solved_within_1": 4, "rate_unlimited_pct": 47.83, "rate_within_2_pct": 34.78, "rate_within_1_pct": 17.39 }, { "task_id": "8f215267", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 7, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 77.78 }, { "task_id": "8f3a5a89", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 3, "solved_within_1": 2, "rate_unlimited_pct": 50.0, "rate_within_2_pct": 37.5, "rate_within_1_pct": 25.0 }, { "task_id": "9385bd28", "attempts": 11, "distinct_sessions": 9, "test_pairs": 2, "solved_unlimited": 10, "solved_within_2": 8, "solved_within_1": 7, "rate_unlimited_pct": 90.91, "rate_within_2_pct": 72.73, "rate_within_1_pct": 63.64 }, { "task_id": "97d7923e", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 4, "solved_within_1": 4, "rate_unlimited_pct": 50.0, "rate_within_2_pct": 40.0, "rate_within_1_pct": 40.0 }, { "task_id": "981571dc", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 66.67, "rate_within_1_pct": 55.56 }, { "task_id": "9aaea919", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 6, "solved_within_1": 2, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 66.67, "rate_within_1_pct": 22.22 }, { "task_id": "9bbf930d", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 3, "solved_within_1": 3, "rate_unlimited_pct": 44.44, "rate_within_2_pct": 33.33, "rate_within_1_pct": 33.33 }, { "task_id": "a251c730", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 5, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 77.78, "rate_within_1_pct": 55.56 }, { "task_id": "a25697e4", "attempts": 13, "distinct_sessions": 9, "test_pairs": 2, "solved_unlimited": 11, "solved_within_2": 9, "solved_within_1": 7, "rate_unlimited_pct": 84.62, "rate_within_2_pct": 69.23, "rate_within_1_pct": 53.85 }, { "task_id": "a32d8b75", "attempts": 19, "distinct_sessions": 17, "test_pairs": 2, "solved_unlimited": 8, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 42.11, "rate_within_2_pct": 31.58, "rate_within_1_pct": 26.32 }, { "task_id": "a395ee82", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 7, "solved_within_1": 5, "rate_unlimited_pct": 88.89, "rate_within_2_pct": 77.78, "rate_within_1_pct": 55.56 }, { "task_id": "a47bf94d", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 8, "rate_unlimited_pct": 90.0, "rate_within_2_pct": 90.0, "rate_within_1_pct": 80.0 }, { "task_id": "a6f40cea", "attempts": 4, "distinct_sessions": 4, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 50.0, "rate_within_1_pct": 25.0 }, { "task_id": "aa4ec2a5", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 6, "solved_within_1": 6, "rate_unlimited_pct": 80.0, "rate_within_2_pct": 60.0, "rate_within_1_pct": 60.0 }, { "task_id": "b0039139", "attempts": 16, "distinct_sessions": 15, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 3, "rate_unlimited_pct": 37.5, "rate_within_2_pct": 37.5, "rate_within_1_pct": 18.75 }, { "task_id": "b10624e5", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 6, "solved_within_2": 4, "solved_within_1": 3, "rate_unlimited_pct": 66.67, "rate_within_2_pct": 44.44, "rate_within_1_pct": 33.33 }, { "task_id": "b5ca7ac4", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 6, "solved_within_2": 4, "solved_within_1": 3, "rate_unlimited_pct": 66.67, "rate_within_2_pct": 44.44, "rate_within_1_pct": 33.33 }, { "task_id": "b99e7126", "attempts": 8, "distinct_sessions": 8, "test_pairs": 1, "solved_unlimited": 6, "solved_within_2": 4, "solved_within_1": 4, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 50.0, "rate_within_1_pct": 50.0 }, { "task_id": "b9e38dc0", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 3, "solved_within_1": 3, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 33.33, "rate_within_1_pct": 33.33 }, { "task_id": "bf45cf4b", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 5, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 55.56 }, { "task_id": "c4d067a0", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 6, "solved_within_1": 4, "rate_unlimited_pct": 77.78, "rate_within_2_pct": 66.67, "rate_within_1_pct": 44.44 }, { "task_id": "c7f57c3e", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 44.44, "rate_within_2_pct": 22.22, "rate_within_1_pct": 11.11 }, { "task_id": "cb2d8a2c", "attempts": 13, "distinct_sessions": 13, "test_pairs": 2, "solved_unlimited": 10, "solved_within_2": 8, "solved_within_1": 5, "rate_unlimited_pct": 76.92, "rate_within_2_pct": 61.54, "rate_within_1_pct": 38.46 }, { "task_id": "cbebaa4b", "attempts": 16, "distinct_sessions": 12, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 7, "solved_within_1": 7, "rate_unlimited_pct": 56.25, "rate_within_2_pct": 43.75, "rate_within_1_pct": 43.75 }, { "task_id": "d35bdbdc", "attempts": 18, "distinct_sessions": 10, "test_pairs": 3, "solved_unlimited": 7, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 38.89, "rate_within_2_pct": 38.89, "rate_within_1_pct": 33.33 }, { "task_id": "d59b0160", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 4, "solved_within_1": 4, "rate_unlimited_pct": 44.44, "rate_within_2_pct": 44.44, "rate_within_1_pct": 44.44 }, { "task_id": "d8e07eb2", "attempts": 20, "distinct_sessions": 15, "test_pairs": 2, "solved_unlimited": 13, "solved_within_2": 8, "solved_within_1": 5, "rate_unlimited_pct": 65.0, "rate_within_2_pct": 40.0, "rate_within_1_pct": 25.0 }, { "task_id": "da515329", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 3, "solved_within_1": 2, "rate_unlimited_pct": 33.33, "rate_within_2_pct": 33.33, "rate_within_1_pct": 22.22 }, { "task_id": "db0c5428", "attempts": 6, "distinct_sessions": 6, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 2, "solved_within_1": 1, "rate_unlimited_pct": 66.67, "rate_within_2_pct": 33.33, "rate_within_1_pct": 16.67 }, { "task_id": "db695cfb", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 3, "solved_within_1": 0, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 33.33, "rate_within_1_pct": 0.0 }, { "task_id": "dbff022c", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 3, "solved_within_1": 1, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 33.33, "rate_within_1_pct": 11.11 }, { "task_id": "dd6b8c4b", "attempts": 14, "distinct_sessions": 12, "test_pairs": 2, "solved_unlimited": 9, "solved_within_2": 7, "solved_within_1": 6, "rate_unlimited_pct": 64.29, "rate_within_2_pct": 50.0, "rate_within_1_pct": 42.86 }, { "task_id": "de809cff", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 8, "solved_within_2": 5, "solved_within_1": 2, "rate_unlimited_pct": 80.0, "rate_within_2_pct": 50.0, "rate_within_1_pct": 20.0 }, { "task_id": "dfadab01", "attempts": 13, "distinct_sessions": 13, "test_pairs": 1, "solved_unlimited": 7, "solved_within_2": 5, "solved_within_1": 4, "rate_unlimited_pct": 53.85, "rate_within_2_pct": 38.46, "rate_within_1_pct": 30.77 }, { "task_id": "e12f9a14", "attempts": 18, "distinct_sessions": 14, "test_pairs": 2, "solved_unlimited": 5, "solved_within_2": 5, "solved_within_1": 3, "rate_unlimited_pct": 27.78, "rate_within_2_pct": 27.78, "rate_within_1_pct": 16.67 }, { "task_id": "e3721c99", "attempts": 15, "distinct_sessions": 10, "test_pairs": 2, "solved_unlimited": 12, "solved_within_2": 10, "solved_within_1": 6, "rate_unlimited_pct": 80.0, "rate_within_2_pct": 66.67, "rate_within_1_pct": 40.0 }, { "task_id": "e376de54", "attempts": 3, "distinct_sessions": 3, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 2, "solved_within_1": 2, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 66.67, "rate_within_1_pct": 66.67 }, { "task_id": "e8686506", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 9, "solved_within_2": 9, "solved_within_1": 9, "rate_unlimited_pct": 100.0, "rate_within_2_pct": 100.0, "rate_within_1_pct": 100.0 }, { "task_id": "e87109e9", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 4, "solved_within_1": 4, "rate_unlimited_pct": 40.0, "rate_within_2_pct": 40.0, "rate_within_1_pct": 40.0 }, { "task_id": "edb79dae", "attempts": 9, "distinct_sessions": 9, "test_pairs": 1, "solved_unlimited": 5, "solved_within_2": 3, "solved_within_1": 3, "rate_unlimited_pct": 55.56, "rate_within_2_pct": 33.33, "rate_within_1_pct": 33.33 }, { "task_id": "eee78d87", "attempts": 4, "distinct_sessions": 4, "test_pairs": 1, "solved_unlimited": 3, "solved_within_2": 3, "solved_within_1": 2, "rate_unlimited_pct": 75.0, "rate_within_2_pct": 75.0, "rate_within_1_pct": 50.0 }, { "task_id": "f931b4a8", "attempts": 25, "distinct_sessions": 20, "test_pairs": 2, "solved_unlimited": 6, "solved_within_2": 6, "solved_within_1": 5, "rate_unlimited_pct": 24.0, "rate_within_2_pct": 24.0, "rate_within_1_pct": 20.0 }, { "task_id": "fc7cae8d", "attempts": 10, "distinct_sessions": 10, "test_pairs": 1, "solved_unlimited": 4, "solved_within_2": 3, "solved_within_1": 1, "rate_unlimited_pct": 40.0, "rate_within_2_pct": 30.0, "rate_within_1_pct": 10.0 } ], "task_rows_note": "The 115 rows every headline number is computed from. The macro figure for a rule is the unweighted mean of that rule's rate column." }