{ "schema": "gobstopper-public-aggregate-study-v1", "study_date": "2026-09-19", "measurement": "Projected current-context tokens in offline replay. Medians include no-plan cases. Literal-string retention is not task accuracy.", "source_commit": "6c2f32a60bf7aec3c38d10853abd8f4d8e9be059", "binary_sha256": "cef977498f235560109b2930bb7ca06f4f1c3ca4c633c1435e0314043624d3ec", "combined": { "selected_sessions": 729, "strategy_evaluations": 3645, "evaluation_failures": 0, "compacted_plans": 92, "compacted_no_plan": 637, "compacted_median_reduction_including_no_plan_percent": 0 }, "cohorts": [ { "name": "initial_discovery", "selected_sessions": 615, "codex_subagent_sessions": 576, "claude_sessions": 39, "wall_seconds": 135.348, "aggregates": [ { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "all_selected", "selected_samples": 512, "successful_evals": 512, "plans": 2, "no_plan": 510, "failures": 0, "primary_reduction_samples": 512, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 21.177650033049204, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.568678321169653, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 64256842, "sum_projected_reclaimed": 43987, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.491999999999997 }, { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "native_compacted", "selected_samples": 73, "successful_evals": 73, "plans": 0, "no_plan": 73, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10927998, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.232 }, { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "no_native_compaction", "selected_samples": 439, "successful_evals": 439, "plans": 2, "no_plan": 437, "failures": 0, "primary_reduction_samples": 439, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 21.177650033049204, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.568678321169653, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 53328844, "sum_projected_reclaimed": 43987, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.393 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "all_selected", "selected_samples": 512, "successful_evals": 512, "plans": 2, "no_plan": 510, "failures": 0, "primary_reduction_samples": 512, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 20.710569464692416, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.228014450937128, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 64256842, "sum_projected_reclaimed": 42945, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.6795 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "native_compacted", "selected_samples": 73, "successful_evals": 73, "plans": 0, "no_plan": 73, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10927998, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 26.911 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "no_native_compaction", "selected_samples": 439, "successful_evals": 439, "plans": 2, "no_plan": 437, "failures": 0, "primary_reduction_samples": 439, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 20.710569464692416, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.228014450937128, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 53328844, "sum_projected_reclaimed": 42945, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.574 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "all_selected", "selected_samples": 512, "successful_evals": 512, "plans": 2, "no_plan": 510, "failures": 0, "primary_reduction_samples": 512, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 20.710569464692416, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.228014450937128, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 64256842, "sum_projected_reclaimed": 42945, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.5425 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "native_compacted", "selected_samples": 73, "successful_evals": 73, "plans": 0, "no_plan": 73, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10927998, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 26.388 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "no_native_compaction", "selected_samples": 439, "successful_evals": 439, "plans": 2, "no_plan": 437, "failures": 0, "primary_reduction_samples": 439, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 20.710569464692416, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.228014450937128, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 53328844, "sum_projected_reclaimed": 42945, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.596 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "all_selected", "selected_samples": 512, "successful_evals": 512, "plans": 2, "no_plan": 510, "failures": 0, "primary_reduction_samples": 512, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 20.710569464692416, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.228014450937128, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 64256842, "sum_projected_reclaimed": 42945, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.552500000000002 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "native_compacted", "selected_samples": 73, "successful_evals": 73, "plans": 0, "no_plan": 73, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10927998, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.042 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "no_native_compaction", "selected_samples": 439, "successful_evals": 439, "plans": 2, "no_plan": 437, "failures": 0, "primary_reduction_samples": 439, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 20.710569464692416, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 22.228014450937128, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 53328844, "sum_projected_reclaimed": 42945, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 2, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 2, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 131, "literal_probes_retained": 96, "nonzero_tail_probe_samples": 1, "tail_probes_total": 1, "tail_probes_retained": 1, "median_wall_ms": 26.483 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "all_selected", "selected_samples": 512, "successful_evals": 512, "plans": 0, "no_plan": 512, "failures": 0, "primary_reduction_samples": 512, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 64256842, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 26.763 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "native_compacted", "selected_samples": 73, "successful_evals": 73, "plans": 0, "no_plan": 73, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10927998, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.173 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "no_native_compaction", "selected_samples": 439, "successful_evals": 439, "plans": 0, "no_plan": 439, "failures": 0, "primary_reduction_samples": 439, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 53328844, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 26.714 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "all_selected", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.003999999999998 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "no_native_compaction", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.003999999999998 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "all_selected", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 26.7375 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "no_native_compaction", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 26.7375 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "all_selected", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.0775 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "no_native_compaction", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.0775 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "all_selected", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.2835 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "no_native_compaction", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.2835 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "all_selected", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.3935 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "no_native_compaction", "selected_samples": 64, "successful_evals": 64, "plans": 0, "no_plan": 64, "failures": 0, "primary_reduction_samples": 64, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2642735, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.3935 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "elide", "stratum": "all_selected", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 15.004381020073037, "median_reduction_planned_only_percent": 16.671596174701765, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.367089134956746, "median_descriptive_resampling_95pct_interval": [ 9.070076001381612, 16.971015665506815 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1303006, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1111, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 169.347 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "elide", "stratum": "native_compacted", "selected_samples": 15, "successful_evals": 15, "plans": 11, "no_plan": 4, "failures": 0, "primary_reduction_samples": 15, "median_reduction_including_no_plan_percent": 12.83808943502251, "median_reduction_planned_only_percent": 16.971015665506815, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.367089134956746, "median_descriptive_resampling_95pct_interval": [ 0.0, 16.971015665506815 ], "sum_context_before": 6442971, "sum_projected_reclaimed": 790374, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 11, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 11, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 722, "literal_probes_retained": 716, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 213.107 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "elide", "stratum": "no_native_compaction", "selected_samples": 8, "successful_evals": 8, "plans": 6, "no_plan": 2, "failures": 0, "primary_reduction_samples": 8, "median_reduction_including_no_plan_percent": 16.018560883083587, "median_reduction_planned_only_percent": 16.54661120376032, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 19.653220435733992, "median_descriptive_resampling_95pct_interval": [ 6.8608055001029475, 17.37934764307617 ], "sum_context_before": 3445067, "sum_projected_reclaimed": 512632, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 6, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 6, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 398, "literal_probes_retained": 395, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 82.617 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "elide", "stratum": "no_observed_parent", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 15.004381020073037, "median_reduction_planned_only_percent": 16.671596174701765, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.367089134956746, "median_descriptive_resampling_95pct_interval": [ 9.070076001381612, 16.971015665506815 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1303006, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1111, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 169.347 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "scored", "stratum": "all_selected", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 14.848747579716184, "median_reduction_planned_only_percent": 16.558007723595143, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 8.930895205272556, 16.767217222167275 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1287493, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1112, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 192.123 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "scored", "stratum": "native_compacted", "selected_samples": 15, "successful_evals": 15, "plans": 11, "no_plan": 4, "failures": 0, "primary_reduction_samples": 15, "median_reduction_including_no_plan_percent": 12.385328864967333, "median_reduction_planned_only_percent": 16.767217222167275, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 0.0, 16.767217222167275 ], "sum_context_before": 6442971, "sum_projected_reclaimed": 780680, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 11, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 11, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 722, "literal_probes_retained": 716, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 238.263 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "scored", "stratum": "no_native_compaction", "selected_samples": 8, "successful_evals": 8, "plans": 6, "no_plan": 2, "failures": 0, "primary_reduction_samples": 8, "median_reduction_including_no_plan_percent": 15.61647758692471, "median_reduction_planned_only_percent": 16.16251607900002, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 19.481178447975136, "median_descriptive_resampling_95pct_interval": [ 6.724138445516694, 17.006892962647388 ], "sum_context_before": 3445067, "sum_projected_reclaimed": 506813, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 6, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 6, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 398, "literal_probes_retained": 396, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 85.9425 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "scored", "stratum": "no_observed_parent", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 14.848747579716184, "median_reduction_planned_only_percent": 16.558007723595143, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 8.930895205272556, 16.767217222167275 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1287493, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1112, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 192.123 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "cache_aware", "stratum": "all_selected", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 14.848747579716184, "median_reduction_planned_only_percent": 16.558007723595143, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 8.930895205272556, 16.767217222167275 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1287493, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1112, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 188.897 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "cache_aware", "stratum": "native_compacted", "selected_samples": 15, "successful_evals": 15, "plans": 11, "no_plan": 4, "failures": 0, "primary_reduction_samples": 15, "median_reduction_including_no_plan_percent": 12.385328864967333, "median_reduction_planned_only_percent": 16.767217222167275, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 0.0, 16.767217222167275 ], "sum_context_before": 6442971, "sum_projected_reclaimed": 780680, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 11, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 11, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 722, "literal_probes_retained": 716, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 263.322 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "cache_aware", "stratum": "no_native_compaction", "selected_samples": 8, "successful_evals": 8, "plans": 6, "no_plan": 2, "failures": 0, "primary_reduction_samples": 8, "median_reduction_including_no_plan_percent": 15.61647758692471, "median_reduction_planned_only_percent": 16.16251607900002, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 19.481178447975136, "median_descriptive_resampling_95pct_interval": [ 6.724138445516694, 17.006892962647388 ], "sum_context_before": 3445067, "sum_projected_reclaimed": 506813, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 6, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 6, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 398, "literal_probes_retained": 396, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 117.524 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "cache_aware", "stratum": "no_observed_parent", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 14.848747579716184, "median_reduction_planned_only_percent": 16.558007723595143, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 8.930895205272556, 16.767217222167275 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1287493, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1112, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 188.897 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "compacted", "stratum": "all_selected", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 14.848747579716184, "median_reduction_planned_only_percent": 16.558007723595143, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 8.930895205272556, 16.767217222167275 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1287493, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1112, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 199.891 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "compacted", "stratum": "native_compacted", "selected_samples": 15, "successful_evals": 15, "plans": 11, "no_plan": 4, "failures": 0, "primary_reduction_samples": 15, "median_reduction_including_no_plan_percent": 12.385328864967333, "median_reduction_planned_only_percent": 16.767217222167275, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 0.0, 16.767217222167275 ], "sum_context_before": 6442971, "sum_projected_reclaimed": 780680, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 11, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 11, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 722, "literal_probes_retained": 716, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 239.535 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "compacted", "stratum": "no_native_compaction", "selected_samples": 8, "successful_evals": 8, "plans": 6, "no_plan": 2, "failures": 0, "primary_reduction_samples": 8, "median_reduction_including_no_plan_percent": 15.61647758692471, "median_reduction_planned_only_percent": 16.16251607900002, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 19.481178447975136, "median_descriptive_resampling_95pct_interval": [ 6.724138445516694, 17.006892962647388 ], "sum_context_before": 3445067, "sum_projected_reclaimed": 506813, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 6, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 6, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 398, "literal_probes_retained": 396, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 88.0675 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "compacted", "stratum": "no_observed_parent", "selected_samples": 23, "successful_evals": 23, "plans": 17, "no_plan": 6, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 14.848747579716184, "median_reduction_planned_only_percent": 16.558007723595143, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 23.084683119024167, "median_descriptive_resampling_95pct_interval": [ 8.930895205272556, 16.767217222167275 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 1287493, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 17, "planned_verification_delta_unavailable": 0, "post_error_samples": 1, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 17, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1120, "literal_probes_retained": 1112, "nonzero_tail_probe_samples": 4, "tail_probes_total": 8, "tail_probes_retained": 8, "median_wall_ms": 199.891 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "dedupe", "stratum": "all_selected", "selected_samples": 23, "successful_evals": 23, "plans": 0, "no_plan": 23, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 79.645 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "dedupe", "stratum": "native_compacted", "selected_samples": 15, "successful_evals": 15, "plans": 0, "no_plan": 15, "failures": 0, "primary_reduction_samples": 15, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 6442971, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 96.74 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "dedupe", "stratum": "no_native_compaction", "selected_samples": 8, "successful_evals": 8, "plans": 0, "no_plan": 8, "failures": 0, "primary_reduction_samples": 8, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 3445067, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 40.2485 }, { "provider": "claude_code", "cohort": "high_context", "strategy": "dedupe", "stratum": "no_observed_parent", "selected_samples": 23, "successful_evals": 23, "plans": 0, "no_plan": 23, "failures": 0, "primary_reduction_samples": 23, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 9888038, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 79.645 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "all_selected", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.6095 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.6095 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "no_observed_parent", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.6095 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "all_selected", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.008499999999998 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.008499999999998 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "no_observed_parent", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.008499999999998 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "all_selected", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 29.258 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 29.258 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "no_observed_parent", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 29.258 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "all_selected", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.015 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.015 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "no_observed_parent", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 28.015 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "all_selected", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.875 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.875 }, { "provider": "claude_code", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "no_observed_parent", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 248772, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 1, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.875 } ] }, { "name": "archived_codex_roots", "selected_sessions": 114, "high_context_sessions": 73, "below_trigger_controls": 41, "wall_seconds": 179.355, "aggregates": [ { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "all_selected", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 37.031442638209626, "median_reduction_planned_only_percent": 37.031442638209626, "min_reduction_including_no_plan_percent": 3.24497128469177, "max_reduction_including_no_plan_percent": 69.7919450444017, "median_descriptive_resampling_95pct_interval": [ 35.75936967180997, 40.88381330685203 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4063491, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3608, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 259.296 }, { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "native_compacted", "selected_samples": 57, "successful_evals": 57, "plans": 57, "no_plan": 0, "failures": 0, "primary_reduction_samples": 57, "median_reduction_including_no_plan_percent": 36.87112682422185, "median_reduction_planned_only_percent": 36.87112682422185, "min_reduction_including_no_plan_percent": 7.901854883183502, "max_reduction_including_no_plan_percent": 69.7919450444017, "median_descriptive_resampling_95pct_interval": [ 34.68706691572151, 40.55299993104739 ], "sum_context_before": 8612085, "sum_projected_reclaimed": 3173100, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 57, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 57, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 3721, "literal_probes_retained": 2818, "nonzero_tail_probe_samples": 23, "tail_probes_total": 88, "tail_probes_retained": 88, "median_wall_ms": 400.627 }, { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 16, "no_plan": 0, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 44.59360282447285, "median_reduction_planned_only_percent": 44.59360282447285, "min_reduction_including_no_plan_percent": 3.24497128469177, "max_reduction_including_no_plan_percent": 62.555170199100836, "median_descriptive_resampling_95pct_interval": [ 28.309167640989635, 48.31931692513438 ], "sum_context_before": 2185222, "sum_projected_reclaimed": 890391, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 16, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 16, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1041, "literal_probes_retained": 790, "nonzero_tail_probe_samples": 8, "tail_probes_total": 34, "tail_probes_retained": 34, "median_wall_ms": 70.6495 }, { "provider": "codex", "cohort": "high_context", "strategy": "elide", "stratum": "no_observed_parent", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 37.031442638209626, "median_reduction_planned_only_percent": 37.031442638209626, "min_reduction_including_no_plan_percent": 3.24497128469177, "max_reduction_including_no_plan_percent": 69.7919450444017, "median_descriptive_resampling_95pct_interval": [ 35.75936967180997, 40.88381330685203 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4063491, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3608, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 259.296 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "all_selected", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 36.436228842966436, "median_reduction_planned_only_percent": 36.436228842966436, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 35.26480773507577, 40.25046893964471 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4012086, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3664, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 264.066 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "native_compacted", "selected_samples": 57, "successful_evals": 57, "plans": 57, "no_plan": 0, "failures": 0, "primary_reduction_samples": 57, "median_reduction_including_no_plan_percent": 36.046523306850936, "median_reduction_planned_only_percent": 36.046523306850936, "min_reduction_including_no_plan_percent": 7.387144495312459, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 34.322291180955645, 39.890069839143415 ], "sum_context_before": 8612085, "sum_projected_reclaimed": 3131762, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 57, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 57, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 3721, "literal_probes_retained": 2868, "nonzero_tail_probe_samples": 23, "tail_probes_total": 88, "tail_probes_retained": 88, "median_wall_ms": 374.172 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 16, "no_plan": 0, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 43.80797883415849, "median_reduction_planned_only_percent": 43.80797883415849, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 62.16005138086063, "median_descriptive_resampling_95pct_interval": [ 27.98351791318231, 47.82124150228463 ], "sum_context_before": 2185222, "sum_projected_reclaimed": 880324, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 16, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 16, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1041, "literal_probes_retained": 796, "nonzero_tail_probe_samples": 8, "tail_probes_total": 34, "tail_probes_retained": 34, "median_wall_ms": 72.8275 }, { "provider": "codex", "cohort": "high_context", "strategy": "scored", "stratum": "no_observed_parent", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 36.436228842966436, "median_reduction_planned_only_percent": 36.436228842966436, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 35.26480773507577, 40.25046893964471 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4012086, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3664, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 264.066 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "all_selected", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 36.436228842966436, "median_reduction_planned_only_percent": 36.436228842966436, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 35.26480773507577, 40.25046893964471 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4012086, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3664, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 249.372 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "native_compacted", "selected_samples": 57, "successful_evals": 57, "plans": 57, "no_plan": 0, "failures": 0, "primary_reduction_samples": 57, "median_reduction_including_no_plan_percent": 36.046523306850936, "median_reduction_planned_only_percent": 36.046523306850936, "min_reduction_including_no_plan_percent": 7.387144495312459, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 34.322291180955645, 39.890069839143415 ], "sum_context_before": 8612085, "sum_projected_reclaimed": 3131762, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 57, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 57, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 3721, "literal_probes_retained": 2868, "nonzero_tail_probe_samples": 23, "tail_probes_total": 88, "tail_probes_retained": 88, "median_wall_ms": 363.214 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 16, "no_plan": 0, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 43.80797883415849, "median_reduction_planned_only_percent": 43.80797883415849, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 62.16005138086063, "median_descriptive_resampling_95pct_interval": [ 27.98351791318231, 47.82124150228463 ], "sum_context_before": 2185222, "sum_projected_reclaimed": 880324, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 16, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 16, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1041, "literal_probes_retained": 796, "nonzero_tail_probe_samples": 8, "tail_probes_total": 34, "tail_probes_retained": 34, "median_wall_ms": 74.83 }, { "provider": "codex", "cohort": "high_context", "strategy": "cache_aware", "stratum": "no_observed_parent", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 36.436228842966436, "median_reduction_planned_only_percent": 36.436228842966436, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 35.26480773507577, 40.25046893964471 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4012086, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3664, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 249.372 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "all_selected", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 36.436228842966436, "median_reduction_planned_only_percent": 36.436228842966436, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 35.26480773507577, 40.25046893964471 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4012086, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3664, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 258.673 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "native_compacted", "selected_samples": 57, "successful_evals": 57, "plans": 57, "no_plan": 0, "failures": 0, "primary_reduction_samples": 57, "median_reduction_including_no_plan_percent": 36.046523306850936, "median_reduction_planned_only_percent": 36.046523306850936, "min_reduction_including_no_plan_percent": 7.387144495312459, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 34.322291180955645, 39.890069839143415 ], "sum_context_before": 8612085, "sum_projected_reclaimed": 3131762, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 57, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 57, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 3721, "literal_probes_retained": 2868, "nonzero_tail_probe_samples": 23, "tail_probes_total": 88, "tail_probes_retained": 88, "median_wall_ms": 373.238 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 16, "no_plan": 0, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 43.80797883415849, "median_reduction_planned_only_percent": 43.80797883415849, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 62.16005138086063, "median_descriptive_resampling_95pct_interval": [ 27.98351791318231, 47.82124150228463 ], "sum_context_before": 2185222, "sum_projected_reclaimed": 880324, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 16, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 16, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 1041, "literal_probes_retained": 796, "nonzero_tail_probe_samples": 8, "tail_probes_total": 34, "tail_probes_retained": 34, "median_wall_ms": 64.8275 }, { "provider": "codex", "cohort": "high_context", "strategy": "compacted", "stratum": "no_observed_parent", "selected_samples": 73, "successful_evals": 73, "plans": 73, "no_plan": 0, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 36.436228842966436, "median_reduction_planned_only_percent": 36.436228842966436, "min_reduction_including_no_plan_percent": 2.9606472968002944, "max_reduction_including_no_plan_percent": 69.43839619376405, "median_descriptive_resampling_95pct_interval": [ 35.26480773507577, 40.25046893964471 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 4012086, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 73, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 73, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 4762, "literal_probes_retained": 3664, "nonzero_tail_probe_samples": 31, "tail_probes_total": 122, "tail_probes_retained": 122, "median_wall_ms": 258.673 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "all_selected", "selected_samples": 73, "successful_evals": 73, "plans": 1, "no_plan": 72, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 8.584897925719439, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 8.584897925719439, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 10471, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 1, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 1, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 66, "literal_probes_retained": 66, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 98.941 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "native_compacted", "selected_samples": 57, "successful_evals": 57, "plans": 1, "no_plan": 56, "failures": 0, "primary_reduction_samples": 57, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 8.584897925719439, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 8.584897925719439, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 8612085, "sum_projected_reclaimed": 10471, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 1, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 1, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 66, "literal_probes_retained": 66, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 120.072 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "no_native_compaction", "selected_samples": 16, "successful_evals": 16, "plans": 0, "no_plan": 16, "failures": 0, "primary_reduction_samples": 16, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2185222, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 32.1595 }, { "provider": "codex", "cohort": "high_context", "strategy": "dedupe", "stratum": "no_observed_parent", "selected_samples": 73, "successful_evals": 73, "plans": 1, "no_plan": 72, "failures": 0, "primary_reduction_samples": 73, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": 8.584897925719439, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 8.584897925719439, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 10797307, "sum_projected_reclaimed": 10471, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 1, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": 0, "new_warning_count": 0, "nonzero_probe_samples": 1, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 66, "literal_probes_retained": 66, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 98.941 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "all_selected", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 47.142 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "native_compacted", "selected_samples": 22, "successful_evals": 22, "plans": 0, "no_plan": 22, "failures": 0, "primary_reduction_samples": 22, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 1139072, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 97.893 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "no_native_compaction", "selected_samples": 19, "successful_evals": 19, "plans": 0, "no_plan": 19, "failures": 0, "primary_reduction_samples": 19, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 892126, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.884 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "elide", "stratum": "no_observed_parent", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 47.142 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "all_selected", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 52.644 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "native_compacted", "selected_samples": 22, "successful_evals": 22, "plans": 0, "no_plan": 22, "failures": 0, "primary_reduction_samples": 22, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 1139072, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 101.252 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "no_native_compaction", "selected_samples": 19, "successful_evals": 19, "plans": 0, "no_plan": 19, "failures": 0, "primary_reduction_samples": 19, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 892126, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.607 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "scored", "stratum": "no_observed_parent", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 52.644 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "all_selected", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 52.879 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "native_compacted", "selected_samples": 22, "successful_evals": 22, "plans": 0, "no_plan": 22, "failures": 0, "primary_reduction_samples": 22, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 1139072, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 100.8835 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "no_native_compaction", "selected_samples": 19, "successful_evals": 19, "plans": 0, "no_plan": 19, "failures": 0, "primary_reduction_samples": 19, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 892126, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.698 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "cache_aware", "stratum": "no_observed_parent", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 52.879 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "all_selected", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 49.26 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "native_compacted", "selected_samples": 22, "successful_evals": 22, "plans": 0, "no_plan": 22, "failures": 0, "primary_reduction_samples": 22, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 1139072, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 104.40700000000001 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "no_native_compaction", "selected_samples": 19, "successful_evals": 19, "plans": 0, "no_plan": 19, "failures": 0, "primary_reduction_samples": 19, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 892126, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.763 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "compacted", "stratum": "no_observed_parent", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 49.26 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "all_selected", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 50.604 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "native_compacted", "selected_samples": 22, "successful_evals": 22, "plans": 0, "no_plan": 22, "failures": 0, "primary_reduction_samples": 22, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 1139072, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 100.69 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "no_native_compaction", "selected_samples": 19, "successful_evals": 19, "plans": 0, "no_plan": 19, "failures": 0, "primary_reduction_samples": 19, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 892126, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 27.004 }, { "provider": "codex", "cohort": "below_trigger_control", "strategy": "dedupe", "stratum": "no_observed_parent", "selected_samples": 41, "successful_evals": 41, "plans": 0, "no_plan": 41, "failures": 0, "primary_reduction_samples": 41, "median_reduction_including_no_plan_percent": 0.0, "median_reduction_planned_only_percent": null, "min_reduction_including_no_plan_percent": 0.0, "max_reduction_including_no_plan_percent": 0.0, "median_descriptive_resampling_95pct_interval": [ 0.0, 0.0 ], "sum_context_before": 2031198, "sum_projected_reclaimed": 0, "baseline_unavailable_samples": 0, "baseline_error_samples": 0, "planned_verification_delta_available": 0, "planned_verification_delta_unavailable": 0, "post_error_samples": 0, "new_error_count": null, "new_warning_count": null, "nonzero_probe_samples": 0, "zero_probe_samples": 0, "probe_score_unavailable_plans": 0, "literal_probes_total": 0, "literal_probes_retained": 0, "nonzero_tail_probe_samples": 0, "tail_probes_total": 0, "tail_probes_retained": 0, "median_wall_ms": 50.604 } ] } ], "privacy": "Aggregate statistics only. Private transcripts, session identifiers, paths, per-session hashes, manifests and per-session results are not published.", "limitations": [ "One person and one Mac; sessions may share prompts, projects or unobserved ancestry.", "No provider continuation, billing, semantic judge or remote model was evaluated.", "The measured extractor had a nominal 64-probe cap but could emit 65\u201367 probes; the reported denominators are actual counts.", "Structural deltas compare severity/code counts, not finding identity or proof of safe provider resume.", "Resampling intervals describe the selected stratum only; they are not population confidence intervals.", "Cohort strata overlap: do not add native-compacted, no-native-compaction and no-observed-parent rows together." ] }