{
  "schema_version": 1,
  "analysis": "post_hoc_diagnostics_of_frozen_development_predictions",
  "cases": 50,
  "benchmark": {
    "commit": "9b6a766712583fec8d3182957260b1123fbfa146",
    "development_sha256": "afe74265475f1897161f54ac91395ded74d839e43236e8f4d189d91f9b40651a",
    "evaluator_sha256": "1236dbcca70129f6200c2703dd67649dbfa760a2a7f74402fa1d3d956b97b2ae",
    "prediction_sha256": "0b43ad574543cb128fa51470432467a34998c675b945665bb44141d6e04bca39"
  },
  "bootstrap": {
    "unit": "paired development case",
    "resamples": 20000,
    "seed": 20260917,
    "interval": "percentile",
    "confidence_level": 0.95,
    "interpretation": "Describes sensitivity to development-case composition; it is not a population-generalization interval."
  },
  "paired_comparisons": {
    "normalized_exact_match_rate": {
      "stable_mean": 0.0,
      "repeat_2024_mean": 0.0,
      "effect_definition": "stable_minus_repeat_2024",
      "mean_effect_positive_favors_stable": 0.0,
      "paired_case_bootstrap_95_ci_low": 0.0,
      "paired_case_bootstrap_95_ci_high": 0.0,
      "stable_case_wins": 0,
      "case_ties": 50,
      "stable_case_losses": 0
    },
    "normalized_edit_similarity": {
      "stable_mean": 0.298061,
      "repeat_2024_mean": 0.291966,
      "effect_definition": "stable_minus_repeat_2024",
      "mean_effect_positive_favors_stable": 0.006095,
      "paired_case_bootstrap_95_ci_low": -0.001701,
      "paired_case_bootstrap_95_ci_high": 0.01389,
      "stable_case_wins": 24,
      "case_ties": 14,
      "stable_case_losses": 12
    },
    "token_jaccard_similarity": {
      "stable_mean": 0.142768,
      "repeat_2024_mean": 0.14193,
      "effect_definition": "stable_minus_repeat_2024",
      "mean_effect_positive_favors_stable": 0.000838,
      "paired_case_bootstrap_95_ci_low": -0.002383,
      "paired_case_bootstrap_95_ci_high": 0.004097,
      "stable_case_wins": 16,
      "case_ties": 20,
      "stable_case_losses": 14
    },
    "token_overlap_f1": {
      "stable_mean": 0.307444,
      "repeat_2024_mean": 0.312202,
      "effect_definition": "stable_minus_repeat_2024",
      "mean_effect_positive_favors_stable": -0.004759,
      "paired_case_bootstrap_95_ci_low": -0.0123,
      "paired_case_bootstrap_95_ci_high": 0.002429,
      "stable_case_wins": 13,
      "case_ties": 20,
      "stable_case_losses": 17
    },
    "rouge_l_f1": {
      "stable_mean": 0.227552,
      "repeat_2024_mean": 0.225683,
      "effect_definition": "stable_minus_repeat_2024",
      "mean_effect_positive_favors_stable": 0.001869,
      "paired_case_bootstrap_95_ci_low": -0.002268,
      "paired_case_bootstrap_95_ci_high": 0.005755,
      "stable_case_wins": 19,
      "case_ties": 20,
      "stable_case_losses": 11
    },
    "character_ngram_f1": {
      "stable_mean": 0.292293,
      "repeat_2024_mean": 0.296534,
      "effect_definition": "stable_minus_repeat_2024",
      "mean_effect_positive_favors_stable": -0.004241,
      "paired_case_bootstrap_95_ci_low": -0.01224,
      "paired_case_bootstrap_95_ci_high": 0.003618,
      "stable_case_wins": 16,
      "case_ties": 14,
      "stable_case_losses": 20
    },
    "word_count_mae": {
      "stable_mean": 41.64,
      "repeat_2024_mean": 56.68,
      "effect_definition": "repeat_2024_error_minus_stable_error",
      "mean_effect_positive_favors_stable": 15.04,
      "paired_case_bootstrap_95_ci_low": 1.18,
      "paired_case_bootstrap_95_ci_high": 32.36,
      "stable_case_wins": 20,
      "case_ties": 20,
      "stable_case_losses": 10
    },
    "line_count_mae": {
      "stable_mean": 9.76,
      "repeat_2024_mean": 6.8,
      "effect_definition": "repeat_2024_error_minus_stable_error",
      "mean_effect_positive_favors_stable": -2.96,
      "paired_case_bootstrap_95_ci_low": -5.54,
      "paired_case_bootstrap_95_ci_high": -0.42,
      "stable_case_wins": 15,
      "case_ties": 12,
      "stable_case_losses": 23
    },
    "source_similarity_mae": {
      "stable_mean": 0.678215,
      "repeat_2024_mean": 0.774317,
      "effect_definition": "repeat_2024_error_minus_stable_error",
      "mean_effect_positive_favors_stable": 0.096102,
      "paired_case_bootstrap_95_ci_low": 0.066064,
      "paired_case_bootstrap_95_ci_high": 0.129348,
      "stable_case_wins": 30,
      "case_ties": 20,
      "stable_case_losses": 0
    }
  },
  "extractive_limit_diagnostics": {
    "future_unique_token_novelty_fraction": {
      "mean": 0.731727,
      "median": 0.739048,
      "case_bootstrap_95_ci_low": 0.697982,
      "case_bootstrap_95_ci_high": 0.765607,
      "minimum": 0.48,
      "maximum": 0.956522
    },
    "future_token_occurrence_unavailable_fraction": {
      "mean": 0.650621,
      "median": 0.651845,
      "case_bootstrap_95_ci_low": 0.592663,
      "case_bootstrap_95_ci_high": 0.706149,
      "minimum": 0.202247,
      "maximum": 0.969697
    },
    "source_unique_token_retention_fraction": {
      "mean": 0.249892,
      "median": 0.233202,
      "case_bootstrap_95_ci_low": 0.22006,
      "case_bootstrap_95_ci_high": 0.280466,
      "minimum": 0.040984,
      "maximum": 0.492063
    },
    "oracle_unique_token_jaccard_ceiling": {
      "mean": 0.268273,
      "median": 0.260952,
      "case_bootstrap_95_ci_low": 0.234441,
      "case_bootstrap_95_ci_high": 0.302025,
      "minimum": 0.043478,
      "maximum": 0.52
    },
    "oracle_bag_of_words_f1_ceiling": {
      "mean": 0.484206,
      "median": 0.516491,
      "case_bootstrap_95_ci_low": 0.422094,
      "case_bootstrap_95_ci_high": 0.546136,
      "minimum": 0.058824,
      "maximum": 0.8875
    },
    "oracle_source_order_subsequence_rouge_l_f1_ceiling": {
      "mean": 0.374081,
      "median": 0.336461,
      "case_bootstrap_95_ci_low": 0.318434,
      "case_bootstrap_95_ci_high": 0.431571,
      "minimum": 0.029851,
      "maximum": 0.779412
    }
  },
  "oracle_warning": "Oracle ceilings use observed follow-ups and are unattainable prospective diagnostics, not prediction results."
}