{
  "status": "Published supporting methodology record, version 1",
  "study_date": "2026-10-14",
  "ownership": "Developer-run study; report and diagnostic review AI-assisted; no independent adjudication",
  "sample": {
    "responses": 67,
    "questions": 39,
    "handwritten": 65,
    "typeset": 2,
    "original_core": 54,
    "reserve_enrichment": 13,
    "reference_score_groups": {
      "low": 11,
      "middle": 13,
      "high": 43
    }
  },
  "configuration": {
    "version": "economics-a-heldout-v1",
    "frozen_at_utc": "2026-10-14T11:19:37.094684+00:00",
    "model": "gpt-6.1-sol",
    "reasoning_effort": "low",
    "provider_api": "OpenAI Responses",
    "store": false,
    "temperature": "omitted",
    "max_output_tokens": {
      "below_10_marks": 6000,
      "10_marks_and_above": 12000
    },
    "image_detail": "auto",
    "first_attempts_per_case": 1,
    "automatic_retries": 0,
    "timeout_seconds": 180,
    "fallback_model": null,
    "feedback_repairs": false,
    "wrapper_source_commit": "e2695bd08dfa215f85371ef3088a89379cc0326f",
    "wrapper_provenance": "Registered Satellite integration source; isolated request-builder benchmark, not a deployed end-to-end product test",
    "raw_provider_response_required": true,
    "model_revision_policy": "Record actual returned model identifier and response ID on every result; alias may change. Do not claim exact future reproducibility of a provider alias.",
    "validation": "Validate schema, numeric mark bounds and total consistency; retain invalid/incomplete results as failures, do not silently rerun",
    "question_materials": "Original question/context and scheme page images plus scheme text; mark only designated question. Unselected questions sometimes share pages.",
    "inputs": "Original embedded scan pixels, saved as PNG; no enhancement, transcription, generative editing or answer rewriting"
  },
  "configuration_selection_limit": "Current Live catalogue default, explicitly selected low effort for this study; no claim of exhaustive Live policy resolution",
  "prompt_hashes": [
    {
      "title": "Edexcel Economic A Level: 25 Marker (Papers 1 and 2)",
      "sha256": "117d695708ac7d607e7dba985b6eca85f72fd5786d187b5ec95449be7453547b"
    },
    {
      "title": "Edexcel Economics A Level: 4 Markers (Diagram Drawing)",
      "sha256": "58cf771d558e1a79e87b0946b229e703807fdd433fdbc29d9c62792b27e8369f"
    },
    {
      "title": "Edexcel Economics A Level: 12 Marker",
      "sha256": "f5bcda1b0299f26c134471e6a8cec126ef326da134a65265df6a67c9acb2759e"
    },
    {
      "title": "Edexcel Economics A Level: 25 Marker (Paper 3)",
      "sha256": "930ed20385887234d6276e1a9afcd63684ba37c1a055a27af9d42d13210a8a36"
    },
    {
      "title": "Edexcel Economics A Level: 5 Marker",
      "sha256": "f90f3fe7f16e25276040c1c63f3d333e9fb3ae2828eb884657de26e4623b61c4"
    },
    {
      "title": "Edexcel Economics A Level: 15 Marker",
      "sha256": "6c21b0cf65e60de9c08e11777e832dbd7fa0f0178c2944ab2f2e7dc4c3774de0"
    },
    {
      "title": "Edexcel Economics A Level: 10 Marker",
      "sha256": "e296d639755b54066ff1a94104fc4890c6adb351be36bbd342771e17d30a4f93"
    },
    {
      "title": "Edexcel Economics A Level: 8 marker",
      "sha256": "518609ab4c55a858153696a7cb88688ce2daee624cdb476217f0ddef460d8c54"
    },
    {
      "title": "Edexcel Economics A Level: 4 marker (K=2, An=1, Ap=1)",
      "sha256": "8d75c1b75be065a349ec1f1e9978f9d39cce37fed7511030214d588a9857fdde"
    }
  ],
  "frozen_manifest_sha256": "e101a20105d644a888b93a43fadd886aabff0b30217854edd676f579b4fe2940",
  "level_analysis_timing": {
    "recorded_at_utc": "2026-10-14T11:30:53.050571+00:00",
    "timing": "Added during the first run after 12 results had been saved; marking instructions, requests, settings and original analysis plan remained unchanged",
    "additional_analysis": [
      "Exact examiner-level agreement and level differences for KAA and evaluation separately where numerical reference components and official scheme level boundaries are available",
      "Level-crossing disagreements highlighted even where raw-mark difference is small",
      "Descriptive low/middle/high score-band agreement shown separately from examiner levels",
      "Unidentifiable reference/AI component levels reported as unavailable; no invented overall essay level from total"
    ],
    "limitations": "This is an explicitly disclosed analysis amendment during data collection, not part of the original pre-run freeze.",
    "results_saved_when_recorded": 12
  },
  "level_correction": {
    "date": "2026-10-14",
    "reason": "Post-run review found explicit published KAA Level 2 and evaluation Level 1 for A038 on 2022 Paper 2 report PDF page 41. These were absent from the frozen structured register.",
    "original_cases": 25,
    "corrected_cases": 26,
    "summary": {
      "KAA": {
        "component_pairs": 26,
        "same_level": 20,
        "one_level_apart": 6,
        "two_or_more_levels_apart": 0,
        "ai_higher": 3,
        "ai_lower": 3
      },
      "evaluation": {
        "component_pairs": 26,
        "same_level": 16,
        "one_level_apart": 9,
        "two_or_more_levels_apart": 1,
        "ai_higher": 7,
        "ai_lower": 3
      }
    },
    "original_frozen_data_and_results_unchanged": true,
    "all_mark_metrics_unchanged": true,
    "reference_components_not_replaced": true
  },
  "analysis_plan": {
    "primary_population": "All 67 frozen cases; no exclusions after seeing AI output",
    "first_attempt_only": true,
    "metrics": [
      "Exact agreement",
      "Within one mark",
      "Within two marks",
      "Mean absolute error in raw marks by tariff",
      "Signed error AI minus examiner",
      "Mean absolute error in percentage points of available marks"
    ],
    "strata": [
      "Maximum mark",
      "Low/middle/high reference score band",
      "Original core versus reserve enrichment",
      "Handwritten versus typeset"
    ],
    "failures": "Keep missing, invalid, incomplete and timed-out cases in all-case agreement denominators as non-agreements; additionally report valid-result-only errors with explicit denominator. Never turn a provider failure into a zero mark.",
    "sensitivity_exclusions": [
      "2022-P1-Q6c-E2",
      "2018-P1-Q6b-E2",
      "2018-P1-Q6d-E1"
    ],
    "uncertainty": "Exploratory only. If intervals are calculated, bootstrap question clusters, not individual responses.",
    "tuning": "No prompt/model/parameter changes after first result; any change requires a new named study version and disclosure of prior exposure.",
    "replicability": "Publish exact case/source identifiers and configuration hashes with results; underlying model training exposure unknown; model alias may drift."
  },
  "calibration_exclusions": [
    [
      "4-mark explain",
      "Tuition-fee cap £9,250; differing bus-ticket PED for students/adults",
      "Current Live/Development and 29 September snapshot"
    ],
    [
      "4-mark diagram",
      "Loss-making luxury brownie baker",
      "Current Live/Development and snapshot"
    ],
    [
      "5-mark",
      "Market structure of UK streaming services",
      "Current Live/Development and snapshot"
    ],
    [
      "8-mark",
      "Cross-elasticity between UK streaming services",
      "Current Live/Development and snapshot"
    ],
    [
      "10-mark",
      "Video-game contestability; Netflix rationality; UK fiscal deficit and national debt",
      "Current Live/Development and snapshot"
    ],
    [
      "12-mark",
      "Private/external benefits of BBC Bitesize and Oak National Academy",
      "Current Live/Development and snapshot"
    ],
    [
      "15-mark",
      "Streaming price discrimination; UK growth policies with output-gap extract",
      "Current Live/Development and snapshot"
    ],
    [
      "25-mark P1/P2",
      "Road-transport carbon emissions London–Glasgow; protectionism/global economy; hotel energy costs",
      "Current Live/Development and snapshot"
    ],
    [
      "25-mark P3",
      "Road-transport emissions; protectionism; illustrative HFSS-tax fragment",
      "Current Live/Development and snapshot"
    ],
    [
      "2024 P1 Q6c, Q6d, Q6e, Q7, Q8",
      "Electronics scale economies; merger concerns; electronic waste; hotel energy costs; fragrance contestability",
      "Prior Cost Analysis evaluations; Q7 also calibration. Exclude all answers to these questions."
    ]
  ],
  "replication_limit": "Full internal prompt/request/output-specification package retained but not publicly released; provider alias may change; this record is insufficient by itself for exact marking-call replication",
  "ofqual_values": "Approximate digitisation of Figure 10, PDF page 22; descriptive context, not matched human-AI comparison",
  "ofqual_source": "https://assets.publishing.service.gov.uk/media/5bfc2418ed915d11846608ae/Marking_consistencies_-_FINAL64493.pdf",
  "publication_date": "2026-10-14",
  "review_method": "AI-assisted diagnostic review of 18 cases with access to both examiner and TeachEdge awards; not blind or independent adjudication."
}