{
  "schemaVersion": 1,
  "updated": "2026-10-22",
  "title": "GEAR paper results",
  "source": {
    "type": "user-provided-paper-screenshot",
    "date": "2026-10-22",
    "precision": "Displayed percentages rounded to one decimal place. Delta calculation bases are recorded per comparison."
  },
  "units": {
    "metrics": "percent",
    "suppliedDeltas": "percentage points"
  },
  "display": {
    "mainTableDecimalPlaces": 1,
    "deltaPolicy": "Retain the deltas supplied in the latest paper table; do not recompute them from rounded display values."
  },
  "metricDefinitions": {
    "passRatePercent": "Percentage of tasks passing all scored checks.",
    "objectiveCompletionPercent": "Mean fraction of scored assertions satisfied per task, expressed as a percentage."
  },
  "rows": [
    {
      "id": "luna-medium-base",
      "model": "GPT 5.6 Luna",
      "modelId": "gpt-5.6-luna",
      "effort": "medium",
      "harness": "DSH",
      "variant": "Base",
      "metrics": {
        "marketing": {
          "passRatePercent": 27.0,
          "objectiveCompletionPercent": 75.4
        },
        "support": {
          "passRatePercent": 17.0,
          "objectiveCompletionPercent": 72.9
        },
        "coding": {
          "passRatePercent": 52.8
        }
      }
    },
    {
      "id": "luna-medium-ours",
      "model": "GPT 5.6 Luna",
      "modelId": "gpt-5.6-luna",
      "effort": "medium",
      "harness": "DSH",
      "variant": "Ours",
      "metrics": {
        "marketing": {
          "passRatePercent": 40.0,
          "objectiveCompletionPercent": 83.9
        },
        "support": {
          "passRatePercent": 25.0,
          "objectiveCompletionPercent": 81.3
        },
        "coding": {
          "passRatePercent": 62.9
        }
      }
    },
    {
      "id": "astra-max-base",
      "model": "GPT 6 Astra",
      "modelId": "gpt-6-astra",
      "effort": "max",
      "harness": "Codex",
      "variant": "Base",
      "metrics": {
        "marketing": {
          "passRatePercent": 57.0,
          "objectiveCompletionPercent": 84.1
        },
        "support": {
          "passRatePercent": 35.0,
          "objectiveCompletionPercent": 88.4
        },
        "coding": {
          "passRatePercent": 86.7
        }
      }
    },
    {
      "id": "astra-max-ours",
      "model": "GPT 6 Astra",
      "modelId": "gpt-6-astra",
      "effort": "max",
      "harness": "DSH",
      "variant": "Ours",
      "metrics": {
        "marketing": {
          "passRatePercent": 61.0,
          "objectiveCompletionPercent": 86.3
        },
        "support": {
          "passRatePercent": 39.0,
          "objectiveCompletionPercent": 88.7
        },
        "coding": {
          "passRatePercent": 87.6
        }
      }
    }
  ],
  "comparisons": [
    {
      "baselineRow": "luna-medium-base",
      "optimizedRow": "luna-medium-ours",
      "suppliedDeltas": {
        "marketing": {
          "passRatePercentagePoints": 13.0,
          "objectiveCompletionPercentagePoints": 8.5
        },
        "support": {
          "passRatePercentagePoints": 8.0,
          "objectiveCompletionPercentagePoints": 8.4
        },
        "coding": {
          "passRatePercentagePoints": 10.1
        }
      },
      "basis": "User-supplied paper table; deltas computed from unrounded values."
    },
    {
      "baselineRow": "astra-max-base",
      "optimizedRow": "astra-max-ours",
      "suppliedDeltas": {
        "marketing": {
          "passRatePercentagePoints": 4.0,
          "objectiveCompletionPercentagePoints": 2.2
        },
        "support": {
          "passRatePercentagePoints": 4.0,
          "objectiveCompletionPercentagePoints": 0.3
        },
        "coding": {
          "passRatePercentagePoints": 0.9
        }
      },
      "basis": "Deltas retained as reported in the corrected user-supplied paper screenshot. Its caption states that deltas are calculated before rounding; Astra Coding is reported as +0.9 percentage points."
    }
  ],
  "supplementaryEvaluations": [
    {
      "id": "luna-max-ours",
      "model": "GPT 5.6 Luna",
      "modelId": "gpt-5.6-luna",
      "effort": "max",
      "harness": "DSH",
      "variant": "Ours",
      "metrics": {
        "marketing": {
          "passRatePercent": 53.0,
          "objectiveCompletionPercent": 88.8802379242704
        },
        "support": {
          "passRatePercent": 27.0,
          "objectiveCompletionPercent": 82.4
        },
        "coding": {
          "passRatePercent": 84.26
        }
      },
      "sources": {
        "marketing": {
          "file": "marketing-results.json",
          "field": "effortComparison[1]"
        },
        "support": {
          "type": "user-provided-clarification",
          "date": "2026-10-22",
          "note": "Luna max Support task pass rate is 27%; objective completion is 0.824, recorded here as 82.4%."
        },
        "coding": {
          "file": "coding-results.json",
          "field": "max.passRate"
        }
      },
      "notes": [
        "Separate max-effort evaluation; comparisons with Luna medium change reasoning effort.",
        "Marketing and Coding retain the existing precise local source values; Support incorporates the user clarification."
      ]
    }
  ],
  "scope": {
    "marketing": {
      "benchmark": "AutomationBench public Marketing",
      "tasks": 100,
      "validationMode": "shared-set-research",
      "independentHeldOut": false,
      "detailsFile": "marketing-results.json"
    },
    "support": {
      "benchmarkVersion": null,
      "scoredTaskCount": null,
      "independentHeldOut": null,
      "detailsFile": "support-results.json"
    },
    "coding": {
      "benchmark": "Terminal-Bench 2.1",
      "scoredTaskCount": null,
      "independentHeldOut": null,
      "detailsFile": "coding-results.json"
    }
  },
  "historicalDataFiles": [
    "marketing-results.json",
    "support-results.json",
    "coding-results.json"
  ],
  "notes": [
    "Main-table metrics and supplied deltas reproduce the corrected user-supplied paper screenshot. Terminal-Bench revisions are recorded in updates. Historical values and audit metadata remain available in the domain files.",
    "Supplied deltas are percentage-point gains, not relative percentage improvements. The corrected paper table reports Luna Coding 52.8% to 62.9% (+10.1 points) and Astra Coding 86.7% to 87.6% (+0.9 points).",
    "Marketing used the same 100 public tasks for optimization and evaluation; these values do not establish independent held-out generalization.",
    "The new Support and Coding Astra aggregates are user-supplied paper results. No missing benchmark version, task count, per-task audit, confidence interval, or validation status is inferred.",
    "The Luna medium baseline-to-optimized comparison includes the earlier harness optimization and the staged continuation; it must not be attributed entirely to Staged GEPA."
  ],
  "updates": [
    {
      "date": "2026-10-22",
      "source": "user-provided-correction",
      "row": "astra-max-ours",
      "metric": "coding.passRatePercent",
      "previousValue": 87.6,
      "value": 89.3,
      "deltaPercentagePoints": 2.6
    },
    {
      "date": "2026-10-22",
      "source": "user-provided-paper-screenshot",
      "row": "astra-max-ours",
      "metric": "coding.passRatePercent",
      "previousValue": 89.3,
      "value": 87.6,
      "previousDeltaPercentagePoints": 2.6,
      "deltaPercentagePoints": 1.1,
      "basis": "Latest paper table reports Codex 86.7%, DSH (Ours) 87.6%, and a supplied +1.1 percentage-point delta. Values are transcribed as reported; no unrounded inputs are inferred."
    },
    {
      "date": "2026-10-22",
      "source": "user-provided-corrected-paper-screenshot",
      "row": "luna-medium-base",
      "metric": "coding.passRatePercent",
      "previousValue": 52.9,
      "value": 52.8,
      "basis": "Corrected Table 1 reports the Luna medium baseline as 52.8%; the optimized value remains 62.9% and the supplied gain remains +10.1 percentage points."
    },
    {
      "date": "2026-10-22",
      "source": "user-provided-corrected-paper-screenshot",
      "baselineRow": "astra-max-base",
      "optimizedRow": "astra-max-ours",
      "metric": "coding.passRatePercentagePoints",
      "previousValue": 1.1,
      "value": 0.9,
      "basis": "Corrected Table 1 reports the Astra Coding gain as +0.9 percentage points; baseline and optimized pass rates remain 86.7% and 87.6%."
    }
  ]
}