{
  "benchmark": "subscription-only-two-tier-grok43-grok45-gpt55",
  "generated_at": "2026-07-08T21:58:44.756998+00:00",
  "subscription_only": true,
  "routes": [
    {
      "label": "GPT-5.5",
      "provider": "openai-codex",
      "model": "gpt-5.5",
      "subscription_route_verified": true,
      "direct_api_key_used": false,
      "api_key_env_stripped": true,
      "smoke_ok": true,
      "wall_seconds": 9.92
    },
    {
      "label": "Grok 4.3",
      "provider": "xai-oauth",
      "model": "grok-4.3",
      "subscription_route_verified": true,
      "direct_api_key_used": false,
      "api_key_env_stripped": true,
      "smoke_ok": true,
      "wall_seconds": 7.98
    },
    {
      "label": "Grok 4.5",
      "provider": "xai-oauth",
      "model": "grok-4.5",
      "subscription_route_verified": true,
      "direct_api_key_used": false,
      "api_key_env_stripped": true,
      "smoke_ok": true,
      "wall_seconds": 9.57
    }
  ],
  "notes": [
    "All scored model calls used subscription/OAuth providers with direct API-key env vars stripped.",
    "Initial relative-path tool probe was excluded as harness-invalid because Hermes child file tools used configured local cwd, not subprocess cwd; replaced by absolute-path tool canary.",
    "Terminal-Bench was intentionally not run because Harbor/Terminal-Bench would use API-key/OpenAI-compatible endpoints, violating the subscription-only constraint."
  ],
  "summaries": {
    "tier1_direct_strict": {
      "GPT-5.5": {
        "passed": 6,
        "total": 7,
        "timeouts": 0,
        "wall_seconds_total": 66.2,
        "fails": [
          {
            "task_id": "d03_log_triage",
            "notes": "json mismatches={'first_error': {'expected': 'E42', 'got': 'E42 disk full'}}",
            "wall_seconds": 7.76,
            "returncode": 0
          }
        ],
        "pass_rate": 0.857,
        "median_success_wall": 8.8
      },
      "Grok 4.3": {
        "passed": 6,
        "total": 7,
        "timeouts": 0,
        "wall_seconds_total": 75.95,
        "fails": [
          {
            "task_id": "d03_log_triage",
            "notes": "json mismatches={'unique_error_count': {'expected': 3, 'got': 4}, 'first_error': {'expected': 'E42', 'got': 'E42 disk full'}}",
            "wall_seconds": 14.75,
            "returncode": 0
          }
        ],
        "pass_rate": 0.857,
        "median_success_wall": 9.27
      },
      "Grok 4.5": {
        "passed": 6,
        "total": 7,
        "timeouts": 0,
        "wall_seconds_total": 64.91,
        "fails": [
          {
            "task_id": "d03_log_triage",
            "notes": "json mismatches={'first_error': {'expected': 'E42', 'got': 'E42 disk full'}}",
            "wall_seconds": 14.93,
            "returncode": 0
          }
        ],
        "pass_rate": 0.857,
        "median_success_wall": 8.33
      }
    },
    "tier1_tool_abs": {
      "GPT-5.5": {
        "passed": 5,
        "total": 5,
        "timeouts": 0,
        "wall_seconds_total": 54.22,
        "fails": [],
        "pass_rate": 1.0,
        "median_success_wall": 10.96
      },
      "Grok 4.3": {
        "passed": 5,
        "total": 5,
        "timeouts": 0,
        "wall_seconds_total": 60.84,
        "fails": [],
        "pass_rate": 1.0,
        "median_success_wall": 12.47
      },
      "Grok 4.5": {
        "passed": 5,
        "total": 5,
        "timeouts": 0,
        "wall_seconds_total": 51.57,
        "fails": [],
        "pass_rate": 1.0,
        "median_success_wall": 9.31
      }
    },
    "tier2_promotion_lite": {
      "GPT-5.5": {
        "passed": 3,
        "total": 3,
        "timeouts": 0,
        "wall_seconds_total": 61.22,
        "fails": [],
        "pass_rate": 1.0,
        "median_success_wall": 22.54
      },
      "Grok 4.3": {
        "passed": 3,
        "total": 3,
        "timeouts": 0,
        "wall_seconds_total": 56.37,
        "fails": [],
        "pass_rate": 1.0,
        "median_success_wall": 18.97
      },
      "Grok 4.5": {
        "passed": 3,
        "total": 3,
        "timeouts": 0,
        "wall_seconds_total": 46.65,
        "fails": [],
        "pass_rate": 1.0,
        "median_success_wall": 13.77
      }
    },
    "combined_scored": {
      "GPT-5.5": {
        "passed": 14,
        "total": 15,
        "timeouts": 0,
        "wall_seconds_total": 181.64,
        "fails": [
          {
            "task_id": "d03_log_triage",
            "notes": "json mismatches={'first_error': {'expected': 'E42', 'got': 'E42 disk full'}}",
            "wall_seconds": 7.76,
            "returncode": 0
          }
        ],
        "pass_rate": 0.933,
        "median_success_wall": 10.99
      },
      "Grok 4.3": {
        "passed": 14,
        "total": 15,
        "timeouts": 0,
        "wall_seconds_total": 193.16,
        "fails": [
          {
            "task_id": "d03_log_triage",
            "notes": "json mismatches={'unique_error_count': {'expected': 3, 'got': 4}, 'first_error': {'expected': 'E42', 'got': 'E42 disk full'}}",
            "wall_seconds": 14.75,
            "returncode": 0
          }
        ],
        "pass_rate": 0.933,
        "median_success_wall": 11.59
      },
      "Grok 4.5": {
        "passed": 14,
        "total": 15,
        "timeouts": 0,
        "wall_seconds_total": 163.13,
        "fails": [
          {
            "task_id": "d03_log_triage",
            "notes": "json mismatches={'first_error': {'expected': 'E42', 'got': 'E42 disk full'}}",
            "wall_seconds": 14.93,
            "returncode": 0
          }
        ],
        "pass_rate": 0.933,
        "median_success_wall": 9.45
      }
    },
    "harness_invalid_relative_tool_probe": {
      "GPT-5.5": {
        "passed": 0,
        "total": 1,
        "timeouts": 1,
        "wall_seconds_total": 240.02,
        "fails": [
          {
            "task_id": "t01_file_read",
            "notes": "expected phrase 'TOKEN: TOKEN-74B'",
            "wall_seconds": 240.02,
            "returncode": 124
          }
        ],
        "pass_rate": 0.0,
        "median_success_wall": null
      },
      "Grok 4.3": {
        "passed": 0,
        "total": 1,
        "timeouts": 0,
        "wall_seconds_total": 17.18,
        "fails": [
          {
            "task_id": "t01_file_read",
            "notes": "expected phrase 'TOKEN: TOKEN-74B'",
            "wall_seconds": 17.18,
            "returncode": 0
          }
        ],
        "pass_rate": 0.0,
        "median_success_wall": null
      },
      "Grok 4.5": {
        "passed": 0,
        "total": 1,
        "timeouts": 1,
        "wall_seconds_total": 240.02,
        "fails": [
          {
            "task_id": "t01_file_read",
            "notes": "expected phrase 'TOKEN: TOKEN-74B'",
            "wall_seconds": 240.02,
            "returncode": 124
          }
        ],
        "pass_rate": 0.0,
        "median_success_wall": null
      }
    }
  },
  "scored_set": "Tier 1 direct strict + corrected absolute-path file/tool canary + Tier 2 promotion-lite; invalid relative-path file probe excluded."
}