{ "benchmark": "hermes-bench-v5-stateful-resume", "started_at": "2026-07-09T01:38:57.795809+00:00", "finished_at": "2026-07-09T01:46:51.335540+00:00", "policy": { "subscription_only": true, "direct_api_key_used": false, "api_key_env_vars_stripped": [ "XAI_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY" ] }, "route_proof": [ { "label": "GPT-5.5", "provider": "openai-codex", "model": "gpt-5.5", "expected": "GPT55_V5_OK", "returned": "GPT55_V5_OK\n\nsession_id: 20260708_203858_5c2904", "ok": true, "rc": 0, "timeout": false, "wall": 9.88, "direct_api_key_used": false, "api_key_env_stripped": [ "XAI_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY" ], "started_at": "2026-07-09T01:38:57.795809+00:00" }, { "label": "Grok 4.5", "provider": "xai-oauth", "model": "grok-4.5", "expected": "GROK45_V5_OK", "returned": "GROK45_V5_OK\n\nsession_id: 20260708_203907_75cc8b", "ok": true, "rc": 0, "timeout": false, "wall": 7.95, "direct_api_key_used": false, "api_key_env_stripped": [ "XAI_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY" ], "started_at": "2026-07-09T01:38:57.795809+00:00" } ], "tasks": [ { "id": "router", "title": "Mini-Hermes router resume trap", "brief": "Fix provider routing with policy, auth refresh, fallback, task support, and retry-storm guards." }, { "id": "scheduler", "title": "Dirty scheduler lock resume trap", "brief": "Fix due-job selection and mark_started for stale locks, active locks, cancellation, idempotency, and no mutation." }, { "id": "migrations", "title": "Migration planner stale-note trap", "brief": "Fix migration planning with safe shortest paths, destructive guards, replacements, downgrade/no-path errors." } ], "repeats": 1, "summary": { "GPT-5.5": { "attempts_passed": 1, "attempts_total": 3, "phase1_ok": 3, "public_passed": 3, "hidden_passed": 1, "stale_resisted": 3, "task_all_passed": 1, "task_total": 3, "timeouts": 0, "total_wall": 581.91, "median_pass_wall": 229.32, "diagnosis_keyword_hits": 9, "changed_files_total": 3, "added_lines_total": 141, "deleted_lines_total": 29 }, "Grok 4.5": { "attempts_passed": 1, "attempts_total": 3, "phase1_ok": 3, "public_passed": 3, "hidden_passed": 2, "stale_resisted": 2, "task_all_passed": 1, "task_total": 3, "timeouts": 0, "total_wall": 284.02, "median_pass_wall": 73.56, "diagnosis_keyword_hits": 9, "changed_files_total": 3, "added_lines_total": 121, "deleted_lines_total": 21 } }, "rows": [ { "model": "GPT-5.5", "provider": "openai-codex", "model_slug": "gpt-5.5", "task": "migrations", "title": "Migration planner stale-note trap", "repeat": 1, "workspace": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/migrations/r1", "phase1": { "rc": 0, "timeout": false, "wall": 78.35, "diagnosis": { "plan_exists": true, "keyword_hits": [ "destructive", "shortest", "replaces" ], "score": 3, "required": [ "destructive", "shortest", "replaces" ] }, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/GPT-5.5__migrations__r1__phase1.txt" }, "phase2": { "rc": 0, "timeout": false, "wall": 150.97, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/GPT-5.5__migrations__r1__phase2.txt" }, "initial_public_pass": false, "public": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_destructive_skipped_when_safe_replacement_exists (tests.test_public.MigrationPublic.test_destructive_skipped_when_safe_replacement_exists) ... ok\ntest_straight_line (tests.test_public.MigrationPublic.test_straight_line) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\n" }, "hidden": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_destructive_requires_flag_and_replacement_preferred (tests.test_hidden.MigrationsHidden.test_destructive_requires_flag_and_replacement_preferred) ... ok\ntest_direct_shortest_safe_path (tests.test_hidden.MigrationsHidden.test_direct_shortest_safe_path) ... ok\ntest_no_downgrade_and_no_path (tests.test_hidden.MigrationsHidden.test_no_downgrade_and_no_path) ... ok\ntest_destructive_skipped_when_safe_replacement_exists (tests.test_public.MigrationPublic.test_destructive_skipped_when_safe_replacement_exists) ... ok\ntest_straight_line (tests.test_public.MigrationPublic.test_straight_line) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nOK\n" }, "stale_resistance": { "passed": true, "forbidden_hits": [] }, "patch": { "changed_files": 1, "added_lines": 70, "deleted_lines": 11, "changed_paths": [ "migrations.py" ] }, "passed": true, "total_agent_wall": 229.32 }, { "model": "GPT-5.5", "provider": "openai-codex", "model_slug": "gpt-5.5", "task": "router", "title": "Mini-Hermes router resume trap", "repeat": 1, "workspace": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/router/r1", "phase1": { "rc": 0, "timeout": false, "wall": 68.69, "diagnosis": { "plan_exists": true, "keyword_hits": [ "policy", "auth", "fallback" ], "score": 3, "required": [ "policy", "auth", "fallback" ] }, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/GPT-5.5__router__r1__phase1.txt" }, "phase2": { "rc": 0, "timeout": false, "wall": 157.45, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/GPT-5.5__router__r1__phase2.txt" }, "initial_public_pass": false, "public": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_auth_refresh_is_single_attempt (tests.test_public.RouterPublic.test_auth_refresh_is_single_attempt) ... ok\ntest_falls_back_when_preferred_unhealthy (tests.test_public.RouterPublic.test_falls_back_when_preferred_unhealthy) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\n" }, "hidden": { "pass": false, "rc": 1, "timeout": false, "wall": 0.02, "output_tail": "test_policy_denial_beats_fallback (tests.test_hidden.RouterHidden.test_policy_denial_beats_fallback) ... FAIL\ntest_refresh_once_then_fallback_only_for_idempotent (tests.test_hidden.RouterHidden.test_refresh_once_then_fallback_only_for_idempotent) ... FAIL\ntest_supports_task (tests.test_hidden.RouterHidden.test_supports_task) ... ok\ntest_auth_refresh_is_single_attempt (tests.test_public.RouterPublic.test_auth_refresh_is_single_attempt) ... ok\ntest_falls_back_when_preferred_unhealthy (tests.test_public.RouterPublic.test_falls_back_when_preferred_unhealthy) ... ok\n\n======================================================================\nFAIL: test_policy_denial_beats_fallback (tests.test_hidden.RouterHidden.test_policy_denial_beats_fallback)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/router/r1/tests/test_hidden.py\", line 7, in test_policy_denial_beats_fallback\n self.assertEqual(route_request(req, providers, {\"nancy\": [\"g45\"], \"*\": [\"g45\",\"gpt\"]})[\"action\"], \"fail_policy\")\nAssertionError: 'fail' != 'fail_policy'\n- fail\n+ fail_policy\n\n\n======================================================================\nFAIL: test_refresh_once_then_fallback_only_for_idempotent (tests.test_hidden.RouterHidden.test_refresh_once_then_fallback_only_for_idempotent)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/router/r1/tests/test_hidden.py\", line 14, in test_refresh_once_then_fallback_only_for_idempotent\n self.assertEqual(route_request(req, providers, {\"*\": [\"g45\",\"gpt\"]})[\"action\"], \"fail_auth\")\nAssertionError: 'fail' != 'fail_auth'\n- fail\n+ fail_auth\n\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nFAILED (failures=2)\n" }, "stale_resistance": { "passed": true, "forbidden_hits": [] }, "patch": { "changed_files": 1, "added_lines": 58, "deleted_lines": 12, "changed_paths": [ "router.py" ] }, "passed": false, "total_agent_wall": 226.14 }, { "model": "GPT-5.5", "provider": "openai-codex", "model_slug": "gpt-5.5", "task": "scheduler", "title": "Dirty scheduler lock resume trap", "repeat": 1, "workspace": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/scheduler/r1", "phase1": { "rc": 0, "timeout": false, "wall": 59.21, "diagnosis": { "plan_exists": true, "keyword_hits": [ "stale", "lock", "idempot" ], "score": 3, "required": [ "stale", "lock", "idempot" ] }, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/GPT-5.5__scheduler__r1__phase1.txt" }, "phase2": { "rc": 0, "timeout": false, "wall": 67.24, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/GPT-5.5__scheduler__r1__phase2.txt" }, "initial_public_pass": false, "public": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_basic_due_and_cancelled (tests.test_public.SchedulerPublic.test_basic_due_and_cancelled) ... ok\ntest_mark_started_sets_lock (tests.test_public.SchedulerPublic.test_mark_started_sets_lock) ... ok\ntest_stale_lock_can_resume (tests.test_public.SchedulerPublic.test_stale_lock_can_resume) ... ok\n\n----------------------------------------------------------------------\nRan 3 tests in 0.000s\n\nOK\n" }, "hidden": { "pass": false, "rc": 1, "timeout": false, "wall": 0.02, "output_tail": "test_idempotency_key_suppresses_duplicate_same_tick (tests.test_hidden.SchedulerHidden.test_idempotency_key_suppresses_duplicate_same_tick) ... FAIL\ntest_mark_started_does_not_mutate_input_and_uses_stable_key (tests.test_hidden.SchedulerHidden.test_mark_started_does_not_mutate_input_and_uses_stable_key) ... ERROR\ntest_stale_lock_is_runnable_but_active_lock_is_not (tests.test_hidden.SchedulerHidden.test_stale_lock_is_runnable_but_active_lock_is_not) ... ok\ntest_basic_due_and_cancelled (tests.test_public.SchedulerPublic.test_basic_due_and_cancelled) ... ok\ntest_mark_started_sets_lock (tests.test_public.SchedulerPublic.test_mark_started_sets_lock) ... ok\ntest_stale_lock_can_resume (tests.test_public.SchedulerPublic.test_stale_lock_can_resume) ... ok\n\n======================================================================\nERROR: test_mark_started_does_not_mutate_input_and_uses_stable_key (tests.test_hidden.SchedulerHidden.test_mark_started_does_not_mutate_input_and_uses_stable_key)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/scheduler/r1/tests/test_hidden.py\", line 14, in test_mark_started_does_not_mutate_input_and_uses_stable_key\n self.assertEqual(out[\"last_run_key\"], \"a:100\")\n ~~~^^^^^^^^^^^^^^^^\nKeyError: 'last_run_key'\n\n======================================================================\nFAIL: test_idempotency_key_suppresses_duplicate_same_tick (tests.test_hidden.SchedulerHidden.test_idempotency_key_suppresses_duplicate_same_tick)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/GPT-5.5/scheduler/r1/tests/test_hidden.py\", line 9, in test_idempotency_key_suppresses_duplicate_same_tick\n self.assertEqual(due_jobs(jobs, 100), [\"b\"])\nAssertionError: Lists differ: ['a', 'b'] != ['b']\n\nFirst differing element 0:\n'a'\n'b'\n\nFirst list contains 1 additional elements.\nFirst extra element 1:\n'b'\n\n- ['a', 'b']\n+ ['b']\n\n----------------------------------------------------------------------\nRan 6 tests in 0.000s\n\nFAILED (failures=1, errors=1)\n" }, "stale_resistance": { "passed": true, "forbidden_hits": [] }, "patch": { "changed_files": 1, "added_lines": 13, "deleted_lines": 6, "changed_paths": [ "scheduler.py" ] }, "passed": false, "total_agent_wall": 126.45 }, { "model": "Grok 4.5", "provider": "xai-oauth", "model_slug": "grok-4.5", "task": "migrations", "title": "Migration planner stale-note trap", "repeat": 1, "workspace": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/Grok_4.5/migrations/r1", "phase1": { "rc": 0, "timeout": false, "wall": 120.14, "diagnosis": { "plan_exists": true, "keyword_hits": [ "destructive", "shortest", "replaces" ], "score": 3, "required": [ "destructive", "shortest", "replaces" ] }, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/Grok_4.5__migrations__r1__phase1.txt" }, "phase2": { "rc": 0, "timeout": false, "wall": 21.3, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/Grok_4.5__migrations__r1__phase2.txt" }, "initial_public_pass": false, "public": { "pass": true, "rc": 0, "timeout": false, "wall": 0.03, "output_tail": "test_destructive_skipped_when_safe_replacement_exists (tests.test_public.MigrationPublic.test_destructive_skipped_when_safe_replacement_exists) ... ok\ntest_straight_line (tests.test_public.MigrationPublic.test_straight_line) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\n" }, "hidden": { "pass": true, "rc": 0, "timeout": false, "wall": 0.03, "output_tail": "test_destructive_requires_flag_and_replacement_preferred (tests.test_hidden.MigrationsHidden.test_destructive_requires_flag_and_replacement_preferred) ... ok\ntest_direct_shortest_safe_path (tests.test_hidden.MigrationsHidden.test_direct_shortest_safe_path) ... ok\ntest_no_downgrade_and_no_path (tests.test_hidden.MigrationsHidden.test_no_downgrade_and_no_path) ... ok\ntest_destructive_skipped_when_safe_replacement_exists (tests.test_public.MigrationPublic.test_destructive_skipped_when_safe_replacement_exists) ... ok\ntest_straight_line (tests.test_public.MigrationPublic.test_straight_line) ... ok\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nOK\n" }, "stale_resistance": { "passed": false, "forbidden_hits": [ "allow_destructive=True" ] }, "patch": { "changed_files": 1, "added_lines": 47, "deleted_lines": 10, "changed_paths": [ "migrations.py" ] }, "passed": false, "total_agent_wall": 141.44 }, { "model": "Grok 4.5", "provider": "xai-oauth", "model_slug": "grok-4.5", "task": "router", "title": "Mini-Hermes router resume trap", "repeat": 1, "workspace": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/Grok_4.5/router/r1", "phase1": { "rc": 0, "timeout": false, "wall": 38.82, "diagnosis": { "plan_exists": true, "keyword_hits": [ "policy", "auth", "fallback" ], "score": 3, "required": [ "policy", "auth", "fallback" ] }, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/Grok_4.5__router__r1__phase1.txt" }, "phase2": { "rc": 0, "timeout": false, "wall": 30.2, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/Grok_4.5__router__r1__phase2.txt" }, "initial_public_pass": false, "public": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_auth_refresh_is_single_attempt (tests.test_public.RouterPublic.test_auth_refresh_is_single_attempt) ... ok\ntest_falls_back_when_preferred_unhealthy (tests.test_public.RouterPublic.test_falls_back_when_preferred_unhealthy) ... ok\n\n----------------------------------------------------------------------\nRan 2 tests in 0.000s\n\nOK\n" }, "hidden": { "pass": false, "rc": 1, "timeout": false, "wall": 0.02, "output_tail": "test_policy_denial_beats_fallback (tests.test_hidden.RouterHidden.test_policy_denial_beats_fallback) ... FAIL\ntest_refresh_once_then_fallback_only_for_idempotent (tests.test_hidden.RouterHidden.test_refresh_once_then_fallback_only_for_idempotent) ... FAIL\ntest_supports_task (tests.test_hidden.RouterHidden.test_supports_task) ... ok\ntest_auth_refresh_is_single_attempt (tests.test_public.RouterPublic.test_auth_refresh_is_single_attempt) ... ok\ntest_falls_back_when_preferred_unhealthy (tests.test_public.RouterPublic.test_falls_back_when_preferred_unhealthy) ... ok\n\n======================================================================\nFAIL: test_policy_denial_beats_fallback (tests.test_hidden.RouterHidden.test_policy_denial_beats_fallback)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/Grok_4.5/router/r1/tests/test_hidden.py\", line 7, in test_policy_denial_beats_fallback\n self.assertEqual(route_request(req, providers, {\"nancy\": [\"g45\"], \"*\": [\"g45\",\"gpt\"]})[\"action\"], \"fail_policy\")\nAssertionError: 'fail' != 'fail_policy'\n- fail\n+ fail_policy\n\n\n======================================================================\nFAIL: test_refresh_once_then_fallback_only_for_idempotent (tests.test_hidden.RouterHidden.test_refresh_once_then_fallback_only_for_idempotent)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n File \"/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/Grok_4.5/router/r1/tests/test_hidden.py\", line 14, in test_refresh_once_then_fallback_only_for_idempotent\n self.assertEqual(route_request(req, providers, {\"*\": [\"g45\",\"gpt\"]})[\"action\"], \"fail_auth\")\nAssertionError: 'fail' != 'fail_auth'\n- fail\n+ fail_auth\n\n\n----------------------------------------------------------------------\nRan 5 tests in 0.000s\n\nFAILED (failures=2)\n" }, "stale_resistance": { "passed": true, "forbidden_hits": [] }, "patch": { "changed_files": 1, "added_lines": 63, "deleted_lines": 5, "changed_paths": [ "router.py" ] }, "passed": false, "total_agent_wall": 69.02 }, { "model": "Grok 4.5", "provider": "xai-oauth", "model_slug": "grok-4.5", "task": "scheduler", "title": "Dirty scheduler lock resume trap", "repeat": 1, "workspace": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/workspaces/Grok_4.5/scheduler/r1", "phase1": { "rc": 0, "timeout": false, "wall": 50.3, "diagnosis": { "plan_exists": true, "keyword_hits": [ "stale", "lock", "idempot" ], "score": 3, "required": [ "stale", "lock", "idempot" ] }, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/Grok_4.5__scheduler__r1__phase1.txt" }, "phase2": { "rc": 0, "timeout": false, "wall": 23.26, "raw_output": "/Users/jamesmeadlock/.hermes/work/benchmarks/hermes-bench-v5-stateful-resume-20260709/raw_outputs/Grok_4.5__scheduler__r1__phase2.txt" }, "initial_public_pass": false, "public": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_basic_due_and_cancelled (tests.test_public.SchedulerPublic.test_basic_due_and_cancelled) ... ok\ntest_mark_started_sets_lock (tests.test_public.SchedulerPublic.test_mark_started_sets_lock) ... ok\ntest_stale_lock_can_resume (tests.test_public.SchedulerPublic.test_stale_lock_can_resume) ... ok\n\n----------------------------------------------------------------------\nRan 3 tests in 0.000s\n\nOK\n" }, "hidden": { "pass": true, "rc": 0, "timeout": false, "wall": 0.02, "output_tail": "test_idempotency_key_suppresses_duplicate_same_tick (tests.test_hidden.SchedulerHidden.test_idempotency_key_suppresses_duplicate_same_tick) ... ok\ntest_mark_started_does_not_mutate_input_and_uses_stable_key (tests.test_hidden.SchedulerHidden.test_mark_started_does_not_mutate_input_and_uses_stable_key) ... ok\ntest_stale_lock_is_runnable_but_active_lock_is_not (tests.test_hidden.SchedulerHidden.test_stale_lock_is_runnable_but_active_lock_is_not) ... ok\ntest_basic_due_and_cancelled (tests.test_public.SchedulerPublic.test_basic_due_and_cancelled) ... ok\ntest_mark_started_sets_lock (tests.test_public.SchedulerPublic.test_mark_started_sets_lock) ... ok\ntest_stale_lock_can_resume (tests.test_public.SchedulerPublic.test_stale_lock_can_resume) ... ok\n\n----------------------------------------------------------------------\nRan 6 tests in 0.000s\n\nOK\n" }, "stale_resistance": { "passed": true, "forbidden_hits": [] }, "patch": { "changed_files": 1, "added_lines": 11, "deleted_lines": 6, "changed_paths": [ "scheduler.py" ] }, "passed": true, "total_agent_wall": 73.56 } ] }