hermes-agent/tests/hermes_cli/test_gpt6_tiers_registration.py
kshitijk4poor bf87ae620e feat(models): add gpt-6.1-sol (OpenRouter + direct OpenAI API)
OpenAI released GPT-6.1 Sol on 2026-09-29. It is live on OpenRouter
(openai/gpt-6.1-sol, -pro) and the direct API; not yet on Nous Portal
or on Codex OAuth for our test account.

- Catalog: OPENROUTER_MODELS + openai-api picker; kept off the nous
  list via _OPENROUTER_ONLY until the Portal serves it. Manifest regenerated.
- Context: 1.05M direct API, 272K Codex fallback; 272K compaction
  auto-raise on the Codex route.
- Pricing: $2/$10, cached input $0.10 (5%, not 6 Sol's 10%), cache write
  $2.50, 2x/1.5x above 272K; -pro aliased to the base row.
- Reasoning: none/minimal return 400 on 6.1 Sol (unlike 6 Sol), so it uses
  Astra's low..max ladder without Astra's account gating. OpenRouter's
  catalog wrongly lists 'none' for it, so the OpenRouter profile omits a
  disable when the OpenAI ladder has no 'none'.
2026-09-29 23:25:26 +05:30

80 lines
4.4 KiB
Python

"""Behavior contracts for the GPT-6 Sol/Terra/Luna registration (the 5.6 tier successors).
Invariant tests only, no list snapshots. They pin what would silently regress:
1. `/model gpt` still lands on the flagship: Astra outranks Sol, Sol outranks
Terra/Luna, and every GPT-6 tier outranks its 5.6 predecessor.
2. The Codex OAuth `-900k` opt-in machinery treats the gpt-6 tiers exactly like
the 5.6 ones: picker synthesis, dated snapshots, wire stripping, the
compaction auto-raise on the base slug (and not on the variant), and the
gpt-5.6 effort ladder (``max`` allowed).
"""
from agent.auxiliary_client import _compression_threshold_for_model
from agent.model_metadata import (
_verified_codex_ctx_for_slug,
is_codex_900k_base,
strip_codex_context_variant_suffix,
)
from agent.reasoning_effort import CODEX_GPT56_EFFORTS, codex_supported_efforts
from hermes_cli.codex_models import _finalize_codex_models
from hermes_cli.model_switch import _model_sort_key
GPT6_TIERS = ("gpt-6-sol", "gpt-6-luna") # terra: never published by OpenAI, not on OpenRouter/Codex (2026-09-22)
def test_model_gpt_resolves_flagship_across_gpt6_tiers():
models = ["gpt-6-luna", "gpt-5.6-sol", "gpt-6-sol", "gpt-6-astra"]
models.sort(key=lambda m: _model_sort_key(m, "gpt"))
assert models[:2] == ["gpt-6-astra", "gpt-6-sol"]
assert models.index("gpt-6-luna") < models.index("gpt-5.6-sol")
def test_gpt6_tiers_share_the_codex_900k_contract_with_56():
ids = _finalize_codex_models(["gpt-5.5"]) # forward-compat synthesizes the tiers from 5.5
for base in GPT6_TIERS:
assert ids.index(f"{base}-900k") == ids.index(base) + 1, base
assert f"{base}-pro-900k" not in ids
assert is_codex_900k_base(f"{base}-2026-09-22"), base # dated snapshots inherit eligibility
assert strip_codex_context_variant_suffix(f"openai/{base}-900k") == f"openai/{base}"
assert _verified_codex_ctx_for_slug(f"{base}-900k") == _verified_codex_ctx_for_slug("gpt-5.6-sol-900k")
assert _compression_threshold_for_model(base, provider="openai-codex") == \
_compression_threshold_for_model("gpt-5.6-sol", provider="openai-codex")
assert _compression_threshold_for_model(f"{base}-900k", provider="openai-codex") is None
assert codex_supported_efforts(f"openai/{base}") == CODEX_GPT56_EFFORTS
def test_gpt61_sol_takes_astra_ladder_without_astra_gating():
"""``none`` 400s on gpt-6.1-sol (live 2026-09-29), but it is not account-gated like Astra."""
from agent.reasoning_effort import CODEX_ASTRA_EFFORTS, is_astra_model
for slug in ("gpt-6.1-sol", "openai/gpt-6.1-sol-pro", "gpt-6.1-sol-2026-09-29"):
assert codex_supported_efforts(slug) == CODEX_ASTRA_EFFORTS, slug
assert not is_astra_model(slug), slug
assert "none" in codex_supported_efforts("gpt-6-sol")
def test_openrouter_omits_disable_the_openai_ladder_rejects(monkeypatch):
"""OpenRouter's catalog advertises ``none`` for both Sol generations; only 6.1 must omit the disable."""
import hermes_cli.models_reasoning_caps as caps_mod
from providers import get_provider_profile
monkeypatch.setattr(caps_mod, "openrouter_model_reasoning_capabilities", lambda model: {
"supports_reasoning": True, "mandatory": False,
"supported_efforts": ["max", "xhigh", "high", "medium", "low", "none"]})
p = get_provider_profile("openrouter")
off = {"enabled": False}
body, _ = p.build_api_kwargs_extras(reasoning_config=off, supports_reasoning=True, model="openai/gpt-6.1-sol")
assert "reasoning" not in body
body, _ = p.build_api_kwargs_extras(reasoning_config=off, supports_reasoning=True, model="openai/gpt-6-sol")
assert body["reasoning"] == off
def test_gpt61_sol_resolves_context_and_pricing_like_its_tier():
from agent.model_metadata import DEFAULT_CONTEXT_LENGTHS, _CODEX_OAUTH_CONTEXT_FALLBACK
from agent.usage_pricing import _OFFICIAL_DOCS_PRICING
assert DEFAULT_CONTEXT_LENGTHS["gpt-6.1-sol"] == DEFAULT_CONTEXT_LENGTHS["gpt-6-sol"]
assert _CODEX_OAUTH_CONTEXT_FALLBACK["gpt-6.1-sol"] == _CODEX_OAUTH_CONTEXT_FALLBACK["gpt-6-sol"]
base = _OFFICIAL_DOCS_PRICING[("openai", "gpt-6.1-sol")]
assert _OFFICIAL_DOCS_PRICING[("openai", "gpt-6.1-sol-pro")] is base
assert base.cache_read_cost_per_million == base.input_cost_per_million / 20 # 5%, not 6 Sol's 10%
assert not is_codex_900k_base("gpt-6.1-sol") # not verified above 272K on Codex