{
  "title": "SpatialScout",
  "subtitle": "Can VLM Agents Actively Explore and Map 3D Scenes?",
  "scene_count": 80,
  "object_count": 3140,
  "turn_budget": 100,
  "scene_sources": [
    {
      "name": "ProcTHOR",
      "count": 30,
      "type": "Residential layouts"
    },
    {
      "name": "iTHOR",
      "count": 24,
      "type": "Kitchens, bedrooms, living rooms & bathrooms"
    },
    {
      "name": "RoboTHOR",
      "count": 12,
      "type": "Furnished rooms"
    },
    {
      "name": "ArchitecTHOR",
      "count": 8,
      "type": "Apartments"
    },
    {
      "name": "ReplicaCAD",
      "count": 6,
      "type": "Apartments"
    }
  ],
  "models": [
    {
      "model": "GPT-6 Astra (xhigh)",
      "valid_tasks": 80,
      "all_mean_iou": 0.33163843381806063,
      "all_gt_0": 0.6778571428571428,
      "all_gt_50": 0.3514732142857143,
      "all_gt_90": 0.01625,
      "submitted_mean_iou": 0.4115990687879953,
      "submitted_gt_0": 0.8386130222156101,
      "submitted_gt_50": 0.43686479324159916,
      "submitted_gt_90": 0.02094100792860589,
      "cost_usd": 3.058349912500003,
      "cost_tasks": 80,
      "micro_iou": 0.3310340604094972,
      "color": "#2563eb",
      "icon": "openai.png"
    },
    {
      "model": "GPT-6 Astra (medium)",
      "valid_tasks": 78,
      "all_mean_iou": 0.312179063706045,
      "all_gt_0": 0.6778846153846154,
      "all_gt_50": 0.3157509157509158,
      "all_gt_90": 0.012912087912087911,
      "submitted_mean_iou": 0.37354883965954105,
      "submitted_gt_0": 0.8083083298451804,
      "submitted_gt_50": 0.37904371500274275,
      "submitted_gt_90": 0.01565122749318406,
      "cost_usd": 2.474226576923077,
      "cost_tasks": 52,
      "micro_iou": 0.3117946347342451,
      "color": "#6489f2",
      "icon": "openai.png"
    },
    {
      "model": "GPT-6 Sol (xhigh)",
      "valid_tasks": 71,
      "all_mean_iou": 0.2087012430135957,
      "all_gt_0": 0.5242957746478873,
      "all_gt_50": 0.1874748490945674,
      "all_gt_90": 0.004627766599597585,
      "submitted_mean_iou": 0.2886685986304944,
      "submitted_gt_0": 0.7185618171423707,
      "submitted_gt_50": 0.26202469884147106,
      "submitted_gt_90": 0.006508224857544916,
      "cost_usd": 0.905050304225351,
      "cost_tasks": 71,
      "micro_iou": 0.20829849224183197,
      "color": "#1b4965",
      "icon": "openai.png"
    },
    {
      "model": "Claude Opus 5.5",
      "valid_tasks": 80,
      "all_mean_iou": 0.4171414392878302,
      "all_gt_0": 0.7741964285714286,
      "all_gt_50": 0.47183035714285715,
      "all_gt_90": 0.032544642857142855,
      "submitted_mean_iou": 0.46662076095207994,
      "submitted_gt_0": 0.864851438902905,
      "submitted_gt_50": 0.528244327123913,
      "submitted_gt_90": 0.036243429174416184,
      "cost_usd": 27.89701837037037,
      "cost_tasks": 27,
      "micro_iou": 0.41711029186946613,
      "color": "#c36d47",
      "icon": "claude-color.png"
    },
    {
      "model": "Claude Sonnet 5.5",
      "valid_tasks": 80,
      "all_mean_iou": 0.3227836423997075,
      "all_gt_0": 0.6719196428571428,
      "all_gt_50": 0.33625,
      "all_gt_90": 0.013973214285714287,
      "submitted_mean_iou": 0.39197900387676243,
      "submitted_gt_0": 0.8121711304324661,
      "submitted_gt_50": 0.41033807182123283,
      "submitted_gt_90": 0.01684765574511759,
      "cost_usd": 11.26667265,
      "cost_tasks": 80,
      "micro_iou": 0.32256763634203955,
      "color": "#d59b78",
      "icon": "claude-color.png"
    },
    {
      "model": "DS V4.1 Flash",
      "valid_tasks": 80,
      "all_mean_iou": 0.034960842560099445,
      "all_gt_0": 0.19075892857142857,
      "all_gt_50": 0.009375,
      "all_gt_90": 0.0,
      "submitted_mean_iou": 0.06268343987199623,
      "submitted_gt_0": 0.3338064596056906,
      "submitted_gt_50": 0.01886177937484643,
      "submitted_gt_90": 0.0,
      "cost_usd": 0.026639910499999996,
      "cost_tasks": 80,
      "micro_iou": 0.03481555045827425,
      "color": "#4164e9",
      "icon": "deepseek-color.png"
    },
    {
      "model": "Qwen3.8-Max",
      "valid_tasks": 77,
      "all_mean_iou": 0.0711626918687117,
      "all_gt_0": 0.25422077922077924,
      "all_gt_50": 0.047495361781076065,
      "all_gt_90": 0.0,
      "submitted_mean_iou": 0.12910599891902722,
      "submitted_gt_0": 0.45305755077300364,
      "submitted_gt_50": 0.09050075023533308,
      "submitted_gt_90": 0.0,
      "cost_usd": 1.0918712207792207,
      "cost_tasks": 77,
      "micro_iou": 0.07069707875599503,
      "color": "#8768d8",
      "icon": "qwen-color.png"
    },
    {
      "model": "Kimi K3",
      "valid_tasks": 14,
      "all_mean_iou": 0.04032252291548199,
      "all_gt_0": 0.188265306122449,
      "all_gt_50": 0.02729591836734694,
      "all_gt_90": 0.0,
      "submitted_mean_iou": 0.06521886171043875,
      "submitted_gt_0": 0.301186167179956,
      "submitted_gt_50": 0.05150160398607604,
      "submitted_gt_90": 0.0,
      "cost_usd": 1.3991885583571428,
      "cost_tasks": 14,
      "micro_iou": 0.040200728249882016,
      "color": "#252b35",
      "icon": "kimi.png"
    }
  ],
  "metric_definition": "Equal-weight mean over valid tasks of each per-task metric (macro average). Valid trajectories end with finish_mapping finished=true. Missing GT objects have zero IoU. For zero submitted objects, submitted-only per-task metrics are set to zero.",
  "cost_definition": "Mean estimated USD per task using OpenRouter standard non-batch prices retrieved in the saved pricing snapshot. Calculate per request, including cached reads, cache writes, and applicable long-context tiers; reasoning tokens are included in output. Average only valid tasks with complete usage. No imputation or zero filling. No gateway markup or credit purchase fees.",
  "pricing_source": "https://openrouter.ai/api/v1/models",
  "pricing_retrieved_utc": "2026-10-19T03:29:17.529141+00:00",
  "cost_plot_aggregation": "All-GT object-weighted (micro) IoU, matching the original published cost plot. The result table uses task-weighted macro IoU.",
  "snapshot_note": "Existing result and cost snapshots; no model runs or cost estimates were refreshed for this webpage."
}