{
  "schema": "assure.sentinel-real-runtime-evaluation.v1",
  "claim": "precursor / early-warning detection under controlled public/synthetic experiments",
  "confidence": "UNCALIBRATED",
  "split_frozen_at": "2026-09-03T23:50:00Z",
  "thresholds_frozen_before_execution": true,
  "held_out_framework": "smolagents",
  "deterministic_replay_agreement": {
    "definition": "offline canonical receipt-preimage reconstruction",
    "agreements": 24,
    "total": 24,
    "rate": 1
  },
  "evaluation": {
    "protocol": "assure.sentinel-evaluation.v0",
    "detectorVersions": {
      "BASELINE_DEVIATION": "0.1.0",
      "PEER_DEVIATION": "0.1.0",
      "RETRY_ERROR_DETERIORATION": "0.1.0",
      "LOOP_REVISIT_RISK": "0.1.0",
      "TOOL_CALL_BURST": "0.1.0",
      "PERMISSION_NOVELTY": "0.1.0",
      "SEQUENCE_NOVELTY": "0.1.0",
      "LATENCY_COST_DETERIORATION": "0.1.0",
      "CONTRADICTION_REVERSAL": "0.1.0",
      "BLAST_RADIUS_WARNING": "0.1.0"
    },
    "partitionCounts": {
      "train": 0,
      "development": 18,
      "held-out": 6
    },
    "corpusTotals": {
      "runs": 24,
      "failures": 17
    },
    "heldOut": {
      "runs": 6,
      "failures": 4,
      "truePositives": 1,
      "falsePositives": 2,
      "trueNegatives": 0,
      "falseNegatives": 3,
      "precision": 0.3333333333333333,
      "recall": 0.25,
      "falsePositiveRate": 1,
      "medianLeadTime": 3,
      "p95LeadTime": 3
    },
    "failuresDetected": 1,
    "failuresMissed": 3,
    "precision": 0.3333333333333333,
    "recall": 0.25,
    "falsePositiveRate": 1,
    "leadTimes": [3],
    "warningTimeDistribution": {
      "count": 1,
      "min": 3,
      "median": 3,
      "p95": 3,
      "max": 3
    },
    "calibration": {
      "state": "NOT_APPLICABLE_UNCALIBRATED_V0",
      "reason": "v0 warnings are deterministic thresholds and have no probability calibration claim"
    },
    "detectorStability": {
      "deterministicReplay": true,
      "versionSet": ["0.1.0"]
    },
    "crossModelTransfer": {
      "unfamiliarModels": ["framework-native-deterministic/no-network-mock/1.27.0.dev0"],
      "recall": 0.25
    },
    "crossTaskTransfer": {
      "unfamiliarTasks": [
        "open-agent-runtime-v1/code-success",
        "open-agent-runtime-v1/tool-success"
      ],
      "recall": null
    },
    "crossAgentTransfer": {
      "sourceAgents": ["browser-use", "crewai", "mini-swe-agent"],
      "unfamiliarAgents": ["smolagents"],
      "recall": 0.25
    },
    "perDetector": {
      "BASELINE_DEVIATION": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "PEER_DEVIATION": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "RETRY_ERROR_DETERIORATION": {
        "runs": 6,
        "failures": 4,
        "truePositives": 1,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 3,
        "precision": 1,
        "recall": 0.25,
        "falsePositiveRate": 0,
        "medianLeadTime": 2,
        "p95LeadTime": 2
      },
      "LOOP_REVISIT_RISK": {
        "runs": 6,
        "failures": 4,
        "truePositives": 1,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 3,
        "precision": 1,
        "recall": 0.25,
        "falsePositiveRate": 0,
        "medianLeadTime": 1,
        "p95LeadTime": 1
      },
      "TOOL_CALL_BURST": {
        "runs": 6,
        "failures": 4,
        "truePositives": 1,
        "falsePositives": 2,
        "trueNegatives": 0,
        "falseNegatives": 3,
        "precision": 0.3333333333333333,
        "recall": 0.25,
        "falsePositiveRate": 1,
        "medianLeadTime": 3,
        "p95LeadTime": 3
      },
      "PERMISSION_NOVELTY": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "SEQUENCE_NOVELTY": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "LATENCY_COST_DETERIORATION": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "CONTRADICTION_REVERSAL": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "BLAST_RADIUS_WARNING": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 0,
        "trueNegatives": 2,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 0,
        "medianLeadTime": null,
        "p95LeadTime": null
      }
    },
    "perAgent": {
      "browser-use": {
        "runs": 6,
        "failures": 4,
        "truePositives": 1,
        "falsePositives": 1,
        "trueNegatives": 1,
        "falseNegatives": 3,
        "precision": 0.5,
        "recall": 0.25,
        "falsePositiveRate": 0.5,
        "medianLeadTime": 2,
        "p95LeadTime": 2
      },
      "crewai": {
        "runs": 6,
        "failures": 4,
        "truePositives": 0,
        "falsePositives": 2,
        "trueNegatives": 0,
        "falseNegatives": 4,
        "precision": 0,
        "recall": 0,
        "falsePositiveRate": 1,
        "medianLeadTime": null,
        "p95LeadTime": null
      },
      "mini-swe-agent": {
        "runs": 6,
        "failures": 5,
        "truePositives": 2,
        "falsePositives": 1,
        "trueNegatives": 0,
        "falseNegatives": 3,
        "precision": 0.6666666666666666,
        "recall": 0.4,
        "falsePositiveRate": 1,
        "medianLeadTime": 2,
        "p95LeadTime": 2
      },
      "smolagents": {
        "runs": 6,
        "failures": 4,
        "truePositives": 1,
        "falsePositives": 2,
        "trueNegatives": 0,
        "falseNegatives": 3,
        "precision": 0.3333333333333333,
        "recall": 0.25,
        "falsePositiveRate": 1,
        "medianLeadTime": 3,
        "p95LeadTime": 3
      }
    }
  },
  "evaluation_receipt": "sha256:b7420c8add90979b4b14d2a0ac5408740d335cb55ec4a22a73e45b2c16c4a8b6"
}