Runloop · Schema

Runloop Benchmark

A BenchmarkDefinitionView represents a grouped set of Scenarios that together form a Benchmark.

AIAI AgentsCoding AgentsSandboxesDevboxesCode ExecutionEvaluationBenchmarksSWE-BenchMCPSnapshotsmicroVMEnterpriseSOC 2

Properties

Name Type Description
id string The ID of the Benchmark.
name string The name of the Benchmark.
scenarioIds array List of Scenario IDs that make up the benchmark.
metadata object User defined metadata to attach to the benchmark for organization.
required_environment_variables array Required environment variables used to run the benchmark. If any required environment variables are missing, the benchmark will fail to start.
required_secret_names array Required secrets used to run the benchmark. If any required secrets are missing, the benchmark will fail to start.
is_public boolean Whether this benchmark is public.
attribution string Attribution information for the benchmark.
description string Detailed description of the benchmark.
status object Whether the benchmark is active or archived. Archived benchmarks are excluded from listings and cannot be run.
View JSON Schema on GitHub

JSON Schema

runloop-benchmark-schema.json Raw ↑ 
{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://raw.githubusercontent.com/api-evangelist/runloop-ai/main/json-schema/runloop-benchmark-schema.json",
  "title": "Runloop Benchmark",
  "description": "A BenchmarkDefinitionView represents a grouped set of Scenarios that together form a Benchmark.",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "id": {
      "type": "string",
      "description": "The ID of the Benchmark."
    },
    "name": {
      "type": "string",
      "description": "The name of the Benchmark."
    },
    "scenarioIds": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "List of Scenario IDs that make up the benchmark."
    },
    "metadata": {
      "type": "object",
      "additionalProperties": {
        "type": "string"
      },
      "description": "User defined metadata to attach to the benchmark for organization."
    },
    "required_environment_variables": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Required environment variables used to run the benchmark. If any required environment variables are missing, the benchmark will fail to start."
    },
    "required_secret_names": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Required secrets used to run the benchmark. If any required secrets are missing, the benchmark will fail to start."
    },
    "is_public": {
      "type": "boolean",
      "description": "Whether this benchmark is public."
    },
    "attribution": {
      "type": "string",
      "description": "Attribution information for the benchmark."
    },
    "description": {
      "type": "string",
      "description": "Detailed description of the benchmark."
    },
    "status": {
      "$ref": "#/$defs/BenchmarkStatus",
      "description": "Whether the benchmark is active or archived. Archived benchmarks are excluded from listings and cannot be run."
    }
  },
  "required": [
    "id",
    "name",
    "scenarioIds",
    "metadata",
    "status"
  ],
  "$defs": {
    "BenchmarkStatus": {
      "type": "string",
      "enum": [
        "active",
        "archived"
      ]
    }
  }
}