Evaluation

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "#/components/schemas/Evaluation",
  "title": "Evaluation",
  "type": "object",
  "properties": {
    "id": {
      "type": "string",
      "format": "uuid",
      "readOnly": true
    },
    "name": {
      "type": "string",
      "description": "Name of the evaluation.",
      "maxLength": 400
    },
    "description": {
      "type": "string",
      "description": "Optional description of what this evaluation checks."
    },
    "enabled": {
      "type": "boolean",
      "description": "Whether the evaluation runs automatically on new $ai_generation events."
    },
    "status": {
      "allOf": [
        {
          "$ref": "#/components/schemas/EvaluationStatusEnum"
        }
      ],
      "readOnly": true
    },
    "status_reason": {
      "readOnly": true,
      "nullable": true,
      "oneOf": [
        {
          "$ref": "#/components/schemas/StatusReasonEnum"
        },
        {
          "$ref": "#/components/schemas/NullEnum"
        }
      ]
    },
    "evaluation_type": {
      "allOf": [
        {
          "$ref": "#/components/schemas/EvaluationTypeEnum"
        }
      ],
      "description": "'llm_judge' uses an LLM to score outputs against a prompt; 'hog' runs deterministic Hog code.\n\n* `llm_judge` - LLM as a judge\n* `hog` - Hog"
    },
    "evaluation_config": {
      "oneOf": [
        {
          "type": "object",
          "title": "LLM judge config",
          "required": [
            "prompt"
          ],
          "properties": {
            "prompt": {
              "type": "string",
              "description": "Evaluation criteria for the LLM judge. Describe what makes a good vs bad response.",
              "minLength": 1
            }
          },
          "additionalProperties": false
        },
        {
          "type": "object",
          "title": "Hog config",
          "required": [
            "source"
          ],
          "properties": {
            "source": {
              "type": "string",
              "description": "Hog source code. Must return true (pass), false (fail), or null for N/A.",
              "minLength": 1
            }
          },
          "additionalProperties": false
        }
      ],
      "description": "Configuration dict. For 'llm_judge': {prompt}. For 'hog': {source}."
    },
    "output_type": {
      "allOf": [
        {
          "$ref": "#/components/schemas/OutputTypeEnum"
        }
      ],
      "description": "Output format. Currently only 'boolean' is supported.\n\n* `boolean` - Boolean (Pass/Fail)"
    },
    "output_config": {
      "type": "object",
      "properties": {
        "allows_na": {
          "type": "boolean",
          "description": "Whether the evaluation can return N/A for non-applicable generations.",
          "default": false
        }
      },
      "additionalProperties": false,
      "description": "Output config. For 'boolean' output_type: {allows_na} to permit N/A results."
    },
    "conditions": {
      "description": "Optional trigger conditions to filter which events are evaluated. OR between condition sets, AND within each."
    },
    "model_configuration": {
      "allOf": [
        {
          "$ref": "#/components/schemas/ModelConfiguration"
        }
      ],
      "nullable": true
    },
    "created_at": {
      "type": "string",
      "format": "date-time",
      "readOnly": true
    },
    "updated_at": {
      "type": "string",
      "format": "date-time",
      "readOnly": true
    },
    "created_by": {
      "allOf": [
        {
          "$ref": "#/components/schemas/UserBasic"
        }
      ],
      "readOnly": true
    },
    "deleted": {
      "type": "boolean",
      "description": "Set to true to soft-delete the evaluation."
    }
  },
  "required": [
    "created_at",
    "created_by",
    "evaluation_type",
    "id",
    "name",
    "output_type",
    "status",
    "status_reason",
    "updated_at"
  ]
}