{
  "/v1/evals": {
    "post": {
      "tags": [
        "OpenAI Evals API"
      ],
      "summary": "Create Eval",
      "description": "Create a new evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals\"       -H \"Authorization: Bearer your-key\"       -H \"Content-Type: application/json\"       -d '{\n    \"name\": \"Test Eval\",\n    \"data_source_config\": {\"type\": \"file\", \"file_id\": \"file-abc123\"},\n    \"testing_criteria\": {\"graders\": [{\"type\": \"llm_as_judge\"}]}\n  }'\n```\n\nReturns: Eval object with id, status, timestamps, etc.",
      "operationId": "create_eval_v1_evals_post",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/Eval"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    },
    "get": {
      "tags": [
        "OpenAI Evals API"
      ],
      "summary": "List Evals",
      "description": "List evaluations with pagination.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals?limit=10\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: ListEvalsResponse with list of evaluations",
      "operationId": "list_evals_v1_evals_get",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "limit",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "integer"
              },
              {
                "type": "null"
              }
            ],
            "default": 20,
            "title": "Limit"
          }
        },
        {
          "name": "after",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "After"
          }
        },
        {
          "name": "before",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Before"
          }
        },
        {
          "name": "order",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Order"
          }
        },
        {
          "name": "order_by",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Order By"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/ListEvalsResponse"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    }
  },
  "/v1/evals/{eval_id}": {
    "get": {
      "tags": [
        "OpenAI Evals API"
      ],
      "summary": "Get Eval",
      "description": "Get a specific evaluation by ID.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: Eval object",
      "operationId": "get_eval_v1_evals__eval_id__get",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/Eval"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    },
    "post": {
      "tags": [
        "OpenAI Evals API"
      ],
      "summary": "Update Eval",
      "description": "Update an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123\"       -H \"Authorization: Bearer your-key\"       -H \"Content-Type: application/json\"       -d '{\"name\": \"Updated Name\"}'\n```\n\nReturns: Updated Eval object",
      "operationId": "update_eval_v1_evals__eval_id__post",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/Eval"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    },
    "delete": {
      "tags": [
        "OpenAI Evals API"
      ],
      "summary": "Delete Eval",
      "description": "Delete an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X DELETE \"http://localhost:4000/v1/evals/eval_123\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: DeleteEvalResponse with deletion confirmation",
      "operationId": "delete_eval_v1_evals__eval_id__delete",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/DeleteEvalResponse"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    }
  },
  "/v1/evals/{eval_id}/cancel": {
    "post": {
      "tags": [
        "OpenAI Evals API"
      ],
      "summary": "Cancel Eval",
      "description": "Cancel a running evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/cancel\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: CancelEvalResponse with cancellation confirmation",
      "operationId": "cancel_eval_v1_evals__eval_id__cancel_post",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/CancelEvalResponse"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    }
  },
  "/v1/evals/{eval_id}/runs": {
    "post": {
      "tags": [
        "OpenAI Evals API - Runs"
      ],
      "summary": "Create Run",
      "description": "Create a new run for an evaluation.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n- Pass model via body: `{\"model\": \"gpt-4-account-1\"}`\n- Pass model via completion.model: `{\"completion\": {\"model\": \"gpt-4-account-1\"}}`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/runs\"       -H \"Authorization: Bearer your-key\"       -H \"Content-Type: application/json\"       -d '{\n    \"data_source\": {\"type\": \"dataset\", \"dataset_id\": \"dataset_123\"},\n    \"completion\": {\"model\": \"gpt-4\", \"temperature\": 0.7}\n  }'\n```\n\nReturns: Run object with id, status, timestamps, etc.",
      "operationId": "create_run_v1_evals__eval_id__runs_post",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/Run"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    },
    "get": {
      "tags": [
        "OpenAI Evals API - Runs"
      ],
      "summary": "List Runs",
      "description": "List all runs for an evaluation with pagination.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123/runs?limit=10\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: ListRunsResponse with list of runs",
      "operationId": "list_runs_v1_evals__eval_id__runs_get",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "limit",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "integer"
              },
              {
                "type": "null"
              }
            ],
            "default": 20,
            "title": "Limit"
          }
        },
        {
          "name": "after",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "After"
          }
        },
        {
          "name": "before",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Before"
          }
        },
        {
          "name": "order",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Order"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/ListRunsResponse"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    }
  },
  "/v1/evals/{eval_id}/runs/{run_id}": {
    "get": {
      "tags": [
        "OpenAI Evals API - Runs"
      ],
      "summary": "Get Run",
      "description": "Get a specific run by ID.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl \"http://localhost:4000/v1/evals/eval_123/runs/run_456\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: Run object with full details",
      "operationId": "get_run_v1_evals__eval_id__runs__run_id__get",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "run_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Run Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/Run"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    },
    "post": {
      "tags": [
        "OpenAI Evals API - Runs"
      ],
      "summary": "Cancel Run",
      "description": "Cancel a running run.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl -X POST \"http://localhost:4000/v1/evals/eval_123/runs/run_456/cancel\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: CancelRunResponse with cancellation confirmation",
      "operationId": "cancel_run_v1_evals__eval_id__runs__run_id__post",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "run_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Run Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/CancelRunResponse"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    },
    "delete": {
      "tags": [
        "OpenAI Evals API - Runs"
      ],
      "summary": "Delete Run",
      "description": "Delete a run.\n\nModel-based routing (for multi-account support):\n- Pass model via header: `x-litellm-model: gpt-4-account-1`\n- Pass model via query: `?model=gpt-4-account-1`\n\nExample usage:\n```bash\ncurl -X DELETE \"http://localhost:4000/v1/evals/eval_123/runs/run_456\"       -H \"Authorization: Bearer your-key\"\n```\n\nReturns: RunDeleteResponse with deletion confirmation",
      "operationId": "delete_run_v1_evals__eval_id__runs__run_id__delete",
      "security": [
        {
          "APIKeyHeader": []
        }
      ],
      "parameters": [
        {
          "name": "eval_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Eval Id"
          }
        },
        {
          "name": "run_id",
          "in": "path",
          "required": true,
          "schema": {
            "type": "string",
            "title": "Run Id"
          }
        },
        {
          "name": "custom_llm_provider",
          "in": "query",
          "required": false,
          "schema": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": "openai",
            "title": "Custom Llm Provider"
          }
        }
      ],
      "responses": {
        "200": {
          "description": "Successful Response",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/RunDeleteResponse"
              }
            }
          }
        },
        "422": {
          "description": "Validation Error",
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/HTTPValidationError"
              }
            }
          }
        }
      }
    }
  }
}