> ## Documentation Index
> Fetch the complete documentation index at: https://azure-foundry.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.

# Run a grader.



## OpenAPI

````yaml api-reference/openai-v1.yaml post /fine_tuning/alpha/graders/run
openapi: 3.0.0
info:
  title: Azure AI Foundry Models Service
  license:
    name: MIT
    url: https://github.com/openai/openai-openapi/blob/master/LICENSE
  version: v1
servers:
  - url: '{endpoint}/openai/v1'
    description: Azure AI Foundry Models APIs
    variables:
      endpoint:
        default: ''
        description: >-
          A supported Azure AI Foundry Models APIs endpoint, including protocol
          and hostname.

          For example:

          https://westus.api.cognitive.microsoft.com).
security:
  - ApiKeyAuth: []
  - ApiKeyAuth_: []
  - OAuth2Auth:
      - https://cognitiveservices.azure.com/.default
tags:
  - name: Chat
  - name: Embeddings
  - name: Evals
  - name: Files
  - name: Fine-tuning
  - name: Models
  - name: Responses
  - name: Vector Stores
paths:
  /fine_tuning/alpha/graders/run:
    post:
      tags:
        - Fine-tuning
      summary: Run a grader.
      operationId: runGrader
      parameters:
        - name: api-version
          in: query
          required: false
          description: >-
            The explicit Azure AI Foundry Models API version to use for this
            request.

            `v1` if not otherwise specified.
          schema:
            $ref: '#/components/schemas/AzureAIFoundryModelsApiVersion'
            default: v1
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/OpenAI.RunGraderRequest'
      responses:
        '200':
          description: The request has succeeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/OpenAI.RunGraderResponse'
        default:
          description: An unexpected error response.
          headers:
            apim-request-id:
              required: false
              description: A request ID used for troubleshooting purposes.
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AzureErrorResponse'
components:
  schemas:
    AzureAIFoundryModelsApiVersion:
      type: string
      enum:
        - v1
        - preview
    OpenAI.RunGraderRequest:
      type: object
      required:
        - grader
        - model_sample
      properties:
        grader:
          anyOf:
            - $ref: '#/components/schemas/OpenAI.GraderStringCheck'
            - $ref: '#/components/schemas/OpenAI.GraderTextSimilarity'
            - $ref: '#/components/schemas/OpenAI.GraderPython'
            - $ref: '#/components/schemas/OpenAI.GraderScoreModel'
            - $ref: '#/components/schemas/OpenAI.GraderMulti'
          description: The grader used for the fine-tuning job.
        item:
          description: >-
            The dataset item provided to the grader. This will be used to
            populate

            the `item` namespace. See [the guide](/docs/guides/graders) for more
            details.
        model_sample:
          type: string
          description: >-
            The model sample to be evaluated. This value will be used to
            populate

            the `sample` namespace. See [the guide](/docs/guides/graders) for
            more details.

            The `output_json` variable will be populated if the model sample is
            a

            valid JSON string.
    OpenAI.RunGraderResponse:
      type: object
      required:
        - reward
        - metadata
        - sub_rewards
        - model_grader_token_usage_per_model
      properties:
        reward:
          type: number
          format: float
        metadata:
          type: object
          properties:
            name:
              type: string
            type:
              type: string
            errors:
              type: object
              properties:
                formula_parse_error:
                  type: boolean
                sample_parse_error:
                  type: boolean
                truncated_observation_error:
                  type: boolean
                unresponsive_reward_error:
                  type: boolean
                invalid_variable_error:
                  type: boolean
                other_error:
                  type: boolean
                python_grader_server_error:
                  type: boolean
                python_grader_server_error_type:
                  type: string
                  nullable: true
                python_grader_runtime_error:
                  type: boolean
                python_grader_runtime_error_details:
                  type: string
                  nullable: true
                model_grader_server_error:
                  type: boolean
                model_grader_refusal_error:
                  type: boolean
                model_grader_parse_error:
                  type: boolean
                model_grader_server_error_details:
                  type: string
                  nullable: true
              required:
                - formula_parse_error
                - sample_parse_error
                - truncated_observation_error
                - unresponsive_reward_error
                - invalid_variable_error
                - other_error
                - python_grader_server_error
                - python_grader_server_error_type
                - python_grader_runtime_error
                - python_grader_runtime_error_details
                - model_grader_server_error
                - model_grader_refusal_error
                - model_grader_parse_error
                - model_grader_server_error_details
            execution_time:
              type: number
              format: float
            scores: {}
            token_usage:
              type: integer
              format: int32
              nullable: true
            sampled_model_name:
              type: string
              nullable: true
          required:
            - name
            - type
            - errors
            - execution_time
            - scores
            - token_usage
            - sampled_model_name
        sub_rewards: {}
        model_grader_token_usage_per_model: {}
    AzureErrorResponse:
      type: object
      properties:
        error:
          type: object
          properties:
            code:
              type: string
              description: The distinct, machine-generated identifier for the error.
            message:
              type: string
              description: A human-readable message associated with the error.
            param:
              type: string
              description: >-
                If applicable, the request input parameter associated with the
                error
            type:
              type: string
              enum:
                - error
              description: The object type, always 'error.'
            inner_error: {}
          description: The error details.
    OpenAI.GraderStringCheck:
      type: object
      required:
        - type
        - name
        - input
        - reference
        - operation
      properties:
        type:
          type: string
          enum:
            - string_check
          description: The object type, which is always `string_check`.
        name:
          type: string
          description: The name of the grader.
        input:
          type: string
          description: The input text. This may include template strings.
        reference:
          type: string
          description: The reference text. This may include template strings.
        operation:
          type: string
          enum:
            - eq
            - ne
            - like
            - ilike
          description: >-
            The string check operation to perform. One of `eq`, `ne`, `like`, or
            `ilike`.
      allOf:
        - $ref: '#/components/schemas/OpenAI.Grader'
      description: >-
        A StringCheckGrader object that performs a string comparison between
        input and reference using a specified operation.
    OpenAI.GraderTextSimilarity:
      type: object
      required:
        - type
        - name
        - input
        - reference
        - evaluation_metric
      properties:
        type:
          type: string
          enum:
            - text_similarity
          description: The type of grader.
        name:
          type: string
          description: The name of the grader.
        input:
          type: string
          description: The text being graded.
        reference:
          type: string
          description: The text being graded against.
        evaluation_metric:
          type: string
          enum:
            - fuzzy_match
            - bleu
            - gleu
            - meteor
            - rouge_1
            - rouge_2
            - rouge_3
            - rouge_4
            - rouge_5
            - rouge_l
          description: >-
            The evaluation metric to use. One of `fuzzy_match`, `bleu`, `gleu`,
            `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, or
            `rouge_l`.
      allOf:
        - $ref: '#/components/schemas/OpenAI.Grader'
      description: >-
        A TextSimilarityGrader object which grades text based on similarity
        metrics.
    OpenAI.GraderPython:
      type: object
      required:
        - type
        - name
        - source
      properties:
        type:
          type: string
          enum:
            - python
          description: The object type, which is always `python`.
        name:
          type: string
          description: The name of the grader.
        source:
          type: string
          description: The source code of the python script.
        image_tag:
          type: string
          description: The image tag to use for the python script.
      allOf:
        - $ref: '#/components/schemas/OpenAI.Grader'
      description: A PythonGrader object that runs a python script on the input.
    OpenAI.GraderScoreModel:
      type: object
      required:
        - type
        - name
        - model
        - input
      properties:
        type:
          type: string
          enum:
            - score_model
          description: The object type, which is always `score_model`.
        name:
          type: string
          description: The name of the grader.
        model:
          type: string
          description: The model to use for the evaluation.
        sampling_params:
          description: The sampling parameters for the model.
        input:
          type: array
          items:
            $ref: '#/components/schemas/OpenAI.EvalItem'
          description: The input text. This may include template strings.
        range:
          type: array
          items:
            type: number
            format: float
          description: The range of the score. Defaults to `[0, 1]`.
      allOf:
        - $ref: '#/components/schemas/OpenAI.Grader'
      description: >-
        A ScoreModelGrader object that uses a model to assign a score to the
        input.
    OpenAI.GraderMulti:
      type: object
      required:
        - type
        - name
        - graders
        - calculate_output
      properties:
        type:
          type: string
          enum:
            - multi
          description: The object type, which is always `multi`.
        name:
          type: string
          description: The name of the grader.
        graders:
          type: object
          additionalProperties:
            anyOf:
              - $ref: '#/components/schemas/OpenAI.GraderStringCheck'
              - $ref: '#/components/schemas/OpenAI.GraderTextSimilarity'
              - $ref: '#/components/schemas/OpenAI.GraderScoreModel'
        calculate_output:
          type: string
          description: A formula to calculate the output based on grader results.
      allOf:
        - $ref: '#/components/schemas/OpenAI.Grader'
      description: >-
        A MultiGrader object combines the output of multiple graders to produce
        a single score.
    OpenAI.Grader:
      type: object
      required:
        - type
      properties:
        type:
          $ref: '#/components/schemas/OpenAI.GraderType'
      discriminator:
        propertyName: type
        mapping:
          label_model: '#/components/schemas/OpenAI.GraderLabelModel'
          text_similarity: '#/components/schemas/OpenAI.GraderTextSimilarity'
          python: '#/components/schemas/OpenAI.GraderPython'
          score_model: '#/components/schemas/OpenAI.GraderScoreModel'
          multi: '#/components/schemas/OpenAI.GraderMulti'
    OpenAI.EvalItem:
      type: object
      required:
        - role
        - content
      properties:
        role:
          type: string
          enum:
            - user
            - assistant
            - system
            - developer
          description: >-
            The role of the message input. One of `user`, `assistant`, `system`,
            or

            `developer`.
        content:
          anyOf:
            - type: string
            - $ref: '#/components/schemas/OpenAI.EvalItemContent'
          description: Text inputs to the model - can contain template strings.
        type:
          type: string
          enum:
            - message
          description: The type of the message input. Always `message`.
      description: >-
        A message input to the model with a role indicating instruction
        following

        hierarchy. Instructions given with the `developer` or `system` role take

        precedence over instructions given with the `user` role. Messages with
        the

        `assistant` role are presumed to have been generated by the model in
        previous

        interactions.
    OpenAI.GraderType:
      type: string
      enum:
        - string_check
        - text_similarity
        - score_model
        - label_model
        - python
        - multi
    OpenAI.EvalItemContent:
      type: object
      required:
        - type
      properties:
        type:
          $ref: '#/components/schemas/OpenAI.EvalItemContentType'
      discriminator:
        propertyName: type
        mapping:
          input_text: '#/components/schemas/OpenAI.EvalItemContentInputText'
          output_text: '#/components/schemas/OpenAI.EvalItemContentOutputText'
    OpenAI.EvalItemContentType:
      type: string
      enum:
        - input_text
        - output_text
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: api-key
    ApiKeyAuth_:
      type: apiKey
      in: header
      name: authorization
    OAuth2Auth:
      type: oauth2
      flows:
        implicit:
          authorizationUrl: https://login.microsoftonline.com/common/oauth2/v2.0/authorize
          scopes:
            https://cognitiveservices.azure.com/.default: ''

````