eval-action/action.yml at main · step-security/eval-action · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
name: "Braintrust eval"
description: "Automatically run evals on your AI project using Braintrust."
author: 'step-security'

# Add your action's branding here. This will appear on the GitHub Marketplace.
branding:
  icon: "activity"
  color: "blue"

# Define your inputs here.
inputs:
  api_key:
    description: "Your Braintrust API key"
    required: true
  root:
    description: "Root directory from which to run evals"
    required: false
    default: "."
  paths:
    description: "A list of paths (and glob patterns) to include in the evals"
    required: false
    default: "."
  runtime:
    description: "The runtime to use for evals. Valid values: node, python, go."
    required: true
  package_manager:
    description:
      "The package manager to use for evals. Valid values: npm or pnpm for node,
      pip or uv for python, or go for Go."
    required: false
    default: ""
  use_proxy:
    description:
      "Whether to use the Braintrust proxy (to cache LLM calls). Set to 'true'
      or 'false'."
    required: true
    default: "true"
  terminate_on_failure:
    description:
      "Whether to terminate the evaluation process when an error occurs. Set to
      'true' or 'false'."
    required: false
    default: "false"
  report_scores:
    description:
      "A comma- or newline-separated list of score names to include in the PR
      comment. By default, all scores are included."
    required: false
    default: ""
  report_metrics:
    description:
      "A comma- or newline-separated list of metric names to include in the PR
      comment. By default, all metrics are included."
    required: false
    default: ""
  github_token:
    description: "Your GitHub token"
    required: true
    default: ${{ github.token }}
  step_key:
    description:
      "A unique key to identify this step. Do not change this unless you know
      what you're doing."
    required: true
    default: ${{ github.workflow_ref }}-${{ github.action }}

runs:
  using: node24
  main: eval/dist/index.js