letta-evals/examples/multi-grader-gate/suite.logical-and.yaml at main · jasonlarkin/letta-evals · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
name: ascii-art-logical-and-gate
description: Logical AND gate requiring both high quality (avg_score >= 0.6) AND valid ASCII (accuracy >= 60%)
dataset: dataset.csv
target:
  kind: letta_agent
  agent_file: ascii-art-agent.af
  base_url: https://api.letta.com/
graders:
  quality:
    kind: model_judge
    display_name: "rubric score"
    prompt_path: rubric.txt
    model: gpt-5-mini
    temperature: 0.0
    provider: openai
    max_retries: 3
    timeout: 120.0
    extractor: last_assistant
    rubric_vars:
      - reference_ascii
  ascii_only:
    kind: tool
    display_name: "ascii character check"
    function: ascii_printable_only
    extractor: last_assistant
gate:
  kind: logical
  operator: and
  conditions:
    - metric_key: quality
      aggregation: avg_score
      op: gte
      value: 0.6
    - metric_key: ascii_only
      aggregation: accuracy
      pass_threshold: 1
      op: gte
      value: 60