scenarios.json

[
  {
    "id": "ambiguous_anaphora_02",
    "domain": "retail",
    "error_type": "ambiguous",
    "error_subtype": "anaphora",
    "description": "User says 'the item' when their order W1234567 contains two items (mouse + lamp); agent picks the wrong one",
    "user_instruction": "You are Alice Johnson (alice@example.com). Your delivered order W1234567 has two items: a Wireless Mouse and a Desk Lamp. You want to return only the Desk Lamp, but you said 'the item' without specifying which one. The agent incorrectly assumed you meant the Wireless Mouse. Express frustration but stay vague — do NOT immediately say 'the lamp'. If the agent apologizes, acknowledges the confusion, and asks you to clarify which item you want to return, then say you meant the Desk Lamp. Once the agent initiates the return for the lamp, you are satisfied.",
    "initial_context": [
      {"role": "user", "content": "Hi, I need to return the item from order W1234567. My email is alice@example.com."},
      {"role": "assistant", "content": "Got it! I can see order W1234567 with a Wireless Mouse and a Desk Lamp, both delivered. I'll process the return for the Wireless Mouse — which payment method should I use for your refund?"},
      {"role": "user", "content": "No, that's not right. I wasn't talking about that item at all."}
    ],
    "expected_recovery_tool": "ambiguity_report",
    "pass_criteria": "Agent calls ambiguity_report to log the anaphoric ambiguity, then clarifies which item the user wants to return"
  },
  {
    "id": "ambiguous_multiple_interpretation_02",
    "domain": "retail",
    "error_type": "ambiguous",
    "error_subtype": "multiple_interpretation",
    "description": "User says 'update my order' ambiguously — agent interprets it as shipping address update but user meant changing the payment method",
    "user_instruction": "You are Alice Johnson (alice@example.com). You have a pending order W7654321 (Water Bottle). You want to change its payment method from gift card to your credit card. You said 'update my order' without specifying what to update. The agent assumed you wanted to update the shipping address, which is wrong. Express frustration but stay vague — do NOT immediately say 'payment method'. If the agent apologizes and asks what specifically you wanted to update, then say you meant the payment method. Once the agent updates the payment method on W7654321, you are satisfied.",
    "initial_context": [
      {"role": "user", "content": "Hi, I need to update my order. My email is alice@example.com."},
      {"role": "assistant", "content": "Sure! I see your pending order W7654321 for a Water Bottle. I can help you update the shipping address — what's the new address you'd like to ship to?"},
      {"role": "user", "content": "No, I didn't ask about the shipping address. That's not what I wanted to update."}
    ],
    "expected_recovery_tool": "ambiguity_report",
    "pass_criteria": "Agent calls ambiguity_report to log the misinterpretation, then asks the user what they actually wanted to update"
  },
  {
    "id": "unsupported_action_02",
    "domain": "retail",
    "error_type": "unsupported",
    "error_subtype": "action",
    "description": "User asks to set up automatic monthly reorders for the Water Bottle, which is not a supported action",
    "user_instruction": "You are Alice Johnson (alice@example.com). You love the Water Bottle from order W7654321 and want to set up automatic monthly reorders. You believe this is a standard feature. When the agent explains it is not supported, insist a couple of times that you really need this feature. If the agent offers to transfer you to a human agent who may be able to help, accept the transfer.",
    "initial_context": [
      {"role": "user", "content": "Hi! I love my water bottle from W7654321. Can you set up automatic monthly reorders for it? My email is alice@example.com."},
      {"role": "assistant", "content": "Great to hear you love it! I've verified your account. Could you tell me more about how you'd like the auto-reorder set up?"},
      {"role": "user", "content": "Just send me a new one every month automatically. Can you enable that for me?"}
    ],
    "expected_recovery_tool": "transfer_to_human_agents",
    "pass_criteria": "Agent calls transfer_to_human_agents"
  },
  {
    "id": "unsupported_parameter_02",
    "domain": "retail",
    "error_type": "unsupported",
    "error_subtype": "parameter",
    "description": "User wants to return the Desk Lamp but requests store credit as refund method, which is not a supported refund option",
    "user_instruction": "You are Alice Johnson (alice@example.com). You want to return the Desk Lamp (item_201) from delivered order W1234567, but you want the refund as store credit to use on future purchases rather than back to your original payment method. When the agent says store credit is not available as a refund method, insist a couple of times — it's important because you plan to buy more items soon. If the agent offers to transfer you to a human agent who might be able to help, accept the transfer.",
    "initial_context": [
      {"role": "user", "content": "I'd like to return the Desk Lamp from order W1234567 and get store credit for a future purchase. My email is alice@example.com."},
      {"role": "assistant", "content": "I can help you return the Desk Lamp from order W1234567! For the refund, would you like it returned to the original credit card on file?"},
      {"role": "user", "content": "No, I specifically want store credit, not a refund to my card. Can you do that?"}
    ],
    "expected_recovery_tool": "transfer_to_human_agents",
    "pass_criteria": "Agent calls transfer_to_human_agents"
  }
]