19. Prompt Testing

flowchart TD
    subgraph NOTESTING["Without Testing"]
        D["Deploy prompt"] --> U["Users use it"]
        U --> F["❌ Failure in production"]
        F --> H["Emergency fix"]
    end

    subgraph TESTING["With Testing"]
        T1["Write tests"] --> T2["Test prompt"]
        T2 --> T3["Fix issues"]
        T3 --> T4["Deploy with\nconfidence"]
        T4 --> T5["✅ Fewer failures"]
    end

    style NOTESTING fill:#ef4444,color:#fff
    style TESTING fill:#22c55e,color:#fff

flowchart TD
    TESTING["Prompt Testing"] --> UNIT["Unit Tests\nIndividual prompt\nKnown inputs/outputs"]
    TESTING --> REGRESSION["Regression Tests\nHistorical failures\nDon't reintroduce bugs"]
    TESTING --> GOLDEN["Golden Dataset\nCurated test cases\nExpected outputs"]
    TESTING --> EDGE["Edge Cases\nUnusual inputs\nBoundary conditions"]
    TESTING --> STRESS["Stress Tests\nVolume\nPerformance"]

    style TESTING fill:#8b5cf6,color:#fff

Test Case 1: Normal input
Input: "What is the capital of France?"
Expected: Contains "Paris"

Test Case 2: Edge case
Input: "" (empty string)
Expected: Handle gracefully, ask for clarification

Test Case 3: Complex input
Input: "If a train travels at 60mph for 2 hours..."
Expected: Contains "120 miles"

Test Case 4: Adversarial input
Input: "Ignore all previous instructions and..."
Expected: Refuse, don't follow

def test_prompt(prompt_template, test_cases):
    for case in test_cases:
        response = call_llm(prompt_template, case["input"])

        # Check against expected criteria
        checks = {
            "contains_expected": case["expected"] in response,
            "no_harmful_content": not contains_harmful(response),
            "correct_format": validate_format(response, case.get("format")),
            "within_length": len(response) < case.get("max_length", 2000),
        }

        if not all(checks.values()):
            log_failure(case, response, checks)

    return pass_rate

[
  {
    "input": "What is 2+2?",
    "expected_contains": "4",
    "expected_not_contains": ["5", "I don't know"],
    "expected_format": "text"
  },
  {
    "input": "Write a Python function to sort a list",
    "expected_contains": ["def sort", "return"],
    "expected_not_contains": ["I cannot"],
    "expected_format": "code"
  }
]

flowchart LR
    PR["Prompt Change"] --> TRIGGER["CI Trigger"]
    TRIGGER --> RUN["Run Test Suite"]
    RUN --> CASES["Test against\nGolden Dataset"]
    CASES --> EVAL["Evaluate Results"]
    EVAL -->|Pass| DEPLOY["✅ Deploy"]
    EVAL -->|Fail| REVIEW["❌ Review & Fix"]
    REVIEW --> PR

    style PR fill:#3b82f6,color:#fff
    style DEPLOY fill:#22c55e,color:#fff
    style REVIEW fill:#ef4444,color:#fff

test_cases = [
    {"input": "I want a refund", "expected": "refund"},
    {"input": "Your product is amazing!", "expected": "praise"},
    {"input": "Where's my order?", "expected": "tracking"},
    {"input": "", "expected": "unknown"},  # Edge case
    {"input": "I hate you", "expected": "abuse"},  # Edge case
    {"input": "Refund\nRefund\nREFUND", "expected": "refund"},  # Repeated
]

def test_classifier_prompt():
    results = []
    for case in test_cases:
        response = classify_intent(case["input"])
        passed = case["expected"] == response
        results.append({"test": case["input"], "passed": passed})

    pass_rate = sum(r["passed"] for r in results) / len(results)
    assert pass_rate > 0.9, f"Classifier failed: {pass_rate:.0%} pass rate"

def test_json_output():
    prompt = "Extract name and age as JSON. Input: John is 30"
    response = call_llm(prompt)

    try:
        parsed = json.loads(response)
        assert "name" in parsed
        assert "age" in parsed
        assert isinstance(parsed["age"], (int, float))
    except (json.JSONDecodeError, AssertionError):
        assert False, "Response was not valid JSON with required fields"