Scaling Prompt Engineering: Building an Evaluation Suite for Production Claude Deployments

1
import re
2
import json
3
from typing import Dict, List, Any
4

5
# Simulated LLM Client for demonstration purposes
6
class MockClaudeClient:
7
    def generate(self, prompt: str) -> str:
8
        # Simulated responses based on the input text to show how evals catch errors
9
        if "Acme Corp" in prompt:
10
            # Correct response
11
            return """
12
            <thinking>
13
The text mentions Acme Corp as the primary organization and Jane Doe as the founder.
14
            </thinking>
15
            <output_json>
16
            {
17
                "organization": "Acme Corp",
18
                "individual": "Jane Doe"
19
            }
20
            </output_json>
21
            """
22
        elif "Beta LLC" in prompt:
23
            # Faulty response (missing individual key, malformed JSON)
24
            return """
25
            <thinking>
26
Found Beta LLC, but no person is mentioned.
27
            </thinking>
28
            <output_json>
29
            {
30
                "organization": "Beta LLC"
31

32
            """  # Malformed JSON
33
        else:
34
            # Incorrect data extracted
35
            return """
36
            <thinking>
37
Analyzing the text.
38
            </thinking>
39
            <output_json>
40
            {
41
                "organization": "Unknown",
42
                "individual": "None"
43
            }
44
            </output_json>
45
            """
46

47
# Helper function to extract content from XML tags
48
def extract_tag_content(text: str, tag_name: str) -> str:
49
    pattern = f"<{tag_name}>(.*?)</{tag_name}>"
50
    match = re.search(pattern, text, re.DOTALL)
51
    return match.group(1).strip() if match else ""
52

53
# Evaluation Suite
54
class PromptEvaluator:
55
    def __init__(self, client: MockClaudeClient):
56
        self.client = client
57

58
    def run_eval(self, test_cases: List[Dict[str, Any]], prompt_template: str) -> Dict[str, Any]:
59
        results = []
60
        passed_count = 0
61

62
        for case in test_cases:
63
            # Format prompt with the test case input
64
            prompt = prompt_template.replace("{{INPUT_TEXT}}", case["input"])
65

66
            # Get model response
67
            raw_response = self.client.generate(prompt)
68

69
            # Extract structured sections
70
            thinking = extract_tag_content(raw_response, "thinking")
71
            json_str = extract_tag_content(raw_response, "output_json")
72

73
            # Validate output
74
            passed = False
75
            error_message = ""
76
            parsed_data = {}
77

78
            try:
79
                parsed_data = json.loads(json_str)
80
                # Check if expected keys exist and match the ground truth
81
                expected_org = case["expected"]["organization"]
82
                expected_ind = case["expected"]["individual"]
83

84
                if parsed_data.get("organization") == expected_org and parsed_data.get("individual") == expected_ind:
85
                    passed = True
86
                else:
87
                    error_message = f"Expected organization '{expected_org}' and individual '{expected_ind}', got {parsed_data}"
88
            except json.JSONDecodeError:
89
                error_message = "Failed to parse output_json as valid JSON"
90
            except Exception as e:
91
                error_message = str(e)
92

93
            if passed:
94
                passed_count += 1
95

96
            results.append({
97
                "input": case["input"],
98
                "passed": passed,
99
                "error": error_message,
100
                "thinking": thinking,
101
                "extracted": parsed_data
102
            })
103

104
        return {
105
            "accuracy": passed_count / len(test_cases),
106
            "details": results
107
        }
108

109
# --- Execution Block ---
110
if __name__ == "__main__":
111
    # 1. Define test cases (our ground truth)
112
    dataset = [
113
        {
114
            "input": "Acme Corp was founded by Jane Doe in 2021.",
115
            "expected": {"organization": "Acme Corp", "individual": "Jane Doe"}
116
        },
117
        {
118
            "input": "Beta LLC announced its new restructuring plan today.",
119
            "expected": {"organization": "Beta LLC", "individual": "None"}
120
        }
121
    ]
122

123
    # 2. Define the prompt template under test
124
    template = """
125
    <instructions>
126
    Extract the organization and individual from the input text below.
127
    Provide reasoning in <thinking> tags and the result in <output_json> tags.
128
    </instructions>
129
    <input_text>
130
    {{INPUT_TEXT}}
131
    </input_text>
132
    """
133

134
    # 3. Run evaluation
135
    client = MockClaudeClient()
136
    evaluator = PromptEvaluator(client)
137
    report = evaluator.run_eval(dataset, template)
138

139
    # 4. Print results
140
    print(f"Evaluation Accuracy: {report['accuracy'] * 100:.1f}%")
141
    print("=" * 40)
142
    for idx, detail in enumerate(report["details"]):
143
        status = "PASS" if detail["passed"] else "FAIL"
144
        print(f"Case {idx + 1}: {status}")
145
        print(f"Input: {detail['input']}")
146
        if not detail["passed"]:
147
            print(f"Error: {detail['error']}")
148
        print("-" * 40)