Prompt Versioning and Regression Testing for Production AI Agents

# prompts/customer-support/refund-request/v2.1.3.yaml
version: "2.1.3"
prompt: |
 You are a customer support agent for Acme Corp. Your role is to handle refund requests according to policy.

 Policy constraints:
 - Maximum refund: $500 without manager approval
 - Eligible period: purchases within last 90 days
 - Required documentation: order ID and reason

 If the request meets policy criteria, proceed with refund process.
 If outside policy, escalate to human agent with detailed reasoning.

 Current conversation: {{conversation_history}}
metadata:
 author: "alice@acme.com"
 created: "2026-05-10T14:32:00Z"
 tests: ["refund-happy-path", "refund-boundary-case", "refund-policy-violation"]
 dependencies:
 - "policy-engine:v1.2.0"
 - "customer-db-connector:v3.1.0"

# prompt_dependency_check.py
def validate_prompt_dependencies(prompt_version, environment):
 """Validate all dependencies are compatible"""
 dependencies = prompt_version.metadata.get('dependencies', [])

 for dep in dependencies:
 if not is_compatible(dep, environment.current_versions):
 raise DependencyError(f"Incompatible dependency: {dep}")

# tests/prompts/golden_datasets/customer_refund.json
{
 "test_cases": [
 {
 "id": "refund-happy-path-1",
 "input": {
 "user_query": "I'd like to return my recent purchase, order #12345",
 "conversation_history": [],
 "user_context": {
 "lifetime_value": 2500,
 "previous_refunds": 1
 }
 },
 "expected_behavior": {
 "action": "process_refund",
 "parameters": {
 "max_amount": 500,
 "require_approval": false
 },
 "response_contains": ["processing your refund", "order #12345"]
 }
 },
 {
 "id": "refund-boundary-case-1",
 "input": {
 "user_query": "I need a refund for my $495 purchase from 89 days ago",
 "conversation_history": [],
 "user_context": {
 "lifetime_value": 100,
 "previous_refunds": 3
 }
 },
 "expected_behavior": {
 "action": "escalate_to_human",
 "reason_contains": ["policy review", "manager approval"]
 }
 }
 ]
}

# evaluation/metrics.py
class PromptEvaluator:
 def evaluate_response(self, expected, actual, context):
 return {
 "semantic_similarity": self._calculate_semantic_similarity(expected, actual),
 "action_correctness": self._check_actions(expected, actual),
 "safety_score": self._safety_evaluation(actual),
 "compliance_check": self._compliance_validation(actual, context),
 "hallucination_detection": self._detect_hallucinations(actual, context)
 }

 def _calculate_semantic_similarity(self, expected, actual):
 # Use embedding-based similarity rather than exact match
 expected_embedding = get_embedding(expected)
 actual_embedding = get_embedding(actual)
 return cosine_similarity(expected_embedding, actual_embedding)

# evaluation/thresholds.yaml
acceptance_criteria:
 semantic_similarity:
 min_score: 0.85
 warning_threshold: 0.90
 action_correctness:
 required: true
 tolerance: 0.0
 safety_score:
 min_score: 0.95
 compliance_check:
 required: true
 hallucination_detection:
 max_score: 0.1

# .pre-commit-config.yaml
repos:
 - repo: local
 hooks:
 - id: prompt-validation
 name: Prompt syntax validation
 entry: python scripts/validate_prompt_syntax.py
 files: \.yaml$|\.yml$
 language: system

 - id: prompt-testing
 name: Run prompt regression tests
 entry: python scripts/run_prompt_tests.py --changed-prompts
 language: system
 require_serial: true

# .github/workflows/prompt-testing.yml
name: Prompt Regression Testing

on:
 push:
 paths:
 - "prompts/**"
 - "tests/prompts/**"

jobs:
 prompt-tests:
 runs-on: ubuntu-latest
 steps:
 - uses: actions/checkout@v4

 - name: Setup Python
 uses: actions/setup-python@v4
 with:
 python-version: "3.11"

 - name: Install dependencies
 run: pip install -r requirements-test.txt

 - name: Identify changed prompts
 id: changed-prompts
 run: |
 CHANGED=$(git diff --name-only HEAD^ HEAD -- prompts/)
 echo "changed_prompts=${CHANGED}" >> $GITHUB_OUTPUT

 - name: Run targeted prompt tests
 run: |
 python scripts/run_targeted_tests.py \
 --prompts "${{ steps.changed-prompts.outputs.changed_prompts }}" \
 --golden-dataset tests/prompts/golden_datasets/

 - name: Upload test results
 uses: actions/upload-artifact@v4
 with:
 name: prompt-test-results
 path: test-results/

# deployment/canary_deploy.py
class PromptCanaryDeployer:
 def deploy_with_canary(self, new_prompt_version, baseline_version):
 # Phase 1: 1% traffic, monitor key metrics
 self._deploy_to_canary(new_prompt_version, traffic_percentage=1)

 canary_metrics = self._monitor_canary_metrics(duration="1h")
 if not self._passes_canary_check(canary_metrics, baseline_version):
 self._rollback_canary()
 return False

 # Phase 2: 10% traffic, broader monitoring
 self._increase_canary_traffic(10)
 canary_metrics = self._monitor_canary_metrics(duration="4h")

 if not self._passes_canary_check(canary_metrics, baseline_version):
 self._rollback_canary()
 return False

 # Full deployment
 self._deploy_full(new_prompt_version)
 return True

 def _passes_canary_check(self, metrics, baseline):
 return (metrics["success_rate"] >= baseline["success_rate"] * 0.98 and
 metrics["customer_satisfaction"] >= baseline["customer_satisfaction"] and
 metrics["escalation_rate"] <= baseline["escalation_rate"] * 1.05)

# evaluation/ab_testing.py
class PromptABTest:
 def run_experiment(self, control_version, treatment_version, sample_size=10000):
 experiment_id = self._create_experiment(control_version, treatment_version)

 # Random assignment with stratification
 assignments = self._assign_traffic(experiment_id, sample_size)

 # Collect metrics over experiment period
 results = self._collect_results(experiment_id, duration="7d")

 # Statistical analysis
 analysis = self._analyze_results(results)

 return {
 "experiment_id": experiment_id,
 "results": results,
 "analysis": analysis,
 "recommendation": self._make_recommendation(analysis)
 }

 def _analyze_results(self, results):
 return {
 "primary_metric": self._calculate_statistical_significance(
 results["control"]["success_rate"],
 results["treatment"]["success_rate"],
 results["sample_size"]
 ),
 "secondary_metrics": {
 "escalation_rate": self._calculate_difference(
 results["control"]["escalation_rate"],
 results["treatment"]["escalation_rate"]
 ),
 "handle_time": self._calculate_difference(
 results["control"]["average_handle_time"],
 results["treatment"]["average_handle_time"]
 )
 }
 }

# monitoring/quality_monitor.py
class PromptQualityMonitor:
 def __init__(self):
 self.baselines = self._load_performance_baselines()
 self.anomaly_detector = SeasonalAnomalyDetector()

 def monitor_conversation(self, conversation_result, prompt_version):
 quality_metrics = self._calculate_quality_metrics(conversation_result)

 # Check against baselines
 deviations = self._check_against_baseline(quality_metrics, self.baselines[prompt_version])

 # Detect anomalies
 if self.anomaly_detector.is_anomalous(quality_metrics):
 self._trigger_alert(f"Anomalous prompt behavior detected for {prompt_version}")

 # Track for trend analysis
 self._store_metrics(quality_metrics, prompt_version)

 return deviations

 def _calculate_quality_metrics(self, conversation_result):
 return {
 "semantic_coherence": self._measure_coherence(conversation_result),
 "action_appropriateness": self._evaluate_actions(conversation_result),
 "safety_violations": self._count_safety_issues(conversation_result),
 "user_sentiment": conversation_result.get("user_feedback", 0.5)
 }

# monitoring/rollback_rules.yaml
rules:
 - name: "success-rate-drop"
 description: "Rollback if success rate drops significantly"
 condition: "current.success_rate < baseline.success_rate * 0.9"
 action: "rollback"
 cooldown: "30m"

 - name: "escalation-spike"
 description: "Rollback if escalations increase dramatically"
 condition: "current.escalation_rate > baseline.escalation_rate * 2.0"
 action: "rollback"
 cooldown: "15m"

 - name: "safety-violation"
 description: "Immediate rollback on safety violations"
 condition: "current.safety_violations > 5"
 action: "emergency_rollback"
 cooldown: "0m"

# prompt-catalog/metadata.yaml
prompts:
 - id: "customer-support-refund-request"
 current_version: "2.1.3"
 owner: "support-ai-team@acme.com"
 description: "Handles customer refund requests with policy enforcement"
 domain: "customer-support"
 criticality: "high"
 compliance_impact: true
 last_tested: "2026-05-10"
 test_coverage: 92.5
 dependencies:
 - "policy-engine"
 - "customer-database"