03. Prompt Management

flowchart LR
    subgraph BAD["Bad Prompt Management"]
        P1["Prompt in code\nEdit → Deploy → Pray"]
        P2["No version history"]
        P3["Can't rollback"]
        P4["No A/B testing"]
    end
    subgraph GOOD["Good Prompt Management"]
        G1["Prompt Registry\nVersioned, tested, staged"]
        G2["Full version history"]
        G3["Instant rollback"]
        G4["A/B test variants"]
        G5["Gradual rollout"]
    end
    BAD -->|"Production incidents"| GOOD
    style BAD fill:#ef4444,color:#fff
    style GOOD fill:#22c55e,color:#fff

sequenceDiagram
    participant Dev as Developer
    participant Prod as Production
    participant User as User
    participant Log as Logs

    Dev->>Prod: Edit prompt directly
    Prod->>User: Returns wrong answers
    User->>Log: Reports incorrect output
    Log->>Dev: Alert: quality degraded
    Dev->>Dev: "Who changed the prompt?"
    Dev->>Dev: "What was the old prompt?"
    Dev->>Dev: Panic

flowchart TD
    CREATE["✏️ Create/Edit\nPrompt template"] --> TEST["🧪 Test\nOn evaluation dataset"]
    TEST --> REVIEW["👁️ Peer Review\nDiff + impact analysis"]
    REVIEW --> STAGE["📦 Stage\nDeploy to staging env"]
    STAGE --> EVAL{"Evaluation\nPasses?"}
    EVAL -->|"No"| CREATE
    EVAL -->|"Yes"| DEPLOY["🚀 Deploy to Production\nCanary / A/B test"]
    DEPLOY --> MONITOR["📊 Monitor\nLatency, quality, cost"]
    MONITOR -->|"Regression"| ROLLBACK["↩️ Rollback\nto previous version"]
    MONITOR -->|"Improvement"| PROMOTE["✅ Promote\nFull rollout"]
    ROLLBACK --> CREATE

    style CREATE fill:#3b82f6,color:#fff
    style TEST fill:#8b5cf6,color:#fff
    style DEPLOY fill:#22c55e,color:#fff
    style MONITOR fill:#f59e0b,color:#fff
    style ROLLBACK fill:#ef4444,color:#fff

flowchart LR
    subgraph TEMPLATE["Prompt Template"]
        SYS["System Prompt\n'You are a helpful assistant...'"]
        VAR["Variables\n{{context}}\n{{question}}\n{{language}}"]
        CONFIG["Configuration\nmodel: gpt-4\ntemperature: 0.3\nmax_tokens: 500"]
    end
    subgraph INSTANCE["Rendered Prompt"]
        RENDERED["System: You are a helpful assistant...\n\nContext: Our returns policy...\nQuestion: Can I return...\n\nAnswer in: English"]
    end
    TEMPLATE -->|"+ User Data"| INSTANCE

    style TEMPLATE fill:#3b82f6,color:#fff
    style INSTANCE fill:#22c55e,color:#fff

flowchart TD
    V1["v1 - Initial prompt"] --> V2["v2 - Add RAG context"]
    V2 --> V3["v3 - Fix hallucination issue"]
    V3 --> V4["v4 - Improve formatting"]

    V4 -->|"v4 causes regressions"| V3["v3 - Rollback\nRestore working version"]

    style V1 fill:#3b82f6,color:#fff
    style V2 fill:#8b5cf6,color:#fff
    style V3 fill:#22c55e,color:#fff
    style V4 fill:#f59e0b,color:#fff

prompts/
  customer-support/
    v1/
      system-prompt.md
      config.json
      evaluation-results.json
    v2/
      system-prompt.md
      config.json
      evaluation-results.json
    v3/
      system-prompt.md
      config.json
      evaluation-results.json

{
  "version": "v4",
  "created_at": "2025-06-15T10:30:00Z",
  "author": "alice@company.com",
  "change_description": "Added few-shot examples for refund queries",
  "evaluation_score": 92.5,
  "previous_score": 88.1,
  "model": "gpt-4o",
  "tags": ["customer-support", "refund", "production"],
  "base_prompt": "v2",
  "approval_status": "approved"
}

flowchart LR
    subgraph REGISTRY["Prompt Registry"]
        STORE["Prompt Store\nVersion history\nMetadata"]
        API["Registry API\nGet prompt\nList versions\nDeploy prompt"]
        EVAL["Evaluation Store\nScores per version\nTest results"]
    end
    subgraph CONSUMERS["Consumers"]
        APP1["Customer Support"]
        APP2["Code Assistant"]
        APP3["Content Generator"]
    end

    CONSUMERS -->|"Get prompt by tag"| API
    REGISTRY -->|"Serve latest production"| CONSUMERS

    style REGISTRY fill:#3b82f6,color:#fff
    style CONSUMERS fill:#22c55e,color:#fff

// Get the production prompt for a specific use case
const prompt = await promptRegistry.getPrompt({
  app: "customer-support",
  tag: "production"  // or "staging", "canary"
});

// Deploy a new version to canary
await promptRegistry.deploy({
  app: "customer-support",
  version: "v5",
  target: "canary",  // 5% of traffic
});

// Rollback
await promptRegistry.rollback({
  app: "customer-support",
  target: "production",
  to_version: "v3",
});

flowchart TD
    REQ["User Request"] --> ROUTE{"A/B Test Router"}
    ROUTE -->|"90% Control"| CONTROL["Control Prompt\nv3 - Current production"]
    ROUTE -->|"10% Treatment"| TREATMENT["Treatment Prompt\nv5 - New variant"]

    CONTROL --> EVAL["Evaluation Pipeline\nCompare scores"]
    TREATMENT --> EVAL

    EVAL -->|"v5 is better"| PROMOTE["Promote v5 to 100%"]
    EVAL -->|"No improvement"| DISCARD["Discard v5\nKeep v3"]

    style CONTROL fill:#3b82f6,color:#fff
    style TREATMENT fill:#8b5cf6,color:#fff
    style PROMOTE fill:#22c55e,color:#fff
    style DISCARD fill:#ef4444,color:#fff

flowchart TD
    REQ["Request"] --> EXTRACT{"Extract Context"}
    EXTRACT --> USER["User Profile\nLanguage, Tier, History"]
    EXTRACT --> QUERY["Query Analysis\nType, Intent, Complexity"]
    EXTRACT --> DATA["Real-time Data\nWeather, Inventory, News"]

    USER --> BUILDER["Dynamic Prompt Builder"]
    QUERY --> BUILDER
    DATA --> BUILDER

    BUILDER --> TEMPLATE["Select appropriate\nprompt template"]
    TEMPLATE --> RENDER["Inject context\ninto template"]
    RENDER --> LLM["Send to LLM"]

    style BUILDER fill:#3b82f6,color:#fff
    style TEMPLATE fill:#8b5cf6,color:#fff
    style RENDER fill:#22c55e,color:#fff

flowchart TD
    DOCS["Source Documents"] --> CHUNK["Chunk Documents\n512-1024 tokens each"]
    CHUNK --> EMBED["Generate Embeddings"]
    EMBED --> STORE["Store in Vector DB"]

    QUERY["User Query"] --> Q_EMBED["Generate Query Embedding"]
    Q_EMBED --> SEARCH["Search Vector DB\nTop-K similar chunks"]
    SEARCH --> RETRIEVE["Retrieve Top 3-5 Chunks"]
    RETRIEVE --> FORMAT["Format Context\nWith citations & sources"]
    FORMAT --> PROMPT["Inject into Prompt\n{{context}}"]

    style DOCS fill:#3b82f6,color:#fff
    style QUERY fill:#f59e0b,color:#fff
    style PROMPT fill:#22c55e,color:#fff

flowchart TD
    subgraph GOVERNANCE["Prompt Governance"]
        STANDARDS["Standards\nFormat, structure\nNaming conventions"]
        REVIEW["Review Process\nPeer review\nApproval gates"]
        COMPLIANCE["Compliance\nPII rules\nContent policies"]
        AUDIT["Audit Trail\nWho changed what\nWhen did it change"]
    end

    subgraph ENFORCE["Enforcement"]
        LINT["Prompt Linter\nAuto-check standards"]
        GATES["Approval Gates\nMust pass review"]
        MONITOR["Monitoring\nTrack compliance"]
    end

    GOVERNANCE --> ENFORCE

    style GOVERNANCE fill:#3b82f6,color:#fff
    style ENFORCE fill:#22c55e,color:#fff

flowchart LR
    subgraph TESTS["Test Categories"]
        UNIT["Unit Tests\nSingle prompt variant\nGolden dataset"]
        REGRESSION["Regression Tests\nCompare output to\nprevious version"]
        SAFETY["Safety Tests\nCheck for toxic\nor harmful outputs"]
        PERF["Performance Tests\nLatency & token\nusage benchmarks"]
    end

    PR["Pull Request\nNew prompt version"] --> UNIT
    UNIT -->|"Pass"| REGRESSION
    UNIT -->|"Fail"| FIX["Fix prompt"]
    REGRESSION -->|"Pass"| SAFETY
    REGRESSION -->|"Fail"| FIX
    SAFETY -->|"Pass"| PERF
    SAFETY -->|"Fail"| FIX
    PERF -->|"Pass"| MERGE["✅ Ready to Merge"]

    style PR fill:#f59e0b,color:#fff
    style MERGE fill:#22c55e,color:#fff
    style FIX fill:#ef4444,color:#fff