02. Build a Perplexity Clone

flowchart TD
    subgraph FRONTEND["Frontend"]
        UI["Search UI\nNext.js"]
        SSE["SSE Client\nStreaming"]
    end
    subgraph ORCH["Orchestration"]
        GW["API Gateway"]
        SEARCH["Search Service\nQuery + Crawl"]
        RANK["Ranking Service\nRelevance scoring"]
        EXTRACT["Content Extraction\nPage parsing"]
    end
    subgraph AI["AI Layer"]
        SUMMARIZE["Summarization\nAnswer generation"]
        CITE["Citation Engine\nSource linking"]
        FOLLOWUP["Follow-up\nContext management"]
    end
    subgraph DATA["Data Layer"]
        PG["PostgreSQL"]
        QDRANT["Qdrant\nEmbeddings"]
        REDIS["Redis\nCache"]
        S3["Page Cache"]
    end
    subgraph EXTERNAL["External"]
        BING["Bing API"]
        GOOGLE["Google Search"]
        SERP["SerpAPI"]
    end

    UI --> GW
    GW --> SEARCH
    SEARCH --> BING
    SEARCH --> GOOGLE
    SEARCH --> SERP
    SEARCH --> EXTRACT
    EXTRACT --> RANK
    RANK --> QDRANT
    RANK --> SUMMARIZE
    SUMMARIZE --> CITE
    CITE --> UI
    SUMMARIZE --> REDIS
    SEARCH --> REDIS
    EXTRACT --> S3

    style FRONTEND fill:#3b82f6,color:#fff
    style ORCH fill:#8b5cf6,color:#fff
    style AI fill:#22c55e,color:#fff
    style DATA fill:#f59e0b,color:#fff
    style EXTERNAL fill:#ef4444,color:#fff

sequenceDiagram
    participant U as User
    participant S as Search Service
    participant Web as Web Search
    participant Crawl as Crawler
    participant Rank as Ranking
    participant LLM as LLM
    participant Cache as Cache

    U->>S: "What is the latest AI news?"
    S->>Cache: Check cache
    Cache-->>S: Miss

    S->>Web: Search multiple engines
    Web-->>S: Raw results (URLs + snippets)

    S->>Crawl: Fetch top 10 pages
    Crawl->>Crawl: Extract content, strip HTML
    Crawl-->>S: Cleaned content

    S->>Rank: Score relevance
    Rank->>Rank: Embed query + docs, cosine similarity
    Rank-->>S: Ranked passages

    S->>LLM: Generate answer from top passages
    LLM->>LLM: Answer + citations
    LLM-->>S: Generated answer

    S->>Cache: Store result (TTL: 1 hour)
    S-->>U: Answer with citations

    U->>S: "What about advancements in robotics?"
    S->>S: Use previous context
    S->>LLM: Follow-up with context
    LLM-->>S: Context-aware answer

flowchart LR
    QUERY["User Query"] --> MULTI["Multi-Engine Search"]
    MULTI --> BING["Bing Search\nAPI"]
    MULTI --> GOOGLE["Google Custom\nSearch"]
    MULTI --> SERP["SerpAPI\nGoogle results"]

    BING --> MERGE["Merge + Deduplicate"]
    GOOGLE --> MERGE
    SERP --> MERGE

    MERGE --> RANK["Rank by\nrelevance score"]
    RANK --> TOP_K["Top-K results\n(5-10 pages)"]

    style MULTI fill:#3b82f6,color:#fff
    style MERGE fill:#f59e0b,color:#fff
    style TOP_K fill:#22c55e,color:#fff

flowchart TD
    URL["Web Page URL"] --> FETCH["HTTP Fetch\nUser-agent headers"]
    FETCH --> PARSE["HTML Parse\nBeautifulSoup/Readability"]
    PARSE --> CLEAN["Clean Content\nRemove ads, nav, scripts"]
    CLEAN --> CHUNK["Chunk Content\n512 token segments"]
    CHUNK --> EMBED["Generate Embeddings\nFor each chunk"]
    EMBED --> INDEX["Store in Vector DB\nWith source metadata"]

    style FETCH fill:#3b82f6,color:#fff
    style CLEAN fill:#22c55e,color:#fff
    style INDEX fill:#f59e0b,color:#fff

flowchart LR
    PASSAGES["Retrieved Passages\n10-20 chunks"] --> CROSS["Cross-Encoder\nRe-ranking model"]
    QUERY["User Query"] --> CROSS
    CROSS --> SCORES["Relevance Scores\n0.0 - 1.0"]
    SCORES --> TOP["Top 3-5 Passages\nFor answer generation"]

    style CROSS fill:#8b5cf6,color:#fff
    style TOP fill:#22c55e,color:#fff

flowchart TD
    ANSWER["Generated Answer"] --> CLAIMS["Extract Claims\nSentence splitting"]
    CLAIMS --> MATCH["Match to Sources\nSemantic similarity"]
    MATCH --> VERIFY["Verify Claim\nIs source relevant?"]
    VERIFY -->|"Yes"| CITE["Add Citation\n[1], [2], etc."]
    VERIFY -->|"No"| DROP["Drop unsupported claim"]
    CITE --> FINAL["Final Answer\nWith numbered sources"]

    style CLAIMS fill:#3b82f6,color:#fff
    style CITE fill:#22c55e,color:#fff
    style DROP fill:#ef4444,color:#fff

erDiagram
    USERS ||--o{ SEARCHES : creates
    SEARCHES ||--o{ SEARCH_RESULTS : contains
    SEARCHES ||--o{ FOLLOW_UPS : has
    SEARCH_RESULTS ||--o{ SOURCES : cites
    COLLECTIONS ||--o{ SEARCHES : groups

    USERS {
        uuid id PK
        string email
        string name
        timestamp created_at
    }
    SEARCHES {
        uuid id PK
        uuid user_id FK
        string query
        text answer
        json citations
        int sources_count
        timestamp created_at
    }
    SEARCH_RESULTS {
        uuid id PK
        uuid search_id FK
        string url
        string title
        string snippet
        float relevance_score
    }
    SOURCES {
        uuid id PK
        string url
        string domain
        string title
        text cached_content
        timestamp crawled_at
    }
    FOLLOW_UPS {
        uuid id PK
        uuid search_id FK
        string query
        text answer
        int turn_number
    }
    COLLECTIONS {
        uuid id PK
        uuid user_id FK
        string name
        string description
        timestamp created_at
    }

flowchart TD
    subgraph PROD["Production"]
        CF["CloudFront\nCDN"]
        ALB["Load Balancer"]

        subgraph ECS["ECS Fargate"]
            FE["Frontend\nNext.js"]
            API["API\nFastAPI"]
            WORKER["Crawler Workers\nCelery"]
            RANK_SVC["Ranking\nService"]
        end

        subgraph DATA["Data"]
            RDS["Aurora\nPostgreSQL"]
            ELASTICACHE["Redis\nCache + Queue"]
            S3["Page Cache"]
            QDRANT_CLOUD["Qdrant\nVector DB"]
        end
    end

    CF --> ALB
    ALB --> FE
    ALB --> API
    API --> WORKER
    API --> RANK_SVC
    API --> RDS
    API --> ELASTICACHE
    WORKER --> S3
    API --> QDRANT_CLOUD

    style PROD fill:#1e293b,color:#fff
    style ECS fill:#3b82f6,color:#fff
    style DATA fill:#f59e0b,color:#fff