03. Build a NotebookLM Clone

flowchart TD
    subgraph FRONTEND["Frontend"]
        UI["Document Library UI"]
        VIEWER["Document Viewer"]
        NOTEBOOK["Notebook Interface"]
        AUDIO["Audio Player"]
    end
    subgraph INGEST["Ingestion Pipeline"]
        UPLOAD["Upload Service"]
        PARSE["Document Parser\nPDF/Word/HTML"]
        YOUTUBE["YouTube Transcriber"]
        OCR["OCR for Images"]
    end
    subgraph PROCESS["Processing"]
        CHUNK["Chunking Service"]
        EMBED["Embedding Service"]
        INDEX["Vector Index"]
        SUMMARIZE["Summarization"]
    end
    subgraph GEN["Generation"]
        QA["Q&A Service"]
        AUDIO_GEN["Audio Generator\nElevenLabs"]
        QUIZ["Quiz Generator"]
        FLASHCARD["Flashcard Generator"]
        MINDMAP["Mind Map Generator"]
    end
    subgraph STORE["Storage"]
        S3["Document Store\nS3/GCS"]
        PG["PostgreSQL\nMetadata"]
        VECTOR["Vector DB\nPinecone"]
    end

    UI --> UPLOAD
    UPLOAD --> PARSE
    UPLOAD --> YOUTUBE
    UPLOAD --> OCR
    PARSE --> CHUNK
    CHUNK --> EMBED
    EMBED --> INDEX
    CHUNK --> SUMMARIZE
    INDEX --> QA
    INDEX --> QUIZ
    INDEX --> FLASHCARD
    SUMMARIZE --> AUDIO_GEN
    SUMMARIZE --> MINDMAP
    UPLOAD --> S3
    CHUNK --> PG
    INDEX --> VECTOR

    style FRONTEND fill:#3b82f6,color:#fff
    style INGEST fill:#f59e0b,color:#fff
    style PROCESS fill:#8b5cf6,color:#fff
    style GEN fill:#22c55e,color:#fff
    style STORE fill:#6366f1,color:#fff

sequenceDiagram
    participant U as User
    participant API as API
    participant Parser as Document Parser
    participant Queue as Celery Queue
    participant Worker as Worker
    participant Vector as Vector DB
    participant AI as LLM

    U->>API: Upload PDF (100 pages)
    API->>Parser: Parse document
    Parser->>Parser: Extract text, images, tables
    Parser-->>API: Parsed document

    API->>Queue: Enqueue processing job

    Queue->>Worker: Dequeue job
    Worker->>Worker: Chunk document into segments
    Worker->>Vector: Generate embeddings
    Worker->>AI: Generate summary
    Worker->>AI: Extract key concepts
    Worker-->>API: Processing complete

    API-->>U: Document ready for interaction

    Note over U,API: 15-30 seconds for 100-page PDF

    U->>API: "Summarize the key findings"
    API->>Vector: Retrieve relevant chunks
    Vector-->>API: Top-K chunks
    API->>AI: Generate summary with sources
    AI-->>API: Summary with citations
    API-->>U: Display summary

flowchart TD
    DOCS["Source Documents"] --> SUMM["Generate Summary\nKey points extraction"]
    SUMM --> SCRIPT["Create Podcast Script\nHost 1 + Host 2 dialogue"]
    SCRIPT --> TTS1["TTS Voice 1\nElevenLabs API"]
    SCRIPT --> TTS2["TTS Voice 2\nElevenLabs API"]
    TTS1 --> MERGE["Audio Mixing\nOverlap + transitions"]
    TTS2 --> MERGE
    MERGE --> FINAL["Final Audio\nMP3/Stream"]

    SUMM --> MUSIC["Background Music\nLicense-free tracks"]
    MUSIC --> MERGE

    style SCRIPT fill:#3b82f6,color:#fff
    style TTS1 fill:#22c55e,color:#fff
    style TTS2 fill:#8b5cf6,color:#fff
    style FINAL fill:#f59e0b,color:#fff

flowchart LR
    TEXT["Document Text"] --> CONCEPTS["Extract Concepts\nLLM identifies 10-15 key concepts"]
    CONCEPTS --> RELATIONS["Find Relationships\nHierarchy + connections"]
    RELATIONS --> STRUCTURE["Build Tree Structure\nRoot → Branches → Leaves"]
    STRUCTURE --> RENDER["Render Mind Map\nD3.js interactive"]

    style CONCEPTS fill:#3b82f6,color:#fff
    style RENDER fill:#22c55e,color:#fff

erDiagram
    USERS ||--o{ DOCUMENTS : owns
    USERS ||--o{ NOTEBOOKS : creates
    DOCUMENTS ||--o{ DOCUMENT_CHUNKS : contains
    NOTEBOOKS ||--o{ NOTES : has
    NOTEBOOKS ||--o{ AUDIO_OVERVIEWS : has

    USERS {
        uuid id PK
        string email
        string name
    }
    DOCUMENTS {
        uuid id PK
        uuid user_id FK
        string title
        string file_type
        int page_count
        string status
        text summary
        timestamp created_at
    }
    DOCUMENT_CHUNKS {
        uuid id PK
        uuid document_id FK
        int chunk_index
        text content
        vector embedding
        int token_count
    }
    NOTEBOOKS {
        uuid id PK
        uuid user_id FK
        string title
        uuid[] document_ids
    }
    NOTES {
        uuid id PK
        uuid notebook_id FK
        text content
        json source_citations
    }
    AUDIO_OVERVIEWS {
        uuid id PK
        uuid notebook_id FK
        string audio_url
        int duration_seconds
        text transcript
    }

flowchart TD
    subgraph CI_CD["CI/CD"]
        BUILD["Build Containers"]
        TEST["Test Pipeline"]
    end
    subgraph PROD["Production (GCP)"]
        subgraph GKE["GKE Cluster"]
            FE["Frontend\nNext.js"]
            API["API\nFastAPI"]
            WORKERS["Workers\nCelery"]
            AUDIO["Audio Gen\nGPU pods"]
        end
        subgraph DATA["Data Services"]
            CLOUD_SQL["Cloud SQL\nPostgreSQL"]
            MEMORYSTORE["Memorystore\nRedis"]
            STORAGE["Cloud Storage"]
        end
        AI_PLATFORM["Vertex AI\nLLM APIs"]
    end

    BUILD --> GKE
    TEST --> GKE
    FE --> API
    API --> WORKERS
    API --> AUDIO
    API --> CLOUD_SQL
    API --> MEMORYSTORE
    WORKERS --> STORAGE
    API --> AI_PLATFORM

    style PROD fill:#1e293b,color:#fff
    style GKE fill:#3b82f6,color:#fff
    style DATA fill:#f59e0b,color:#fff