feat: implement backend for generate wiki

api/api.py

@@ -1,10 +1,12 @@
-from fastapi import FastAPI, WebSocket
+from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from api.language_config import router as language_config_router
+from api.processed_projects import router as processed_projects_router
 from api.simple_chat import router as simple_chat_router
 from api.websocket_wiki import handle_websocket_chat
+from api.wiki import router as wiki_router
 from api.wiki_cache import router as wiki_cache_router
-from utils.logger import logger
 
 app = FastAPI()
 app.add_middleware(
@@ -18,3 +20,6 @@
 app.add_websocket_route("/ws/chat", handle_websocket_chat)
 app.include_router(wiki_cache_router)
 app.include_router(simple_chat_router)
+app.include_router(language_config_router)
+app.include_router(processed_projects_router)
+app.include_router(wiki_router)

api/config.py

@@ -0,0 +1,13 @@
+def load_language_config():
+    default_config = {
+        "supported_languages": {
+            "en": "English",
+            "vi": "Vietnamese (Tiếng Việt)",
+        },
+        "default": "en",
+    }
+    return default_config
+
+
+configs = {}
+configs["language_config"] = load_language_config()

api/language_config.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+
+from api.config import configs
+
+router = APIRouter(prefix="/lang/config", tags=["Language Configuration"])
+
+
+@router.get("")
+async def get_language_config():
+    return configs["language_config"]

api/main.py

@@ -1,9 +1,11 @@
-import uvicorn
 import os
 import sys
-from utils.logger import logger
+
+import uvicorn
 from dotenv import load_dotenv
 
+from utils.logger import logger
+
 load_dotenv()
 
 if __name__ == "__main__":
@@ -13,5 +15,5 @@
         "api.api:app",
         host="0.0.0.0",
         port=int(os.environ.get("PORT", 8001)),
-        reload=False,
+        reload=True,
     )

api/models.py

@@ -1,8 +1,13 @@
-from typing import List, Optional, Dict
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, Field
 
 
+def to_camel(string: str) -> str:
+    parts = string.split("_")
+    return parts[0] + "".join(word.capitalize() for word in parts[1:])
+
+
 class ChatMessage(BaseModel):
     role: str  # 'user' or 'assistant'
     content: str
@@ -68,6 +73,10 @@ class WikiPage(BaseModel):
     importance: str
     related_pages: List[str]
 
+    class Config:
+        alias_generator = to_camel
+        populate_by_name = True
+
 
 class WikiStructureModel(BaseModel):
     """
@@ -99,3 +108,29 @@ class WikiCacheRequest(BaseModel):
     repo_type: str
     wiki_structure: WikiStructureModel
     generated_pages: Dict[str, WikiPage]
+
+
+class ProcessedProjectEntry(BaseModel):
+    id: str
+    owner: str
+    repo: str
+    name: str
+    repo_type: str
+    submitted_at: int  # timestampt
+    language: str
+
+
+class WikiTaskRequest(BaseModel):
+    owner: str
+    repo: str
+    repo_url: str
+    repo_info: Dict[str, Any]
+    token: Optional[str] = None
+
+
+class WikiTaskStatus(BaseModel):
+    status: str
+    message: str = ""
+    progress: Optional[set[str]] = None
+    error: Optional[str] = None
+    result: Optional[WikiCacheData] = None

api/processed_projects.py

@@ -0,0 +1,42 @@
+import asyncio
+import os
+from typing import List
+
+from fastapi import APIRouter, HTTPException
+
+from api.models import ProcessedProjectEntry
+from utils.constants import WIKI_CACHE_DIR
+from utils.logger import logger
+
+router = APIRouter(prefix="/api/processed_projects", tags=["Processed Projects"])
+
+
+@router.get("", response_model=List[ProcessedProjectEntry])
+async def get_processed_projects():
+    project_entries: List[ProcessedProjectEntry] = []
+    try:
+        filenames = await asyncio.to_thread(os.listdir, WIKI_CACHE_DIR)
+        for filename in filenames:
+            if filename.endswith("_wiki_cache.json"):
+                file_path = os.path.join(WIKI_CACHE_DIR, filename)
+                stats = await asyncio.to_thread(os.stat, file_path)
+                parts = filename.replace("_wiki_cache.json", "").split("_")
+                if len(parts) == 3:
+                    owner, repo, repo_type = parts
+                    project_entries.append(
+                        ProcessedProjectEntry(
+                            id=f"{filename}",
+                            owner=owner,
+                            repo=repo,
+                            name=f"{owner}/{repo}",
+                            repo_type=repo_type,
+                            submitted_at=int(stats.st_mtime * 1000),
+                            language="en",
+                        )
+                    )
+
+        project_entries.sort(key=lambda x: x.submitted_at, reverse=True)
+        return project_entries
+    except Exception as e:
+        logger.error(f"Error getting processed projects: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

api/rag.py

@@ -1,16 +1,18 @@
-import adalflow
-import adalflow.core
 import traceback
-from utils.logger import logger
 from dataclasses import dataclass, field
+from typing import List, Union
 from uuid import uuid4
+
+import adalflow
+import adalflow.core
 from adalflow import GoogleGenAIClient, OllamaClient
-from utils.localdb_manager import LocalDBManager
 from adalflow.components.retriever.faiss_retriever import (
     FAISSRetriever,
     RetrieverOutput,
 )
-from typing import Union, List
+
+from utils.localdb_manager import LocalDBManager
+from utils.logger import logger
 
 
 @dataclass
@@ -213,7 +215,7 @@ def ollama_embedder(query: Union[str, list]):
             },
             model_client=GoogleGenAIClient(),
             model_kwargs={
-                "model": "gemini-2.0-flash",
+                "model": "gemini-2.5-flash",
                 "temperature": 0.7,
                 "top_p": 0.8,
                 "top_k": 40,

api/simple_chat.py

@@ -38,7 +38,7 @@ async def chat_completions_stream(request: ChatCompletionRequest):
                         f"Input too large: {tokens} tokens. Skipping request."
                     )
         try:
-            request_rag = RAG(provider="google", model="gemini-2.0-flash")
+            request_rag = RAG(provider="google", model="gemini-2.5-flash")
             request_rag.prepare_retriever(request.repo_url, "github")
         except Exception as e:
             traceback.print_exc()
@@ -164,7 +164,7 @@ async def chat_completions_stream(request: ChatCompletionRequest):
 
         prompt += f"<query>\n{query}\n</query>\n\nAssistant: "
         model = genai.GenerativeModel(
-            model_name="gemini-2.0-flash",
+            model_name="gemini-2.5-flash",
             generation_config={
                 "temperature": 0.7,
                 "top_p": 0.8,

api/wiki.py

@@ -0,0 +1,65 @@
+import uuid
+from http.client import HTTPException
+from typing import Any
+
+from fastapi import APIRouter, BackgroundTasks
+
+from api.models import WikiTaskRequest, WikiTaskStatus
+from utils.logger import logger
+from utils.repository_structure import RepositoryStructureFetcher
+
+router = APIRouter(prefix="/api/wiki", tags=["Wiki"])
+
+tasks = {}  # replace with Redis
+
+
+def update_task_status(
+    task_id: str,
+    status: str,
+    message: str = "",
+    result: Any = None,
+    progress: set = set(),
+):
+    tasks[task_id]["status"] = status
+    tasks[task_id]["message"] = message
+    tasks[task_id]["result"] = result
+    tasks[task_id]["progress"] = progress
+
+
+@router.post("/generate")
+async def generate_wiki(wiki: WikiTaskRequest, background_tasks: BackgroundTasks):
+    task_id = str(uuid.uuid4())
+    tasks[task_id] = {
+        "status": "started",
+        "message": "Generating wiki...",
+        "progress": set(),
+        "error": None,
+        "result": None,
+    }
+    background_tasks.add_task(generate_wiki, wiki, task_id)
+    return {"task_id": task_id}
+
+
+async def generate_wiki(wiki: WikiTaskRequest, task_id: str):
+    try:
+        fetcher = RepositoryStructureFetcher(
+            repo_info=wiki.repo_info,
+            repo_url=wiki.repo_url,
+            owner=wiki.owner,
+            repo=wiki.repo,
+            token=wiki.token,
+        )
+        await fetcher.fetch_repository_structure(update_task_status, task_id)
+        # tasks[task_id]["status"] = "success"
+        # tasks[task_id]["result"] = fetcher.wiki_structure
+        # tasks[task_id]["message"] = "Wiki generated successfully"
+    except Exception as e:
+        tasks[task_id]["status"] = "error"
+        tasks[task_id]["error"] = str(e)
+
+
+@router.get("/status/{task_id}", response_model=WikiTaskStatus)
+async def get_task_status(task_id: str):
+    if task_id not in tasks:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return tasks[task_id]

api/wiki_cache.py

@@ -1,18 +1,15 @@
 import json
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
-from adalflow.utils import get_adalflow_default_root_path
 from fastapi import APIRouter, HTTPException
 
-from api.models import WikiCacheData, WikiCacheRequest, WikiPage, WikiStructureModel
+from api.models import WikiCacheData, WikiCacheRequest
+from utils.constants import WIKI_CACHE_DIR
 from utils.logger import logger
 
 router = APIRouter(prefix="/api/wiki_cache", tags=["Wiki Cache"])
 
-WIKI_CACHE_DIR = os.path.join("./.cache", "wiki_cache")
-os.makedirs(WIKI_CACHE_DIR, exist_ok=True)
-
 
 def get_wiki_cache_path(owner: str, repo: str, repo_type: str) -> str:
     filename = f"{owner}_{repo}_{repo_type}_wiki_cache.json"
@@ -28,6 +25,7 @@ async def read_wiki_cache_data(
     """
     # This is a placeholder for actual data retrieval logic
     cache_path = get_wiki_cache_path(owner, repo, repo_type)
+    logger.info(f"Reading wiki cache data from {cache_path}")
     if os.path.exists(cache_path):
         try:
             with open(cache_path, "r", encoding="utf-8") as file:
@@ -54,7 +52,7 @@ async def write_wiki_cache_data(data: WikiCacheRequest) -> bool:
         return False
 
 
-@router.get("/", response_model=Optional[WikiCacheData])
+@router.get("", response_model=Optional[WikiCacheData])
 async def get_wiki_cache(
     owner: str, repo: str, repo_type: str
 ) -> Optional[WikiCacheData]:
@@ -69,7 +67,7 @@ async def get_wiki_cache(
         return None
 
 
-@router.post("/")
+@router.post("")
 async def update_wiki_cache(request_data: WikiCacheRequest) -> Dict[str, Any]:
     """
     Update the wiki cache data for a specific repository.

utils/constants.py

@@ -1,2 +1,7 @@
+import os
+
 MAX_EMBEDDING_TOKEN = 8192
 TARGET_SERVER_BASE_URL = "http://localhost:8001"
+
+WIKI_CACHE_DIR = os.path.join("./.cache", "wiki_cache")
+os.makedirs(WIKI_CACHE_DIR, exist_ok=True)