diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml new file mode 100644 index 0000000..2efad39 --- /dev/null +++ b/.github/workflows/ci-backend.yml @@ -0,0 +1,177 @@ +name: Backend CI + +on: + push: + branches: [ master, develop ] + pull_request: + branches: [ master, develop ] + +jobs: + test: + runs-on: ubuntu-latest + environment: Test CI/CD + env: + APP_SECRET_KEY: 'test-secret-key-for-ci' + DB_NAME: ch8rtests + DB_USER: testuser + DB_PASSWORD: testpass + DB_HOST: localhost + PORT: 5432 + TEST_DB_NAME: ch8rtests + TEST_DB_USER: testuser + TEST_DB_PASSWORD: testpass + TEST_DB_HOST: localhost + TEST_DB_PORT: 5432 + DJANGO_SETTINGS_MODULE: config.test_settings + CONNECT_TO_LOCAL_VECTOR_DB: 'true' + QDRANT_LOCAL_HOST: localhost + QDRANT_LOCAL_PORT: 6333 + SECRET_ENCRYPTION_KEY: 'r5kAsRtMlZuP0b6UeRXdI9QqL_0uTTvTkT7jXsmkGxk=' + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY || 'test-key' }} + QDRANT_CLOUD_API_KEY: ${{ secrets.QDRANT_CLOUD_API_KEY || 'test-key' }} + MAILERSEND_API_KEY: ${{ secrets.MAILERSEND_API_KEY || 'test-key' }} + DISCORD_SIGNUP_WEBHOOK_URL: 'https://example.com' + WIDGET_URL: 'https://widget.ch8r.com' + API_BASE_URL: 'http://localhost:8000/api' + FRONTEND_URL: 'http://localhost:3000' + CLOSED_ALPHA_SIGN_UPS: '["test@example.com"]' + REQUIRE_ACCOUNT_APPROVAL: 'False' + + services: + postgres: + image: postgres:15 + env: + POSTGRES_DB: ch8rtests + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + qdrant: + image: qdrant/qdrant:v1.11.0 + ports: + - 6333:6333 + - 6334:6334 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + cd backend + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Django system checks + run: | + cd backend + python manage.py check + + - name: Collect static files + run: | + cd backend + python manage.py collectstatic --noinput + + - name: Run Django migrations + run: | + cd backend + python manage.py migrate --settings=config.test_settings + + - name: Run backend tests + run: | + cd backend + pytest --cov=core --cov-report=term-missing --cov-fail-under=60 --json-report --json-report-file=test-results.json + + - name: Discord notification on success + if: success() + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + + // Read test results from JSON file + let testResults = 'Tests completed successfully'; + try { + const testData = JSON.parse(fs.readFileSync('backend/test-results.json', 'utf8')); + const passed = testData.summary.passed || 0; + const failed = testData.summary.failed || 0; + testResults = `Tests passed: ${passed}, Failed: ${failed}`; + } catch (error) { + console.log('Could not read test results:', error.message); + testResults = 'Tests completed successfully'; + } + + const payload = { + embeds: [{ + title: 'Backend CI - Tests Passed', + description: `Branch: **${process.env.GITHUB_REF_NAME}**\nCommit: [${process.env.GITHUB_SHA.substring(0, 7)}](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/commit/${process.env.GITHUB_SHA})\n${testResults}`, + color: 3066993, + timestamp: new Date().toISOString() + }] + }; + + const webhookUrl = process.env.DISCORD_TEST_WEBHOOK_URL; + console.log('Discord webhook URL available:', !!webhookUrl); + if (webhookUrl) { + console.log('Sending Discord notification...'); + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + console.log('Discord response status:', response.status); + if (!response.ok) { + console.error('Failed to send Discord notification:', response.status, response.statusText); + } else { + console.log('Discord notification sent successfully'); + } + } else { + console.log('Discord webhook URL not found'); + } + env: + DISCORD_TEST_WEBHOOK_URL: ${{ secrets.DISCORD_TEST_WEBHOOK_URL }} + + - name: Discord notification on failure + if: failure() + uses: actions/github-script@v6 + with: + script: | + const payload = { + embeds: [{ + title: 'Backend CI - Tests Failed', + description: `Branch: **${process.env.GITHUB_REF_NAME}**\nCommit: [${process.env.GITHUB_SHA.substring(0, 7)}](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/commit/${process.env.GITHUB_SHA})\nTests failed. Check the logs for details.`, + color: 15158332, + timestamp: new Date().toISOString() + }] + }; + + const webhookUrl = process.env.DISCORD_TEST_WEBHOOK_URL; + console.log('Discord webhook URL available:', !!webhookUrl); + if (webhookUrl) { + console.log('Sending Discord notification...'); + const response = await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + console.log('Discord response status:', response.status); + if (!response.ok) { + console.error('Failed to send Discord notification:', response.status, response.statusText); + } else { + console.log('Discord notification sent successfully'); + } + } else { + console.log('Discord webhook URL not found'); + } + env: + DISCORD_TEST_WEBHOOK_URL: ${{ secrets.DISCORD_TEST_WEBHOOK_URL }} diff --git a/.gitignore b/.gitignore index ad3fabc..0b69ec3 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,7 @@ __pycache__/ .vscode/ .DS_Store qdrant_storage -.env \ No newline at end of file +.env +.kiro/ +backend/.hypothesis +.hypothesis \ No newline at end of file diff --git a/backend/.env.example b/backend/.env.example index d598391..5ee9934 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,4 +1,6 @@ APP_SECRET_KEY='app_secret_key' +DEBUG=False +ALLOWED_HOSTS=localhost,127.0.0.1 GEMINI_API_KEY=GEMINI_KEY DB_NAME=chatterbox DB_USER=postgres @@ -29,3 +31,10 @@ API_BASE_URL=http://localhost:8000/api FRONTEND_URL=http://localhost:3000 CLOSED_ALPHA_SIGN_UPS='["test@example.com"]' +REQUIRE_ACCOUNT_APPROVAL=False + +TEST_DB_NAME=ch8rtests +TEST_DB_USER=anish +TEST_DB_PASSWORD=Anish@1996 +TEST_DB_HOST=localhost +TEST_DB_PORT=5432 diff --git a/backend/config/celery.py b/backend/config/celery.py index 340e36d..ee7cdea 100644 --- a/backend/config/celery.py +++ b/backend/config/celery.py @@ -7,3 +7,4 @@ app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() +app.autodiscover_tasks(['core.tasks.github_tasks']) diff --git a/backend/config/settings.py b/backend/config/settings.py index 3dba965..6c358a5 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -9,9 +9,9 @@ SECRET_KEY = os.environ.get('APP_SECRET_KEY') CLOSED_ALPHA_SIGN_UPS = os.environ.get('CLOSED_ALPHA_SIGN_UPS') -DEBUG = True +DEBUG = os.environ.get('DEBUG', 'False').lower() == 'true' -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = os.environ.get('ALLOWED_HOSTS', '').split(',') if os.environ.get('ALLOWED_HOSTS') else [] ASGI_APPLICATION = "config.asgi.application" @@ -26,6 +26,7 @@ 'rest_framework.authtoken', 'channels', 'core', + 'github_data', 'corsheaders', 'rest_framework_api_key', ] @@ -71,6 +72,13 @@ 'PASSWORD': os.getenv('PASSWORD'), 'HOST': os.getenv('DB_HOST'), 'PORT': os.getenv('PORT'), + 'TEST': { + 'NAME': os.getenv('TEST_DB_NAME', 'test_db'), + 'USER': os.getenv('TEST_DB_USER'), + 'PASSWORD': os.getenv('TEST_DB_PASSWORD'), + 'HOST': os.getenv('TEST_DB_HOST', 'localhost'), + 'PORT': os.getenv('TEST_DB_PORT', '5432'), + } } } @@ -106,7 +114,9 @@ 'rest_framework.parsers.MultiPartParser', 'rest_framework.parsers.FormParser', 'rest_framework.parsers.JSONParser', - ] + ], + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', + 'PAGE_SIZE': 20, } CORS_ALLOWED_ORIGINS = [ @@ -164,3 +174,5 @@ API_BASE_URL = os.getenv("API_BASE_URL") FRONTEND_URL = os.getenv("FRONTEND_URL") DISCORD_SIGNUP_WEBHOOK_URL = os.getenv("DISCORD_SIGNUP_WEBHOOK_URL") + +REQUIRE_ACCOUNT_APPROVAL = os.getenv("REQUIRE_ACCOUNT_APPROVAL", "False").lower() == "true" diff --git a/backend/config/test_settings.py b/backend/config/test_settings.py new file mode 100644 index 0000000..2503967 --- /dev/null +++ b/backend/config/test_settings.py @@ -0,0 +1,30 @@ +import os +from .settings import * + +DEBUG = False + +CONNECT_TO_LOCAL_VECTOR_DB = False + +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': os.getenv('TEST_DB_NAME', 'test_db'), + 'USER': os.getenv('TEST_DB_USER'), + 'PASSWORD': os.getenv('TEST_DB_PASSWORD'), + 'HOST': os.getenv('TEST_DB_HOST', 'localhost'), + 'PORT': os.getenv('TEST_DB_PORT', '5432'), + } +} + +EMAIL_BACKEND = 'django.core.mail.backends.locmem.LocMemBackend' + +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + } +} diff --git a/backend/core/admin.py b/backend/core/admin.py index 8c38f3f..e69de29 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/backend/core/admin/__init__.py b/backend/core/admin/__init__.py new file mode 100644 index 0000000..2b33e7e --- /dev/null +++ b/backend/core/admin/__init__.py @@ -0,0 +1 @@ +from .github_admin import * diff --git a/backend/core/admin/github_admin.py b/backend/core/admin/github_admin.py new file mode 100644 index 0000000..d606a58 --- /dev/null +++ b/backend/core/admin/github_admin.py @@ -0,0 +1,333 @@ +from django.contrib import admin +from django.utils.html import format_html +from django.urls import reverse +from django.utils.safestring import mark_safe + +from core.models.github_data import ( + GitHubRepository, GitHubIssue, GitHubIssueComment, GitHubPullRequest, + GitHubPRComment, GitHubPRFile, GitHubDiscussion, GitHubDiscussionComment, + GitHubWikiPage, GitHubRepositoryFile +) + + +@admin.register(GitHubRepository) +class GitHubRepositoryAdmin(admin.ModelAdmin): + """Admin interface for GitHub repositories""" + list_display = [ + 'full_name', 'app_integration', 'ingestion_status', + 'last_ingested_at', 'is_private', 'created_at' + ] + list_filter = [ + 'ingestion_status', 'is_private', 'created_at', 'app_integration' + ] + search_fields = ['full_name', 'name', 'repo_owner', 'description'] + readonly_fields = ['uuid', 'created_at', 'updated_at'] + ordering = ['-created_at'] + + fieldsets = ( + ('Repository Information', { + 'fields': ('full_name', 'name', 'repo_owner', 'description', 'url') + }), + ('Configuration', { + 'fields': ('is_private', 'default_branch', 'app_integration') + }), + ('Ingestion Status', { + 'fields': ('ingestion_status', 'last_ingested_at') + }), + ('Metadata', { + 'fields': ('uuid', 'created_at', 'updated_at'), + 'classes': ('collapse',) + }) + ) + + def get_queryset(self, request): + return super().get_queryset(request).select_related('app_integration__application') + + +class GitHubIssueCommentInline(admin.TabularInline): + """Inline admin for issue comments""" + model = GitHubIssueComment + extra = 0 + readonly_fields = ['github_id', 'author', 'created_at'] + fields = ['github_id', 'author', 'body_preview', 'created_at'] + + def body_preview(self, obj): + if obj.body: + return obj.body[:100] + '...' if len(obj.body) > 100 else obj.body + return '' + body_preview.short_description = 'Body Preview' + + +@admin.register(GitHubIssue) +class GitHubIssueAdmin(admin.ModelAdmin): + """Admin interface for GitHub issues""" + list_display = [ + 'issue_number', 'title', 'repository', 'state', 'author', + 'created_at', 'comment_count' + ] + list_filter = ['state', 'created_at', 'repository'] + search_fields = ['title', 'body', 'author'] + readonly_fields = ['uuid', 'github_id', 'created_at', 'updated_at'] + ordering = ['-created_at'] + inlines = [GitHubIssueCommentInline] + + fieldsets = ( + ('Issue Information', { + 'fields': ('github_id', 'number', 'title', 'state', 'repository') + }), + ('Content', { + 'fields': ('body', 'author', 'author_association') + }), + ('Metadata', { + 'fields': ( + 'assignees', 'labels', 'milestone', 'locked', + 'created_at', 'updated_at', 'closed_at' + ), + 'classes': ('collapse',) + }), + ('External Links', { + 'fields': ('url',) + }) + ) + + def issue_number(self, obj): + return f"#{obj.number}" + issue_number.short_description = 'Issue #' + + def comment_count(self, obj): + return obj.comments.count() + comment_count.short_description = 'Comments' + + def get_queryset(self, request): + return super().get_queryset(request).select_related('repository') + + +class GitHubPRCommentInline(admin.TabularInline): + """Inline admin for PR comments""" + model = GitHubPRComment + extra = 0 + readonly_fields = ['github_id', 'author', 'created_at'] + fields = ['github_id', 'author', 'body_preview', 'created_at'] + + def body_preview(self, obj): + if obj.body: + return obj.body[:100] + '...' if len(obj.body) > 100 else obj.body + return '' + body_preview.short_description = 'Body Preview' + + +class GitHubPRFileInline(admin.TabularInline): + """Inline admin for PR files""" + model = GitHubPRFile + extra = 0 + readonly_fields = ['filename', 'status', 'additions', 'deletions'] + fields = ['filename', 'status', 'additions', 'deletions', 'changes'] + ordering = ['filename'] + + +@admin.register(GitHubPullRequest) +class GitHubPullRequestAdmin(admin.ModelAdmin): + """Admin interface for GitHub pull requests""" + list_display = [ + 'pr_number', 'title', 'repository', 'state', 'author', + 'merged', 'created_at', 'comment_count', 'file_count' + ] + list_filter = ['state', 'merged', 'created_at', 'repository'] + search_fields = ['title', 'body', 'author'] + readonly_fields = ['uuid', 'github_id', 'created_at', 'updated_at'] + ordering = ['-created_at'] + inlines = [GitHubPRCommentInline, GitHubPRFileInline] + + fieldsets = ( + ('PR Information', { + 'fields': ('github_id', 'number', 'title', 'state', 'repository') + }), + ('Content', { + 'fields': ('body', 'author', 'author_association') + }), + ('Branch Information', { + 'fields': ('head_branch', 'base_branch', 'merged', 'merged_at') + }), + ('Statistics', { + 'fields': ('additions', 'deletions', 'changed_files') + }), + ('Metadata', { + 'fields': ( + 'assignees', 'reviewers', 'labels', 'milestone', + 'created_at', 'updated_at', 'closed_at' + ), + 'classes': ('collapse',) + }), + ('External Links', { + 'fields': ('url',) + }) + ) + + def pr_number(self, obj): + return f"#{obj.number}" + pr_number.short_description = 'PR #' + + def comment_count(self, obj): + return obj.comments.count() + comment_count.short_description = 'Comments' + + def file_count(self, obj): + return obj.files.count() + file_count.short_description = 'Files' + + def get_queryset(self, request): + return super().get_queryset(request).select_related('repository') + + +@admin.register(GitHubDiscussion) +class GitHubDiscussionAdmin(admin.ModelAdmin): + """Admin interface for GitHub discussions""" + list_display = [ + 'discussion_number', 'title', 'repository', 'category_name', + 'author', 'upvote_count', 'created_at', 'comment_count' + ] + list_filter = ['created_at', 'repository'] + search_fields = ['title', 'body', 'author'] + readonly_fields = ['uuid', 'github_id', 'created_at', 'updated_at'] + ordering = ['-created_at'] + + fieldsets = ( + ('Discussion Information', { + 'fields': ('github_id', 'number', 'title', 'repository') + }), + ('Content', { + 'fields': ('body', 'author', 'author_association') + }), + ('Category', { + 'fields': ('category',) + }), + ('Interaction', { + 'fields': ('upvote_count', 'answer_chosen_at', 'answer_chosen_by') + }), + ('Metadata', { + 'fields': ('created_at', 'updated_at', 'last_edited_at'), + 'classes': ('collapse',) + }), + ('External Links', { + 'fields': ('url',) + }) + ) + + def discussion_number(self, obj): + return f"#{obj.number}" + discussion_number.short_description = 'Discussion #' + + def category_name(self, obj): + if obj.category and isinstance(obj.category, dict): + return obj.category.get('name', 'N/A') + return 'N/A' + category_name.short_description = 'Category' + + def comment_count(self, obj): + return obj.comments.count() + comment_count.short_description = 'Comments' + + def get_queryset(self, request): + return super().get_queryset(request).select_related('repository') + + +@admin.register(GitHubWikiPage) +class GitHubWikiPageAdmin(admin.ModelAdmin): + """Admin interface for GitHub wiki pages""" + list_display = ['title', 'repository', 'last_modified', 'created_at'] + list_filter = ['last_modified', 'created_at', 'repository'] + search_fields = ['title', 'content'] + readonly_fields = ['uuid', 'sha', 'created_at', 'updated_at'] + ordering = ['title'] + + fieldsets = ( + ('Wiki Page Information', { + 'fields': ('title', 'repository') + }), + ('Content', { + 'fields': ('content',) + }), + ('Metadata', { + 'fields': ('sha', 'html_url', 'download_url', 'last_modified'), + 'classes': ('collapse',) + }) + ) + + def get_queryset(self, request): + return super().get_queryset(request).select_related('repository') + + +@admin.register(GitHubRepositoryFile) +class GitHubRepositoryFileAdmin(admin.ModelAdmin): + """Admin interface for GitHub repository files""" + list_display = ['name', 'path', 'repository', 'size', 'content_type', 'last_modified'] + list_filter = ['content_type', 'last_modified', 'created_at', 'repository'] + search_fields = ['name', 'path', 'content'] + readonly_fields = ['uuid', 'sha', 'size', 'created_at', 'updated_at'] + ordering = ['path'] + + fieldsets = ( + ('File Information', { + 'fields': ('name', 'path', 'repository') + }), + ('Content', { + 'fields': ('content', 'content_type', 'encoding') + }), + ('Metadata', { + 'fields': ('sha', 'size', 'html_url', 'download_url', 'last_modified'), + 'classes': ('collapse',) + }) + ) + + def get_queryset(self, request): + return super().get_queryset(request).select_related('repository') +@admin.register(GitHubIssueComment) +class GitHubIssueCommentAdmin(admin.ModelAdmin): + """Admin interface for GitHub issue comments""" + list_display = ['github_id', 'issue', 'author', 'created_at'] + list_filter = ['created_at', 'author_association'] + search_fields = ['body', 'author'] + readonly_fields = ['uuid', 'github_id', 'created_at', 'updated_at'] + ordering = ['-created_at'] + + def get_queryset(self, request): + return super().get_queryset(request).select_related('issue__repository') + + +@admin.register(GitHubPRComment) +class GitHubPRCommentAdmin(admin.ModelAdmin): + """Admin interface for GitHub PR comments""" + list_display = ['github_id', 'pull_request', 'author', 'created_at'] + list_filter = ['created_at', 'author_association'] + search_fields = ['body', 'author'] + readonly_fields = ['uuid', 'github_id', 'created_at', 'updated_at'] + ordering = ['-created_at'] + + def get_queryset(self, request): + return super().get_queryset(request).select_related('pull_request__repository') + + +@admin.register(GitHubPRFile) +class GitHubPRFileAdmin(admin.ModelAdmin): + """Admin interface for GitHub PR files""" + list_display = ['filename', 'pull_request', 'status', 'additions', 'deletions'] + list_filter = ['status', 'pull_request__repository'] + search_fields = ['filename', 'patch'] + readonly_fields = ['uuid', 'created_at', 'updated_at'] + ordering = ['filename'] + + def get_queryset(self, request): + return super().get_queryset(request).select_related('pull_request__repository') + + +@admin.register(GitHubDiscussionComment) +class GitHubDiscussionCommentAdmin(admin.ModelAdmin): + """Admin interface for GitHub discussion comments""" + list_display = ['github_id', 'discussion', 'author', 'created_at', 'upvote_count'] + list_filter = ['created_at', 'author_association'] + search_fields = ['body', 'author'] + readonly_fields = ['uuid', 'github_id', 'created_at', 'updated_at'] + ordering = ['-created_at'] + + def get_queryset(self, request): + return super().get_queryset(request).select_related('discussion__repository') diff --git a/backend/core/agent_response_schema.py b/backend/core/agent_response_schema.py index 91d7264..bcc18c8 100644 --- a/backend/core/agent_response_schema.py +++ b/backend/core/agent_response_schema.py @@ -10,7 +10,12 @@ class ResponseStatus(str, Enum): POTENTIALLY_IRRELEVANT = "POTENTIALLY_IRRELEVANT" class SupportAgentResponse(BaseModel): - answer: str = Field(..., description="The support agent's reply to the user query.") + answer: str = Field( + ..., + description=( + "The assistant's response in GitHub Flavored Markdown. " + ) + ) status: ResponseStatus = Field(..., description="The result of the response analysis or next step needed.") escalation: bool = Field(..., description="Indicates whether the query should be escalated to human support.") reason_for_escalation: str = Field(..., description="Optional explanation for escalation. Can be an empty string.") \ No newline at end of file diff --git a/backend/core/consts.py b/backend/core/consts.py index 2d301bf..e833897 100644 --- a/backend/core/consts.py +++ b/backend/core/consts.py @@ -1,8 +1,21 @@ -REGISTERED_USER_ID_PREFIX = "reg" +DASHBOARD_USER_ID_PREFIX = "dashboard" LIVE_UPDATES_PREFIX = "live" AI_ROLE_AI_AGENT="assistant" AI_ROLE_HUMAN_AGENT="assistant" AI_ROLE_USER="user" AI_ROLE_SYSTEM="system" -AI_ROLE_UNKNOWN="unknown" \ No newline at end of file +AI_ROLE_UNKNOWN="unknown" + +SUPPORTED_AI_PROVIDERS = [ + { + 'id': 'gemini', + 'label': 'Google Gemini', + 'base_url': 'https://generativelanguage.googleapis.com/v1beta' + }, + { + 'id': 'custom', + 'label': 'Custom Provider', + 'base_url': '' + } +] diff --git a/backend/core/consumers.py b/backend/core/consumers.py index 0b20a9d..92f3827 100644 --- a/backend/core/consumers.py +++ b/backend/core/consumers.py @@ -1,7 +1,13 @@ +import logging + from channels.generic.websocket import AsyncJsonWebsocketConsumer from core.consts import LIVE_UPDATES_PREFIX +logger = logging.getLogger(__name__) + +VALID_CLIENT_ID_PREFIXES = ('widget_', 'dashboard_') + class LiveUpdatesConsumer(AsyncJsonWebsocketConsumer): async def connect(self): @@ -12,6 +18,14 @@ async def connect(self): await self.close() return + if not self.client_id.startswith(VALID_CLIENT_ID_PREFIXES): + logger.warning( + "LiveUpdatesConsumer: rejected connection with invalid client_id prefix: %s", + self.client_id, + ) + await self.close() + return + await self.channel_layer.group_add(self.group_name, self.channel_name) await self.accept() @@ -31,4 +45,12 @@ async def send_kb_updates(self, event): await self.send_json({ "type": "kb_updates", "data": event["data"], + }) + + async def send_unread_update(self, event): + await self.send_json({ + "type": "unread_update", + "chatroom_uuid": event["chatroom_uuid"], + "has_unread": event["has_unread"], + "sender_identifier": event["sender_identifier"], }) \ No newline at end of file diff --git a/backend/core/fields.py b/backend/core/fields.py new file mode 100644 index 0000000..2e517ac --- /dev/null +++ b/backend/core/fields.py @@ -0,0 +1,16 @@ +from django.db import models +from cryptography.fernet import Fernet +from config import settings + +fernet = Fernet(settings.SECRET_ENCRYPTION_KEY) + +class EncryptedCharField(models.CharField): + def get_prep_value(self, value): + if value: + return fernet.encrypt(value.encode()).decode() + return value + + def from_db_value(self, value, expression, connection): + if value: + return fernet.decrypt(value.encode()).decode() + return value diff --git a/backend/core/llm_client_utils.py b/backend/core/llm_client_utils.py index 3250f15..b9c077b 100644 --- a/backend/core/llm_client_utils.py +++ b/backend/core/llm_client_utils.py @@ -1,12 +1,9 @@ -from core.agent_response_schema import SupportAgentResponse from core.consts import AI_ROLE_AI_AGENT, AI_ROLE_USER, AI_ROLE_UNKNOWN, AI_ROLE_SYSTEM -from openai.types.shared_params import ResponseFormatJSONSchema - AI_ROLE_MAP = { "agent_llm": AI_ROLE_AI_AGENT, - "reg_": AI_ROLE_USER, - "anon_": AI_ROLE_USER + "dashboard_": AI_ROLE_USER, + "widget_": AI_ROLE_USER, } def messages_to_llm_conversation(messages_queryset): @@ -16,21 +13,6 @@ def messages_to_llm_conversation(messages_queryset): conversation.append({"role": role, "content": msg.message}) return conversation -def get_agent_response_schema(schema_type: str): - if schema_type == "support_response": - schema_dict = SupportAgentResponse.model_json_schema() - return ResponseFormatJSONSchema( - type="json_schema", - json_schema={ - "name": "support_agent_response", - "description": "Schema for the support agent's reply and escalation decision.", - "schema": schema_dict, - "strict": True - } - ) - else: - return None - def add_instructions_to_convo( conversation, instructions, diff --git a/backend/core/management/commands/cleanup_app_ai_provider_duplicates.py b/backend/core/management/commands/cleanup_app_ai_provider_duplicates.py new file mode 100644 index 0000000..bbe0dac --- /dev/null +++ b/backend/core/management/commands/cleanup_app_ai_provider_duplicates.py @@ -0,0 +1,39 @@ +from django.core.management.base import BaseCommand +from django.db.models import Count +from core.models.app_ai_provider import AppAIProvider + + +class Command(BaseCommand): + help = 'Clean up duplicate AppAIProvider records, keeping only the most recent for each app/context/capability combination' + + def handle(self, *args, **options): + duplicates = AppAIProvider.objects.values('application', 'context', 'capability') \ + .annotate(count=Count('id')) \ + .filter(count__gt=1) + + total_deleted = 0 + + for duplicate_group in duplicates: + records = AppAIProvider.objects.filter( + application=duplicate_group['application'], + context=duplicate_group['context'], + capability=duplicate_group['capability'] + ).order_by('-updated_at') + + records_to_delete = records[1:] + count_deleted = records_to_delete.count() + + if count_deleted > 0: + self.stdout.write( + f'Deleting {count_deleted} duplicate records for app={duplicate_group["application"]}, ' + f'context={duplicate_group["context"]}, capability={duplicate_group["capability"]}' + ) + records_to_delete.delete() + total_deleted += count_deleted + + if total_deleted > 0: + self.stdout.write( + self.style.SUCCESS(f'Successfully deleted {total_deleted} duplicate records') + ) + else: + self.stdout.write('No duplicate records found') diff --git a/backend/core/management/commands/cleanup_stuck_ingestions.py b/backend/core/management/commands/cleanup_stuck_ingestions.py new file mode 100644 index 0000000..22ee338 --- /dev/null +++ b/backend/core/management/commands/cleanup_stuck_ingestions.py @@ -0,0 +1,66 @@ +from django.core.management.base import BaseCommand +from django.utils import timezone +from core.models.github_data import GitHubRepository +import logging + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Clean up stuck GitHub repository ingestions' + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be cleaned up without making changes', + ) + parser.add_argument( + '--minutes', + type=int, + default=30, + help='Mark ingestions as failed if they have been running for more than this many minutes', + ) + + def handle(self, *args, **options): + dry_run = options['dry_run'] + minutes = options['minutes'] + + cutoff_time = timezone.now() - timezone.timedelta(minutes=minutes) + + stuck_repos = GitHubRepository.objects.filter( + ingestion_status='running', + updated_at__lt=cutoff_time + ) + + if not stuck_repos.exists(): + self.stdout.write( + self.style.SUCCESS(f'No stuck ingestions found (cutoff: {minutes} minutes)') + ) + return + + self.stdout.write( + self.style.WARNING(f'Found {stuck_repos.count()} stuck ingestions:') + ) + + for repo in stuck_repos: + age_minutes = (timezone.now() - repo.updated_at).total_seconds() / 60 + self.stdout.write( + f' - {repo.full_name} (ID: {repo.id}, ' + f'running for {age_minutes:.1f} minutes, ' + f'last updated: {repo.updated_at})' + ) + + if dry_run: + self.stdout.write( + self.style.SUCCESS('Dry run completed. No changes made.') + ) + return + + updated_count = stuck_repos.update(ingestion_status='failed') + + self.stdout.write( + self.style.SUCCESS( + f'Marked {updated_count} stuck ingestions as failed' + ) + ) diff --git a/backend/core/management/commands/test_github_migration.py b/backend/core/management/commands/test_github_migration.py new file mode 100644 index 0000000..83a80e5 --- /dev/null +++ b/backend/core/management/commands/test_github_migration.py @@ -0,0 +1,163 @@ +from django.core.management.base import BaseCommand +from django.conf import settings +import logging +import sys +import time + +from core.models import AppIntegration, Integration +from core.utils.github_migration_helper import GitHubMigrationHelper + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Test GitHub API migration from REST to GraphQL' + + def add_arguments(self, parser): + parser.add_argument( + '--owner', + type=str, + required=True, + help='Repository owner' + ) + parser.add_argument( + '--repo', + type=str, + required=True, + help='Repository name' + ) + parser.add_argument( + '--integration-id', + type=int, + help='Integration ID (if not provided, will search for GitHub integration)' + ) + parser.add_argument( + '--generate-report', + action='store_true', + help='Generate detailed migration report' + ) + parser.add_argument( + '--limit', + type=int, + default=50, + help='Number of items to test (default: 50)' + ) + + def handle(self, *args, **options): + owner = options['owner'] + repo = options['repo'] + limit = options['limit'] + generate_report = options['generate_report'] + integration_id = options.get('integration_id') + + try: + app_integration = self._get_github_integration(integration_id) + if not app_integration: + self.stdout.write( + self.style.ERROR('GitHub integration not found') + ) + return + + helper = GitHubMigrationHelper(app_integration) + + self.stdout.write( + self.style.SUCCESS(f'Testing GitHub API migration for {owner}/{repo}') + ) + + self.stdout.write('\n=== Testing Issues Performance ===') + issue_results = helper.compare_issue_ingestion_performance(owner, repo, limit) + self._print_results('Issues', issue_results) + + self.stdout.write('\n=== Testing Pull Requests Performance ===') + pr_results = helper.compare_pr_ingestion_performance(owner, repo, limit) + self._print_results('Pull Requests', pr_results) + + if generate_report: + self.stdout.write('\n=== Generating Migration Report ===') + report = helper.generate_migration_report(owner, repo) + + report_filename = f'github_migration_report_{owner}_{repo}_{int(time.time())}.md' + with open(report_filename, 'w') as f: + f.write(report) + + self.stdout.write( + self.style.SUCCESS(f'Report saved to: {report_filename}') + ) + + self.stdout.write('\n=== Summary ===') + if issue_results.get('improvement'): + imp = issue_results['improvement'] + self.stdout.write( + f'Issues: {imp["time_reduction_percent"]}% faster, {imp["speed_multiplier"]}x speedup' + ) + + if pr_results.get('improvement'): + imp = pr_results['improvement'] + self.stdout.write( + f'PRs: {imp["time_reduction_percent"]}% faster, {imp["speed_multiplier"]}x speedup' + ) + + except Exception as e: + self.stdout.write( + self.style.ERROR(f'Error: {e}') + ) + logger.exception('Migration test failed') + finally: + if 'helper' in locals(): + helper.close() + + def _get_github_integration(self, integration_id=None): + """Get GitHub integration""" + try: + if integration_id: + return AppIntegration.objects.get(id=integration_id) + + integration = Integration.objects.filter( + integration_type='github' + ).first() + + if not integration: + return None + + return AppIntegration.objects.filter( + integration=integration + ).first() + + except Exception as e: + logger.error(f"Failed to get GitHub integration: {e}") + return None + + def _print_results(self, entity_type, results): + self.stdout.write(f'\n{entity_type} Results:') + + if 'error' in results.get('rest_api', {}): + self.stdout.write(f" REST API: ERROR - {results['rest_api']['error']}") + else: + rest = results['rest_api'] + self.stdout.write( + f" REST API: {rest['duration_seconds']}s, " + f"{rest['api_calls']} calls, " + f"{rest.get('issues_processed', rest.get('prs_processed', 0))} items" + ) + + if 'error' in results.get('graphql', {}): + self.stdout.write(f" GraphQL: ERROR - {results['graphql']['error']}") + else: + graphql = results['graphql'] + self.stdout.write( + f" GraphQL: {graphql['duration_seconds']}s, " + f"{graphql['api_calls']} calls, " + f"{graphql.get('issues_processed', graphql.get('prs_processed', 0))} items" + ) + + if results.get('improvement'): + imp = results['improvement'] + self.stdout.write( + self.style.SUCCESS( + f" Improvement: {imp['time_reduction_percent']}% faster, " + f"{imp['speed_multiplier']}x speedup, " + f"{imp['api_call_reduction']} fewer API calls" + ) + ) + else: + self.stdout.write(" No improvement data available") diff --git a/backend/core/middleware.py b/backend/core/middleware.py index fcb90bf..fb68c61 100644 --- a/backend/core/middleware.py +++ b/backend/core/middleware.py @@ -2,6 +2,7 @@ from core.models import AccountStatus from rest_framework.authtoken.models import Token from django.http import JsonResponse +from django.conf import settings class AccountStatusMiddleware(MiddlewareMixin): @@ -16,6 +17,9 @@ class AccountStatusMiddleware(MiddlewareMixin): ] def process_request(self, request): + if not getattr(settings, 'REQUIRE_ACCOUNT_APPROVAL', False): + return None + if any(request.path.startswith(path) for path in self.EXCLUDED_PATHS): return None diff --git a/backend/core/migrations/0012_basemodel_remove_appintegration_uuid.py b/backend/core/migrations/0012_basemodel_remove_appintegration_uuid.py new file mode 100644 index 0000000..4bd8929 --- /dev/null +++ b/backend/core/migrations/0012_basemodel_remove_appintegration_uuid.py @@ -0,0 +1,28 @@ +# Generated by Django 5.2.3 on 2026-02-27 17:34 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0011_alter_accountstatus_id'), + ] + + operations = [ + migrations.CreateModel( + name='BaseModel', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + ), + migrations.RemoveField( + model_name='appintegration', + name='uuid', + ), + ] diff --git a/backend/core/migrations/0013_aiprovider_delete_basemodel.py b/backend/core/migrations/0013_aiprovider_delete_basemodel.py new file mode 100644 index 0000000..848064d --- /dev/null +++ b/backend/core/migrations/0013_aiprovider_delete_basemodel.py @@ -0,0 +1,37 @@ +# Generated by Django 5.2.3 on 2026-02-27 17:39 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_basemodel_remove_appintegration_uuid'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='AIProvider', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(blank=True, max_length=255, null=True)), + ('provider', models.CharField(max_length=255)), + ('key', models.CharField(max_length=1000)), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.DeleteModel( + name='BaseModel', + ), + ] diff --git a/backend/core/migrations/0014_rename_key_aiprovider_provider_api_key_and_more.py b/backend/core/migrations/0014_rename_key_aiprovider_provider_api_key_and_more.py new file mode 100644 index 0000000..290983b --- /dev/null +++ b/backend/core/migrations/0014_rename_key_aiprovider_provider_api_key_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.3 on 2026-02-27 17:48 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0013_aiprovider_delete_basemodel'), + ] + + operations = [ + migrations.RenameField( + model_name='aiprovider', + old_name='key', + new_name='provider_api_key', + ), + migrations.AddField( + model_name='aiprovider', + name='base_url', + field=models.CharField(default=django.utils.timezone.now, max_length=100), + preserve_default=False, + ), + migrations.AddField( + model_name='aiprovider', + name='is_builtin', + field=models.BooleanField(blank=True, default=False), + ), + ] diff --git a/backend/core/migrations/0015_alter_aiprovider_provider_api_key.py b/backend/core/migrations/0015_alter_aiprovider_provider_api_key.py new file mode 100644 index 0000000..4ee9047 --- /dev/null +++ b/backend/core/migrations/0015_alter_aiprovider_provider_api_key.py @@ -0,0 +1,19 @@ +# Generated by Django 5.2.3 on 2026-02-27 18:06 + +import core.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0014_rename_key_aiprovider_provider_api_key_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='aiprovider', + name='provider_api_key', + field=core.fields.EncryptedCharField(max_length=1000), + ), + ] diff --git a/backend/core/migrations/0016_appaiprovider.py b/backend/core/migrations/0016_appaiprovider.py new file mode 100644 index 0000000..766fc49 --- /dev/null +++ b/backend/core/migrations/0016_appaiprovider.py @@ -0,0 +1,34 @@ +# Generated by Django 5.2.3 on 2026-03-01 18:41 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0015_alter_aiprovider_provider_api_key'), + ] + + operations = [ + migrations.CreateModel( + name='AppAIProvider', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('context', models.CharField(default='default', max_length=50)), + ('capability', models.CharField(default='text', max_length=50)), + ('priority', models.PositiveIntegerField(default=100)), + ('external_model_id', models.CharField(blank=True, max_length=255, null=True)), + ('is_active', models.BooleanField(default=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('ai_provider', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='application_configs', to='core.aiprovider')), + ('application', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='ai_provider_configs', to='core.application')), + ], + options={ + 'ordering': ['context', 'capability', 'priority'], + }, + ), + ] diff --git a/backend/core/migrations/0017_migrate_base_url_to_metadata.py b/backend/core/migrations/0017_migrate_base_url_to_metadata.py new file mode 100644 index 0000000..38f763d --- /dev/null +++ b/backend/core/migrations/0017_migrate_base_url_to_metadata.py @@ -0,0 +1,27 @@ +# Generated by Django 5.2.3 on 2026-03-04 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0016_appaiprovider'), + ] + + operations = [ + migrations.AlterModelOptions( + name='aiprovider', + options={'ordering': ['created_at']}, + ), + migrations.AddField( + model_name='appaiprovider', + name='metadata', + field=models.JSONField(blank=True, null=True), + ), + migrations.AlterField( + model_name='appaiprovider', + name='context', + field=models.CharField(max_length=50), + ), + ] diff --git a/backend/core/migrations/0018_migrate_base_url_to_metadata_data.py b/backend/core/migrations/0018_migrate_base_url_to_metadata_data.py new file mode 100644 index 0000000..35308d0 --- /dev/null +++ b/backend/core/migrations/0018_migrate_base_url_to_metadata_data.py @@ -0,0 +1,41 @@ +# Generated manually for AI Provider Configurability Enhancement + +from django.db import migrations, models + + +def migrate_base_url_to_metadata(apps, schema_editor): + """ + Migrate existing base_url values to metadata field for backward compatibility. + """ + AIProvider = apps.get_model('core', 'AIProvider') + + for provider in AIProvider.objects.all(): + if provider.base_url and not provider.metadata: + provider.metadata = {'base_url': provider.base_url} + provider.save() + + +def reverse_migrate_base_url_to_metadata(apps, schema_editor): + """ + Reverse migration: extract base_url from metadata if present. + """ + AIProvider = apps.get_model('core', 'AIProvider') + + for provider in AIProvider.objects.all(): + if provider.metadata and isinstance(provider.metadata, dict) and 'base_url' in provider.metadata: + provider.base_url = provider.metadata['base_url'] + provider.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0017_migrate_base_url_to_metadata'), + ] + + operations = [ + migrations.RunPython( + migrate_base_url_to_metadata, + reverse_migrate_base_url_to_metadata, + ), + ] diff --git a/backend/core/migrations/0019_remove_aiprovider_base_url.py b/backend/core/migrations/0019_remove_aiprovider_base_url.py new file mode 100644 index 0000000..d10b5a3 --- /dev/null +++ b/backend/core/migrations/0019_remove_aiprovider_base_url.py @@ -0,0 +1,17 @@ +# Generated manually for AI Provider Configurability Enhancement - Remove base_url field + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0018_migrate_base_url_to_metadata_data'), + ] + + operations = [ + migrations.RemoveField( + model_name='aiprovider', + name='base_url', + ), + ] diff --git a/backend/core/migrations/0020_aiprovider_base_url.py b/backend/core/migrations/0020_aiprovider_base_url.py new file mode 100644 index 0000000..855f23b --- /dev/null +++ b/backend/core/migrations/0020_aiprovider_base_url.py @@ -0,0 +1,20 @@ +# Generated by Django 5.2.3 on 2026-03-04 17:49 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0019_remove_aiprovider_base_url'), + ] + + operations = [ + migrations.AddField( + model_name='aiprovider', + name='base_url', + field=models.CharField(default=django.utils.timezone.now, max_length=100), + preserve_default=False, + ), + ] diff --git a/backend/core/migrations/0021_remove_aiprovider_base_url.py b/backend/core/migrations/0021_remove_aiprovider_base_url.py new file mode 100644 index 0000000..df607be --- /dev/null +++ b/backend/core/migrations/0021_remove_aiprovider_base_url.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.3 on 2026-03-04 17:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0020_aiprovider_base_url'), + ] + + operations = [ + migrations.RemoveField( + model_name='aiprovider', + name='base_url', + ), + ] diff --git a/backend/core/migrations/0022_aiprovidermodels.py b/backend/core/migrations/0022_aiprovidermodels.py new file mode 100644 index 0000000..05f89c9 --- /dev/null +++ b/backend/core/migrations/0022_aiprovidermodels.py @@ -0,0 +1,34 @@ +# Generated by Django 5.2.3 on 2026-03-07 06:28 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0021_remove_aiprovider_base_url'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='AIProviderModels', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('models_data', models.JSONField(blank=True, null=True)), + ('ai_provider', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='models', to='core.aiprovider')), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['created_at'], + 'unique_together': {('ai_provider',)}, + }, + ), + ] diff --git a/backend/core/migrations/0023_githubrepository_githubpullrequest_githubissue_and_more.py b/backend/core/migrations/0023_githubrepository_githubpullrequest_githubissue_and_more.py new file mode 100644 index 0000000..7bbb29c --- /dev/null +++ b/backend/core/migrations/0023_githubrepository_githubpullrequest_githubissue_and_more.py @@ -0,0 +1,354 @@ +# Generated by Django 5.2.3 on 2026-03-13 19:14 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0022_aiprovidermodels'), + ] + + operations = [ + migrations.CreateModel( + name='GitHubRepository', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('name', models.CharField(max_length=255)), + ('repo_owner', models.CharField(max_length=255)), + ('full_name', models.CharField(max_length=511, unique=True)), + ('description', models.TextField(blank=True)), + ('url', models.URLField()), + ('is_private', models.BooleanField(default=False)), + ('default_branch', models.CharField(default='main', max_length=100)), + ('last_ingested_at', models.DateTimeField(blank=True, null=True)), + ('ingestion_status', models.CharField(choices=[('pending', 'Pending'), ('running', 'Running'), ('completed', 'Completed'), ('failed', 'Failed')], default='pending', max_length=20)), + ('metadata', models.JSONField(blank=True, default=dict)), + ('app_integration', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='github_repositories', to='core.appintegration')), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='GitHubPullRequest', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('github_id', models.IntegerField()), + ('number', models.IntegerField()), + ('title', models.CharField(max_length=1024)), + ('body', models.TextField(blank=True)), + ('state', models.CharField(choices=[('open', 'Open'), ('closed', 'Closed'), ('merged', 'Merged')], max_length=20)), + ('author', models.CharField(max_length=255)), + ('author_association', models.CharField(blank=True, max_length=50)), + ('assignees', models.JSONField(blank=True, default=list)), + ('reviewers', models.JSONField(blank=True, default=list)), + ('labels', models.JSONField(blank=True, default=list)), + ('milestone', models.JSONField(blank=True, null=True)), + ('head_branch', models.CharField(max_length=255)), + ('base_branch', models.CharField(max_length=255)), + ('merged', models.BooleanField(default=False)), + ('merged_at', models.DateTimeField(blank=True, null=True)), + ('merge_commit_sha', models.CharField(blank=True, max_length=40)), + ('additions', models.IntegerField(default=0)), + ('deletions', models.IntegerField(default=0)), + ('changed_files', models.IntegerField(default=0)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('closed_at', models.DateTimeField(blank=True, null=True)), + ('url', models.URLField()), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pull_requests', to='core.githubrepository')), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='GitHubIssue', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('github_id', models.IntegerField()), + ('number', models.IntegerField()), + ('title', models.CharField(max_length=1024)), + ('body', models.TextField(blank=True)), + ('state', models.CharField(choices=[('open', 'Open'), ('closed', 'Closed')], max_length=20)), + ('author', models.CharField(max_length=255)), + ('author_association', models.CharField(blank=True, max_length=50)), + ('assignees', models.JSONField(blank=True, default=list)), + ('labels', models.JSONField(blank=True, default=list)), + ('milestone', models.JSONField(blank=True, null=True)), + ('locked', models.BooleanField(default=False)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('closed_at', models.DateTimeField(blank=True, null=True)), + ('url', models.URLField()), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='issues', to='core.githubrepository')), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='GitHubDiscussion', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('github_id', models.IntegerField()), + ('number', models.IntegerField()), + ('title', models.CharField(max_length=1024)), + ('body', models.TextField()), + ('author', models.CharField(max_length=255)), + ('author_association', models.CharField(blank=True, max_length=50)), + ('category', models.JSONField(blank=True, default=dict)), + ('answer_chosen_at', models.DateTimeField(blank=True, null=True)), + ('answer_chosen_by', models.CharField(blank=True, max_length=255)), + ('upvote_count', models.IntegerField(default=0)), + ('viewer_has_upvoted', models.BooleanField(default=False)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('last_edited_at', models.DateTimeField(blank=True, null=True)), + ('url', models.URLField()), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='discussions', to='core.githubrepository')), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.CreateModel( + name='GitHubRepositoryFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('path', models.CharField(max_length=1024)), + ('name', models.CharField(max_length=255)), + ('content', models.TextField(blank=True)), + ('sha', models.CharField(max_length=40)), + ('size', models.IntegerField(default=0)), + ('content_type', models.CharField(blank=True, max_length=100)), + ('encoding', models.CharField(blank=True, max_length=20)), + ('html_url', models.URLField(blank=True)), + ('download_url', models.URLField(blank=True)), + ('last_modified', models.DateTimeField()), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='core.githubrepository')), + ], + options={ + 'ordering': ['path'], + }, + ), + migrations.CreateModel( + name='GitHubWikiPage', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('title', models.CharField(max_length=1024)), + ('content', models.TextField(blank=True)), + ('sha', models.CharField(max_length=40)), + ('html_url', models.URLField()), + ('download_url', models.URLField(blank=True)), + ('last_modified', models.DateTimeField()), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='wiki_pages', to='core.githubrepository')), + ], + options={ + 'ordering': ['title'], + }, + ), + migrations.CreateModel( + name='GitHubDiscussionComment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('github_id', models.IntegerField()), + ('body', models.TextField()), + ('author', models.CharField(max_length=255)), + ('author_association', models.CharField(blank=True, max_length=50)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('last_edited_at', models.DateTimeField(blank=True, null=True)), + ('upvote_count', models.IntegerField(default=0)), + ('viewer_has_upvoted', models.BooleanField(default=False)), + ('discussion', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='core.githubdiscussion')), + ('parent_comment', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='replies', to='core.githubdiscussioncomment')), + ], + options={ + 'ordering': ['created_at'], + 'indexes': [models.Index(fields=['discussion', 'created_at'], name='core_github_discuss_1e6319_idx'), models.Index(fields=['parent_comment'], name='core_github_parent__814852_idx'), models.Index(fields=['github_id'], name='core_github_github__f6d622_idx')], + 'unique_together': {('discussion', 'github_id')}, + }, + ), + migrations.CreateModel( + name='GitHubIssueComment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('github_id', models.IntegerField()), + ('body', models.TextField()), + ('author', models.CharField(max_length=255)), + ('author_association', models.CharField(blank=True, max_length=50)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('url', models.URLField()), + ('issue', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='core.githubissue')), + ], + options={ + 'ordering': ['created_at'], + 'indexes': [models.Index(fields=['issue', 'created_at'], name='core_github_issue_i_4ce323_idx'), models.Index(fields=['github_id'], name='core_github_github__03ac27_idx')], + 'unique_together': {('issue', 'github_id')}, + }, + ), + migrations.CreateModel( + name='GitHubPRFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('filename', models.CharField(max_length=1024)), + ('status', models.CharField(choices=[('added', 'Added'), ('modified', 'Modified'), ('removed', 'Removed'), ('renamed', 'Renamed')], max_length=20)), + ('additions', models.IntegerField(default=0)), + ('deletions', models.IntegerField(default=0)), + ('changes', models.IntegerField(default=0)), + ('patch', models.TextField(blank=True)), + ('blob_url', models.URLField(blank=True)), + ('raw_url', models.URLField(blank=True)), + ('contents_url', models.URLField(blank=True)), + ('pull_request', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='core.githubpullrequest')), + ], + options={ + 'ordering': ['filename'], + 'indexes': [models.Index(fields=['pull_request', 'status'], name='core_github_pull_re_99615e_idx'), models.Index(fields=['filename'], name='core_github_filenam_87a746_idx')], + 'unique_together': {('pull_request', 'filename')}, + }, + ), + migrations.CreateModel( + name='GitHubPRComment', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('metadata', models.JSONField(blank=True, null=True)), + ('github_id', models.IntegerField()), + ('body', models.TextField()), + ('author', models.CharField(max_length=255)), + ('author_association', models.CharField(blank=True, max_length=50)), + ('created_at', models.DateTimeField()), + ('updated_at', models.DateTimeField()), + ('url', models.URLField()), + ('pull_request', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', to='core.githubpullrequest')), + ], + options={ + 'ordering': ['created_at'], + 'indexes': [models.Index(fields=['pull_request', 'created_at'], name='core_github_pull_re_df6c54_idx'), models.Index(fields=['github_id'], name='core_github_github__9e3d67_idx')], + 'unique_together': {('pull_request', 'github_id')}, + }, + ), + migrations.AddIndex( + model_name='githubrepository', + index=models.Index(fields=['full_name'], name='core_github_full_na_b75f30_idx'), + ), + migrations.AddIndex( + model_name='githubrepository', + index=models.Index(fields=['app_integration'], name='core_github_app_int_58dafa_idx'), + ), + migrations.AddIndex( + model_name='githubrepository', + index=models.Index(fields=['ingestion_status'], name='core_github_ingesti_aa41e2_idx'), + ), + migrations.AddIndex( + model_name='githubpullrequest', + index=models.Index(fields=['repository', 'state'], name='core_github_reposit_b95ca8_idx'), + ), + migrations.AddIndex( + model_name='githubpullrequest', + index=models.Index(fields=['github_id'], name='core_github_github__297ae4_idx'), + ), + migrations.AddIndex( + model_name='githubpullrequest', + index=models.Index(fields=['created_at'], name='core_github_created_a16c0e_idx'), + ), + migrations.AddIndex( + model_name='githubpullrequest', + index=models.Index(fields=['merged'], name='core_github_merged_931fb9_idx'), + ), + migrations.AlterUniqueTogether( + name='githubpullrequest', + unique_together={('repository', 'github_id')}, + ), + migrations.AddIndex( + model_name='githubissue', + index=models.Index(fields=['repository', 'state'], name='core_github_reposit_9298e7_idx'), + ), + migrations.AddIndex( + model_name='githubissue', + index=models.Index(fields=['github_id'], name='core_github_github__dd2959_idx'), + ), + migrations.AddIndex( + model_name='githubissue', + index=models.Index(fields=['created_at'], name='core_github_created_477ea1_idx'), + ), + migrations.AlterUniqueTogether( + name='githubissue', + unique_together={('repository', 'github_id')}, + ), + migrations.AddIndex( + model_name='githubdiscussion', + index=models.Index(fields=['repository', 'category'], name='core_github_reposit_ec6d8d_idx'), + ), + migrations.AddIndex( + model_name='githubdiscussion', + index=models.Index(fields=['github_id'], name='core_github_github__b437c0_idx'), + ), + migrations.AddIndex( + model_name='githubdiscussion', + index=models.Index(fields=['created_at'], name='core_github_created_4d527a_idx'), + ), + migrations.AlterUniqueTogether( + name='githubdiscussion', + unique_together={('repository', 'github_id')}, + ), + migrations.AddIndex( + model_name='githubrepositoryfile', + index=models.Index(fields=['repository', 'path'], name='core_github_reposit_57e56b_idx'), + ), + migrations.AddIndex( + model_name='githubrepositoryfile', + index=models.Index(fields=['name'], name='core_github_name_835748_idx'), + ), + migrations.AddIndex( + model_name='githubrepositoryfile', + index=models.Index(fields=['last_modified'], name='core_github_last_mo_3ab257_idx'), + ), + migrations.AlterUniqueTogether( + name='githubrepositoryfile', + unique_together={('repository', 'path')}, + ), + migrations.AddIndex( + model_name='githubwikipage', + index=models.Index(fields=['repository', 'title'], name='core_github_reposit_d6a576_idx'), + ), + migrations.AddIndex( + model_name='githubwikipage', + index=models.Index(fields=['last_modified'], name='core_github_last_mo_c0980f_idx'), + ), + migrations.AlterUniqueTogether( + name='githubwikipage', + unique_together={('repository', 'title')}, + ), + ] diff --git a/backend/core/migrations/0023_message_ai_provider_id_message_model.py b/backend/core/migrations/0023_message_ai_provider_id_message_model.py new file mode 100644 index 0000000..63e9bc3 --- /dev/null +++ b/backend/core/migrations/0023_message_ai_provider_id_message_model.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.3 on 2026-03-12 17:12 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0022_aiprovidermodels'), + ] + + operations = [ + migrations.AddField( + model_name='message', + name='ai_provider_id', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.aiprovider'), + ), + migrations.AddField( + model_name='message', + name='model', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/backend/core/migrations/0024_rename_ai_provider_id_message_ai_provider.py b/backend/core/migrations/0024_rename_ai_provider_id_message_ai_provider.py new file mode 100644 index 0000000..a492eca --- /dev/null +++ b/backend/core/migrations/0024_rename_ai_provider_id_message_ai_provider.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.3 on 2026-03-12 17:15 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0023_message_ai_provider_id_message_model'), + ] + + operations = [ + migrations.RenameField( + model_name='message', + old_name='ai_provider_id', + new_name='ai_provider', + ), + ] diff --git a/backend/core/migrations/0025_chatroom_ai_provider_chatroom_model.py b/backend/core/migrations/0025_chatroom_ai_provider_chatroom_model.py new file mode 100644 index 0000000..ec00cfb --- /dev/null +++ b/backend/core/migrations/0025_chatroom_ai_provider_chatroom_model.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.3 on 2026-03-13 17:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0024_rename_ai_provider_id_message_ai_provider'), + ] + + operations = [ + migrations.AddField( + model_name='chatroom', + name='ai_provider', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.aiprovider'), + ), + migrations.AddField( + model_name='chatroom', + name='model', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/backend/core/migrations/0025_update_github_ids_to_bigint.py b/backend/core/migrations/0025_update_github_ids_to_bigint.py new file mode 100644 index 0000000..3a91600 --- /dev/null +++ b/backend/core/migrations/0025_update_github_ids_to_bigint.py @@ -0,0 +1,42 @@ +# Generated by Django 5.2.3 on 2026-03-14 04:35 + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0023_githubrepository_githubpullrequest_githubissue_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='githubissuecomment', + name='github_id', + field=models.BigIntegerField(), + ), + migrations.AlterField( + model_name='githubpullrequest', + name='github_id', + field=models.BigIntegerField(), + ), + migrations.AlterField( + model_name='githubissue', + name='github_id', + field=models.BigIntegerField(), + ), + migrations.AlterField( + model_name='githubdiscussion', + name='github_id', + field=models.BigIntegerField(), + ), + migrations.AlterField( + model_name='githubdiscussioncomment', + name='github_id', + field=models.BigIntegerField(), + ), + migrations.AlterField( + model_name='githubprcomment', + name='github_id', + field=models.BigIntegerField(), + ), + ] diff --git a/backend/core/migrations/0026_remove_chatroom_chat_type.py b/backend/core/migrations/0026_remove_chatroom_chat_type.py new file mode 100644 index 0000000..c2ddc60 --- /dev/null +++ b/backend/core/migrations/0026_remove_chatroom_chat_type.py @@ -0,0 +1,15 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_chatroom_ai_provider_chatroom_model'), + ] + + operations = [ + migrations.RunSQL( + sql='ALTER TABLE core_chatroom DROP COLUMN IF EXISTS chat_type;', + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/backend/core/migrations/0027_chatroomparticipant_has_unread.py b/backend/core/migrations/0027_chatroomparticipant_has_unread.py new file mode 100644 index 0000000..2ad41c3 --- /dev/null +++ b/backend/core/migrations/0027_chatroomparticipant_has_unread.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.3 on 2026-03-15 17:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0026_remove_chatroom_chat_type'), + ] + + operations = [ + migrations.AddField( + model_name='chatroomparticipant', + name='has_unread', + field=models.BooleanField(default=False), + ), + ] diff --git a/backend/core/migrations/0028_remove_chatroomparticipant_unread_count.py b/backend/core/migrations/0028_remove_chatroomparticipant_unread_count.py new file mode 100644 index 0000000..f61df4c --- /dev/null +++ b/backend/core/migrations/0028_remove_chatroomparticipant_unread_count.py @@ -0,0 +1,15 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0027_chatroomparticipant_has_unread'), + ] + + operations = [ + migrations.RunSQL( + sql="SELECT 1", # column already dropped directly; this is a no-op + reverse_sql="SELECT 1", + ), + ] diff --git a/backend/core/migrations/0029_message_is_internal.py b/backend/core/migrations/0029_message_is_internal.py new file mode 100644 index 0000000..0fccb77 --- /dev/null +++ b/backend/core/migrations/0029_message_is_internal.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0028_remove_chatroomparticipant_unread_count'), + ] + + operations = [ + migrations.AddField( + model_name='message', + name='is_internal', + field=models.BooleanField(default=False), + ), + ] diff --git a/backend/core/migrations/0030_chatroom_mode.py b/backend/core/migrations/0030_chatroom_mode.py new file mode 100644 index 0000000..b86aaab --- /dev/null +++ b/backend/core/migrations/0030_chatroom_mode.py @@ -0,0 +1,20 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0029_message_is_internal'), + ] + + operations = [ + migrations.AddField( + model_name='chatroom', + name='mode', + field=models.CharField( + choices=[('ai', 'AI Mode'), ('direct', 'Direct Mode')], + default='ai', + max_length=10, + ), + ), + ] diff --git a/backend/core/migrations/0031_migrate_participant_identifier_prefixes.py b/backend/core/migrations/0031_migrate_participant_identifier_prefixes.py new file mode 100644 index 0000000..87ddfb9 --- /dev/null +++ b/backend/core/migrations/0031_migrate_participant_identifier_prefixes.py @@ -0,0 +1,35 @@ +from django.db import migrations + + +def migrate_participant_prefixes(apps, schema_editor): + ChatroomParticipant = apps.get_model('core', 'ChatroomParticipant') + + # Process in batches of 1000 + for participant in ChatroomParticipant.objects.iterator(chunk_size=1000): + uid = participant.user_identifier + + if uid.startswith('anon_'): + participant.user_identifier = 'widget_' + uid[len('anon_'):] + participant.save(update_fields=['user_identifier']) + elif uid.startswith('reg_'): + participant.user_identifier = 'dashboard_' + uid[len('reg_'):] + participant.save(update_fields=['user_identifier']) + elif uid.startswith('reg:'): + participant.user_identifier = 'dashboard_' + uid[len('reg:'):] + participant.save(update_fields=['user_identifier']) + # Already migrated (widget_ or dashboard_) — skip + + +def noop(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0030_chatroom_mode'), + ] + + operations = [ + migrations.RunPython(migrate_participant_prefixes, noop), + ] diff --git a/backend/core/migrations/0032_rename_chatroom_mode_column.py b/backend/core/migrations/0032_rename_chatroom_mode_column.py new file mode 100644 index 0000000..458ff76 --- /dev/null +++ b/backend/core/migrations/0032_rename_chatroom_mode_column.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.3 on 2026-03-16 18:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0031_migrate_participant_identifier_prefixes'), + ] + + operations = [ + migrations.AlterField( + model_name='chatroom', + name='mode', + field=models.CharField(choices=[('ai', 'AI Mode'), ('direct', 'Direct Mode')], db_column='chat_mode', default='ai', max_length=10), + ), + ] diff --git a/backend/core/migrations/0033_message_platform_ai_mode.py b/backend/core/migrations/0033_message_platform_ai_mode.py new file mode 100644 index 0000000..661853e --- /dev/null +++ b/backend/core/migrations/0033_message_platform_ai_mode.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.3 on 2026-03-20 17:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0032_rename_chatroom_mode_column'), + ] + + operations = [ + migrations.AddField( + model_name='message', + name='ai_mode', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='message', + name='platform', + field=models.CharField(choices=[('dashboard', 'Dashboard'), ('widget', 'Widget')], default='dashboard', max_length=20), + ), + ] diff --git a/backend/core/migrations/0034_remove_chatroom_mode.py b/backend/core/migrations/0034_remove_chatroom_mode.py new file mode 100644 index 0000000..5833ce1 --- /dev/null +++ b/backend/core/migrations/0034_remove_chatroom_mode.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.3 on 2026-03-20 18:24 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0033_message_platform_ai_mode'), + ] + + operations = [ + migrations.RemoveField( + model_name='chatroom', + name='mode', + ), + ] diff --git a/backend/core/migrations/0035_merge_20260323_1011.py b/backend/core/migrations/0035_merge_20260323_1011.py new file mode 100644 index 0000000..309e3bb --- /dev/null +++ b/backend/core/migrations/0035_merge_20260323_1011.py @@ -0,0 +1,14 @@ +# Generated by Django 5.2.3 on 2026-03-23 10:11 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_update_github_ids_to_bigint'), + ('core', '0034_remove_chatroom_mode'), + ] + + operations = [ + ] diff --git a/backend/core/models/__init__.py b/backend/core/models/__init__.py index 15bd2dc..e736333 100644 --- a/backend/core/models/__init__.py +++ b/backend/core/models/__init__.py @@ -1,3 +1,4 @@ +from .base_model import BaseModel from .application import Application from .chatroom import ChatRoom from .chatroom_participant import ChatroomParticipant @@ -12,4 +13,12 @@ from .llm_model import LLMModel from .app_model import AppModel from .app_integration import AppIntegration -from .account_status import AccountStatus \ No newline at end of file +from .account_status import AccountStatus +from .ai_provider import AIProvider +from .app_ai_provider import AppAIProvider +from .ai_provider_models import AIProviderModels +from .github_data import ( + GitHubRepository, GitHubIssue, GitHubIssueComment, GitHubPullRequest, + GitHubPRComment, GitHubPRFile, GitHubDiscussion, GitHubDiscussionComment, + GitHubWikiPage, GitHubRepositoryFile +) diff --git a/backend/core/models/ai_provider.py b/backend/core/models/ai_provider.py new file mode 100644 index 0000000..4dd5571 --- /dev/null +++ b/backend/core/models/ai_provider.py @@ -0,0 +1,18 @@ +from django.contrib.auth.models import User +from django.db import models + +from core.fields import EncryptedCharField + +from .base_model import BaseModel + +class AIProvider(BaseModel): + name = models.CharField(max_length=255, null=True, blank=True) + provider = models.CharField(max_length=255) + provider_api_key = EncryptedCharField(max_length=1000) + is_builtin = models.BooleanField(default=False, blank=True) + metadata = models.JSONField(blank=True, null=True) + + creator = models.ForeignKey(User, on_delete=models.CASCADE) + + class Meta: + ordering = ['created_at'] diff --git a/backend/core/models/ai_provider_models.py b/backend/core/models/ai_provider_models.py new file mode 100644 index 0000000..e1a61ce --- /dev/null +++ b/backend/core/models/ai_provider_models.py @@ -0,0 +1,16 @@ +from django.contrib.auth.models import User +from django.db import models + +from .ai_provider import AIProvider +from .base_model import BaseModel + + +class AIProviderModels(BaseModel): + ai_provider = models.ForeignKey(AIProvider, on_delete=models.CASCADE, related_name='models') + models_data = models.JSONField(blank=True, null=True) + + creator = models.ForeignKey(User, on_delete=models.CASCADE) + + class Meta: + ordering = ['created_at'] + unique_together = ['ai_provider'] \ No newline at end of file diff --git a/backend/core/models/app_ai_provider.py b/backend/core/models/app_ai_provider.py new file mode 100644 index 0000000..ab3f51a --- /dev/null +++ b/backend/core/models/app_ai_provider.py @@ -0,0 +1,50 @@ +import uuid +from django.db import models + + +class AppAIProvider(models.Model): + id = models.AutoField(primary_key=True) + uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + + application = models.ForeignKey( + "Application", + on_delete=models.CASCADE, + related_name="ai_provider_configs" + ) + ai_provider = models.ForeignKey( + "AIProvider", + on_delete=models.CASCADE, + related_name="application_configs" + ) + + context = models.CharField(max_length=50) + capability = models.CharField(max_length=50, default='text') + priority = models.PositiveIntegerField(default=100) + external_model_id = models.CharField(max_length=255, blank=True, null=True) + metadata = models.JSONField(blank=True, null=True) + + is_active = models.BooleanField(default=True) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ['context', 'capability', 'priority'] + + def __str__(self): + return f"{self.application.name} - {self.ai_provider.name} ({self.context}:{self.capability})" + + def save(self, *args, **kwargs): + if not self.priority or self.priority == 100: + existing_configs = AppAIProvider.objects.filter( + application=self.application, + context=self.context, + capability=self.capability + ).exclude(id=self.id).order_by('-priority') + + if existing_configs.exists(): + self.priority = existing_configs.first().priority + 100 + else: + self.priority = 100 + + super().save(*args, **kwargs) diff --git a/backend/core/models/app_integration.py b/backend/core/models/app_integration.py index e09d4cb..f00cc60 100644 --- a/backend/core/models/app_integration.py +++ b/backend/core/models/app_integration.py @@ -1,10 +1,8 @@ from django.db import models from core.models import Application, Integration -import uuid class AppIntegration(models.Model): id = models.AutoField(primary_key=True) - uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) application = models.ForeignKey( Application, on_delete=models.CASCADE, related_name="app_integrations" ) diff --git a/backend/core/models/application.py b/backend/core/models/application.py index 0852bd3..00263db 100644 --- a/backend/core/models/application.py +++ b/backend/core/models/application.py @@ -12,14 +12,3 @@ class Application(models.Model): def __str__(self): return self.name - - def get_model_by_type(self, model_type): - app_model = ( - self.model_configs.filter( - Q(llm_model__is_default=True) | Q(llm_model__owner=self.owner), - llm_model__model_type=model_type - ) - .select_related("llm_model") - .first() - ) - return app_model.llm_model if app_model else None diff --git a/backend/core/models/base_model.py b/backend/core/models/base_model.py new file mode 100644 index 0000000..f783951 --- /dev/null +++ b/backend/core/models/base_model.py @@ -0,0 +1,13 @@ +from django.db import models +import uuid + +class BaseModel(models.Model): + uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + + metadata = models.JSONField(blank=True, null=True) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + abstract = True diff --git a/backend/core/models/chatroom.py b/backend/core/models/chatroom.py index 0807b24..8dc9450 100644 --- a/backend/core/models/chatroom.py +++ b/backend/core/models/chatroom.py @@ -7,6 +7,8 @@ class ChatRoom(models.Model): uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) name = models.CharField(max_length=255) application = models.ForeignKey('Application', on_delete=models.CASCADE, related_name='chatrooms') + ai_provider = models.ForeignKey('AIProvider', on_delete=models.SET_NULL, null=True, blank=True) + model = models.CharField(max_length=255, null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) def __str__(self): diff --git a/backend/core/models/chatroom_participant.py b/backend/core/models/chatroom_participant.py index f1590ea..ca3bb5a 100644 --- a/backend/core/models/chatroom_participant.py +++ b/backend/core/models/chatroom_participant.py @@ -14,6 +14,7 @@ class ChatroomParticipant(models.Model): user_identifier = models.CharField(max_length=255) metadata = models.JSONField(blank=True, null=True) role = models.CharField(max_length=15, choices=ROLE_CHOICES, default='user') + has_unread = models.BooleanField(default=False) def __str__(self): return f"Participant {self.user_identifier} in ChatRoom {self.chatroom_id}" \ No newline at end of file diff --git a/backend/core/models/github_data.py b/backend/core/models/github_data.py new file mode 100644 index 0000000..2bfcf4d --- /dev/null +++ b/backend/core/models/github_data.py @@ -0,0 +1,343 @@ +from django.db import models +from django.contrib.auth.models import User +from core.models.base_model import BaseModel +import uuid + + +class GitHubRepository(BaseModel): + """GitHub repository configuration for ingestion""" + name = models.CharField(max_length=255) + repo_owner = models.CharField(max_length=255) + full_name = models.CharField(max_length=511, unique=True) + description = models.TextField(blank=True) + url = models.URLField() + is_private = models.BooleanField(default=False) + default_branch = models.CharField(max_length=100, default='main') + last_ingested_at = models.DateTimeField(null=True, blank=True) + ingestion_status = models.CharField( + max_length=20, + choices=[ + ('pending', 'Pending'), + ('running', 'Running'), + ('completed', 'Completed'), + ('failed', 'Failed'), + ], + default='pending' + ) + metadata = models.JSONField(default=dict, blank=True) + app_integration = models.ForeignKey( + 'AppIntegration', + on_delete=models.CASCADE, + related_name='github_repositories' + ) + + class Meta: + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['full_name']), + models.Index(fields=['app_integration']), + models.Index(fields=['ingestion_status']), + ] + + def __str__(self): + return self.full_name + + +class GitHubIssue(BaseModel): + """GitHub issue data""" + github_id = models.BigIntegerField() + number = models.IntegerField() + title = models.CharField(max_length=1024) + body = models.TextField(blank=True) + state = models.CharField(max_length=20, choices=[('open', 'Open'), ('closed', 'Closed')]) + author = models.CharField(max_length=255) + author_association = models.CharField(max_length=50, blank=True) + assignees = models.JSONField(default=list, blank=True) + labels = models.JSONField(default=list, blank=True) + milestone = models.JSONField(null=True, blank=True) + locked = models.BooleanField(default=False) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + closed_at = models.DateTimeField(null=True, blank=True) + url = models.URLField() + repository = models.ForeignKey( + GitHubRepository, + on_delete=models.CASCADE, + related_name='issues' + ) + + class Meta: + unique_together = ['repository', 'github_id'] + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['repository', 'state']), + models.Index(fields=['github_id']), + models.Index(fields=['created_at']), + ] + + def __str__(self): + return f"{self.repository.full_name}#{self.number}: {self.title}" + + +class GitHubIssueComment(BaseModel): + """Comments on GitHub issues""" + github_id = models.BigIntegerField() + body = models.TextField() + author = models.CharField(max_length=255) + author_association = models.CharField(max_length=50, blank=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + url = models.URLField() + issue = models.ForeignKey( + GitHubIssue, + on_delete=models.CASCADE, + related_name='comments' + ) + + class Meta: + unique_together = ['issue', 'github_id'] + ordering = ['created_at'] + indexes = [ + models.Index(fields=['issue', 'created_at']), + models.Index(fields=['github_id']), + ] + + def __str__(self): + return f"Comment on {self.issue.repository.full_name}#{self.issue.number}" + + +class GitHubPullRequest(BaseModel): + """GitHub pull request data""" + github_id = models.BigIntegerField() + number = models.IntegerField() + title = models.CharField(max_length=1024) + body = models.TextField(blank=True) + state = models.CharField( + max_length=20, + choices=[('open', 'Open'), ('closed', 'Closed'), ('merged', 'Merged')] + ) + author = models.CharField(max_length=255) + author_association = models.CharField(max_length=50, blank=True) + assignees = models.JSONField(default=list, blank=True) + reviewers = models.JSONField(default=list, blank=True) + labels = models.JSONField(default=list, blank=True) + milestone = models.JSONField(null=True, blank=True) + head_branch = models.CharField(max_length=255) + base_branch = models.CharField(max_length=255) + merged = models.BooleanField(default=False) + merged_at = models.DateTimeField(null=True, blank=True) + merge_commit_sha = models.CharField(max_length=40, blank=True) + additions = models.IntegerField(default=0) + deletions = models.IntegerField(default=0) + changed_files = models.IntegerField(default=0) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + closed_at = models.DateTimeField(null=True, blank=True) + url = models.URLField() + repository = models.ForeignKey( + GitHubRepository, + on_delete=models.CASCADE, + related_name='pull_requests' + ) + + class Meta: + unique_together = ['repository', 'github_id'] + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['repository', 'state']), + models.Index(fields=['github_id']), + models.Index(fields=['created_at']), + models.Index(fields=['merged']), + ] + + def __str__(self): + return f"PR {self.repository.full_name}#{self.number}: {self.title}" + + +class GitHubPRComment(BaseModel): + """Comments on GitHub pull requests""" + github_id = models.BigIntegerField() + body = models.TextField() + author = models.CharField(max_length=255) + author_association = models.CharField(max_length=50, blank=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + url = models.URLField() + pull_request = models.ForeignKey( + GitHubPullRequest, + on_delete=models.CASCADE, + related_name='comments' + ) + + class Meta: + unique_together = ['pull_request', 'github_id'] + ordering = ['created_at'] + indexes = [ + models.Index(fields=['pull_request', 'created_at']), + models.Index(fields=['github_id']), + ] + + def __str__(self): + return f"Comment on PR {self.pull_request.repository.full_name}#{self.pull_request.number}" + + +class GitHubPRFile(BaseModel): + """Files changed in GitHub pull requests""" + filename = models.CharField(max_length=1024) + status = models.CharField( + max_length=20, + choices=[('added', 'Added'), ('modified', 'Modified'), ('removed', 'Removed'), ('renamed', 'Renamed')] + ) + additions = models.IntegerField(default=0) + deletions = models.IntegerField(default=0) + changes = models.IntegerField(default=0) + patch = models.TextField(blank=True) + blob_url = models.URLField(blank=True) + raw_url = models.URLField(blank=True) + contents_url = models.URLField(blank=True) + pull_request = models.ForeignKey( + GitHubPullRequest, + on_delete=models.CASCADE, + related_name='files' + ) + + class Meta: + unique_together = ['pull_request', 'filename'] + ordering = ['filename'] + indexes = [ + models.Index(fields=['pull_request', 'status']), + models.Index(fields=['filename']), + ] + + def __str__(self): + return f"{self.filename} in PR {self.pull_request.repository.full_name}#{self.pull_request.number}" + + +class GitHubDiscussion(BaseModel): + """GitHub discussion data""" + github_id = models.BigIntegerField() + number = models.IntegerField() + title = models.CharField(max_length=1024) + body = models.TextField() + author = models.CharField(max_length=255) + author_association = models.CharField(max_length=50, blank=True) + category = models.JSONField(default=dict, blank=True) + answer_chosen_at = models.DateTimeField(null=True, blank=True) + answer_chosen_by = models.CharField(max_length=255, blank=True) + upvote_count = models.IntegerField(default=0) + viewer_has_upvoted = models.BooleanField(default=False) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + last_edited_at = models.DateTimeField(null=True, blank=True) + url = models.URLField() + repository = models.ForeignKey( + GitHubRepository, + on_delete=models.CASCADE, + related_name='discussions' + ) + + class Meta: + unique_together = ['repository', 'github_id'] + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['repository', 'category']), + models.Index(fields=['github_id']), + models.Index(fields=['created_at']), + ] + + def __str__(self): + return f"Discussion {self.repository.full_name}#{self.number}: {self.title}" + + +class GitHubDiscussionComment(BaseModel): + """Comments in GitHub discussions""" + github_id = models.BigIntegerField() + body = models.TextField() + author = models.CharField(max_length=255) + author_association = models.CharField(max_length=50, blank=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + last_edited_at = models.DateTimeField(null=True, blank=True) + upvote_count = models.IntegerField(default=0) + viewer_has_upvoted = models.BooleanField(default=False) + parent_comment = models.ForeignKey( + 'self', + on_delete=models.CASCADE, + null=True, + blank=True, + related_name='replies' + ) + discussion = models.ForeignKey( + GitHubDiscussion, + on_delete=models.CASCADE, + related_name='comments' + ) + + class Meta: + unique_together = ['discussion', 'github_id'] + ordering = ['created_at'] + indexes = [ + models.Index(fields=['discussion', 'created_at']), + models.Index(fields=['parent_comment']), + models.Index(fields=['github_id']), + ] + + def __str__(self): + return f"Comment in Discussion {self.discussion.repository.full_name}#{self.discussion.number}" + + +class GitHubWikiPage(BaseModel): + """GitHub wiki pages""" + title = models.CharField(max_length=1024) + content = models.TextField(blank=True) + sha = models.CharField(max_length=40) + html_url = models.URLField() + download_url = models.URLField(blank=True) + last_modified = models.DateTimeField() + repository = models.ForeignKey( + GitHubRepository, + on_delete=models.CASCADE, + related_name='wiki_pages' + ) + + class Meta: + unique_together = ['repository', 'title'] + ordering = ['title'] + indexes = [ + models.Index(fields=['repository', 'title']), + models.Index(fields=['last_modified']), + ] + + def __str__(self): + return f"Wiki page '{self.title}' in {self.repository.full_name}" + + +class GitHubRepositoryFile(BaseModel): + """Repository files (README, Contributing, etc.)""" + path = models.CharField(max_length=1024) + name = models.CharField(max_length=255) + content = models.TextField(blank=True) + sha = models.CharField(max_length=40) + size = models.IntegerField(default=0) + content_type = models.CharField(max_length=100, blank=True) + encoding = models.CharField(max_length=20, blank=True) + html_url = models.URLField(blank=True) + download_url = models.URLField(blank=True) + last_modified = models.DateTimeField() + repository = models.ForeignKey( + GitHubRepository, + on_delete=models.CASCADE, + related_name='files' + ) + + class Meta: + unique_together = ['repository', 'path'] + ordering = ['path'] + indexes = [ + models.Index(fields=['repository', 'path']), + models.Index(fields=['name']), + models.Index(fields=['last_modified']), + ] + + def __str__(self): + return f"File {self.path} in {self.repository.full_name}" diff --git a/backend/core/models/knowledge_base.py b/backend/core/models/knowledge_base.py index 38ea008..bdd9f7c 100644 --- a/backend/core/models/knowledge_base.py +++ b/backend/core/models/knowledge_base.py @@ -15,7 +15,8 @@ class KnowledgeBase(models.Model): SOURCE_CHOICES = [ ('url', 'URL'), ('file', 'File'), - ('text', 'Text') + ('text', 'Text'), + ('github', 'GitHub') ] id = models.AutoField(primary_key=True) diff --git a/backend/core/models/message.py b/backend/core/models/message.py index ca4df27..bae6d63 100644 --- a/backend/core/models/message.py +++ b/backend/core/models/message.py @@ -8,7 +8,19 @@ class Message(models.Model): message = models.TextField() sender_identifier = models.CharField(max_length=255) metadata = models.JSONField(blank=True, null=True) + platform = models.CharField( + max_length=20, + choices=[('dashboard', 'Dashboard'), ('widget', 'Widget')], + default='dashboard', + ) + is_internal = models.BooleanField(default=False) + ai_mode = models.BooleanField(default=False) + ai_provider = models.ForeignKey('AIProvider', on_delete=models.SET_NULL, null=True, blank=True) + model = models.CharField(max_length=255, null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) + class Meta: + ordering = ['created_at'] + def __str__(self): return f"Message {self.uuid} in ChatRoom {self.chatroom.name}" diff --git a/backend/core/qdrant.py b/backend/core/qdrant.py index f712dbc..3652daf 100644 --- a/backend/core/qdrant.py +++ b/backend/core/qdrant.py @@ -11,33 +11,39 @@ logger = logging.getLogger(__name__) load_dotenv() -connect_to_local = os.getenv("CONNECT_TO_LOCAL_VECTOR_DB", "false").lower() == "true" - -if connect_to_local: - print('Connecting to local vector db') - qdrant = QdrantClient( - host=os.getenv("QDRANT_LOCAL_HOST", "localhost"), - port=int(os.getenv("QDRANT_LOCAL_PORT", "6333")), - prefer_grpc=True, - ) +if os.getenv("DISABLE_VECTOR_DB", "false").lower() == "true": + qdrant = None else: - print('Connecting to remote vector db') + connect_to_local = os.getenv("CONNECT_TO_LOCAL_VECTOR_DB", "false").lower() == "true" + + if connect_to_local: + print('Connecting to local vector db') + qdrant = QdrantClient( + host=os.getenv("QDRANT_LOCAL_HOST", "localhost"), + port=int(os.getenv("QDRANT_LOCAL_PORT", "6333")), + prefer_grpc=True, + ) + else: + print('Connecting to remote vector db') - cloud_host = os.getenv("QDRANT_CLOUD_HOST") - cloud_port = os.getenv("QDRANT_CLOUD_PORT", "6333") - api_key = os.getenv("QDRANT_CLOUD_API_KEY") + cloud_host = os.getenv("QDRANT_CLOUD_HOST") + cloud_port = os.getenv("QDRANT_CLOUD_PORT", "6333") + api_key = os.getenv("QDRANT_CLOUD_API_KEY") - full_url = f"{cloud_host}:{cloud_port}" + full_url = f"{cloud_host}:{cloud_port}" - qdrant = QdrantClient( - url=full_url, - api_key=api_key, - prefer_grpc=False, - ) + qdrant = QdrantClient( + url=full_url, + api_key=api_key, + prefer_grpc=False, + ) COLLECTION_NAME = "advq" def init_qdrant(retries=3, delay=2): + if qdrant is None: + return + for attempt in range(retries): try: if not qdrant.collection_exists(COLLECTION_NAME): @@ -66,4 +72,4 @@ def ensure_payload_indexes(): field_schema=PayloadSchemaType.KEYWORD ) except Exception as e: - print(f"Payload index for '{field}' may already exist or failed:", e) \ No newline at end of file + print(f"Payload index for '{field}' may already exist or failed:", e) diff --git a/backend/core/serializers/__init__.py b/backend/core/serializers/__init__.py index 78469d8..7bfabaa 100644 --- a/backend/core/serializers/__init__.py +++ b/backend/core/serializers/__init__.py @@ -11,4 +11,6 @@ from .configure_app import LoadAppConfigurationSerializer, ConfigureAppIntegrationSerializer from .app_integration import AppIntegrationViewSerializer from .app_model import AppModelViewSerializer, ConfigureAppModelsSerializer -from .password import ForgotPasswordSerializer, ResetPasswordSerializer \ No newline at end of file +from .password import ForgotPasswordSerializer, ResetPasswordSerializer +from .ai_provider import AIProviderSerializer +from .app_ai_provider import AppAIProviderSerializer, AppAIProviderCreateSerializer, AppAIProviderUpdateSerializer \ No newline at end of file diff --git a/backend/core/serializers/ai_provider.py b/backend/core/serializers/ai_provider.py new file mode 100644 index 0000000..1e047ba --- /dev/null +++ b/backend/core/serializers/ai_provider.py @@ -0,0 +1,76 @@ +from rest_framework import serializers +from core.models.ai_provider import AIProvider +from core.consts import SUPPORTED_AI_PROVIDERS +from core.utils import extract_and_merge_fields + +class AIProviderSerializer(serializers.ModelSerializer): + class Meta: + model = AIProvider + exclude = ['provider_api_key'] + +class AIProviderCreateSerializer(serializers.ModelSerializer): + base_url = serializers.CharField(max_length=500, required=False, allow_blank=True) + name = serializers.CharField(max_length=500, required=True, allow_blank=False) + + class Meta: + model = AIProvider + fields = [ + 'uuid', 'name', 'provider', 'provider_api_key', + 'base_url', 'creator' + ] + read_only_fields = ['uuid', 'creator'] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.instance is not None: + self.fields['provider_api_key'].required = False + self.fields['provider_api_key'].allow_blank = True + if 'provider' in self.fields: + self.fields['provider'].read_only = True + + def validate_provider(self, value): + supported_provider_ids = [p['id'] for p in SUPPORTED_AI_PROVIDERS] + if value not in supported_provider_ids: + supported_labels = [p['label'] for p in SUPPORTED_AI_PROVIDERS] + raise serializers.ValidationError( + f"Provider '{value}' is not supported. Supported providers are: {', '.join(supported_labels)}" + ) + return value + + def validate(self, attrs): + provider = attrs.get('provider') or (self.instance.provider if self.instance else None) + base_url = attrs.get('base_url') + + if self.instance is None and provider == 'custom': + if base_url is None or not base_url.strip(): + raise serializers.ValidationError({ + 'base_url': 'Custom provider requires base url' + }) + + if self.instance is not None and base_url is not None and provider == 'custom' and not base_url.strip(): + raise serializers.ValidationError({ + 'base_url': 'Custom provider requires base url' + }) + + return attrs + + def create(self, validated_data): + metadata = extract_and_merge_fields(validated_data, ['name', 'provider', 'provider_api_key']) + validated_data['metadata'] = metadata + validated_data['creator'] = self.context['request'].user + + return super().create(validated_data) + + def update(self, instance, validated_data): + api_key = validated_data.pop('provider_api_key', None) + if api_key and isinstance(api_key, str) and api_key.strip(): + instance.provider_api_key = api_key + + metadata = extract_and_merge_fields(validated_data, ['name', 'provider'], instance.metadata or {}) + validated_data['metadata'] = metadata + + for attr, value in validated_data.items(): + setattr(instance, attr, value) + + instance.save() + return instance diff --git a/backend/core/serializers/api_key.py b/backend/core/serializers/api_key.py index fa5f754..796078e 100644 --- a/backend/core/serializers/api_key.py +++ b/backend/core/serializers/api_key.py @@ -1,7 +1,6 @@ from rest_framework import serializers from core.models import ApplicationAPIKey, Application from django.shortcuts import get_object_or_404 -from uuid import UUID import secrets class APIKeySerializer(serializers.ModelSerializer): @@ -13,8 +12,8 @@ class APIKeySerializer(serializers.ModelSerializer): class Meta: model = ApplicationAPIKey - fields = ['name', 'permissions', 'id', 'created'] - read_only_fields = ['api_key', 'created', 'id'] + fields = ['name', 'permissions', 'id', 'created', 'owner'] + read_only_fields = ['api_key', 'created', 'id', 'owner'] def generate_api_key(self): return secrets.token_urlsafe(32) diff --git a/backend/core/serializers/app_ai_provider.py b/backend/core/serializers/app_ai_provider.py new file mode 100644 index 0000000..3f24468 --- /dev/null +++ b/backend/core/serializers/app_ai_provider.py @@ -0,0 +1,80 @@ +from rest_framework import serializers +from core.models.app_ai_provider import AppAIProvider +from core.models.ai_provider import AIProvider +from .ai_provider import AIProviderSerializer + +class AppAIProviderSerializer(serializers.ModelSerializer): + ai_provider = AIProviderSerializer(read_only=True) + + class Meta: + model = AppAIProvider + fields = [ + 'id', 'uuid', 'ai_provider', 'context', 'capability', + 'priority', 'external_model_id', 'is_active', + 'created_at', 'updated_at' + ] + read_only_fields = ['id', 'uuid', 'priority', 'created_at', 'updated_at'] + + +class AppAIProviderCreateSerializer(serializers.ModelSerializer): + ai_provider_id = serializers.IntegerField(write_only=True) + context = serializers.CharField() + capability = serializers.CharField() + external_model_id = serializers.CharField(required=False, allow_blank=True) + + class Meta: + model = AppAIProvider + fields = ['ai_provider_id', 'context', 'capability', 'external_model_id'] + + def to_representation(self, instance): + return AppAIProviderSerializer(instance, context=self.context).data + + def validate_ai_provider_id(self, value): + try: + ai_provider = AIProvider.objects.get(id=value) + if ai_provider.creator != self.context['request'].user and not ai_provider.is_builtin: + raise serializers.ValidationError("You don't own this AI provider") + return value + except AIProvider.DoesNotExist: + raise serializers.ValidationError("AI provider not found") + + def validate(self, data): + return data + + def create(self, validated_data): + ai_provider_id = validated_data.pop('ai_provider_id') + ai_provider = AIProvider.objects.get(id=ai_provider_id) + application = self.context['application'] + + existing_configs = AppAIProvider.objects.filter( + application=application, + context=validated_data['context'], + capability=validated_data['capability'] + ).order_by('-updated_at') + + if existing_configs.exists(): + config_to_keep = existing_configs.first() + configs_to_delete = existing_configs[1:] + + if configs_to_delete: + for config in configs_to_delete: + config.delete() + + config_to_keep.external_model_id = validated_data.get('external_model_id') + config_to_keep.ai_provider = ai_provider + config_to_keep.save() + return config_to_keep + + return AppAIProvider.objects.create( + application=application, + ai_provider=ai_provider, + **validated_data + ) + +class AppAIProviderUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = AppAIProvider + fields = ['external_model_id'] + + def to_representation(self, instance): + return AppAIProviderSerializer(instance, context=self.context).data diff --git a/backend/core/serializers/app_integration.py b/backend/core/serializers/app_integration.py index fd85e88..90e665b 100644 --- a/backend/core/serializers/app_integration.py +++ b/backend/core/serializers/app_integration.py @@ -1,22 +1,39 @@ from rest_framework import serializers +from typing import Dict, Any from core.models.app_integration import AppIntegration from core.serializers.integration import IntegrationViewSerializer + +class AppIntegrationCreateSerializer(serializers.ModelSerializer): + class Meta: + model = AppIntegration + fields = ['application', 'integration'] + + def validate(self, attrs: Dict[str, Any]) -> Dict[str, Any]: + return attrs + + class AppIntegrationViewSerializer(serializers.ModelSerializer): integration = IntegrationViewSerializer(read_only=True) class Meta: model = AppIntegration fields = [ - 'id', 'uuid', + 'id', 'integration', 'metadata', - 'created_at', 'updated_at' + 'created_at', + 'updated_at' ] + read_only_fields = ['id', 'created_at', 'updated_at'] - def to_representation(self, instance): + def to_representation(self, instance: AppIntegration) -> Dict[str, Any]: integration_data = IntegrationViewSerializer(instance.integration).data - integration_data["metadata"] = instance.metadata - integration_data["app_integration_uuid"] = str(instance.uuid) + + integration_data.update({ + 'metadata': instance.metadata or {}, + 'app_integration_uuid': str(instance.id) + }) + return integration_data diff --git a/backend/core/serializers/change_password.py b/backend/core/serializers/change_password.py new file mode 100644 index 0000000..d168ae5 --- /dev/null +++ b/backend/core/serializers/change_password.py @@ -0,0 +1,60 @@ +from rest_framework import serializers +from django.contrib.auth.password_validation import validate_password +from django.core.exceptions import ValidationError +import re + + +class ChangePasswordSerializer(serializers.Serializer): + current_password = serializers.CharField(write_only=True, required=True) + new_password = serializers.CharField(write_only=True, required=True, min_length=8) + confirm_password = serializers.CharField(write_only=True, required=True) + + def validate_current_password(self, value): + user = self.context['request'].user + if not user.check_password(value): + raise serializers.ValidationError("Current password is incorrect") + return value + + def validate_new_password(self, value): + errors = [] + + if len(value) < 8: + errors.append("Password must be at least 8 characters long") + + if not re.search(r'[a-z]', value) or not re.search(r'[A-Z]', value): + errors.append("Password must contain both uppercase and lowercase letters") + + if not re.search(r'[0-9]', value): + errors.append("Password must contain at least one number") + + if not re.search(r'[^a-zA-Z0-9]', value): + errors.append("Password must contain at least one special character") + + if errors: + raise serializers.ValidationError(errors) + + try: + validate_password(value) + except ValidationError as django_error: + if isinstance(django_error.messages, list): + errors.extend(django_error.messages) + else: + errors.append(str(django_error)) + raise serializers.ValidationError(errors) + + return value + + def validate(self, attrs): + if attrs['new_password'] != attrs['confirm_password']: + raise serializers.ValidationError("New passwords don't match") + + if attrs['current_password'] == attrs['new_password']: + raise serializers.ValidationError("New password must be different from current password") + + return attrs + + def save(self): + user = self.context['request'].user + user.set_password(self.validated_data['new_password']) + user.save() + return user diff --git a/backend/core/serializers/chatroom.py b/backend/core/serializers/chatroom.py index 61d2c2d..163d541 100644 --- a/backend/core/serializers/chatroom.py +++ b/backend/core/serializers/chatroom.py @@ -1,22 +1,30 @@ from rest_framework import serializers -from core.models import ChatroomParticipant +from core.models import ChatroomParticipant, AIProvider from core.models.chatroom import ChatRoom from core.serializers import ApplicationViewSerializer from core.serializers.message import ViewMessageSerializer +from core.serializers.ai_provider import AIProviderSerializer class ChatRoomViewSerializer(serializers.ModelSerializer): class Meta: model = ChatRoom - fields = ['uuid', 'name'] + fields = ['uuid', 'name', 'ai_provider', 'model'] class ChatRoomWithMessagesSerializer(serializers.ModelSerializer): application = ApplicationViewSerializer(read_only=True) - messages = ViewMessageSerializer(many=True, read_only=True) + ai_provider = AIProviderSerializer(read_only=True) + ai_model = serializers.CharField(source='model', read_only=True) + messages = serializers.SerializerMethodField() + chatroom = ChatRoomViewSerializer(read_only=True) class Meta: model = ChatRoom - fields = ['uuid', 'name', 'application', 'messages'] + fields = ['uuid', 'name', 'application', 'messages', 'ai_provider', 'ai_model', 'chatroom'] + + def get_messages(self, chatroom): + messages_qs = self.context.get('messages_qs', chatroom.messages.all().order_by('created_at')) + return ViewMessageSerializer(messages_qs, many=True).data class ChatroomParticipantSerializer(serializers.ModelSerializer): class Meta: @@ -25,15 +33,31 @@ class Meta: class ChatRoomPreviewSerializer(serializers.ModelSerializer): last_message = serializers.SerializerMethodField() + has_unread = serializers.SerializerMethodField() class Meta: model = ChatRoom - fields = ['uuid', 'name', 'last_message'] + fields = ['uuid', 'name', 'last_message', 'has_unread'] def get_last_message(self, chatroom): - last_msg = chatroom.messages.order_by('-created_at').first() + user_identifier = self.context.get('user_identifier', '') + is_dashboard = user_identifier.startswith('dashboard_') + qs = chatroom.messages.order_by('-created_at') + if not is_dashboard: + qs = qs.filter(is_internal=False) + last_msg = qs.first() return ViewMessageSerializer(last_msg).data if last_msg else None + def get_has_unread(self, chatroom): + user_identifier = self.context.get('user_identifier') + if not user_identifier: + return False + result = ChatroomParticipant.objects.filter( + chatroom=chatroom, + user_identifier=user_identifier + ).values_list('has_unread', flat=True).first() + return result if result is not None else False + class ChatRoomDetailSerializer(serializers.ModelSerializer): participants = ChatroomParticipantSerializer(many=True, read_only=True) messages = ViewMessageSerializer(many=True, read_only=True) @@ -41,3 +65,12 @@ class ChatRoomDetailSerializer(serializers.ModelSerializer): class Meta: model = ChatRoom fields = ['uuid', 'name', 'participants', 'messages'] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if hasattr(self.instance, 'messages'): + self.fields['messages'] = ViewMessageSerializer( + self.instance.messages.all().order_by('created_at'), + many=True, + read_only=True + ) diff --git a/backend/core/serializers/github_serializers.py b/backend/core/serializers/github_serializers.py new file mode 100644 index 0000000..236db7b --- /dev/null +++ b/backend/core/serializers/github_serializers.py @@ -0,0 +1,195 @@ +from rest_framework import serializers +from core.models.github_data import ( + GitHubRepository, GitHubIssue, GitHubIssueComment, GitHubPullRequest, + GitHubPRComment, GitHubPRFile, GitHubDiscussion, GitHubDiscussionComment, + GitHubWikiPage, GitHubRepositoryFile +) + + +class GitHubRepositorySerializer(serializers.ModelSerializer): + class Meta: + model = GitHubRepository + fields = [ + 'id', 'uuid', 'name', 'repo_owner', 'full_name', 'description', 'url', + 'is_private', 'default_branch', 'last_ingested_at', 'ingestion_status', + 'metadata', 'created_at', 'updated_at' + ] + read_only_fields = ['id', 'uuid', 'created_at', 'updated_at'] + + +class GitHubIssueCommentSerializer(serializers.ModelSerializer): + class Meta: + model = GitHubIssueComment + fields = [ + 'id', 'uuid', 'github_id', 'body', 'author', 'author_association', + 'created_at', 'updated_at', 'url' + ] + read_only_fields = ['id', 'uuid'] + + +class GitHubIssueSerializer(serializers.ModelSerializer): + comments = GitHubIssueCommentSerializer(many=True, read_only=True) + comment_count = serializers.SerializerMethodField() + + class Meta: + model = GitHubIssue + fields = [ + 'id', 'uuid', 'github_id', 'number', 'title', 'body', 'state', + 'author', 'author_association', 'assignees', 'labels', 'milestone', + 'locked', 'created_at', 'updated_at', 'closed_at', 'url', + 'comments', 'comment_count' + ] + read_only_fields = ['id', 'uuid'] + + def get_comment_count(self, obj): + return obj.comments.count() + + +class GitHubPRFileSerializer(serializers.ModelSerializer): + class Meta: + model = GitHubPRFile + fields = [ + 'id', 'uuid', 'filename', 'status', 'additions', 'deletions', + 'changes', 'patch', 'blob_url', 'raw_url', 'contents_url' + ] + read_only_fields = ['id', 'uuid'] + + +class GitHubPRCommentSerializer(serializers.ModelSerializer): + class Meta: + model = GitHubPRComment + fields = [ + 'id', 'uuid', 'github_id', 'body', 'author', 'author_association', + 'created_at', 'updated_at', 'url' + ] + read_only_fields = ['id', 'uuid'] + + +class GitHubPullRequestSerializer(serializers.ModelSerializer): + comments = GitHubPRCommentSerializer(many=True, read_only=True) + files = GitHubPRFileSerializer(many=True, read_only=True) + comment_count = serializers.SerializerMethodField() + file_count = serializers.SerializerMethodField() + + class Meta: + model = GitHubPullRequest + fields = [ + 'id', 'uuid', 'github_id', 'number', 'title', 'body', 'state', + 'author', 'author_association', 'assignees', 'reviewers', 'labels', + 'milestone', 'head_branch', 'base_branch', 'merged', 'merged_at', + 'merge_commit_sha', 'additions', 'deletions', 'changed_files', + 'created_at', 'updated_at', 'closed_at', 'url', + 'comments', 'files', 'comment_count', 'file_count' + ] + read_only_fields = ['id', 'uuid'] + + def get_comment_count(self, obj): + return obj.comments.count() + + def get_file_count(self, obj): + return obj.files.count() + + +class GitHubDiscussionCommentSerializer(serializers.ModelSerializer): + replies = serializers.SerializerMethodField() + + class Meta: + model = GitHubDiscussionComment + fields = [ + 'id', 'uuid', 'github_id', 'body', 'author', 'author_association', + 'created_at', 'updated_at', 'last_edited_at', 'upvote_count', + 'viewer_has_upvoted', 'parent_comment', 'replies' + ] + read_only_fields = ['id', 'uuid'] + + def get_replies(self, obj): + return GitHubDiscussionCommentSerializer(obj.replies.all(), many=True).data + + +class GitHubDiscussionSerializer(serializers.ModelSerializer): + comments = GitHubDiscussionCommentSerializer(many=True, read_only=True) + comment_count = serializers.SerializerMethodField() + + class Meta: + model = GitHubDiscussion + fields = [ + 'id', 'uuid', 'github_id', 'number', 'title', 'body', 'author', + 'author_association', 'category', 'answer_chosen_at', 'answer_chosen_by', + 'upvote_count', 'viewer_has_upvoted', 'created_at', 'updated_at', + 'last_edited_at', 'url', 'comments', 'comment_count' + ] + read_only_fields = ['id', 'uuid'] + + def get_comment_count(self, obj): + return obj.comments.count() + + +class GitHubWikiPageSerializer(serializers.ModelSerializer): + class Meta: + model = GitHubWikiPage + fields = [ + 'id', 'uuid', 'title', 'content', 'sha', 'html_url', + 'download_url', 'last_modified' + ] + read_only_fields = ['id', 'uuid'] + + +class GitHubRepositoryFileSerializer(serializers.ModelSerializer): + class Meta: + model = GitHubRepositoryFile + fields = [ + 'id', 'uuid', 'path', 'name', 'content', 'sha', 'size', + 'content_type', 'encoding', 'html_url', 'download_url', 'last_modified' + ] + read_only_fields = ['id', 'uuid'] + + +class GitHubRepositoryDetailSerializer(GitHubRepositorySerializer): + issues = GitHubIssueSerializer(many=True, read_only=True) + pull_requests = GitHubPullRequestSerializer(many=True, read_only=True) + discussions = GitHubDiscussionSerializer(many=True, read_only=True) + wiki_pages = GitHubWikiPageSerializer(many=True, read_only=True) + files = GitHubRepositoryFileSerializer(many=True, read_only=True) + issue_count = serializers.SerializerMethodField() + pr_count = serializers.SerializerMethodField() + discussion_count = serializers.SerializerMethodField() + wiki_page_count = serializers.SerializerMethodField() + file_count = serializers.SerializerMethodField() + + class Meta(GitHubRepositorySerializer.Meta): + fields = GitHubRepositorySerializer.Meta.fields + [ + 'issues', 'pull_requests', 'discussions', 'wiki_pages', 'files', + 'issue_count', 'pr_count', 'discussion_count', 'wiki_page_count', 'file_count' + ] + + def get_issue_count(self, obj): + return obj.issues.count() + + def get_pr_count(self, obj): + return obj.pull_requests.count() + + def get_discussion_count(self, obj): + return obj.discussions.count() + + def get_wiki_page_count(self, obj): + return obj.wiki_pages.count() + + def get_file_count(self, obj): + return obj.files.count() + + +class GitHubIngestionRequestSerializer(serializers.Serializer): + owner = serializers.CharField(max_length=255) + repo = serializers.CharField(max_length=255) + since = serializers.DateTimeField(required=False, allow_null=True) + app_integration_id = serializers.IntegerField() + + def validate_app_integration_id(self, value): + from core.models import AppIntegration + request = self.context.get('request') + qs = AppIntegration.objects.filter(id=value) + if request: + qs = qs.filter(application__owner=request.user) + if not qs.exists(): + raise serializers.ValidationError("Invalid app_integration_id or access denied.") + return value diff --git a/backend/core/serializers/message.py b/backend/core/serializers/message.py index aff8965..6896384 100644 --- a/backend/core/serializers/message.py +++ b/backend/core/serializers/message.py @@ -1,22 +1,29 @@ import uuid from rest_framework import serializers -from core.models import Message +from core.models import Message, AIProvider class CreateMessageSerializer(serializers.Serializer): chatroom_identifier = serializers.CharField(required=False) sender_identifier = serializers.CharField(required=False) message = serializers.CharField() metadata = serializers.JSONField(required=False) - send_to_user = serializers.BooleanField(required=False, default=False) + is_internal = serializers.BooleanField(required=False, default=False) + ai_mode = serializers.BooleanField(required=False, default=False) + ai_provider = serializers.IntegerField(required=False) + model = serializers.CharField(required=False) - def validate_chatroom_identifier(self, value): - if value == 'new_chat': - return value - try: - return str(uuid.UUID(value)) - except ValueError: - raise serializers.ValidationError('Must be a valid UUID or "new_chat"') + def __init__(self, *args, **kwargs): + self.app_owner = kwargs.pop('app_owner', None) + super().__init__(*args, **kwargs) + + def validate_ai_provider(self, value): + if value is not None: + try: + ai_provider = AIProvider.objects.get(id=value, creator=self.app_owner) + except AIProvider.DoesNotExist: + raise serializers.ValidationError('Invalid AI provider') + return value class ViewMessageSerializer(serializers.ModelSerializer): @@ -31,5 +38,10 @@ class Meta: 'chatroom_identifier', 'message', 'metadata', + 'ai_provider_id', + 'model', + 'is_internal', + 'platform', + 'ai_mode', 'created_at' ] diff --git a/backend/core/services/__init__.py b/backend/core/services/__init__.py index 922f59c..4debb1e 100644 --- a/backend/core/services/__init__.py +++ b/backend/core/services/__init__.py @@ -2,4 +2,8 @@ from .file_extractors import extract_text_from_file from .notifications import notify_users from .encryption import encrypt, decrypt, generate_verification_token, verify_verification_token -from .private_key_encryption import decrypt_with_private_key \ No newline at end of file +from .private_key_encryption import decrypt_with_private_key +from .contracts.ai_provider_contract import AIProviderContract +from .factories.ai_provider_factory import AIProviderFactory +from .providers.ai.gemini_provider import GeminiProvider +from .providers.ai.custom_provider import CustomProvider diff --git a/backend/core/services/abstractions.py b/backend/core/services/abstractions.py new file mode 100644 index 0000000..59870b8 --- /dev/null +++ b/backend/core/services/abstractions.py @@ -0,0 +1,70 @@ +from abc import ABC, abstractmethod +from typing import List, Dict, Any, Optional, Tuple + + +class DataProcessor(ABC): + @abstractmethod + def process_data(self, data: Dict[str, Any]) -> Dict[str, Any]: + pass + + @abstractmethod + def validate_data(self, data: Dict[str, Any]) -> bool: + pass + + +class IngestionService(ABC): + @abstractmethod + def ingest(self, owner: str, repo: str, since: Optional[str] = None) -> None: + pass + + @abstractmethod + def get_status(self) -> str: + pass + + +class AIProviderInterface(ABC): + @abstractmethod + def create_client(self, api_key: str, config: Dict[str, Any]) -> Any: + pass + + @abstractmethod + def validate_connection(self, api_key: str, config: Dict[str, Any]) -> Tuple[bool, Any]: + pass + + @abstractmethod + def get_models(self) -> List[str]: + pass + + +class RepositoryManagerInterface(ABC): + @abstractmethod + def get_or_create_repository(self, owner: str, repo: str) -> Any: + pass + + @abstractmethod + def update_ingestion_status(self, repository: Any, status: str) -> None: + pass + + +class ValidationService(ABC): + @abstractmethod + def validate(self, data: Dict[str, Any]) -> Tuple[bool, Any]: + pass + + @abstractmethod + def get_validation_errors(self, data: Dict[str, Any]) -> List[str]: + pass + + +class EmbeddingService(ABC): + @abstractmethod + def create_embeddings(self, text: str) -> List[float]: + pass + + @abstractmethod + def create_sparse_embeddings(self, text: str) -> Optional[Dict[str, Any]]: + pass + + @abstractmethod + def store_embeddings(self, embeddings: List[Any]) -> bool: + pass diff --git a/backend/core/services/ai_client_service.py b/backend/core/services/ai_client_service.py new file mode 100644 index 0000000..70d1455 --- /dev/null +++ b/backend/core/services/ai_client_service.py @@ -0,0 +1,144 @@ +from typing import Optional, Tuple, Any, Dict + +from core.models import Application, AIProvider, AppAIProvider +from .factories.ai_provider_factory import AIProviderFactory +from .ai_provider_validator import AIProviderValidator + + +class AIClientService: + + def __init__(self): + self.provider_factory = AIProviderFactory() + self.validator = AIProviderValidator() + + def get_client_and_model( + self, + app: Application, + ai_provider_id: Optional[int] = None, + model: Optional[str] = None, + context: str = 'response', + capability: str = 'text' + ) -> Tuple[Optional[Any], Optional[str]]: + """ + Get AI client and model for the given application. + + Args: + app: Application instance + ai_provider_id: Specific AI provider ID (optional) + model: Specific model name (optional) + context: Usage context + capability: Required capability + + Returns: + Tuple of (client_instance, model_name) + """ + provider_config = self._resolve_provider_config(app, ai_provider_id, context, capability) + + if not provider_config: + return None, None + + client = self._create_client(provider_config) + selected_model = model or provider_config.get('model') + + if not selected_model: + selected_model = self._get_default_model(client) + + return client, selected_model + + def _resolve_provider_config( + self, + app: Application, + ai_provider_id: Optional[int], + context: str, + capability: str + ) -> Optional[Dict[str, Any]]: + """Resolve provider configuration based on input parameters""" + if ai_provider_id: + return self._get_provider_by_id(ai_provider_id) + else: + return self._get_app_provider_config(app, context, capability) + + def _get_provider_by_id(self, ai_provider_id: int) -> Optional[Dict[str, Any]]: + """Get provider configuration by ID""" + try: + ai_provider = AIProvider.objects.get(id=ai_provider_id) + return { + 'provider': ai_provider, + 'type': ai_provider.provider, + 'api_key': ai_provider.provider_api_key, + 'config': ai_provider.metadata or {} + } + except AIProvider.DoesNotExist: + return None + + def _get_app_provider_config( + self, + app: Application, + context: str, + capability: str + ) -> Optional[Dict[str, Any]]: + """Get provider configuration from application settings""" + try: + config = self._get_app_provider(app, context, capability) + if not config: + return None + + return { + 'provider': config.ai_provider, + 'type': config.ai_provider.provider, + 'api_key': config.ai_provider.provider_api_key, + 'config': config.ai_provider.metadata or {}, + 'model': config.external_model_id + } + except Exception: + return None + + def _get_app_provider(self, app: Application, context: str, capability: str) -> Optional[AppAIProvider]: + return app.app_ai_providers.filter( + context=context, + capability=capability + ).first() + + def _create_client(self, provider_config: Dict[str, Any]) -> Optional[Any]: + """Create AI client instance""" + try: + return self.provider_factory.create_provider( + provider_type=provider_config['type'], + api_key=provider_config['api_key'], + config=provider_config['config'] + ) + except Exception: + return None + + def _get_default_model(self, client: Any) -> Optional[str]: + """Get default model from client""" + try: + supported_models = client.get_models() + return supported_models[0] if supported_models else None + except Exception: + return None + + def validate_ai_provider( + self, + validated_data: Dict[str, Any], + instance: AIProvider = None + ) -> Tuple[bool, Any]: + """ + Validate AI provider configuration + + Args: + validated_data: Validated data from serializer + instance: Existing instance (for updates) + + Returns: + Tuple of (is_valid, result) + """ + main_data, config_data = self.validator.validate_ai_provider_data(validated_data, instance) + + is_valid, provider_models = self.validator.validate_provider_config( + provider_type=main_data['provider'], + api_key=main_data['provider_api_key'], + config=config_data + ) + + return is_valid, provider_models if is_valid else None diff --git a/backend/core/services/ai_provider_validator.py b/backend/core/services/ai_provider_validator.py new file mode 100644 index 0000000..b15109c --- /dev/null +++ b/backend/core/services/ai_provider_validator.py @@ -0,0 +1,79 @@ +from typing import Tuple, Dict, Any + +from core.models import AIProvider +from .factories.ai_provider_factory import AIProviderFactory + + +class AIProviderValidator: + """Validates AI provider configurations""" + + def __init__(self): + self.provider_factory = AIProviderFactory() + + def validate_provider_config( + self, + provider_type: str, + api_key: str, + config: Dict[str, Any] + ) -> Tuple[bool, Any]: + """ + Validate AI provider configuration + + Args: + provider_type: Type of AI provider + api_key: API key for the provider + config: Additional configuration + + Returns: + Tuple of (is_valid, provider_models) + """ + try: + return self.provider_factory.validate_provider( + provider_type=provider_type, + api_key=api_key, + config=config + ) + except Exception as e: + return False, str(e) + + def validate_ai_provider_data( + self, + validated_data: Dict[str, Any], + instance: AIProvider = None + ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """ + Prepare and validate AI provider data + + Args: + validated_data: Validated serializer data + instance: Existing AI provider instance (for updates) + + Returns: + Tuple of (main_fields, config_data) + """ + main_fields = ['name', 'provider', 'provider_api_key'] + config = {} + + if instance: + current_data = { + 'name': instance.name, + 'provider': instance.provider, + 'provider_api_key': instance.provider_api_key + } + if instance.metadata: + config.update(instance.metadata) + update_data = {**current_data, **validated_data} + if not update_data['provider_api_key']: + update_data['provider_api_key'] = instance.provider_api_key + validation_data = update_data + else: + validation_data = validated_data + + main_data = {} + for field, value in validation_data.items(): + if field in main_fields: + main_data[field] = value + else: + config[field] = str(value).strip() if value is not None else '' + + return main_data, config diff --git a/backend/core/services/contracts/__init__.py b/backend/core/services/contracts/__init__.py new file mode 100644 index 0000000..8f0d13f --- /dev/null +++ b/backend/core/services/contracts/__init__.py @@ -0,0 +1 @@ +from .ai_provider_contract import AIProviderContract \ No newline at end of file diff --git a/backend/core/services/contracts/ai_provider_contract.py b/backend/core/services/contracts/ai_provider_contract.py new file mode 100644 index 0000000..9c9ac3a --- /dev/null +++ b/backend/core/services/contracts/ai_provider_contract.py @@ -0,0 +1,24 @@ +from abc import ABC, abstractmethod +from typing import Optional, Dict, Any + + +class AIProviderContract(ABC): + def __init__(self, api_key: str, config: Optional[Dict[str, Any]] = None): + self.api_key = api_key + self.config = config or {} + + @abstractmethod + def generate_text(self, model: str, contents: str, **kwargs) -> str: + pass + + @abstractmethod + def validate_connection(self) -> tuple[bool, list[Dict[str, Any]]]: + pass + + @abstractmethod + def get_models(self) -> list[Dict[str, Any]]: + pass + + @abstractmethod + def embed(self, model: str, texts: list[str]) -> list[list[float]]: + pass diff --git a/backend/core/services/factories/__init__.py b/backend/core/services/factories/__init__.py new file mode 100644 index 0000000..d0a1138 --- /dev/null +++ b/backend/core/services/factories/__init__.py @@ -0,0 +1 @@ +from .ai_provider_factory import AIProviderFactory \ No newline at end of file diff --git a/backend/core/services/factories/ai_provider_factory.py b/backend/core/services/factories/ai_provider_factory.py new file mode 100644 index 0000000..0113edf --- /dev/null +++ b/backend/core/services/factories/ai_provider_factory.py @@ -0,0 +1,49 @@ +from typing import Optional, Dict, Any + +from ..contracts.ai_provider_contract import AIProviderContract +from ..providers.ai.custom_provider import CustomProvider +from ..providers.ai.gemini_provider import GeminiProvider + + +class AIProviderFactory: + PROVIDER_CLASSES = { + 'gemini': GeminiProvider, + 'custom': CustomProvider, + } + + @staticmethod + def create_provider(provider_type: str, api_key: str, config: Optional[Dict[str, Any]] = None) -> AIProviderContract: + provider_class = AIProviderFactory.PROVIDER_CLASSES.get(provider_type.lower()) + + if provider_class is None: + supported_providers = list(AIProviderFactory.PROVIDER_CLASSES.keys()) + raise ValueError( + f"Unsupported provider type: {provider_type}. " + f"Supported providers: {supported_providers}" + ) + + try: + return provider_class(api_key=api_key, config=config or {}) + except Exception as e: + raise ValueError(f"Failed to create {provider_type} provider: {e}") + + @staticmethod + def validate_provider(provider_type: str, api_key: str, config: Optional[Dict[str, Any]] = None) -> tuple[bool, list[Dict[str, Any]]]: + provider_class = AIProviderFactory.PROVIDER_CLASSES.get(provider_type.lower()) + + if provider_class is None: + supported_providers = list(AIProviderFactory.PROVIDER_CLASSES.keys()) + raise ValueError( + f"Unsupported provider type: {provider_type}. " + f"Supported providers: {supported_providers}" + ) + + try: + provider = provider_class(api_key=api_key, config=config or {}) + return provider.validate_connection() + except Exception as e: + return False, [] + + @staticmethod + def get_supported_providers() -> list[str]: + return list(AIProviderFactory.PROVIDER_CLASSES.keys()) diff --git a/backend/core/services/github_client.py b/backend/core/services/github_client.py new file mode 100644 index 0000000..87d2d72 --- /dev/null +++ b/backend/core/services/github_client.py @@ -0,0 +1,313 @@ +import requests +import logging +from typing import Dict, List, Optional, Any +from urllib.parse import urljoin +import time + +logger = logging.getLogger(__name__) + + +class GitHubAPIClient: + """Production-grade GitHub API client with rate limiting and error handling""" + + BASE_URL = "https://api.github.com" + API_VERSION = "2022-11-28" + + def __init__(self, token: str): + self.token = token + self.session = requests.Session() + self.session.headers.update({ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": self.API_VERSION, + "User-Agent": "Ch8r-GitHub-Ingestion/1.0" + }) + + def _make_request(self, method: str, endpoint: str, **kwargs) -> Dict[str, Any]: + """Make HTTP request with proper error handling and rate limiting""" + url = urljoin(self.BASE_URL, endpoint) + + max_retries = 3 + retry_delay = 1 + timeout = 30 + + for attempt in range(max_retries): + try: + if 'timeout' not in kwargs: + kwargs['timeout'] = timeout + + response = self.session.request(method, url, **kwargs) + + if response.status_code == 403: + rate_limit_remaining = response.headers.get('X-RateLimit-Remaining', '0') + if rate_limit_remaining == '0': + reset_time = int(response.headers.get('X-RateLimit-Reset', time.time() + 60)) + wait_time = max(reset_time - time.time(), 1) + logger.warning(f"Rate limit hit, waiting {wait_time:.1f} seconds") + time.sleep(wait_time) + continue + + if response.status_code >= 400: + error_data = response.json() if response.content else {} + error_msg = error_data.get('message', f'HTTP {response.status_code}') + logger.error(f"GitHub API error: {error_msg}") + + if response.status_code in [401, 404]: + raise requests.exceptions.HTTPError(f"GitHub API error: {error_msg}") + elif response.status_code >= 500: + if attempt < max_retries - 1: + time.sleep(retry_delay * (2 ** attempt)) + continue + raise requests.exceptions.HTTPError(f"GitHub API server error: {error_msg}") + + response.raise_for_status() + return response.json() + + except requests.exceptions.Timeout as e: + logger.warning(f"Request timeout (attempt {attempt + 1}/{max_retries}): {e}") + if attempt < max_retries - 1: + time.sleep(retry_delay * (2 ** attempt)) + continue + logger.error(f"Request timeout after {max_retries} attempts: {e}") + raise + + except requests.exceptions.RequestException as e: + if attempt < max_retries - 1: + logger.warning(f"Request failed, retrying ({attempt + 1}/{max_retries}): {e}") + time.sleep(retry_delay * (2 ** attempt)) + continue + logger.error(f"Request failed after {max_retries} attempts: {e}") + raise + + raise requests.exceptions.RequestException("Max retries exceeded") + + def get_repository_info(self, owner: str, repo: str) -> Dict[str, Any]: + """Get repository information""" + return self._make_request('GET', f'/repos/{owner}/{repo}') + + def get_issues(self, owner: str, repo: str, state: str = 'all', + since: Optional[str] = None, per_page: int = 100) -> List[Dict[str, Any]]: + """Get all issues for a repository""" + issues = [] + page = 1 + + params = { + 'state': state, + 'per_page': per_page, + 'sort': 'created', + 'direction': 'desc' + } + if since: + params['since'] = since + + while True: + params['page'] = page + data = self._make_request('GET', f'/repos/{owner}/{repo}/issues', params=params) + + if not data: + break + + issues.extend([issue for issue in data if 'pull_request' not in issue]) + + if len(data) < per_page: + break + + page += 1 + + return issues + + def get_issue_comments(self, owner: str, repo: str, issue_number: int) -> List[Dict[str, Any]]: + """Get comments for a specific issue""" + comments = [] + page = 1 + per_page = 100 + + while True: + data = self._make_request( + 'GET', + f'/repos/{owner}/{repo}/issues/{issue_number}/comments', + params={'page': page, 'per_page': per_page} + ) + + if not data: + break + + comments.extend(data) + + if len(data) < per_page: + break + + page += 1 + + return comments + + def get_pull_requests(self, owner: str, repo: str, state: str = 'all', + since: Optional[str] = None, per_page: int = 100) -> List[Dict[str, Any]]: + """Get all pull requests for a repository""" + prs = [] + page = 1 + + params = { + 'state': state, + 'per_page': per_page, + 'sort': 'created', + 'direction': 'desc' + } + if since: + params['since'] = since + + while True: + params['page'] = page + data = self._make_request('GET', f'/repos/{owner}/{repo}/pulls', params=params) + + if not data: + break + + prs.extend(data) + + if len(data) < per_page: + break + + page += 1 + + return prs + + def get_pull_request_comments(self, owner: str, repo: str, pr_number: int) -> List[Dict[str, Any]]: + """Get comments for a specific pull request""" + comments = [] + page = 1 + per_page = 100 + + while True: + try: + data = self._make_request( + 'GET', + f'/repos/{owner}/{repo}/pulls/{pr_number}/comments', + params={'page': page, 'per_page': per_page} + ) + + if not data or len(data) == 0: + break + + comments.extend(data) + + if len(data) < per_page: + break + + page += 1 + except Exception as e: + logger.error(f"Failed to fetch PR comments for {owner}/{repo}#{pr_number}, page {page}: {e}") + break + + return comments + + def get_pull_request_files(self, owner: str, repo: str, pr_number: int) -> List[Dict[str, Any]]: + """Get files changed in a pull request""" + files = [] + page = 1 + per_page = 100 + + while True: + try: + data = self._make_request( + 'GET', + f'/repos/{owner}/{repo}/pulls/{pr_number}/files', + params={'page': page, 'per_page': per_page} + ) + + if not data or len(data) == 0: + break + + files.extend(data) + + if len(data) < per_page: + break + + page += 1 + except Exception as e: + logger.error(f"Failed to fetch PR files for {owner}/{repo}#{pr_number}, page {page}: {e}") + break + + return files + + def get_discussions(self, owner: str, repo: str, since: Optional[str] = None, + per_page: int = 100) -> List[Dict[str, Any]]: + """Get all discussions for a repository (requires GraphQL API)""" + discussions = [] + page = 1 + + query = f"repo:{owner}/{repo} is:discussion" + if since: + query += f" created:>{since}" + + params = { + 'q': query, + 'per_page': per_page, + 'sort': 'created', + 'order': 'desc' + } + + while True: + params['page'] = page + data = self._make_request('GET', '/search/issues', params=params) + + if not data.get('items'): + break + + discussions.extend(data['items']) + + if len(data['items']) < per_page: + break + + page += 1 + + return discussions + + def get_discussion_comments(self, owner: str, repo: str, discussion_number: int) -> List[Dict[str, Any]]: + """Get comments for a discussion (requires GraphQL API)""" + logger.warning(f"Discussion comments not implemented via REST API for {owner}/{repo}#{discussion_number}") + return [] + + def get_wiki_pages(self, owner: str, repo: str) -> List[Dict[str, Any]]: + """ + GitHub's REST API does not expose wiki page content directly. + The wiki is a separate git repo at https://github.com/{owner}/{repo}.wiki.git + We return an empty list here — wiki content is not available via REST. + """ + logger.info(f"Wiki pages are not available via GitHub REST API for {owner}/{repo}, skipping.") + return [] + + def get_repository_file(self, owner: str, repo: str, path: str, ref: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Get content of a specific repository file""" + try: + params = {} + if ref: + params['ref'] = ref + + return self._make_request('GET', f'/repos/{owner}/{repo}/contents/{path}', params=params) + except requests.exceptions.HTTPError as e: + if '404' in str(e): + logger.info(f"File not found: {path} in {owner}/{repo}") + return None + raise + + def get_code_comments(self, owner: str, repo: str, path: str, + ref: Optional[str] = None) -> List[Dict[str, Any]]: + """Get code comments for a specific file""" + try: + params = {} + if ref: + params['ref'] = ref + + return self._make_request('GET', f'/repos/{owner}/{repo}/commits', params=params) + except Exception as e: + logger.error(f"Failed to fetch code comments for {owner}/{repo}/{path}: {e}") + return [] + + def get_rate_limit_status(self) -> Dict[str, Any]: + """Get current rate limit status""" + return self._make_request('GET', '/rate_limit') + + def close(self): + """Close the session""" + self.session.close() diff --git a/backend/core/services/github_data_processors.py b/backend/core/services/github_data_processors.py new file mode 100644 index 0000000..09b9bf2 --- /dev/null +++ b/backend/core/services/github_data_processors.py @@ -0,0 +1,128 @@ +import logging +from datetime import datetime +from typing import Dict, List, Any, Optional +from django.utils import timezone + +from core.models.github_data import ( + GitHubRepository, GitHubIssue, GitHubIssueComment, + GitHubPullRequest, GitHubPRComment, GitHubPRFile +) +from core.services.abstractions import DataProcessor + +logger = logging.getLogger(__name__) + + +class BaseDataProcessor(DataProcessor): + def validate_data(self, data: Dict[str, Any]) -> bool: + return isinstance(data, dict) and bool(data) + + @staticmethod + def parse_iso_datetime(date_string: Optional[str]) -> Optional[datetime]: + if not date_string: + return None + try: + return datetime.fromisoformat(date_string.replace('Z', '+00:00')) + except (ValueError, AttributeError) as e: + logger.warning(f"Failed to parse datetime '{date_string}': {e}") + return None + + @staticmethod + def extract_user_login(user_data: Dict[str, Any]) -> str: + return user_data.get('login', '') if user_data else '' + + @staticmethod + def extract_label_names(labels_data: List[Dict[str, Any]]) -> List[str]: + return [label.get('name', '') for label in labels_data if label.get('name')] + + +class RepositoryDataProcessor(BaseDataProcessor): + def process_data(self, repo_data: Dict[str, Any]) -> Dict[str, Any]: + if not self.validate_data(repo_data): + raise ValueError("Invalid repository data") + + return { + 'name': repo_data.get('name', ''), + 'description': repo_data.get('description', ''), + 'url': repo_data.get('html_url', ''), + 'is_private': repo_data.get('private', False), + 'default_branch': repo_data.get('default_branch', 'main') + } + + +class IssueDataProcessor(BaseDataProcessor): + def process_data(self, issue_data: Dict[str, Any]) -> Dict[str, Any]: + return { + 'number': issue_data['number'], + 'title': issue_data['title'], + 'body': issue_data.get('body', '') or '', + 'state': issue_data['state'], + 'author': self.extract_user_login(issue_data.get('user')), + 'author_association': issue_data.get('author_association', ''), + 'assignees': [user['login'] for user in issue_data.get('assignees', [])], + 'labels': self.extract_label_names(issue_data.get('labels', [])), + 'milestone': issue_data.get('milestone'), + 'locked': issue_data.get('locked', False), + 'created_at': self.parse_iso_datetime(issue_data['created_at']), + 'updated_at': self.parse_iso_datetime(issue_data['updated_at']), + 'closed_at': self.parse_iso_datetime(issue_data.get('closed_at')), + 'url': issue_data['html_url'] + } + + def process_comment_data(self, comment_data: Dict[str, Any]) -> Dict[str, Any]: + return { + 'body': comment_data['body'], + 'author': self.extract_user_login(comment_data.get('user')), + 'author_association': comment_data.get('author_association', ''), + 'created_at': self.parse_iso_datetime(comment_data['created_at']), + 'updated_at': self.parse_iso_datetime(comment_data['updated_at']), + 'url': comment_data['html_url'] + } + + +class PullRequestDataProcessor(BaseDataProcessor): + def process_data(self, pr_data: Dict[str, Any]) -> Dict[str, Any]: + return { + 'number': pr_data['number'], + 'title': pr_data['title'], + 'body': pr_data.get('body', '') or '', + 'state': pr_data['state'], + 'author': self.extract_user_login(pr_data.get('user')), + 'author_association': pr_data.get('author_association', ''), + 'assignees': [user['login'] for user in pr_data.get('assignees', [])], + 'labels': self.extract_label_names(pr_data.get('labels', [])), + 'milestone': pr_data.get('milestone'), + 'head_branch': pr_data['head']['ref'] if pr_data.get('head') else '', + 'base_branch': pr_data['base']['ref'] if pr_data.get('base') else '', + 'merged': pr_data.get('merged', False), + 'merged_at': self.parse_iso_datetime(pr_data.get('merged_at')), + 'merge_commit_sha': pr_data.get('merge_commit_sha', ''), + 'additions': pr_data.get('additions', 0), + 'deletions': pr_data.get('deletions', 0), + 'changed_files': pr_data.get('changed_files', 0), + 'created_at': self.parse_iso_datetime(pr_data['created_at']), + 'updated_at': self.parse_iso_datetime(pr_data['updated_at']), + 'closed_at': self.parse_iso_datetime(pr_data.get('closed_at')), + 'url': pr_data['html_url'] + } + + def process_comment_data(self, comment_data: Dict[str, Any]) -> Dict[str, Any]: + return { + 'body': comment_data['body'], + 'author': self.extract_user_login(comment_data.get('user')), + 'author_association': comment_data.get('author_association', ''), + 'created_at': self.parse_iso_datetime(comment_data['created_at']), + 'updated_at': self.parse_iso_datetime(comment_data['updated_at']), + 'url': comment_data['html_url'] + } + + def process_file_data(self, file_data: Dict[str, Any]) -> Dict[str, Any]: + return { + 'status': file_data['status'], + 'additions': file_data.get('additions', 0), + 'deletions': file_data.get('deletions', 0), + 'changes': file_data.get('changes', 0), + 'patch': file_data.get('patch', '') or '', + 'blob_url': file_data.get('blob_url', ''), + 'raw_url': file_data.get('raw_url', ''), + 'contents_url': file_data.get('contents_url', '') + } diff --git a/backend/core/services/github_graphql_client.py b/backend/core/services/github_graphql_client.py new file mode 100644 index 0000000..afd3b7b --- /dev/null +++ b/backend/core/services/github_graphql_client.py @@ -0,0 +1,416 @@ +import logging +import requests +from typing import Dict, List, Optional, Any +from gql import gql, Client +from gql.transport.requests import RequestsHTTPTransport +from gql.transport.exceptions import TransportQueryError, TransportServerError +import time + +logger = logging.getLogger(__name__) + + +class GitHubGraphQLClient: + """GitHub GraphQL API v4 client for bulk operations""" + + def __init__(self, token: str): + self.token = token + self.endpoint = "https://api.github.com/graphql" + + self.transport = RequestsHTTPTransport( + url=self.endpoint, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github.v4+json", + "User-Agent": "Ch8r-GitHub-GraphQL/1.0" + }, + timeout=30, + retries=3 + ) + + self.client = Client(transport=self.transport, fetch_schema_from_transport=False) + + def _execute_query(self, query: str, variables: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """Execute GraphQL query with error handling and rate limiting""" + max_retries = 3 + retry_delay = 1 + + for attempt in range(max_retries): + try: + query_doc = gql(query) + result = self.client.execute(query_doc, variable_values=variables) + return result + + except TransportQueryError as e: + logger.error(f"GraphQL syntax error: {e}") + raise + + except TransportServerError as e: + error_data = str(e) + + if "rate limit" in error_data.lower() or "api rate limit exceeded" in error_data.lower(): + logger.warning(f"Rate limit hit, waiting {retry_delay * (2 ** attempt)} seconds") + time.sleep(retry_delay * (2 ** attempt)) + continue + + if "bad credentials" in error_data.lower() or "unauthorized" in error_data.lower(): + logger.error(f"Authentication error: {e}") + raise + + logger.error(f"GraphQL execution error: {e}") + if attempt < max_retries - 1: + time.sleep(retry_delay * (2 ** attempt)) + continue + raise + + except Exception as e: + logger.warning(f"Request failed, retrying ({attempt + 1}/{max_retries}): {e}") + if attempt < max_retries - 1: + time.sleep(retry_delay * (2 ** attempt)) + continue + logger.error(f"GraphQL request failed after {max_retries} attempts: {e}") + raise + + raise Exception("Max retries exceeded") + + def get_issues_with_comments(self, owner: str, repo: str, + states: List[str] = None, + since: Optional[str] = None, + first: int = 100, + after_cursor: Optional[str] = None) -> Dict[str, Any]: + """ + Get issues with their comments in a single GraphQL query + + Args: + owner: Repository owner + repo: Repository name + states: List of issue states (OPEN, CLOSED, etc.) + since: ISO datetime string for filtering by creation date + first: Number of items per page + after_cursor: Pagination cursor + + Returns: + Dictionary containing issues and pagination info + """ + if states is None: + states = ["OPEN", "CLOSED"] + + state_filter = "[" + ", ".join([f"{state.upper()}" for state in states]) + "]" + + query = """ + query GetIssuesWithComments($owner: String!, $repo: String!, $states: [IssueState!]!, $first: Int!, $after: String, $orderBy: IssueOrder!) { + repository(owner: $owner, name: $repo) { + issues(first: $first, after: $after, states: $states, orderBy: $orderBy) { + pageInfo { + hasNextPage + endCursor + startCursor + } + edges { + node { + id + number + title + body + state + author { + login + } + authorAssociation + assignees(first: 10) { + nodes { + login + } + } + labels(first: 20) { + nodes { + name + } + } + milestone { + title + } + locked + createdAt + updatedAt + closedAt + url + comments(first: 100, orderBy: {field: UPDATED_AT, direction: ASC}) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + id + body + author { + login + } + authorAssociation + createdAt + updatedAt + url + } + } + } + } + } + } + } + } + """ + + variables = { + "owner": owner, + "repo": repo, + "states": states, + "first": first, + "orderBy": { + "field": "CREATED_AT", + "direction": "DESC" + } + } + + if after_cursor: + variables["after"] = after_cursor + + return self._execute_query(query, variables) + + def get_all_issues_with_comments(self, owner: str, repo: str, + states: List[str] = None, + since: Optional[str] = None) -> List[Dict[str, Any]]: + """ + Get all issues with comments, handling pagination automatically + + Returns: + List of all issues with their comments + """ + all_issues = [] + has_next_page = True + after_cursor = None + + while has_next_page: + result = self.get_issues_with_comments( + owner=owner, + repo=repo, + states=states, + since=since, + after_cursor=after_cursor + ) + + repository_data = result.get('repository') + if not repository_data: + break + + issues_data = repository_data.get('issues', {}) + edges = issues_data.get('edges', []) + + for edge in edges: + issue_node = edge.get('node', {}) + if issue_node: + all_issues.append(issue_node) + + page_info = issues_data.get('pageInfo', {}) + has_next_page = page_info.get('hasNextPage', False) + after_cursor = page_info.get('endCursor') + + logger.info(f"Fetched {len(edges)} issues, total so far: {len(all_issues)}") + + logger.info(f"Total issues fetched: {len(all_issues)}") + return all_issues + + def get_all_pull_requests_with_comments(self, owner: str, repo: str, + states: List[str] = None, + since: Optional[str] = None) -> List[Dict[str, Any]]: + """ + Get all pull requests with comments, handling pagination automatically + + Returns: + List of all pull requests with their comments + """ + all_prs = [] + has_next_page = True + after_cursor = None + + while has_next_page: + result = self.get_pull_requests_with_comments( + owner=owner, + repo=repo, + states=states, + since=since, + after_cursor=after_cursor + ) + + repository_data = result.get('repository') + if not repository_data: + break + + prs_data = repository_data.get('pullRequests', {}) + edges = prs_data.get('edges', []) + + for edge in edges: + pr_node = edge.get('node', {}) + if pr_node: + all_prs.append(pr_node) + + page_info = prs_data.get('pageInfo', {}) + has_next_page = page_info.get('hasNextPage', False) + after_cursor = page_info.get('endCursor') + + logger.info(f"Fetched {len(edges)} pull requests, total so far: {len(all_prs)}") + + logger.info(f"Total pull requests fetched: {len(all_prs)}") + return all_prs + + def get_pull_requests_with_comments(self, owner: str, repo: str, + states: List[str] = None, + since: Optional[str] = None, + first: int = 100, + after_cursor: Optional[str] = None) -> Dict[str, Any]: + """ + Get pull requests with their comments in a single GraphQL query + + Args: + owner: Repository owner + repo: Repository name + states: List of PR states (OPEN, CLOSED, MERGED) + since: ISO datetime string for filtering by creation date + first: Number of items per page + after_cursor: Pagination cursor + + Returns: + Dictionary containing pull requests and pagination info + """ + if states is None: + states = ["OPEN", "CLOSED", "MERGED"] + + query = """ + query GetPRsWithComments($owner: String!, $repo: String!, $states: [PullRequestState!]!, $first: Int!, $after: String, $orderBy: IssueOrder!) { + repository(owner: $owner, name: $repo) { + pullRequests(first: $first, after: $after, states: $states, orderBy: $orderBy) { + pageInfo { + hasNextPage + endCursor + startCursor + } + edges { + node { + id + number + title + body + state + author { + login + } + authorAssociation + assignees(first: 10) { + nodes { + login + } + } + labels(first: 20) { + nodes { + name + } + } + milestone { + title + } + headRefName + baseRefName + merged + mergedAt + mergeCommit { + oid + } + additions + deletions + changedFiles + createdAt + updatedAt + closedAt + url + comments(first: 100, orderBy: {field: UPDATED_AT, direction: ASC}) { + pageInfo { + hasNextPage + endCursor + } + edges { + node { + id + body + author { + login + } + authorAssociation + createdAt + updatedAt + url + } + } + } + } + } + } + } + } + """ + + variables = { + "owner": owner, + "repo": repo, + "states": states, + "first": first, + "orderBy": { + "field": "CREATED_AT", + "direction": "DESC" + } + } + + if after_cursor: + variables["after"] = after_cursor + + return self._execute_query(query, variables) + + def get_repository_info(self, owner: str, repo: str) -> Dict[str, Any]: + """Get repository information using GraphQL""" + query = """ + query GetRepositoryInfo($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + id + name + nameWithOwner + description + url + isPrivate + defaultBranchRef { + name + } + createdAt + updatedAt + pushedAt + stargazerCount + forkCount + primaryLanguage { + name + } + licenseInfo { + name + } + } + } + """ + + variables = { + "owner": owner, + "repo": repo + } + + result = self._execute_query(query, variables) + return result.get('repository', {}) + + def close(self): + """Close GraphQL client""" + if hasattr(self.client, 'close_session'): + self.client.close_session() diff --git a/backend/core/services/github_graphql_ingestion.py b/backend/core/services/github_graphql_ingestion.py new file mode 100644 index 0000000..5347a43 --- /dev/null +++ b/backend/core/services/github_graphql_ingestion.py @@ -0,0 +1,458 @@ +import logging +from datetime import datetime +import hashlib +from typing import Dict, List, Optional, Any +from django.utils import timezone +from django.db import transaction + +from core.models.github_data import ( + GitHubRepository, GitHubIssue, GitHubIssueComment, GitHubPullRequest, + GitHubPRComment, GitHubPRFile +) +from core.models import AppIntegration +from core.services.github_graphql_client import GitHubGraphQLClient +from core.services.github_client import GitHubAPIClient +from core.services.ingestion import chunk_text, embed_text, embed_sparse +from core.models import IngestedChunk +from core.qdrant import qdrant, COLLECTION_NAME +from qdrant_client.http.models import PointStruct +from qdrant_client.models import SparseVector +import uuid + +logger = logging.getLogger(__name__) + + +def _extract_numeric_id_from_global_id(global_id: str) -> int: + hash_object = hashlib.md5(global_id.encode()) + hex_digest = hash_object.hexdigest() + return int(hex_digest[:8], 16) + + +class GitHubGraphQLIngestionService: + def __init__(self, app_integration: AppIntegration): + self.app_integration = app_integration + self.graphql_client = None + self.rest_client = None # Still needed for some operations like PR files + self.repository = None + + def _get_graphql_client(self) -> GitHubGraphQLClient: + if not self.graphql_client: + token = self.app_integration.integration.config.get('token') + if not token: + raise ValueError("GitHub token not found in integration config") + self.graphql_client = GitHubGraphQLClient(token) + return self.graphql_client + + def _get_rest_client(self) -> GitHubAPIClient: + if not self.rest_client: + token = self.app_integration.integration.config.get('token') + if not token: + raise ValueError("GitHub token not found in integration config") + self.rest_client = GitHubAPIClient(token) + return self.rest_client + + def _get_or_create_repository(self, owner: str, repo: str) -> GitHubRepository: + full_name = f"{owner}/{repo}" + + repository, created = GitHubRepository.objects.get_or_create( + full_name=full_name, + defaults={ + 'name': repo, + 'repo_owner': owner, + 'app_integration': self.app_integration, + 'ingestion_status': 'pending' + } + ) + + if created: + try: + client = self._get_graphql_client() + repo_info = client.get_repository_info(owner, repo) + + repository.description = repo_info.get('description', '') + repository.url = repo_info.get('url', '') + repository.is_private = repo_info.get('isPrivate', False) + repository.default_branch = repo_info.get('defaultBranchRef', {}).get('name', 'main') + repository.save() + + logger.info(f"Created repository record for {full_name}") + except Exception as e: + logger.error(f"Failed to fetch repository info for {full_name}: {e}") + repository.delete() + raise + + self.repository = repository + return repository + + def _ingest_issues(self, owner: str, repo: str, since: Optional[str] = None): + try: + client = self._get_graphql_client() + + logger.info(f"[GraphQLIngestion] Starting bulk issue ingestion for {owner}/{repo}") + issues_data = client.get_all_issues_with_comments( + owner=owner, + repo=repo, + states=['OPEN', 'CLOSED'], + since=since + ) + + logger.info(f"[GraphQLIngestion] Processing {len(issues_data)} issues from GraphQL") + + for i, issue_data in enumerate(issues_data, 1): + try: + logger.info(f"[GraphQLIngestion] Processing issue {i}/{len(issues_data)}: #{issue_data.get('number', 'unknown')}") + self._ingest_single_issue_from_graphql(issue_data) + except Exception as inner_e: + issue_number = issue_data.get('number', 'unknown') + logger.warning(f"Failed to ingest issue #{issue_number}: {inner_e}") + logger.debug(f"Issue data that failed: {issue_data}") + continue + + logger.info(f"[GraphQLIngestion] Processed {len(issues_data)} issues for {owner}/{repo}") + + logger.info(f"[GraphQLIngestion] Completed issue ingestion for {owner}/{repo}") + + except Exception as e: + logger.error(f"Failed to ingest issues for {owner}/{repo}: {e}") + raise + + def _ingest_single_issue_from_graphql(self, issue_data: Dict[str, Any]): + with transaction.atomic(): + author = issue_data.get('author', {}).get('login', '') if issue_data.get('author') else '' + + assignees = [ + user['login'] for user in + issue_data.get('assignees', {}).get('nodes', []) + ] + + labels = [ + label['name'] for label in + issue_data.get('labels', {}).get('nodes', []) + ] + + milestone = issue_data.get('milestone', {}).get('title') if issue_data.get('milestone') else None + + issue, created = GitHubIssue.objects.update_or_create( + repository=self.repository, + github_id=issue_data['number'], + defaults={ + 'number': issue_data['number'], + 'title': issue_data['title'], + 'body': issue_data.get('body', '') or '', + 'state': issue_data['state'].lower(), + 'author': author, + 'author_association': issue_data.get('authorAssociation', ''), + 'assignees': assignees, + 'labels': labels, + 'milestone': milestone, + 'locked': issue_data.get('locked', False), + 'created_at': self._parse_datetime(issue_data['createdAt']), + 'updated_at': self._parse_datetime(issue_data['updatedAt']), + 'closed_at': self._parse_datetime(issue_data.get('closedAt')), + 'url': issue_data['url'] + } + ) + + comments_data = issue_data.get('comments', {}).get('edges', []) + for comment_edge in comments_data: + comment_data = comment_edge.get('node', {}) + if comment_data: + try: + self._ingest_issue_comment_from_graphql(issue, comment_data) + except Exception as inner_e: + logger.warning(f"Failed to ingest comment {comment_data.get('id', 'unknown')}: {inner_e}") + continue + + logger.debug(f"[GraphQLIngestion] {'Created' if created else 'Updated'} issue #{issue.number} with {len(comments_data)} comments") + + def _ingest_issue_comment_from_graphql(self, issue: GitHubIssue, comment_data: Dict[str, Any]): + author = comment_data.get('author', {}).get('login', '') if comment_data.get('author') else '' + + GitHubIssueComment.objects.update_or_create( + issue=issue, + github_id=_extract_numeric_id_from_global_id(comment_data['id']), + defaults={ + 'body': comment_data['body'], + 'author': author, + 'author_association': comment_data.get('authorAssociation', ''), + 'created_at': self._parse_datetime(comment_data['createdAt']), + 'updated_at': self._parse_datetime(comment_data['updatedAt']), + 'url': comment_data['url'] + } + ) + + def _ingest_pull_requests(self, owner: str, repo: str, since: Optional[str] = None): + try: + graphql_client = self._get_graphql_client() + rest_client = self._get_rest_client() + + logger.info(f"[GraphQLIngestion] Starting bulk PR ingestion for {owner}/{repo}") + + prs_data = graphql_client.get_all_pull_requests_with_comments( + owner=owner, + repo=repo, + states=['OPEN', 'CLOSED', 'MERGED'], + since=since + ) + + logger.info(f"[GraphQLIngestion] Processing {len(prs_data)} pull requests from GraphQL") + + for i, pr_data in enumerate(prs_data, 1): + try: + logger.info(f"[GraphQLIngestion] Processing PR {i}/{len(prs_data)}: #{pr_data.get('number', 'unknown')}") + self._ingest_single_pull_request_from_graphql(pr_data, owner, repo, rest_client) + except Exception as inner_e: + pr_number = pr_data.get('number', 'unknown') + logger.warning(f"Failed to ingest PR #{pr_number}: {inner_e}") + logger.debug(f"PR data that failed: {pr_data}") + continue + + logger.info(f"[GraphQLIngestion] Processed {len(prs_data)} pull requests for {owner}/{repo}") + + logger.info(f"[GraphQLIngestion] Completed PR ingestion for {owner}/{repo}") + + except Exception as e: + logger.error(f"Failed to ingest pull requests for {owner}/{repo}: {e}") + raise + + def _ingest_single_pull_request_from_graphql(self, pr_data: Dict[str, Any], owner: str, repo: str, rest_client: GitHubAPIClient): + pr_number = pr_data.get('number', 'unknown') + + try: + with transaction.atomic(): + author = pr_data.get('author', {}).get('login', '') if pr_data.get('author') else '' + + assignees = [ + user['login'] for user in + pr_data.get('assignees', {}).get('nodes', []) + ] + + reviewers = [] + + labels = [ + label['name'] for label in + pr_data.get('labels', {}).get('nodes', []) + ] + + milestone = pr_data.get('milestone', {}).get('title') if pr_data.get('milestone') else None + + merge_commit_sha = pr_data.get('mergeCommit', {}).get('oid', '') if pr_data.get('mergeCommit') else '' + + pr, created = GitHubPullRequest.objects.update_or_create( + repository=self.repository, + github_id=pr_data['number'], + defaults={ + 'number': pr_data['number'], + 'title': pr_data['title'], + 'body': pr_data.get('body', '') or '', + 'state': pr_data['state'].lower(), + 'author': author, + 'author_association': pr_data.get('authorAssociation', ''), + 'assignees': assignees, + 'reviewers': reviewers, + 'labels': labels, + 'milestone': milestone, + 'head_branch': pr_data.get('headRefName', ''), + 'base_branch': pr_data.get('baseRefName', ''), + 'merged': pr_data.get('merged', False), + 'merged_at': self._parse_datetime(pr_data.get('mergedAt')), + 'merge_commit_sha': merge_commit_sha, + 'additions': pr_data.get('additions', 0), + 'deletions': pr_data.get('deletions', 0), + 'changed_files': pr_data.get('changedFiles', 0), + 'created_at': self._parse_datetime(pr_data['createdAt']), + 'updated_at': self._parse_datetime(pr_data['updatedAt']), + 'closed_at': self._parse_datetime(pr_data.get('closedAt')), + 'url': pr_data['url'] + } + ) + + logger.debug(f"[GraphQLIngestion] {'Created' if created else 'Updated'} PR #{pr_number}") + + comments_data = pr_data.get('comments', {}).get('edges', []) + for comment_edge in comments_data: + comment_data = comment_edge.get('node', {}) + if comment_data: + try: + self._ingest_pr_comment_from_graphql(pr, comment_data) + except Exception as inner_e: + logger.warning(f"Failed to ingest PR comment {comment_data.get('id', 'unknown')}: {inner_e}") + continue + + try: + logger.debug(f"[GraphQLIngestion] Fetching files for PR #{pr_number} via REST") + files = rest_client.get_pull_request_files(owner, repo, pr_data['number']) + logger.debug(f"[GraphQLIngestion] Found {len(files)} files for PR #{pr_number}") + + for file_data in files: + GitHubPRFile.objects.update_or_create( + pull_request=pr, + filename=file_data['filename'], + defaults={ + 'status': file_data['status'], + 'additions': file_data.get('additions', 0), + 'deletions': file_data.get('deletions', 0), + 'changes': file_data.get('changes', 0), + 'patch': file_data.get('patch', '') or '', + 'blob_url': file_data.get('blob_url', ''), + 'raw_url': file_data.get('raw_url', ''), + 'contents_url': file_data.get('contents_url', '') + } + ) + except Exception as e: + logger.warning(f"Failed to ingest files for PR #{pr_number}: {e}") + + logger.debug(f"[GraphQLIngestion] Completed PR #{pr_number}") + + except Exception as e: + logger.error(f"Failed to ingest single PR #{pr_number}: {e}") + raise + + def _ingest_pr_comment_from_graphql(self, pull_request: GitHubPullRequest, comment_data: Dict[str, Any]): + author = comment_data.get('author', {}).get('login', '') if comment_data.get('author') else '' + + GitHubPRComment.objects.update_or_create( + pull_request=pull_request, + github_id=_extract_numeric_id_from_global_id(comment_data['id']), + defaults={ + 'body': comment_data['body'], + 'author': author, + 'author_association': comment_data.get('authorAssociation', ''), + 'created_at': self._parse_datetime(comment_data['createdAt']), + 'updated_at': self._parse_datetime(comment_data['updatedAt']), + 'url': comment_data['url'] + } + ) + + def _create_knowledge_base_content(self) -> str: + """Create knowledge base content from all ingested data""" + content_parts = [] + + if not self.repository: + return "" + + content_parts.append(f"# Repository: {self.repository.full_name}") + if self.repository.description: + content_parts.append(f"Description: {self.repository.description}") + content_parts.append("") + + issues = self.repository.issues.all() + if issues: + content_parts.append("## Issues") + for issue in issues[:50]: + content_parts.append(f"### Issue #{issue.number}: {issue.title}") + content_parts.append(f"State: {issue.state}") + content_parts.append(f"Author: {issue.author}") + if issue.body: + content_parts.append(f"Description: {issue.body[:500]}...") + if issue.labels: + content_parts.append(f"Labels: {', '.join(issue.labels)}") + + comments = issue.comments.all()[:5] + for comment in comments: + content_parts.append(f"Comment by {comment.author}: {comment.body[:200]}...") + content_parts.append("") + + prs = self.repository.pull_requests.all() + if prs: + content_parts.append("## Pull Requests") + for pr in prs[:50]: + content_parts.append(f"### PR #{pr.number}: {pr.title}") + content_parts.append(f"State: {pr.state}") + content_parts.append(f"Author: {pr.author}") + if pr.body: + content_parts.append(f"Description: {pr.body[:500]}...") + if pr.labels: + content_parts.append(f"Labels: {', '.join(pr.labels)}") + content_parts.append("") + + return "\n".join(content_parts) + + def _ingest_to_knowledge_base(self): + from core.models import KnowledgeBase + from core.tasks.kb import send_kb_update + + app = self.app_integration.application + full_name = self.repository.full_name + logger.info(f"[GraphQLIngestion] _ingest_to_knowledge_base: app={app.name}, repo={full_name}") + + kb, created = KnowledgeBase.objects.get_or_create( + application=app, + source_type='github', + path=full_name, + defaults={ + 'metadata': { + 'source': 'github', + 'repository': full_name, + 'content': self._create_knowledge_base_content(), + 'ingestion_method': 'graphql' + }, + 'status': 'pending' + } + ) + logger.info(f"[GraphQLIngestion] KnowledgeBase {'created' if created else 'found'}: uuid={kb.uuid}, status={kb.status}") + + if not created: + logger.info(f"[GraphQLIngestion] Updating existing KB content for {full_name}") + kb.metadata['content'] = self._create_knowledge_base_content() + kb.metadata['ingestion_method'] = 'graphql' + kb.save() + + send_kb_update(kb, 'processing') + + from core.services.ingestion import ingest_kb + logger.info(f"[GraphQLIngestion] Calling ingest_kb for kb={kb.uuid}") + ingest_kb(kb, app) + logger.info(f"[GraphQLIngestion] ingest_kb completed for kb={kb.uuid}, final status={kb.status}") + + send_kb_update(kb, kb.status) + + def _parse_datetime(self, dt_str: Optional[str]) -> Optional[datetime]: + if not dt_str: + return None + + try: + if dt_str.endswith('Z'): + naive_dt = datetime.fromisoformat(dt_str.replace('Z', '')) + return timezone.make_aware(naive_dt) + else: + return datetime.fromisoformat(dt_str) + except Exception as e: + logger.warning(f"Failed to parse datetime: {dt_str}, error: {e}") + return timezone.now() + + def ingest_repository(self, owner: str, repo: str, since: Optional[str] = None): + try: + logger.info(f"[GraphQLIngestion] Starting GraphQL ingestion for {owner}/{repo} (since={since})") + + repository = self._get_or_create_repository(owner, repo) + repository.ingestion_status = 'running' + repository.save() + logger.info(f"[GraphQLIngestion] Repository record ready: id={repository.id}, full_name={repository.full_name}") + + logger.info(f"[GraphQLIngestion] Ingesting issues using GraphQL...") + self._ingest_issues(owner, repo, since) + + logger.info(f"[GraphQLIngestion] Ingesting pull requests using GraphQL...") + self._ingest_pull_requests(owner, repo, since) + + logger.info(f"[GraphQLIngestion] Building knowledge base content...") + self._ingest_to_knowledge_base() + + repository.ingestion_status = 'completed' + repository.last_ingested_at = timezone.now() + repository.save() + logger.info(f"[GraphQLIngestion] Completed GraphQL ingestion for {owner}/{repo}") + + except Exception as e: + logger.error(f"[GraphQLIngestion] Failed GraphQL ingestion for {owner}/{repo}: {e}", exc_info=True) + if self.repository: + self.repository.ingestion_status = 'failed' + self.repository.save() + raise + + finally: + if self.graphql_client: + self.graphql_client.close() + if self.rest_client: + self.rest_client.close() diff --git a/backend/core/services/github_ingestion.py b/backend/core/services/github_ingestion.py new file mode 100644 index 0000000..580205e --- /dev/null +++ b/backend/core/services/github_ingestion.py @@ -0,0 +1,207 @@ +import logging +from typing import Optional +from django.utils import timezone + +from core.models import AppIntegration +from core.services.github_graphql_ingestion import GitHubGraphQLIngestionService +from core.services.github_repository_manager import GitHubRepositoryManager +from core.services.github_issue_ingestion_service import GitHubIssueIngestionService +from core.services.github_pr_ingestion_service import GitHubPRIngestionService +from core.services.ingestion import chunk_text, embed_text, embed_sparse +from core.models import IngestedChunk +from core.qdrant import qdrant, COLLECTION_NAME +from qdrant_client.http.models import PointStruct +from qdrant_client.models import SparseVector +import uuid + +logger = logging.getLogger(__name__) + + +class GitHubDataIngestionService: + def __init__(self, app_integration: AppIntegration, use_graphql: bool = True): + self.app_integration = app_integration + self.use_graphql = use_graphql + self._graphql_service = None + self._repository_manager = None + self.repository = None + + def _get_graphql_service(self) -> GitHubGraphQLIngestionService: + if not self._graphql_service: + self._graphql_service = GitHubGraphQLIngestionService(self.app_integration) + return self._graphql_service + + def _get_repository_manager(self) -> GitHubRepositoryManager: + if not self._repository_manager: + self._repository_manager = GitHubRepositoryManager(self.app_integration) + return self._repository_manager + + def ingest_repository_data(self, owner: str, repo: str, since: Optional[str] = None): + try: + self._get_or_create_repository(owner, repo) + self._update_repository_status('ingesting') + + if self.use_graphql: + self._ingest_with_graphql(owner, repo, since) + else: + self._ingest_with_rest_api(owner, repo, since) + + self._create_and_store_embeddings() + self._update_repository_status('completed') + + logger.info(f"Successfully completed ingestion for {owner}/{repo}") + + except Exception as e: + logger.error(f"Failed to ingest repository {owner}/{repo}: {e}") + self._update_repository_status('failed') + raise + + def _get_or_create_repository(self, owner: str, repo: str): + repository_manager = self._get_repository_manager() + self.repository = repository_manager.get_or_create_repository(owner, repo) + return self.repository + + def _update_repository_status(self, status: str): + if self.repository: + repository_manager = self._get_repository_manager() + repository_manager.update_ingestion_status(self.repository, status) + + def _ingest_with_graphql(self, owner: str, repo: str, since: Optional[str] = None): + graphql_service = self._get_graphql_service() + graphql_service.ingest_repository_data(owner, repo, since) + + def _ingest_with_rest_api(self, owner: str, repo: str, since: Optional[str] = None): + repository_manager = self._get_repository_manager() + github_client = repository_manager.get_github_client() + + issue_service = GitHubIssueIngestionService(github_client, self.repository) + issue_service.ingest_issues(owner, repo, since) + + pr_service = GitHubPRIngestionService(github_client, self.repository) + pr_service.ingest_pull_requests(owner, repo, since) + + def _create_and_store_embeddings(self): + if not self.repository: + logger.warning("No repository available for embedding creation") + return + + try: + content = self._create_knowledge_base_content() + if not content.strip(): + logger.warning("No content available for embedding creation") + return + + chunks = chunk_text(content) + logger.info(f"Created {len(chunks)} chunks from repository content") + + points = [] + for i, chunk in enumerate(chunks): + dense_embedding = embed_text(chunk) + sparse_embedding = embed_sparse(chunk) + + point = PointStruct( + id=str(uuid.uuid4()), + vector=dense_embedding, + payload={ + 'text': chunk, + 'repository_id': str(self.repository.id), + 'repository_name': self.repository.full_name, + 'chunk_index': i, + 'source': 'github_ingestion' + } + ) + + if sparse_embedding: + point.sparse_vector = SparseVector( + indices=sparse_embedding['indices'], + values=sparse_embedding['values'] + ) + + points.append(point) + + if points: + qdrant.upsert( + collection_name=COLLECTION_NAME, + points=points + ) + logger.info(f"Successfully stored {len(points)} embeddings in vector database") + + except Exception as e: + logger.error(f"Failed to create embeddings: {e}") + logger.warning("Continuing without embeddings due to error") + + def _create_knowledge_base_content(self) -> str: + """Create knowledge base content from all ingested data""" + if not self.repository: + return "" + + content_parts = [] + + content_parts.append(f"# Repository: {self.repository.full_name}") + if self.repository.description: + content_parts.append(f"Description: {self.repository.description}") + content_parts.append("") + + self._add_issues_to_content(content_parts) + + self._add_pull_requests_to_content(content_parts) + + return "\n".join(content_parts) + + def _add_issues_to_content(self, content_parts: list): + """Add issues to knowledge base content""" + issues = self.repository.issues.all().order_by('-created_at')[:50] + + if not issues: + return + + content_parts.append("## Issues") + for issue in issues: + content_parts.append(f"### Issue #{issue.number}: {issue.title}") + content_parts.append(f"State: {issue.state}") + content_parts.append(f"Author: {issue.author}") + + if issue.body: + content_parts.append(f"Description: {issue.body[:500]}...") + + if issue.labels: + content_parts.append(f"Labels: {', '.join(issue.labels)}") + + recent_comments = issue.comments.all().order_by('-created_at')[:3] + if recent_comments: + content_parts.append("**Recent Comments:**") + for comment in recent_comments: + content_parts.append(f"- {comment.author}: {comment.body[:200]}...") + + content_parts.append("") + + def _add_pull_requests_to_content(self, content_parts: list): + """Add pull requests to knowledge base content""" + prs = self.repository.pull_requests.all().order_by('-created_at')[:50] + + if not prs: + return + + content_parts.append("## Pull Requests") + for pr in prs: + content_parts.append(f"### PR #{pr.number}: {pr.title}") + content_parts.append(f"State: {pr.state}") + content_parts.append(f"Author: {pr.author}") + + if pr.body: + content_parts.append(f"Description: {pr.body[:500]}...") + + if pr.labels: + content_parts.append(f"Labels: {', '.join(pr.labels)}") + + if pr.merged: + content_parts.append(f"**Merged:** Yes (Commit: {pr.merge_commit_sha[:8]}...)") + else: + content_parts.append("**Merged:** No") + + recent_comments = pr.comments.all().order_by('-created_at')[:3] + if recent_comments: + content_parts.append("**Recent Comments:**") + for comment in recent_comments: + content_parts.append(f"- {comment.author}: {comment.body[:200]}...") + + content_parts.append("") diff --git a/backend/core/services/github_issue_ingestion_service.py b/backend/core/services/github_issue_ingestion_service.py new file mode 100644 index 0000000..0f1aa18 --- /dev/null +++ b/backend/core/services/github_issue_ingestion_service.py @@ -0,0 +1,79 @@ +import logging +from typing import Dict, List, Any +from django.db import transaction + +from core.models.github_data import GitHubIssue, GitHubIssueComment +from core.services.github_client import GitHubAPIClient +from core.services.github_data_processors import IssueDataProcessor + +logger = logging.getLogger(__name__) + + +class GitHubIssueIngestionService: + def __init__(self, github_client: GitHubAPIClient, repository): + self.github_client = github_client + self.repository = repository + self.processor = IssueDataProcessor() + + def ingest_issues(self, owner: str, repo: str, since: str = None): + try: + issues = self.github_client.get_issues(owner, repo, state='all', since=since) + logger.info(f"Found {len(issues)} issues for {owner}/{repo}") + + for issue_data in issues: + self._ingest_single_issue(issue_data, owner, repo) + + except Exception as e: + logger.error(f"Failed to ingest issues for {owner}/{repo}: {e}") + raise + + def _ingest_single_issue(self, issue_data: Dict[str, Any], owner: str, repo: str): + try: + with transaction.atomic(): + issue = self._create_or_update_issue(issue_data) + self._ingest_issue_comments(issue, owner, repo, issue_data['number']) + + except Exception as e: + logger.error(f"Failed to ingest issue #{issue_data.get('number', 'unknown')}: {e}") + raise + + def _create_or_update_issue(self, issue_data: Dict[str, Any]) -> GitHubIssue: + processed_data = self.processor.process_data(issue_data) + + issue, created = GitHubIssue.objects.update_or_create( + repository=self.repository, + github_id=issue_data['id'], + defaults=processed_data + ) + + action = "Created" if created else "Updated" + logger.debug(f"{action} issue #{issue.number}: {issue.title}") + + return issue + + def _ingest_issue_comments(self, issue: GitHubIssue, owner: str, repo: str, issue_number: int): + try: + comments = self.github_client.get_issue_comments(owner, repo, issue_number) + logger.debug(f"Found {len(comments)} comments for issue #{issue_number}") + + for comment_data in comments: + self._create_or_update_comment(issue, comment_data) + + except Exception as e: + logger.warning(f"Failed to ingest comments for issue #{issue_number}: {e}") + + def _create_or_update_comment(self, issue: GitHubIssue, comment_data: Dict[str, Any]): + try: + processed_data = self.processor.process_comment_data(comment_data) + + comment, created = GitHubIssueComment.objects.update_or_create( + issue=issue, + github_id=comment_data['id'], + defaults=processed_data + ) + + if created: + logger.debug(f"Created comment {comment.github_id}") + + except Exception as e: + logger.warning(f"Failed to ingest comment {comment_data.get('id', 'unknown')}: {e}") diff --git a/backend/core/services/github_pr_ingestion_service.py b/backend/core/services/github_pr_ingestion_service.py new file mode 100644 index 0000000..e85f832 --- /dev/null +++ b/backend/core/services/github_pr_ingestion_service.py @@ -0,0 +1,107 @@ +import logging +from typing import Dict, List, Any +from django.db import transaction + +from core.models.github_data import GitHubPullRequest, GitHubPRComment, GitHubPRFile +from core.services.github_client import GitHubAPIClient +from core.services.github_data_processors import PullRequestDataProcessor + +logger = logging.getLogger(__name__) + + +class GitHubPRIngestionService: + def __init__(self, github_client: GitHubAPIClient, repository): + self.github_client = github_client + self.repository = repository + self.processor = PullRequestDataProcessor() + + def ingest_pull_requests(self, owner: str, repo: str, since: str = None): + try: + prs = self.github_client.get_pull_requests(owner, repo, state='all', since=since) + logger.info(f"Found {len(prs)} pull requests for {owner}/{repo}") + + for pr_data in prs: + self._ingest_single_pr(pr_data, owner, repo) + + except Exception as e: + logger.error(f"Failed to ingest pull requests for {owner}/{repo}: {e}") + raise + + def _ingest_single_pr(self, pr_data: Dict[str, Any], owner: str, repo: str): + try: + with transaction.atomic(): + pr = self._create_or_update_pr(pr_data) + self._ingest_pr_comments(pr, owner, repo, pr_data['number']) + self._ingest_pr_files(pr, owner, repo, pr_data['number']) + + except Exception as e: + logger.error(f"Failed to ingest PR #{pr_data.get('number', 'unknown')}: {e}") + raise + + def _create_or_update_pr(self, pr_data: Dict[str, Any]) -> GitHubPullRequest: + processed_data = self.processor.process_data(pr_data) + + pr, created = GitHubPullRequest.objects.update_or_create( + repository=self.repository, + github_id=pr_data['id'], + defaults=processed_data + ) + + action = "Created" if created else "Updated" + logger.debug(f"{action} PR #{pr.number}: {pr.title}") + + return pr + + def _ingest_pr_comments(self, pr: GitHubPullRequest, owner: str, repo: str, pr_number: int): + try: + comments = self.github_client.get_pull_request_comments(owner, repo, pr_number) + logger.debug(f"Found {len(comments)} comments for PR #{pr_number}") + + for comment_data in comments: + self._create_or_update_pr_comment(pr, comment_data) + + except Exception as e: + logger.warning(f"Failed to ingest comments for PR #{pr_number}: {e}") + + def _create_or_update_pr_comment(self, pr: GitHubPullRequest, comment_data: Dict[str, Any]): + try: + processed_data = self.processor.process_comment_data(comment_data) + + comment, created = GitHubPRComment.objects.update_or_create( + pull_request=pr, + github_id=comment_data['id'], + defaults=processed_data + ) + + if created: + logger.debug(f"Created PR comment {comment.github_id}") + + except Exception as e: + logger.warning(f"Failed to ingest PR comment {comment_data.get('id', 'unknown')}: {e}") + + def _ingest_pr_files(self, pr: GitHubPullRequest, owner: str, repo: str, pr_number: int): + try: + files = self.github_client.get_pull_request_files(owner, repo, pr_number) + logger.debug(f"Found {len(files)} files for PR #{pr_number}") + + for file_data in files: + self._create_or_update_pr_file(pr, file_data) + + except Exception as e: + logger.warning(f"Failed to ingest files for PR #{pr_number}: {e}") + + def _create_or_update_pr_file(self, pr: GitHubPullRequest, file_data: Dict[str, Any]): + try: + processed_data = self.processor.process_file_data(file_data) + + file_record, created = GitHubPRFile.objects.update_or_create( + pull_request=pr, + filename=file_data['filename'], + defaults=processed_data + ) + + if created: + logger.debug(f"Created PR file {file_record.filename}") + + except Exception as e: + logger.warning(f"Failed to ingest PR file {file_data.get('filename', 'unknown')}: {e}") diff --git a/backend/core/services/github_repository_manager.py b/backend/core/services/github_repository_manager.py new file mode 100644 index 0000000..0c37ba9 --- /dev/null +++ b/backend/core/services/github_repository_manager.py @@ -0,0 +1,68 @@ +import logging +from typing import Optional +from django.db import transaction +from django.utils import timezone + +from core.models.github_data import GitHubRepository +from core.models import AppIntegration +from core.services.github_client import GitHubAPIClient +from core.services.github_data_processors import RepositoryDataProcessor + +logger = logging.getLogger(__name__) + + +class GitHubRepositoryManager: + def __init__(self, app_integration: AppIntegration): + self.app_integration = app_integration + self._github_client = None + + def get_github_client(self) -> GitHubAPIClient: + if not self._github_client: + token = self.app_integration.integration.config.get('token') + if not token: + raise ValueError("GitHub token not found in integration config") + self._github_client = GitHubAPIClient(token) + return self._github_client + + def get_or_create_repository(self, owner: str, repo: str) -> GitHubRepository: + full_name = f"{owner}/{repo}" + + repository, created = GitHubRepository.objects.get_or_create( + full_name=full_name, + defaults={ + 'name': repo, + 'repo_owner': owner, + 'app_integration': self.app_integration, + 'ingestion_status': 'pending' + } + ) + + if created: + self._enrich_repository_data(repository, owner, repo) + + return repository + + def _enrich_repository_data(self, repository: GitHubRepository, owner: str, repo: str): + try: + client = self.get_github_client() + repo_info = client.get_repository_info(owner, repo) + + processor = RepositoryDataProcessor() + processed_data = processor.process_data(repo_info) + + for field, value in processed_data.items(): + setattr(repository, field, value) + + repository.save() + logger.info(f"Enriched repository record for {repository.full_name}") + + except Exception as e: + logger.error(f"Failed to enrich repository {repository.full_name}: {e}") + repository.delete() + raise + + def update_ingestion_status(self, repository: GitHubRepository, status: str): + repository.ingestion_status = status + repository.last_ingested_at = timezone.now() + repository.save() + logger.info(f"Updated ingestion status for {repository.full_name}: {status}") diff --git a/backend/core/services/ingestion.py b/backend/core/services/ingestion.py index 1133db7..1e5e0ed 100644 --- a/backend/core/services/ingestion.py +++ b/backend/core/services/ingestion.py @@ -2,41 +2,50 @@ from qdrant_client.http.exceptions import UnexpectedResponse import logging +from core.models import IngestedChunk +from core.models.llm_model import LLMModel from core.llm_client import LLMClient -from core.models import IngestedChunk, LLMModel from core.qdrant import qdrant, COLLECTION_NAME from qdrant_client.http.models import PointIdsList, PointStruct from qdrant_client.models import Filter as ModelsFilter, FieldCondition as ModelsFieldCondition, \ MatchValue as ModelsMatchValue from qdrant_client.models import Prefetch, SparseVector +from core.services.ai_client_service import AIClientService import uuid logger = logging.getLogger(__name__) -sparse_model = SparseTextEmbedding(model_name="Qdrant/bm25") +_sparse_model = None +def _get_sparse_model(): + global _sparse_model + if _sparse_model is None: + logger.info("[ingestion] Loading sparse embedding model (Qdrant/bm25)...") + _sparse_model = SparseTextEmbedding(model_name="Qdrant/bm25") + logger.info("[ingestion] Sparse embedding model loaded.") + return _sparse_model -def embed_text(text_chunks: list[str], app) -> list[list[float]]: - embedding_model = app.get_model_by_type(LLMModel.ModelType.EMBEDDING) - try: - client = LLMClient( - base_url=embedding_model.base_url, - api_key=embedding_model.config, - ) - response = client.embed(text_chunks, embedding_model.model_name) - if not response: - return [[] for _ in text_chunks] +def embed_text(text_chunks: list[str], app) -> list[list[float]]: + ai_client_service = AIClientService() + provider, model = ai_client_service.get_client_and_model( + app=app, + context='response', + capability='embedding' + ) + + if not provider or not model: + logger.error("[embed_text] No embedding provider available") + return [[] for _ in text_chunks] - if isinstance(response[0], float): - return [response] - return response + try: + return provider.embed(model, text_chunks) except Exception as e: logger.error(f"[embed_text] Embedding failed: {e}") return [[] for _ in text_chunks] def embed_sparse(text_chunks: list[str]) -> list: - sparse_embeddings = list(sparse_model.embed(text_chunks)) + sparse_embeddings = list(_get_sparse_model().embed(text_chunks)) return [ SparseVector( indices=embedding.indices.tolist(), @@ -62,7 +71,7 @@ def get_chunks(query_text, app, top_k=5): dense_query = dense_embeddings[0] try: - sparse_embeddings = list(sparse_model.embed([query_text]))[0] + sparse_embeddings = list(_get_sparse_model().embed([query_text]))[0] sparse_query = SparseVector( indices=sparse_embeddings.indices.tolist(), values=sparse_embeddings.values.tolist() @@ -117,30 +126,44 @@ def get_chunks(query_text, app, top_k=5): def ingest_kb(kb, app): content = kb.metadata.get('content', '') + logger.info(f"[ingest_kb] Starting for kb={kb.uuid}, source_type={kb.source_type}, content_length={len(content)}") if not content: + logger.warning(f"[ingest_kb] No content found for kb={kb.uuid}, skipping") return + kb.status = 'processing' + kb.save(update_fields=['status']) + chunks = chunk_text(content) + logger.info(f"[ingest_kb] Created {len(chunks)} chunks for kb={kb.uuid}") dense_embeddings = embed_text(text_chunks=chunks, app=app) + non_empty = sum(1 for v in dense_embeddings if v) + logger.info(f"[ingest_kb] Dense embeddings: {non_empty}/{len(chunks)} non-empty for kb={kb.uuid}") sparse_embeddings = embed_sparse(chunks) existing_chunks = IngestedChunk.objects.filter(knowledge_base=kb) qdrant_ids = [str(chunk.uuid) for chunk in existing_chunks] existing_chunks.delete() + logger.info(f"[ingest_kb] Deleted {len(qdrant_ids)} existing chunks for kb={kb.uuid}") try: if qdrant_ids: qdrant.delete( collection_name=COLLECTION_NAME, - points=qdrant_ids + points_selector=qdrant_ids ) except Exception as e: logger.warning(f"Failed to delete vectors from Qdrant: {e}") + upserted = 0 for i, (chunk, dense_vector, sparse_vector) in enumerate(zip(chunks, dense_embeddings, sparse_embeddings)): + if not dense_vector: + logger.warning(f"[ingest_kb] Skipping chunk {i} for kb {kb.uuid}: empty dense embedding") + continue + chunk_uuid = uuid.uuid4() IngestedChunk.objects.create( uuid=chunk_uuid, @@ -169,6 +192,12 @@ def ingest_kb(kb, app): ) ] ) + upserted += 1 + + logger.info(f"[ingest_kb] Upserted {upserted}/{len(chunks)} chunks to Qdrant for kb={kb.uuid}") + kb.status = 'completed' + kb.save(update_fields=['status']) + logger.info(f"[ingest_kb] Finished for kb={kb.uuid}, status=completed") def delete_vectors_from_qdrant(ids): if not ids: @@ -180,4 +209,4 @@ def delete_vectors_from_qdrant(ids): points_selector=PointIdsList(points=ids) ) except Exception as e: - logger.warning(f"Failed to delete vectors from Qdrant: {e}") \ No newline at end of file + logger.warning(f"Failed to delete vectors from Qdrant: {e}") diff --git a/backend/core/services/kb_utils.py b/backend/core/services/kb_utils.py index 9d73d88..f013a9d 100644 --- a/backend/core/services/kb_utils.py +++ b/backend/core/services/kb_utils.py @@ -84,4 +84,4 @@ def parse_kb_from_request(request): return parsed_items def format_text_uri(text_value: str) -> str: - return f"text://{text_value[:50]}" \ No newline at end of file + return f"text://{text_value[:50]}" diff --git a/backend/core/services/providers/__init__.py b/backend/core/services/providers/__init__.py new file mode 100644 index 0000000..dda9abe --- /dev/null +++ b/backend/core/services/providers/__init__.py @@ -0,0 +1,2 @@ +from .ai.gemini_provider import GeminiProvider +from .ai.custom_provider import CustomProvider \ No newline at end of file diff --git a/backend/core/services/providers/ai/__init__.py b/backend/core/services/providers/ai/__init__.py new file mode 100644 index 0000000..56e5361 --- /dev/null +++ b/backend/core/services/providers/ai/__init__.py @@ -0,0 +1,2 @@ +from .gemini_provider import GeminiProvider +from .custom_provider import CustomProvider diff --git a/backend/core/services/providers/ai/custom_provider.py b/backend/core/services/providers/ai/custom_provider.py new file mode 100644 index 0000000..6b7ecd0 --- /dev/null +++ b/backend/core/services/providers/ai/custom_provider.py @@ -0,0 +1,23 @@ +import json +import requests +from typing import Optional, Dict, Any, List +from ...contracts.ai_provider_contract import AIProviderContract + + +class CustomProvider(AIProviderContract): + def __init__(self, api_key: str, config: Optional[Dict[str, Any]] = None): + super().__init__(api_key, config) + + raise NotImplementedError("Not implemented") + + def generate_text(self, model: str, contents: str, **kwargs) -> str: + raise NotImplementedError("Not implemented") + + def validate_connection(self) -> tuple[bool, List[Dict[str, Any]]]: + raise NotImplementedError("Not implemented") + + def get_models(self) -> List[Dict[str, Any]]: + raise NotImplementedError("Not implemented") + + def embed(self, model: str, texts: list[str]) -> list[list[float]]: + raise NotImplementedError("Not implemented") diff --git a/backend/core/services/providers/ai/gemini_provider.py b/backend/core/services/providers/ai/gemini_provider.py new file mode 100644 index 0000000..5bc85fb --- /dev/null +++ b/backend/core/services/providers/ai/gemini_provider.py @@ -0,0 +1,56 @@ +from typing import Optional, Dict, Any +from google import genai +from google.genai.types import GenerateContentConfig +from ...contracts.ai_provider_contract import AIProviderContract +from core.agent_response_schema import SupportAgentResponse + +class GeminiProvider(AIProviderContract): + def __init__(self, api_key: str, config: Optional[Dict[str, Any]] = None): + super().__init__(api_key, config) + + try: + self.client = genai.Client(api_key=api_key) + except Exception as e: + raise ValueError(f"Failed to initialize Gemini client: {e}") + + def generate_text(self, model: str, contents: str, **kwargs) -> SupportAgentResponse: + try: + response = self.client.models.generate_content( + model=model, + contents=contents, + config=GenerateContentConfig( + response_mime_type="application/json", + response_json_schema=SupportAgentResponse.model_json_schema(), + ) + ) + + return SupportAgentResponse.model_validate_json(response.text) + + except Exception as e: + raise ValueError(f"Gemini API error: {e}") + + def validate_connection(self) -> tuple[bool, list[Dict[str, Any]]]: + try: + models = self.get_models() + return True, models + except Exception as e: + return False, [] + + def get_models(self) -> list[Dict[str, Any]]: + try: + models = list(self.client.models.list()) + serializable_models = [] + for model in models: + model_dict = vars(model).copy() + stringified_model = {key: str(value) for key, value in model_dict.items()} + serializable_models.append(stringified_model) + return serializable_models + except Exception as e: + raise ValueError(f"Failed to retrieve models from Gemini API: {e}") + + def embed(self, model: str, texts: list[str]) -> list[list[float]]: + try: + result = self.client.models.embed_content(model=model, contents=texts) + return [e.values for e in result.embeddings] + except Exception as e: + raise ValueError(f"Gemini embedding error: {e}") diff --git a/backend/core/services/unread.py b/backend/core/services/unread.py new file mode 100644 index 0000000..90cff60 --- /dev/null +++ b/backend/core/services/unread.py @@ -0,0 +1,72 @@ +import logging + +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer + +from core.models.chatroom_participant import ChatroomParticipant +from core.consts import LIVE_UPDATES_PREFIX + +logger = logging.getLogger(__name__) + + +def mark_unread_for_participants( + chatroom, sender_identifier: str, is_internal: bool = False +) -> list[str]: + """ + Atomically set has_unread=True on all ChatroomParticipant records + for the given chatroom, excluding the sender. + + When is_internal=True, only dashboard_ participants are updated + (internal messages are not visible to widget users). + + Returns the list of user_identifier values that were updated, + for use in broadcasting unread notifications. + """ + qs = ChatroomParticipant.objects.filter( + chatroom=chatroom + ).exclude( + user_identifier=sender_identifier + ) + if is_internal: + qs = qs.filter(user_identifier__startswith='dashboard_') + qs.update(has_unread=True) + return list(qs.values_list('user_identifier', flat=True)) + + +def broadcast_unread_update( + user_identifier: str, + chatroom_uuid: str, + has_unread: bool, + sender_identifier: str, +) -> None: + """ + Broadcast an unread_update event to the user's WebSocket group. + Logs a warning and continues on failure. + """ + channel_layer = get_channel_layer() + group_name = f"{LIVE_UPDATES_PREFIX}_{user_identifier}" + try: + async_to_sync(channel_layer.group_send)( + group_name, + { + "type": "send_unread_update", + "chatroom_uuid": chatroom_uuid, + "has_unread": has_unread, + "sender_identifier": sender_identifier, + }, + ) + except Exception as e: + logger.warning( + "Failed to broadcast unread update to group %s: %s", group_name, e + ) + + +def mark_read_for_participant(chatroom, user_identifier: str) -> None: + """ + Atomically set has_unread=False on the ChatroomParticipant record + for the given chatroom and user_identifier. + """ + ChatroomParticipant.objects.filter( + chatroom=chatroom, + user_identifier=user_identifier + ).update(has_unread=False) diff --git a/backend/core/tasks/__init__.py b/backend/core/tasks/__init__.py index 83bf924..49e1fec 100644 --- a/backend/core/tasks/__init__.py +++ b/backend/core/tasks/__init__.py @@ -1,4 +1,5 @@ from core.tasks.message import generate_bot_response from core.tasks.kb import process_kb from core.tasks.notification import send_notification_task -from core.tasks.email import send_verification_email_task, send_discord_notification_task \ No newline at end of file +from core.tasks.email import send_verification_email_task, send_discord_notification_task +from core.tasks.github_tasks import ingest_github_repository_task # noqa: F401 — ensures Celery autodiscovery \ No newline at end of file diff --git a/backend/core/tasks/github_tasks.py b/backend/core/tasks/github_tasks.py new file mode 100644 index 0000000..61a900b --- /dev/null +++ b/backend/core/tasks/github_tasks.py @@ -0,0 +1,244 @@ +import logging +from asgiref.sync import async_to_sync +from celery import shared_task +from channels.layers import get_channel_layer +from django.utils import timezone +from datetime import datetime, timedelta + +from core.models import AppIntegration +from core.models.github_data import GitHubRepository +from core.services.github_ingestion import GitHubDataIngestionService +from core.consts import DASHBOARD_USER_ID_PREFIX, LIVE_UPDATES_PREFIX + +logger = logging.getLogger(__name__) + + +def send_github_ingestion_update(repository, ingestion_status): + try: + channel_layer = get_channel_layer() + owner_id = repository.app_integration.application.owner.id + group_name = f"{LIVE_UPDATES_PREFIX}_{DASHBOARD_USER_ID_PREFIX}_{owner_id}" + + async_to_sync(channel_layer.group_send)( + group_name, + { + "type": "send.kb.updates", + "data": { + "id": str(repository.id), + "repository": repository.full_name, + "ingestion_status": ingestion_status, + }, + } + ) + except Exception as e: + logger.warning(f"Failed to send WebSocket update for {repository.full_name}: {e}") + + +@shared_task(bind=True, max_retries=3, default_retry_delay=60) +def ingest_github_repository_task(self, app_integration_id, owner, repo, since=None): + """Celery task for GitHub repository ingestion""" + logger.info(f"[Task] ingest_github_repository_task started: {owner}/{repo} (app_integration={app_integration_id}, retry={self.request.retries})") + try: + app_integration = AppIntegration.objects.select_related( + 'application__owner', 'integration' + ).get(id=app_integration_id) + logger.info(f"[Task] AppIntegration found: id={app_integration_id}, app={app_integration.application.name}") + + ingestion_service = GitHubDataIngestionService(app_integration, use_graphql=True) + ingestion_service.ingest_repository(owner, repo, since) + + logger.info(f"[Task] ingest_repository completed for {owner}/{repo}") + + try: + repository = GitHubRepository.objects.get( + app_integration_id=app_integration_id, + full_name=f'{owner}/{repo}' + ) + send_github_ingestion_update(repository, 'completed') + except GitHubRepository.DoesNotExist: + logger.warning(f"[Task] GitHubRepository not found after ingestion: {owner}/{repo}") + + return { + 'status': 'success', + 'repository': f'{owner}/{repo}', + 'completed_at': timezone.now().isoformat() + } + + except AppIntegration.DoesNotExist: + logger.error(f"[Task] AppIntegration {app_integration_id} not found") + raise + + except Exception as exc: + logger.error(f"[Task] ingest_github_repository_task failed for {owner}/{repo}: {exc}", exc_info=True) + + try: + repository = GitHubRepository.objects.get( + app_integration_id=app_integration_id, + full_name=f'{owner}/{repo}' + ) + repository.ingestion_status = 'failed' + repository.save() + send_github_ingestion_update(repository, 'failed') + + from core.models import KnowledgeBase + from core.tasks.kb import send_kb_update + kb = KnowledgeBase.objects.filter( + application=repository.app_integration.application, + source_type='github', + path=repository.full_name, + ).first() + if kb: + kb.status = 'failed' + kb.save(update_fields=['status']) + send_kb_update(kb, 'failed') + logger.info(f"[Task] Marked KB {kb.uuid} as failed") + except Exception as inner_exc: + logger.warning(f"[Task] Failed to update failure status: {inner_exc}") + + if self.request.retries < self.max_retries: + logger.info(f"[Task] Retrying (attempt {self.request.retries + 1}/{self.max_retries})") + raise self.retry(exc=exc, countdown=60 * (2 ** self.request.retries)) + else: + logger.error(f"[Task] Exhausted retries for {owner}/{repo}") + raise + + +@shared_task(bind=True, max_retries=2, default_retry_delay=30) +def sync_all_github_repositories_task(self, app_integration_id): + """Sync all repositories for a given integration""" + try: + logger.info(f"Starting sync for all GitHub repositories for integration {app_integration_id}") + + app_integration = AppIntegration.objects.get(id=app_integration_id) + repositories = GitHubRepository.objects.filter( + app_integration=app_integration, + ingestion_status='completed' + ) + + results = [] + for repository in repositories: + try: + owner, repo = repository.full_name.split('/') + + since = (timezone.now() - timedelta(days=7)).isoformat() + + ingestion_service = GitHubDataIngestionService(app_integration, use_graphql=True) + ingestion_service.ingest_repository(owner, repo, since) + + results.append({ + 'repository': repository.full_name, + 'status': 'success' + }) + + logger.info(f"Successfully synced {repository.full_name}") + + except Exception as e: + logger.error(f"Failed to sync {repository.full_name}: {e}") + results.append({ + 'repository': repository.full_name, + 'status': 'failed', + 'error': str(e) + }) + + logger.info(f"Completed sync for {len(results)} repositories") + return { + 'status': 'completed', + 'results': results, + 'completed_at': timezone.now().isoformat() + } + + except AppIntegration.DoesNotExist: + logger.error(f"AppIntegration {app_integration_id} not found") + raise + + except Exception as exc: + logger.error(f"GitHub sync task failed: {exc}") + + if self.request.retries < self.max_retries: + raise self.retry(exc=exc, countdown=30 * (2 ** self.request.retries)) + else: + raise + + +@shared_task(bind=True, max_retries=2, default_retry_delay=30) +def cleanup_old_github_data_task(self, days_old=90): + """Clean up old GitHub data to save storage space""" + try: + logger.info(f"Starting cleanup of GitHub data older than {days_old} days") + + cutoff_date = timezone.now() - timedelta(days=days_old) + + old_repositories = GitHubRepository.objects.filter( + last_ingested_at__lt=cutoff_date, + ingestion_status='completed' + ) + + deleted_count = 0 + for repository in old_repositories: + try: + repository_name = repository.full_name + repository.delete() + deleted_count += 1 + logger.info(f"Deleted old repository: {repository_name}") + + except Exception as e: + logger.error(f"Failed to delete repository {repository.full_name}: {e}") + + logger.info(f"Deleted {deleted_count} old GitHub repositories") + return { + 'status': 'completed', + 'deleted_count': deleted_count, + 'cutoff_date': cutoff_date.isoformat() + } + + except Exception as exc: + logger.error(f"GitHub cleanup task failed: {exc}") + + if self.request.retries < self.max_retries: + raise self.retry(exc=exc, countdown=30 * (2 ** self.request.retries)) + else: + raise + + +@shared_task(bind=True, max_retries=1, default_retry_delay=15) +def update_github_repository_status_task(self, repository_id): + """Update repository status and metadata""" + try: + from core.services.github_client import GitHubAPIClient + + repository = GitHubRepository.objects.get(id=repository_id) + owner, repo = repository.full_name.split('/') + + token = repository.app_integration.integration.config.get('token') + if not token: + logger.error("No GitHub token found for repository") + return + + client = GitHubAPIClient(token) + + repo_info = client.get_repository_info(owner, repo) + + repository.description = repo_info.get('description', '') + repository.url = repo_info.get('html_url', '') + repository.is_private = repo_info.get('private', False) + repository.default_branch = repo_info.get('default_branch', 'main') + repository.save() + + logger.info(f"Updated metadata for {repository.full_name}") + return { + 'status': 'success', + 'repository': repository.full_name, + 'updated_at': timezone.now().isoformat() + } + + except GitHubRepository.DoesNotExist: + logger.error(f"Repository {repository_id} not found") + raise + + except Exception as exc: + logger.error(f"Failed to update repository status: {exc}") + + if self.request.retries < self.max_retries: + raise self.retry(exc=exc, countdown=15) + else: + raise diff --git a/backend/core/tasks/kb.py b/backend/core/tasks/kb.py index 9d30e0e..d94eda9 100644 --- a/backend/core/tasks/kb.py +++ b/backend/core/tasks/kb.py @@ -1,32 +1,45 @@ +import logging + from asgiref.sync import async_to_sync from celery import shared_task from channels.layers import get_channel_layer -from core.consts import REGISTERED_USER_ID_PREFIX, LIVE_UPDATES_PREFIX +from core.consts import DASHBOARD_USER_ID_PREFIX, LIVE_UPDATES_PREFIX from core.models import KnowledgeBase from core.models.knowledge_base import KBStatus from core.services import extract_text_from_file, ingest_kb +logger = logging.getLogger(__name__) + def send_kb_update(kb, status): - channel_layer = get_channel_layer() - owner_id = kb.application.owner.id - - group_name = f"{LIVE_UPDATES_PREFIX}_{REGISTERED_USER_ID_PREFIX}_{owner_id}" - - data = { - "id": str(kb.id), - "uuid": str(kb.uuid), - "status": status, - } - - async_to_sync(channel_layer.group_send)( - group_name, - { - "type": "send.kb.updates", - "data": data, + try: + channel_layer = get_channel_layer() + if channel_layer is None: + logger.warning(f"[send_kb_update] Channel layer is None, skipping WS update for kb {kb.uuid}") + return + + owner_id = kb.application.owner.id + group_name = f"{LIVE_UPDATES_PREFIX}_{DASHBOARD_USER_ID_PREFIX}_{owner_id}" + + data = { + "id": str(kb.id), + "uuid": str(kb.uuid), + "status": status, } - ) + + logger.info(f"[send_kb_update] Sending status={status} for kb={kb.uuid} to group={group_name}") + + async_to_sync(channel_layer.group_send)( + group_name, + { + "type": "send.kb.updates", + "data": data, + } + ) + logger.info(f"[send_kb_update] Successfully sent update for kb={kb.uuid}") + except Exception as e: + logger.error(f"[send_kb_update] Failed to send WS update for kb={kb.uuid}: {e}", exc_info=True) def process_kb_item(kb): if kb.source_type == 'file': diff --git a/backend/core/tasks/message.py b/backend/core/tasks/message.py index 90e66b2..df355b4 100644 --- a/backend/core/tasks/message.py +++ b/backend/core/tasks/message.py @@ -1,4 +1,3 @@ -import json import logging from celery import shared_task @@ -8,30 +7,100 @@ from django.db.models import Q from core.consts import LIVE_UPDATES_PREFIX -from core.integrations import get_app_integrations, execute_tool_call -from core.llm_client import LLMClient -from core.llm_client_utils import messages_to_llm_conversation, get_agent_response_schema, add_kb_to_convo, \ +from core.llm_client_utils import messages_to_llm_conversation, add_kb_to_convo, \ add_instructions_to_convo -from core.models import IngestedChunk, Application, LLMModel -from core.models.message import Message -from core.prompts import SMART_ESCALATION_TEMPLATE +from core.models import IngestedChunk, Application, AIProvider, Message from core.serializers.message import ViewMessageSerializer from core.services.ingestion import get_chunks from core.services.template_loader import TemplateLoader -from core.utils import parse_llm_response +from core.services.ai_client_service import AIClientService logger = logging.getLogger(__name__) AGENT_IDENTIFIER = getattr(settings, "DEFAULT_AGENT_IDENTIFIER", "agent_llm_001") +def _send_live_update(bot_message: Message, user_message: Message): + channel_layer = get_channel_layer() + + if user_message.platform == 'widget': + participants = list( + user_message.chatroom.participants.filter( + Q(user_identifier__startswith='widget_') | + Q(user_identifier__startswith='dashboard_') | + Q(role='human_agent') + ).exclude( + Q(role='agent') + ).values_list('user_identifier', flat=True) + ) + else: + participants = list( + user_message.chatroom.participants.filter( + Q(user_identifier__startswith='dashboard_') | Q(role='human_agent') + ).exclude( + Q(role='agent') + ).values_list('user_identifier', flat=True) + ) + + for participant_id in participants: + group_name = f"{LIVE_UPDATES_PREFIX}_{participant_id}" + try: + async_to_sync(channel_layer.group_send)( + group_name, + { + "type": "send.message", + "message": ViewMessageSerializer(bot_message).data, + } + ) + logger.info(f"Live update sent to {group_name}") + except Exception as e: + logger.error(f"Failed to send message to {group_name}: {str(e)}") + logger.error(f"Exception type: {type(e).__name__}") + + if not participants: + logger.warning("No participants found for live update! ") + @shared_task -def generate_bot_response(message_id, app_uuid): +def generate_bot_response(message_id, app_uuid, ai_provider_id=None, model=None): app = Application.objects.get(uuid=app_uuid) user_message = Message.objects.get(id=message_id) chatroom = user_message.chatroom question = user_message.message + ai_client_service = AIClientService() + provider, model = ai_client_service.get_client_and_model( + app=app, + ai_provider_id=ai_provider_id, + model=model, + context='response', + capability='text' + ) + + if not provider or not model: + error_message = "No AI provider configured or available" + answer = error_message + metadata = { + "status": "ERROR", + "escalation": True, + "reason_for_escalation": error_message, + "error_details": error_message, + } + + bot_message = Message.objects.create( + chatroom=chatroom, + sender_identifier=AGENT_IDENTIFIER, + message=answer, + metadata=metadata, + ai_provider_id=ai_provider_id, + model=model, + platform=user_message.platform, + ai_mode=True, + is_internal=user_message.is_internal, + ) + + _send_live_update(bot_message, user_message) + return + has_chunks = IngestedChunk.objects.filter( knowledge_base__application__uuid=app_uuid ).exists() @@ -45,88 +114,35 @@ def generate_bot_response(message_id, app_uuid): prompt_context = { "product_name": app.name, - "product_type": 'Software as a Service (SaaS)', - "tone": "friendly and professional", + "tone": "professional", } - system_instruction = TemplateLoader.render_template('customer_support.j2', prompt_context) - - text_model = app.get_model_by_type(LLMModel.ModelType.TEXT) - client = LLMClient( - base_url=text_model.base_url, - api_key=text_model.config, - ) + system_instruction = TemplateLoader.render_template('prompts/default.j2', prompt_context) + print(system_instruction) conversation = messages_to_llm_conversation(messages) conversation = add_instructions_to_convo(conversation, system_instruction) conversation = add_kb_to_convo(conversation, kb_data) - response_schema = get_agent_response_schema("support_response") - - tools = get_app_integrations(app) - logger.info("Tools: %s ", tools) - logger.info("Conversation: %s", conversation) - tool_call_response = client.chat( - messages=conversation, - model=text_model.model_name, - tools=tools - ) - - logger.info("Tool_call_response: %s", tool_call_response) - tool_results = {} - - for choice in tool_call_response.choices: - msg = choice.message - if msg.tool_calls: - for tool_call in msg.tool_calls: - tool_name = tool_call.function.name - - args = ( - json.loads(tool_call.function.arguments) - if isinstance(tool_call.function.arguments, str) - else tool_call.function.arguments - ) - - tool_results[tool_name] = execute_tool_call(app, tool_name, **args) - logger.info(f"Tool: {tool_name}, Result: {tool_results[tool_name]}") - - conversation.append({ - "role": "assistant", - "type": "function_call_output", - "call_id": tool_call.id, - "content": json.dumps(tool_results[tool_name]), - }) - - llm_response = client.chat( - conversation, - model=text_model.model_name, - response_schema=response_schema - ) - - escalation = False + prompt = "\n".join([f"{msg.get('role', 'user')}: {msg.get('content', '')}" for msg in conversation]) try: - llm_response_data = parse_llm_response(llm_response.choices[0].message.content) - - logger.info("Final LLM Response:\n%s", json.dumps(llm_response_data, indent=2)) - - - answer = llm_response_data.get("answer", "").strip() - status = llm_response_data.get("status", "ERROR").strip() - escalation = llm_response_data.get("escalation", False) - reason = llm_response_data.get("reason_for_escalation", "").strip() - + agent_response = provider.generate_text(model, prompt) + answer = agent_response.answer metadata = { - "status": status, - "escalation": escalation, - "reason_for_escalation": reason, + "status": agent_response.status, + "escalation": agent_response.escalation, + "reason_for_escalation": agent_response.reason_for_escalation, } - - except json.JSONDecodeError: - answer = llm_response.content.strip() + except Exception as e: + logger.error(f"Failed to generate content: {e}") + error_message = str(e) + + answer = error_message metadata = { "status": "ERROR", "escalation": True, - "reason_for_escalation": "Malformed LLM response", + "reason_for_escalation": error_message, + "error_details": error_message, } bot_message = Message.objects.create( @@ -134,37 +150,11 @@ def generate_bot_response(message_id, app_uuid): sender_identifier=AGENT_IDENTIFIER, message=answer, metadata=metadata, + ai_provider_id=ai_provider_id, + model=model, + platform=user_message.platform, + ai_mode=True, + is_internal=user_message.is_internal, ) - channel_layer = get_channel_layer() - participants = list( - user_message.chatroom.participants.exclude( - Q(role='agent') - ).values_list('user_identifier', flat=True) - ) - - for participant_id in participants: - group_name = f"{LIVE_UPDATES_PREFIX}_{participant_id}" - try: - async_to_sync(channel_layer.group_send)( - group_name, - { - "type": "send.message", - "message": ViewMessageSerializer(bot_message).data, - } - ) - except Exception as e: - logger.warning(f"Failed to send message to {group_name}: {str(e)}") - - print(escalation) - if escalation: - from core.services.notifications import notify_users - - context = { - "app": chatroom.application, - "chatroom_uuid": chatroom.uuid, - "user_id": user_message.sender_identifier, - "user_query": user_message.message, - "agent_response": answer, - } - notify_users(chatroom.application, SMART_ESCALATION_TEMPLATE, context) + _send_live_update(bot_message, user_message) diff --git a/backend/core/tests/__init__.py b/backend/core/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/core/tests/conftest.py b/backend/core/tests/conftest.py new file mode 100644 index 0000000..32b387c --- /dev/null +++ b/backend/core/tests/conftest.py @@ -0,0 +1,78 @@ +import os +import django +from django.conf import settings +from pathlib import Path + +if not settings.configured: + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.test_settings') + + base_dir = Path(__file__).resolve().parent.parent.parent + env_path = base_dir / '.env' + if env_path.exists(): + from dotenv import load_dotenv + load_dotenv(env_path) + + django.setup() + +import pytest +from django.test import TestCase +from rest_framework.test import APITestCase, APIClient +from factory.django import DjangoModelFactory +import factory + +def pytest_configure(config): + config.addinivalue_line("markers", "unit: Unit tests") + config.addinivalue_line("markers", "integration: Integration tests") + config.addinivalue_line("markers", "api: API endpoint tests") + config.addinivalue_line("markers", "slow: Slow running tests") + + +@pytest.fixture(autouse=True) +def enable_db_access_for_all_tests(db): + pass + + +@pytest.fixture +def api_client(): + return APIClient() + + +@pytest.fixture +def authenticated_client(api_client, user_factory): + user = user_factory() + api_client.force_authenticate(user=user) + return api_client, user + + +class BaseTestCase(TestCase): + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + + +class BaseAPITestCase(APITestCase): + def setUp(self): + super().setUp() + self.client = APIClient() + + def tearDown(self): + super().tearDown() + + +class BaseServiceTestCase(TestCase): + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + + +class BaseFactory(DjangoModelFactory): + class Meta: + abstract = True + + @classmethod + def _create(cls, model_class, *args, **kwargs): + return super()._create(model_class, *args, **kwargs) diff --git a/backend/core/tests/factories.py b/backend/core/tests/factories.py new file mode 100644 index 0000000..92143dc --- /dev/null +++ b/backend/core/tests/factories.py @@ -0,0 +1,61 @@ +import factory +from django.contrib.auth.models import User +from core.models import ( + Application, + AIProvider, + Integration, + AppIntegration +) + +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = User + + username = factory.Sequence(lambda n: f'testuser{n}') + email = factory.Sequence(lambda n: f'testuser{n}@example.com') + first_name = factory.Faker('first_name') + last_name = factory.Faker('last_name') + is_active = True + +class ApplicationFactory(factory.django.DjangoModelFactory): + class Meta: + model = Application + + owner = factory.SubFactory(UserFactory) + name = factory.Faker('company') + uuid = factory.Faker('uuid4') + +class AIProviderFactory(factory.django.DjangoModelFactory): + class Meta: + model = AIProvider + + name = factory.Faker('company') + provider = factory.Iterator(['openai', 'anthropic', 'google']) + provider_api_key = factory.Faker('password') + metadata = factory.LazyAttribute(lambda obj: {'base_url': 'https://example.com'}) + is_builtin = False + creator = factory.SubFactory(UserFactory) + + +class IntegrationFactory(factory.django.DjangoModelFactory): + class Meta: + model = Integration + + name = factory.Sequence(lambda n: f"Integration {n}") + integration_type = factory.Iterator(['github', 'slack', 'jira']) + config = factory.Lambda(lambda: { + 'token': 'test_token', + 'api_url': 'https://api.example.com' + }) + + +class AppIntegrationFactory(factory.django.DjangoModelFactory): + class Meta: + model = AppIntegration + + application = factory.SubFactory(ApplicationFactory) + integration = factory.SubFactory(IntegrationFactory) + metadata = factory.Lambda(lambda: { + 'enabled': True, + 'sync_frequency': 'daily' + }) diff --git a/backend/core/tests/test_ai_provider.py b/backend/core/tests/test_ai_provider.py new file mode 100644 index 0000000..2922049 --- /dev/null +++ b/backend/core/tests/test_ai_provider.py @@ -0,0 +1,403 @@ +import pytest +from rest_framework import status +from unittest.mock import patch + +from core.models import AIProvider +from core.serializers.ai_provider import AIProviderCreateSerializer +from core.tests.conftest import BaseAPITestCase +from core.tests.factories import UserFactory, AIProviderFactory + + +@pytest.mark.api +class TestAIProviderAPI(BaseAPITestCase): + """Test suite for AI Provider API endpoints.""" + + def setUp(self): + super().setUp() + self.list_url = '/api/ai-providers/' + + def test_list_ai_providers_authenticated_user(self): + """Test that authenticated users can list their AI providers.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + provider1 = AIProviderFactory(creator=user, name="OpenAI Provider") + provider2 = AIProviderFactory(creator=user, name="Anthropic Provider") + + other_user = UserFactory() + other_provider = AIProviderFactory(creator=other_user, name="Other User Provider") + + response = self.client.get(self.list_url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + data = response.json() + self.assertEqual(len(data['results']), 2) + + provider_names = [provider['name'] for provider in data['results']] + self.assertIn("OpenAI Provider", provider_names) + self.assertIn("Anthropic Provider", provider_names) + self.assertNotIn("Other User Provider", provider_names) + + provider_data = data['results'][0] + expected_fields = ['id', 'uuid', 'name', 'provider', 'is_builtin', 'creator', 'created_at', 'updated_at', 'metadata'] + for field in expected_fields: + self.assertIn(field, provider_data) + + self.assertIn('supported_ai_providers', data) + self.assertIsInstance(data['supported_ai_providers'], list) + + def test_list_ai_providers_includes_builtin_and_user_owned(self): + """Test that authenticated users can list their AI providers plus builtin providers.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + user_provider1 = AIProviderFactory(creator=user, name="User OpenAI Provider", is_builtin=False) + user_provider2 = AIProviderFactory(creator=user, name="User Anthropic Provider", is_builtin=False) + + builtin_provider1 = AIProviderFactory(creator=user, name="Builtin OpenAI", is_builtin=True) + builtin_provider2 = AIProviderFactory(creator=user, name="Builtin Claude", is_builtin=True) + + other_user = UserFactory() + other_user_provider = AIProviderFactory(creator=other_user, name="Other User Provider", is_builtin=False) + + response = self.client.get(self.list_url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + data = response.json() + self.assertEqual(len(data['results']), 4) + + provider_names = [provider['name'] for provider in data['results']] + self.assertIn("User OpenAI Provider", provider_names) + self.assertIn("User Anthropic Provider", provider_names) + self.assertIn("Builtin OpenAI", provider_names) + self.assertIn("Builtin Claude", provider_names) + self.assertNotIn("Other User Provider", provider_names) + + builtin_providers = [p for p in data['results'] if p['is_builtin']] + user_owned_providers = [p for p in data['results'] if not p['is_builtin']] + + self.assertEqual(len(builtin_providers), 2) + self.assertEqual(len(user_owned_providers), 2) + + builtin_names = [p['name'] for p in builtin_providers] + self.assertIn("Builtin OpenAI", builtin_names) + self.assertIn("Builtin Claude", builtin_names) + + def test_list_ai_providers_unauthenticated(self): + """Test that unauthenticated users cannot list AI providers.""" + response = self.client.get(self.list_url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_retrieve_other_users_provider(self): + """Test that user A cannot retrieve user B's AI provider by ID.""" + user_a = UserFactory() + user_b = UserFactory() + + provider_b = AIProviderFactory(creator=user_b, name="User B's Provider") + + self.client.force_authenticate(user=user_a) + + detail_url = f'/api/ai-providers/{provider_b.uuid}/' + response = self.client.get(detail_url) + + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_update_other_users_provider(self): + """Test that user A cannot update user B's AI provider.""" + user_a = UserFactory() + user_b = UserFactory() + + provider_b = AIProviderFactory(creator=user_b, name="User B's Provider") + + self.client.force_authenticate(user=user_a) + + detail_url = f'/api/ai-providers/{provider_b.uuid}/' + update_data = {'name': 'Hacked Name'} + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_delete_other_users_provider(self): + """Test that user A cannot delete user B's AI provider.""" + user_a = UserFactory() + user_b = UserFactory() + + provider_b = AIProviderFactory(creator=user_b, name="User B's Provider") + + self.client.force_authenticate(user=user_a) + + detail_url = f'/api/ai-providers/{provider_b.uuid}/' + response = self.client.delete(detail_url) + + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + @patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') + def test_create_ai_provider(self, mock_validate): + """Test that authenticated user can create their own AI provider.""" + mock_validate.return_value = (True, ['gemini-1.5-pro', 'gemini-1.5-flash']) + + user = UserFactory() + self.client.force_authenticate(user=user) + + create_data = { + 'name': 'My Gemini Provider', + 'provider': 'gemini', + 'base_url': 'https://generativelanguage.googleapis.com', + 'provider_api_key': 'sk-test123456789' + } + + response = self.client.post(self.list_url, create_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + data = response.json() + + self.assertEqual(data['ai_provider']['name'], 'My Gemini Provider') + self.assertEqual(data['ai_provider']['provider'], 'gemini') + self.assertEqual(data['ai_provider']['metadata']['base_url'], 'https://generativelanguage.googleapis.com') + self.assertEqual(data['ai_provider']['creator'], user.id) + self.assertTrue(data['validation']['is_valid']) + self.assertEqual(data['validation']['models'], ['gemini-1.5-pro', 'gemini-1.5-flash']) + + provider = AIProvider.objects.get(uuid=data['ai_provider']['uuid']) + self.assertEqual(provider.creator, user) + self.assertEqual(provider.name, 'My Gemini Provider') + self.assertEqual(provider.metadata['base_url'], 'https://generativelanguage.googleapis.com') + + @patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') + def test_update_own_provider(self, mock_validate): + """Test that authenticated user can update their own AI provider.""" + mock_validate.return_value = (True, ['gemini-1.5-pro', 'gemini-1.5-flash']) + + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, name="Original Name") + + detail_url = f'/api/ai-providers/{provider.uuid}/' + update_data = {'name': 'Updated Name'} + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + + self.assertEqual(data['name'], 'Updated Name') + self.assertEqual(data['creator'], user.id) + + provider.refresh_from_db() + self.assertEqual(provider.name, 'Updated Name') + + def test_cannot_update_provider_field(self): + """Test that authenticated user cannot update the provider field of their AI provider.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, provider='openai', name="Original Name") + + detail_url = f'/api/ai-providers/{provider.uuid}/' + update_data = { + 'name': 'Updated Name', + 'provider': 'anthropic' + } + with patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') as mock_validate: + mock_validate.return_value = (True, ['gpt-3.5-turbo']) + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + provider.refresh_from_db() + self.assertEqual(provider.name, 'Updated Name') + self.assertEqual(provider.provider, 'openai') + + def test_delete_own_provider(self): + """Test that authenticated user can delete their own AI provider.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, name="Provider to Delete") + + detail_url = f'/api/ai-providers/{provider.uuid}/' + response = self.client.delete(detail_url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + with self.assertRaises(AIProvider.DoesNotExist): + AIProvider.objects.get(id=provider.id) + + def test_update_without_api_key_does_not_change_api_key(self): + """Test that update request without specifying provider api key does not update provider api key.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, name="Test Provider") + original_api_key = provider.provider_api_key + + detail_url = f'/api/ai-providers/{provider.uuid}/' + update_data = {'name': 'Updated Name'} + with patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') as mock_validate: + mock_validate.return_value = (True, ['gpt-3.5-turbo']) + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + provider.refresh_from_db() + self.assertEqual(provider.name, 'Updated Name') + self.assertEqual(provider.provider_api_key, original_api_key) + + @patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') + def test_update_with_api_key_changes_api_key(self, mock_validate): + """Test that update request with provider api key updates the provider api key.""" + mock_validate.return_value = (True, ['gemini-1.5-pro', 'gemini-1.5-flash']) + + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, name="Test Provider") + original_api_key = provider.provider_api_key + new_api_key = 'new-api-key-12345' + + detail_url = f'/api/ai-providers/{provider.uuid}/' + update_data = {'name': 'Updated Name', 'provider_api_key': new_api_key} + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + provider.refresh_from_db() + self.assertEqual(provider.name, 'Updated Name') + self.assertEqual(provider.provider_api_key, new_api_key) + self.assertNotEqual(provider.provider_api_key, original_api_key) + + def test_update_with_empty_api_key_does_not_change_api_key(self): + """Test that update request with empty string provider api key does not update the provider api key.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, name="Test Provider") + original_api_key = provider.provider_api_key + + detail_url = f'/api/ai-providers/{provider.uuid}/' + update_data = {'name': 'Updated Name', 'provider_api_key': ''} + with patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') as mock_validate: + mock_validate.return_value = (True, ['gpt-3.5-turbo']) + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + provider.refresh_from_db() + self.assertEqual(provider.name, 'Updated Name') + self.assertEqual(provider.provider_api_key, original_api_key) + + def test_update_with_whitespace_api_key_does_not_change_api_key(self): + """Test that update request with whitespace-only provider api key does not update the provider api key.""" + user = UserFactory() + self.client.force_authenticate(user=user) + + provider = AIProviderFactory(creator=user, name="Test Provider") + original_api_key = provider.provider_api_key + + detail_url = f'/api/ai-providers/{provider.uuid}/' + update_data = {'name': 'Updated Name', 'provider_api_key': ' '} + with patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') as mock_validate: + mock_validate.return_value = (True, ['gpt-3.5-turbo']) + response = self.client.patch(detail_url, update_data, format='json') + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + provider.refresh_from_db() + self.assertEqual(provider.name, 'Updated Name') + self.assertEqual(provider.provider_api_key, original_api_key) + + @patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') + def test_api_key_is_encrypted_in_database(self, mock_validate): + """Test that provider_api_key is encrypted when stored in the database.""" + mock_validate.return_value = (True, ['gemini-1.5-pro', 'gemini-1.5-flash']) + + user = UserFactory() + self.client.force_authenticate(user=user) + + api_key = 'test-api-key-12345' + create_data = { + 'name': 'Test Provider', + 'provider': 'gemini', + 'base_url': 'https://generativelanguage.googleapis.com', + 'provider_api_key': api_key + } + + response = self.client.post(self.list_url, create_data, format='json') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + provider = AIProvider.objects.get(uuid=response.json()['ai_provider']['uuid']) + + self.assertEqual(provider.provider_api_key, api_key) + + from django.db import connection + with connection.cursor() as cursor: + cursor.execute("SELECT provider_api_key FROM core_aiprovider WHERE id = %s", [provider.id]) + raw_db_value = cursor.fetchone()[0] + self.assertNotEqual(raw_db_value, api_key) + self.assertTrue(raw_db_value.startswith('gAAAAA')) + + + @patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') + def test_create_with_supported_provider_gemini(self, mock_validate): + """Test that AI provider can be created with supported 'gemini' provider.""" + mock_validate.return_value = (True, ['gemini-1.5-pro', 'gemini-1.5-flash']) + + user = UserFactory() + data = { + 'name': 'My Google Gemini Provider', + 'provider': 'gemini', + 'base_url': 'https://generativelanguage.googleapis.com', + 'provider_api_key': 'test-api-key-12345' + } + + serializer = AIProviderCreateSerializer(data=data, context={'request': type('MockRequest', (), {'user': user})()}) + + assert serializer.is_valid(), f"Serializer should be valid but got errors: {serializer.errors}" + provider = serializer.save() + + assert provider.name == 'My Google Gemini Provider' + assert provider.provider == 'gemini' + assert provider.metadata['base_url'] == 'https://generativelanguage.googleapis.com' + assert provider.creator == user + + @patch('core.services.factories.ai_provider_factory.AIProviderFactory.validate_provider') + def test_create_with_supported_provider_custom(self, mock_validate): + """Test that AI provider can be created with supported 'custom' provider.""" + mock_validate.return_value = (True, ['custom-model-1', 'custom-model-2']) + + user = UserFactory() + data = { + 'name': 'My Custom Provider', + 'provider': 'custom', + 'base_url': 'https://my-custom-api.com', + 'provider_api_key': 'test-api-key-67890' + } + + serializer = AIProviderCreateSerializer(data=data, context={'request': type('MockRequest', (), {'user': user})()}) + + assert serializer.is_valid() + provider = serializer.save() + + assert provider.name == 'My Custom Provider' + assert provider.provider == 'custom' + assert provider.metadata['base_url'] == 'https://my-custom-api.com' + assert provider.creator == user + + def test_create_with_unsupported_provider_fails(self): + """Test that AI provider creation fails with unsupported provider.""" + user = UserFactory() + data = { + 'name': 'My Unsupported Provider', + 'provider': 'unsupported_provider', + 'base_url': 'https://unsupported-api.com', + 'provider_api_key': 'test-api-key-12345' + } + + serializer = AIProviderCreateSerializer(data=data, context={'request': type('MockRequest', (), {'user': user})()}) + + assert not serializer.is_valid() + assert 'provider' in serializer.errors + assert "not supported" in str(serializer.errors['provider'][0]) + assert "Google Gemini" in str(serializer.errors['provider'][0]) diff --git a/backend/core/tests/test_app_ai_provider.py b/backend/core/tests/test_app_ai_provider.py new file mode 100644 index 0000000..c3a9a5a --- /dev/null +++ b/backend/core/tests/test_app_ai_provider.py @@ -0,0 +1,339 @@ +import pytest +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase +from django.contrib.auth.models import User + +from core.models import Application, AIProvider, AppAIProvider + + +class AppAIProviderTest(APITestCase): + def setUp(self): + self.user = User.objects.create_user(username='test', password='test') + self.application = Application.objects.create(owner=self.user, name='Test App') + self.ai_provider = AIProvider.objects.create( + name='Test Provider', + provider='openai', + provider_api_key='test-key', + metadata={'base_url': 'https://api.openai.com'}, + creator=self.user + ) + + def tearDown(self): + AppAIProvider.objects.all().delete() + + def test_create_app_ai_provider(self): + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + data = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'widget', + 'capability': 'text', + 'external_model_id': 'gpt-4' + } + response = self.client.post(url, data) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.data['context'], 'widget') + self.assertEqual(response.data['capability'], 'text') + self.assertEqual(response.data['priority'], 100) + self.assertEqual(response.data['external_model_id'], 'gpt-4') + self.assertEqual(response.data['ai_provider']['id'], self.ai_provider.id) + + def test_list_app_ai_providers(self): + config = AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='text', + external_model_id='gpt-4' + ) + + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + self.assertEqual(response.data['results'][0]['id'], config.id) + + def test_filter_by_context(self): + AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='text' + ) + AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='dashboard', + capability='text' + ) + + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + response = self.client.get(url, {'context': 'widget'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + self.assertEqual(response.data['results'][0]['context'], 'widget') + + def test_filter_by_capability(self): + AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='text' + ) + AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='image' + ) + + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + response = self.client.get(url, {'capability': 'text'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data['results']), 1) + self.assertEqual(response.data['results'][0]['capability'], 'text') + + def test_update_app_ai_provider(self): + config = AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='text', + external_model_id='gpt-4' + ) + + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-detail', kwargs={ + 'application_uuid': self.application.uuid, + 'uuid': config.uuid + }) + data = {'external_model_id': 'gpt-3.5-turbo'} + response = self.client.put(url, data) + self.assertEqual(response.status_code, status.HTTP_200_OK) + config.refresh_from_db() + self.assertEqual(config.external_model_id, 'gpt-3.5-turbo') + + def test_delete_app_ai_provider(self): + config = AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider + ) + + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-detail', kwargs={ + 'application_uuid': self.application.uuid, + 'uuid': config.uuid + }) + response = self.client.delete(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertFalse(AppAIProvider.objects.filter(id=config.id).exists()) + + def test_priority_auto_assignment(self): + config1 = AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='text' + ) + self.assertEqual(config1.priority, 100) + + config2 = AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='widget', + capability='text' + ) + self.assertEqual(config2.priority, 200) + + config3 = AppAIProvider.objects.create( + application=self.application, + ai_provider=self.ai_provider, + context='dashboard', + capability='text' + ) + self.assertEqual(config3.priority, 100) + + def test_unauthorized_access(self): + other_user = User.objects.create_user(username='other', password='other') + other_app = Application.objects.create(owner=other_user, name='Other App') + other_ai_provider = AIProvider.objects.create( + name='Other AI Provider', + provider='openai', + provider_api_key='test', + metadata={'base_url': 'https://api.openai.com'}, + creator=other_user + ) + other_config = AppAIProvider.objects.create( + application=other_app, + ai_provider=other_ai_provider, + context='widget', + capability='text', + external_model_id='gpt-4' + ) + + self.client.force_authenticate(user=self.user) + + url = reverse('application-ai-providers-list', kwargs={'application_uuid': other_app.uuid}) + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + data = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'widget', + 'capability': 'text', + 'external_model_id': 'gpt-4' + } + response = self.client.post(url, data) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + update_url = reverse('application-ai-providers-detail', kwargs={ + 'application_uuid': other_app.uuid, + 'uuid': other_config.uuid + }) + update_data = {'external_model_id': 'gpt-3.5-turbo'} + response = self.client.put(update_url, update_data) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + response = self.client.get(update_url) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + response = self.client.delete(update_url) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_invalid_ai_provider(self): + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + data = { + 'ai_provider_id': 999, + 'context': 'widget', + 'capability': 'text' + } + response = self.client.post(url, data) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_uniqueness_constraint_same_app_context_capability(self): + """Test that only one record exists for the same app/context/capability pair.""" + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + + data1 = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'text', + 'capability': 'response', + 'external_model_id': 'gpt-4' + } + response1 = self.client.post(url, data1) + self.assertEqual(response1.status_code, status.HTTP_201_CREATED) + + data2 = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'text', + 'capability': 'response', + 'external_model_id': 'gpt-3.5-turbo' + } + response2 = self.client.post(url, data2) + self.assertEqual(response2.status_code, status.HTTP_201_CREATED) + + configs = AppAIProvider.objects.filter( + application=self.application, + context='text', + capability='response' + ) + self.assertEqual(configs.count(), 1) + + config = configs.first() + self.assertEqual(config.external_model_id, 'gpt-3.5-turbo') + + def test_multiple_different_context_capability_pairs_allowed(self): + """Test that an app can have multiple records with different context/capability pairs.""" + self.client.force_authenticate(user=self.user) + url = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + + data1 = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'text', + 'capability': 'response', + 'external_model_id': 'gpt-4' + } + response1 = self.client.post(url, data1) + self.assertEqual(response1.status_code, status.HTTP_201_CREATED) + + data2 = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'embedding', + 'capability': 'embedding', + 'external_model_id': 'text-embedding-ada-002' + } + response2 = self.client.post(url, data2) + self.assertEqual(response2.status_code, status.HTTP_201_CREATED) + + data3 = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'widget', + 'capability': 'text', + 'external_model_id': 'gpt-3.5-turbo' + } + response3 = self.client.post(url, data3) + self.assertEqual(response3.status_code, status.HTTP_201_CREATED) + + configs = AppAIProvider.objects.filter(application=self.application) + self.assertEqual(configs.count(), 3) + + text_response = configs.filter(context='text', capability='response') + embedding_embedding = configs.filter(context='embedding', capability='embedding') + widget_text = configs.filter(context='widget', capability='text') + + self.assertEqual(text_response.count(), 1) + self.assertEqual(embedding_embedding.count(), 1) + self.assertEqual(widget_text.count(), 1) + + self.assertEqual(text_response.first().external_model_id, 'gpt-4') + self.assertEqual(embedding_embedding.first().external_model_id, 'text-embedding-ada-002') + self.assertEqual(widget_text.first().external_model_id, 'gpt-3.5-turbo') + + def test_different_apps_can_have_same_context_capability(self): + """Test that different apps can have the same context/capability pairs independently.""" + other_user = User.objects.create_user(username='other_user', password='test') + other_app = Application.objects.create(owner=other_user, name='Other App') + other_ai_provider = AIProvider.objects.create( + name='Other AI Provider', + provider='openai', + provider_api_key='test-key-2', + metadata={'base_url': 'https://api.openai.com'}, + creator=other_user + ) + + self.client.force_authenticate(user=self.user) + url1 = reverse('application-ai-providers-list', kwargs={'application_uuid': self.application.uuid}) + data1 = { + 'ai_provider_id': self.ai_provider.id, + 'context': 'text', + 'capability': 'response', + 'external_model_id': 'gpt-4' + } + response1 = self.client.post(url1, data1) + self.assertEqual(response1.status_code, status.HTTP_201_CREATED) + + self.client.force_authenticate(user=other_user) + url2 = reverse('application-ai-providers-list', kwargs={'application_uuid': other_app.uuid}) + data2 = { + 'ai_provider_id': other_ai_provider.id, + 'context': 'text', + 'capability': 'response', + 'external_model_id': 'gpt-3.5-turbo' + } + response2 = self.client.post(url2, data2) + self.assertEqual(response2.status_code, status.HTTP_201_CREATED) + + app1_configs = AppAIProvider.objects.filter(application=self.application) + app2_configs = AppAIProvider.objects.filter(application=other_app) + + self.assertEqual(app1_configs.count(), 1) + self.assertEqual(app2_configs.count(), 1) + + self.assertEqual(app1_configs.first().external_model_id, 'gpt-4') + self.assertEqual(app2_configs.first().external_model_id, 'gpt-3.5-turbo') diff --git a/backend/core/tests/test_basic.py b/backend/core/tests/test_basic.py new file mode 100644 index 0000000..cd59557 --- /dev/null +++ b/backend/core/tests/test_basic.py @@ -0,0 +1,22 @@ +import pytest + + +class TestBasic: + @pytest.mark.unit + def test_math(self): + assert 2 + 2 == 4 + assert 1 + 1 == 2 + assert 3 * 2 == 6 + + @pytest.mark.unit + def test_strings(self): + assert "hello" + "world" == "helloworld" + assert len("test") == 4 + assert "test".upper() == "TEST" + + @pytest.mark.unit + def test_lists(self): + my_list = [1, 2, 3, 4] + assert len(my_list) == 4 + assert my_list[0] == 1 + assert 5 not in my_list diff --git a/backend/core/tests/test_github_graphql.py b/backend/core/tests/test_github_graphql.py new file mode 100644 index 0000000..78aba11 --- /dev/null +++ b/backend/core/tests/test_github_graphql.py @@ -0,0 +1,298 @@ +""" +Tests for GitHub GraphQL client and ingestion service +""" + +import pytest +from unittest.mock import Mock, patch, MagicMock +from django.test import TestCase +from django.utils import timezone +from datetime import datetime + +from core.services.github_graphql_client import GitHubGraphQLClient +from core.services.github_graphql_ingestion import GitHubGraphQLIngestionService +from core.models import AppIntegration, Integration, GitHubRepository +from core.tests.factories import AppIntegrationFactory, IntegrationFactory + + +class TestGitHubGraphQLClient(TestCase): + """Test GitHub GraphQL client functionality""" + + def setUp(self): + self.token = "test_token" + self.client = GitHubGraphQLClient(self.token) + + def tearDown(self): + if hasattr(self.client, 'close'): + self.client.close() + + @patch('core.services.github_graphql_client.Client.execute') + def test_get_repository_info(self, mock_execute): + """Test repository info retrieval""" + mock_response = { + 'repository': { + 'id': 'R_123', + 'name': 'test-repo', + 'nameWithOwner': 'owner/test-repo', + 'description': 'Test repository', + 'url': 'https://github.com/owner/test-repo', + 'isPrivate': False, + 'defaultBranchRef': { + 'name': 'main' + }, + 'createdAt': '2023-01-01T00:00:00Z', + 'updatedAt': '2023-01-02T00:00:00Z' + } + } + mock_execute.return_value = mock_response + + result = self.client.get_repository_info('owner', 'test-repo') + + self.assertEqual(result['name'], 'test-repo') + self.assertEqual(result['nameWithOwner'], 'owner/test-repo') + self.assertFalse(result['isPrivate']) + self.assertEqual(result['defaultBranchRef']['name'], 'main') + + @patch('core.services.github_graphql_client.Client.execute') + def test_get_issues_with_comments(self, mock_execute): + """Test issues with comments retrieval""" + mock_response = { + 'repository': { + 'issues': { + 'pageInfo': { + 'hasNextPage': False, + 'endCursor': 'cursor123' + }, + 'edges': [ + { + 'node': { + 'id': 'I_123', + 'number': 1, + 'title': 'Test Issue', + 'body': 'Issue body', + 'state': 'OPEN', + 'author': { + 'login': 'testuser' + }, + 'comments': { + 'edges': [ + { + 'node': { + 'id': 'IC_456', + 'body': 'Comment body', + 'author': { + 'login': 'commenter' + }, + 'createdAt': '2023-01-01T00:00:00Z' + } + } + ] + } + } + } + ] + } + } + } + mock_execute.return_value = mock_response + + result = self.client.get_issues_with_comments('owner', 'repo') + + self.assertEqual(len(result['repository']['issues']['edges']), 1) + issue = result['repository']['issues']['edges'][0]['node'] + self.assertEqual(issue['number'], 1) + self.assertEqual(issue['title'], 'Test Issue') + self.assertEqual(len(issue['comments']['edges']), 1) + + @patch('core.services.github_graphql_client.Client.execute') + def test_rate_limit_handling(self, mock_execute): + """Test rate limit handling""" + from gql.exceptions import GraphQLExecutionError + + # Mock rate limit error + mock_execute.side_effect = GraphQLExecutionError("API rate limit exceeded") + + with self.assertRaises(GraphQLExecutionError): + self.client.get_repository_info('owner', 'repo') + + # Verify retry attempts (should be called 3 times) + self.assertEqual(mock_execute.call_count, 3) + + +class TestGitHubGraphQLIngestionService(TestCase): + """Test GitHub GraphQL ingestion service""" + + def setUp(self): + self.integration = IntegrationFactory( + integration_type='github', + config={'token': 'test_token'} + ) + self.app_integration = AppIntegrationFactory( + integration=self.integration + ) + + self.service = GitHubGraphQLIngestionService(self.app_integration) + + def tearDown(self): + if hasattr(self.service, 'graphql_client') and self.service.graphql_client: + self.service.graphql_client.close() + if hasattr(self.service, 'rest_client') and self.service.rest_client: + self.service.rest_client.close() + + @patch('core.services.github_graphql_ingestion.GitHubGraphQLClient') + def test_get_or_create_repository(self, mock_graphql_client_class): + mock_client = Mock() + mock_graphql_client_class.return_value = mock_client + + mock_client.get_repository_info.return_value = { + 'id': 'R_123', + 'name': 'test-repo', + 'description': 'Test repository', + 'url': 'https://github.com/owner/test-repo', + 'isPrivate': False, + 'defaultBranchRef': {'name': 'main'} + } + + service = GitHubGraphQLIngestionService(self.app_integration) + repository = service._get_or_create_repository('owner', 'test-repo') + + self.assertIsInstance(repository, GitHubRepository) + self.assertEqual(repository.name, 'test-repo') + self.assertEqual(repository.repo_owner, 'owner') + self.assertEqual(repository.full_name, 'owner/test-repo') + self.assertFalse(repository.is_private) + self.assertEqual(repository.default_branch, 'main') + + @patch('core.services.github_graphql_ingestion.GitHubGraphQLClient') + def test_ingest_single_issue_from_graphql(self, mock_graphql_client_class): + """Test single issue ingestion from GraphQL data""" + repository = GitHubRepository.objects.create( + name='test-repo', + repo_owner='owner', + full_name='owner/test-repo', + app_integration=self.app_integration + ) + self.service.repository = repository + + issue_data = { + 'id': 'I_123', + 'number': 1, + 'title': 'Test Issue', + 'body': 'Issue body', + 'state': 'OPEN', + 'author': {'login': 'testuser'}, + 'authorAssociation': 'CONTRIBUTOR', + 'assignees': {'nodes': [{'login': 'assignee1'}]}, + 'labels': {'nodes': [{'name': 'bug'}]}, + 'milestone': None, + 'locked': False, + 'createdAt': '2023-01-01T00:00:00Z', + 'updatedAt': '2023-01-02T00:00:00Z', + 'closedAt': None, + 'url': 'https://github.com/owner/test-repo/issues/1', + 'comments': { + 'edges': [ + { + 'node': { + 'id': 'IC_456', + 'body': 'Comment body', + 'author': {'login': 'commenter'}, + 'authorAssociation': 'CONTRIBUTOR', + 'createdAt': '2023-01-01T01:00:00Z', + 'updatedAt': '2023-01-01T01:00:00Z', + 'url': 'https://github.com/owner/test-repo/issues/1#issuecomment-456' + } + } + ] + } + } + + self.service._ingest_single_issue_from_graphql(issue_data) + + from core.models.github_data import GitHubIssue, GitHubIssueComment + issue = GitHubIssue.objects.get(repository=repository, number=1) + self.assertEqual(issue.title, 'Test Issue') + self.assertEqual(issue.state, 'open') + self.assertEqual(issue.author, 'testuser') + self.assertEqual(issue.assignees, ['assignee1']) + self.assertEqual(issue.labels, ['bug']) + + comment = GitHubIssueComment.objects.get(issue=issue) + self.assertEqual(comment.body, 'Comment body') + self.assertEqual(comment.author, 'commenter') + + def test_parse_datetime(self): + dt = self.service._parse_datetime('2023-01-01T00:00:00Z') + self.assertIsInstance(dt, datetime) + self.assertTrue(timezone.is_aware(dt)) + + dt = self.service._parse_datetime(None) + self.assertIsNone(dt) + + +class TestGitHubMigrationHelper(TestCase): + + def setUp(self): + self.integration = IntegrationFactory( + integration_type='github', + config={'token': 'test_token'} + ) + self.app_integration = AppIntegrationFactory( + integration=self.integration + ) + + def test_migration_helper_initialization(self): + from core.utils.github_migration_helper import GitHubMigrationHelper + + helper = GitHubMigrationHelper(self.app_integration) + self.assertEqual(helper.app_integration, self.app_integration) + self.assertIsNone(helper.rest_client) + self.assertIsNone(helper.graphql_client) + + @patch('core.utils.github_migration_helper.GitHubAPIClient') + @patch('core.utils.github_migration_helper.GitHubGraphQLClient') + def test_performance_comparison(self, mock_graphql_client_class, mock_rest_client_class): + from core.utils.github_migration_helper import GitHubMigrationHelper + + mock_rest_client = Mock() + mock_graphql_client = Mock() + mock_rest_client_class.return_value = mock_rest_client + mock_graphql_client_class.return_value = mock_graphql_client + + mock_rest_client.get_issues.return_value = [ + {'id': 1, 'number': 1, 'title': 'Issue 1'}, + {'id': 2, 'number': 2, 'title': 'Issue 2'} + ] + mock_rest_client.get_issue_comments.return_value = [{'id': 101, 'body': 'Comment 1'}] + + mock_graphql_client.get_all_issues_with_comments.return_value = [ + { + 'id': 'I_1', + 'number': 1, + 'title': 'Issue 1', + 'comments': {'edges': [{'node': {'id': 'IC_101', 'body': 'Comment 1'}}]} + }, + { + 'id': 'I_2', + 'number': 2, + 'title': 'Issue 2', + 'comments': {'edges': []} + } + ] + + helper = GitHubMigrationHelper(self.app_integration) + results = helper.compare_issue_ingestion_performance('owner', 'repo', limit=2) + + self.assertIn('repository', results) + self.assertIn('rest_api', results) + self.assertIn('graphql', results) + self.assertIn('improvement', results) + + self.assertEqual(results['rest_api']['api_calls'], 3) # 1 for issues + 2 for comments + self.assertEqual(results['rest_api']['issues_processed'], 2) + + self.assertEqual(results['graphql']['api_calls'], 1) # Single GraphQL query + self.assertEqual(results['graphql']['issues_processed'], 2) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/backend/core/tests/test_github_ingestion.py b/backend/core/tests/test_github_ingestion.py new file mode 100644 index 0000000..a958e5a --- /dev/null +++ b/backend/core/tests/test_github_ingestion.py @@ -0,0 +1,369 @@ +import pytest +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime + +from django.test import TestCase +from django.utils import timezone +from rest_framework.test import APIClient +from rest_framework import status + +from core.models import AppIntegration, Application +from core.models.github_data import GitHubRepository, GitHubIssue +from core.services.github_client import GitHubAPIClient +from core.services.github_ingestion import GitHubDataIngestionService +from core.views.github_ingestion import GitHubIngestionViewSet + + +class GitHubAPIClientTestCase(TestCase): + """Test cases for GitHub API client""" + + def setUp(self): + self.client = GitHubAPIClient("test_token") + + def test_initialization(self): + """Test client initialization""" + self.assertEqual(self.client.token, "test_token") + self.assertIn("Authorization", self.client.session.headers) + self.assertEqual(self.client.session.headers["Authorization"], "Bearer test_token") + + @patch('core.services.github_client.requests.Session.request') + def test_make_request_success(self, mock_request): + """Test successful API request""" + mock_response = Mock() + mock_response.json.return_value = {"id": 123, "title": "Test Issue"} + mock_response.status_code = 200 + mock_response.raise_for_status.return_value = None + mock_request.return_value = mock_response + + result = self.client._make_request('GET', '/test') + + self.assertEqual(result["id"], 123) + self.assertEqual(result["title"], "Test Issue") + mock_request.assert_called_once() + + @patch('core.services.github_client.requests.Session.request') + def test_make_request_rate_limit(self, mock_request): + """Test rate limit handling""" + # First call hits rate limit + mock_rate_limit_response = Mock() + mock_rate_limit_response.status_code = 403 + mock_rate_limit_response.headers = { + 'X-RateLimit-Remaining': '0', + 'X-RateLimit-Reset': str(int(timezone.now().timestamp()) + 60) + } + mock_rate_limit_response.json.return_value = {"message": "Rate limit exceeded"} + + # Second call succeeds + mock_success_response = Mock() + mock_success_response.status_code = 200 + mock_success_response.json.return_value = {"id": 123} + mock_success_response.raise_for_status.return_value = None + + mock_request.side_effect = [mock_rate_limit_response, mock_success_response] + + with patch('time.sleep'): # Skip actual sleep + result = self.client._make_request('GET', '/test') + + self.assertEqual(result["id"], 123) + self.assertEqual(mock_request.call_count, 2) + + @patch('core.services.github_client.requests.Session.request') + def test_get_repository_info(self, mock_request): + """Test getting repository information""" + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "name": "test-repo", + "full_name": "owner/test-repo", + "description": "Test repository", + "private": False, + "default_branch": "main" + } + mock_response.raise_for_status.return_value = None + mock_request.return_value = mock_response + + result = self.client.get_repository_info("owner", "test-repo") + + self.assertEqual(result["full_name"], "owner/test-repo") + self.assertEqual(result["description"], "Test repository") + self.assertFalse(result["private"]) + + +class GitHubDataIngestionServiceTestCase(TestCase): + """Test cases for GitHub data ingestion service""" + + def setUp(self): + self.user = Mock() + self.application = Mock() + self.application.owner = self.user + + self.integration = Mock() + self.integration.config = {"token": "test_token"} + + self.app_integration = Mock() + self.app_integration.integration = self.integration + self.app_integration.app = self.application + self.app_integration.id = 1 + + @patch('core.services.github_ingestion.GitHubAPIClient') + def test_get_or_create_repository_new(self, mock_client_class): + """Test creating a new repository record""" + mock_client = Mock() + mock_client_class.return_value = mock_client + + mock_repo_info = { + "description": "Test repository", + "html_url": "https://github.com/owner/test-repo", + "private": False, + "default_branch": "main" + } + mock_client.get_repository_info.return_value = mock_repo_info + + service = GitHubDataIngestionService(self.app_integration, use_graphql=True) + repository = service._get_or_create_repository("owner", "test-repo") + + self.assertEqual(repository.full_name, "owner/test-repo") + self.assertEqual(repository.name, "test-repo") + self.assertEqual(repository.owner, "owner") + self.assertEqual(repository.description, "Test repository") + self.assertFalse(repository.is_private) + self.assertEqual(repository.default_branch, "main") + + @patch('core.services.github_ingestion.GitHubAPIClient') + def test_ingest_single_issue(self, mock_client_class): + """Test ingesting a single issue""" + mock_client = Mock() + mock_client_class.return_value = mock_client + + # Mock repository + repository = GitHubRepository( + full_name="owner/test-repo", + name="test-repo", + owner="owner", + app_integration=self.app_integration + ) + repository.save() + + service = GitHubDataIngestionService(self.app_integration, use_graphql=True) + service.repository = repository + + issue_data = { + "id": 123, + "number": 1, + "title": "Test Issue", + "body": "This is a test issue", + "state": "open", + "user": {"login": "testuser"}, + "author_association": "NONE", + "assignees": [], + "labels": [{"name": "bug"}], + "milestone": None, + "locked": False, + "created_at": "2023-01-01T00:00:00Z", + "updated_at": "2023-01-01T00:00:00Z", + "closed_at": None, + "html_url": "https://github.com/owner/test-repo/issues/1" + } + + mock_client.get_issue_comments.return_value = [] + + service._ingest_single_issue(issue_data, "owner", "test-repo") + + # Verify issue was created + issue = GitHubIssue.objects.get(repository=repository, github_id=123) + self.assertEqual(issue.title, "Test Issue") + self.assertEqual(issue.state, "open") + self.assertEqual(issue.author, "testuser") + self.assertEqual(issue.labels, ["bug"]) + + +class GitHubIngestionViewSetTestCase(TestCase): + """Test cases for GitHub ingestion API endpoints""" + + def setUp(self): + self.client = APIClient() + self.user = Mock() + self.user.id = 1 + + # Mock authentication + self.client.force_authenticate(user=self.user) + + self.application = Application.objects.create( + name="Test App", + owner=self.user, + description="Test application" + ) + + # This would need to be adapted based on your actual Integration model + # self.integration = Integration.objects.create( + # name="GitHub Integration", + # type="pms", + # provider="github", + # owner=self.user, + # config={"token": "test_token"} + # ) + + # self.app_integration = AppIntegration.objects.create( + # app=self.application, + # integration=self.integration + # ) + + @patch('core.views.github_ingestion.GitHubDataIngestionService') + def test_ingest_repository_success(self, mock_service_class): + """Test successful repository ingestion""" + mock_service = Mock() + mock_service_class.return_value = mock_service + + viewset = GitHubIngestionViewSet() + viewset.request = Mock() + viewset.request.query_params = { + 'owner': 'test-owner', + 'repo': 'test-repo' + } + viewset.request.data = { + 'app_integration_id': 1 + } + viewset.request.user = self.user + + # Mock the AppIntegration lookup + with patch('core.views.github_ingestion.get_object_or_404') as mock_get: + mock_app_integration = Mock() + mock_app_integration.app.owner = self.user + mock_get.return_value = mock_app_integration + + response = viewset.ingest_repository(viewset.request) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['status'], 'success') + self.assertEqual(response.data['repository'], 'test-owner/test-repo') + + def test_ingest_repository_missing_params(self): + """Test ingestion with missing parameters""" + viewset = GitHubIngestionViewSet() + viewset.request = Mock() + viewset.request.query_params = {} # Missing owner and repo + viewset.request.data = {} + + response = viewset.ingest_repository(viewset.request) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn('error', response.data) + + @patch('core.views.github_ingestion.GitHubDataIngestionService') + def test_ingest_repository_failure(self, mock_service_class): + """Test ingestion failure handling""" + mock_service = Mock() + mock_service.ingest_repository.side_effect = Exception("Test error") + mock_service_class.return_value = mock_service + + viewset = GitHubIngestionViewSet() + viewset.request = Mock() + viewset.request.query_params = { + 'owner': 'test-owner', + 'repo': 'test-repo' + } + viewset.request.data = { + 'app_integration_id': 1 + } + viewset.request.user = self.user + + with patch('core.views.github_ingestion.get_object_or_404') as mock_get: + mock_app_integration = Mock() + mock_app_integration.app.owner = self.user + mock_get.return_value = mock_app_integration + + response = viewset.ingest_repository(viewset.request) + + self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR) + self.assertIn('error', response.data) + + +class GitHubDataIntegrityTestCase(TestCase): + """Test data integrity and relationships""" + + def setUp(self): + self.user = Mock() + self.application = Mock() + self.application.owner = self.user + + self.app_integration = Mock() + self.app_integration.app = self.application + self.app_integration.id = 1 + + # Create test repository + self.repository = GitHubRepository.objects.create( + full_name="owner/test-repo", + name="test-repo", + owner="owner", + app_integration=self.app_integration, + url="https://github.com/owner/test-repo" + ) + + def test_issue_creation(self): + """Test issue creation and relationships""" + issue = GitHubIssue.objects.create( + repository=self.repository, + github_id=123, + number=1, + title="Test Issue", + body="Test body", + state="open", + author="testuser", + created_at=timezone.now(), + updated_at=timezone.now(), + url="https://github.com/owner/test-repo/issues/1" + ) + + self.assertEqual(issue.repository, self.repository) + self.assertEqual(str(issue), "owner/test-repo#1: Test Issue") + + def test_pull_request_creation(self): + """Test pull request creation and relationships""" + from core.models.github_data import GitHubPullRequest + + pr = GitHubPullRequest.objects.create( + repository=self.repository, + github_id=456, + number=1, + title="Test PR", + body="Test PR body", + state="open", + author="testuser", + head_branch="feature-branch", + base_branch="main", + merged=False, + created_at=timezone.now(), + updated_at=timezone.now(), + url="https://github.com/owner/test-repo/pull/1" + ) + + self.assertEqual(pr.repository, self.repository) + self.assertEqual(str(pr), "PR owner/test-repo#1: Test PR") + + def test_cascade_delete(self): + """Test cascade deletion behavior""" + # Create issue + issue = GitHubIssue.objects.create( + repository=self.repository, + github_id=123, + number=1, + title="Test Issue", + state="open", + author="testuser", + created_at=timezone.now(), + updated_at=timezone.now(), + url="https://github.com/owner/test-repo/issues/1" + ) + + # Verify issue exists + self.assertTrue(GitHubIssue.objects.filter(id=issue.id).exists()) + + # Delete repository + self.repository.delete() + + # Verify issue is also deleted + self.assertFalse(GitHubIssue.objects.filter(id=issue.id).exists()) + + +if __name__ == '__main__': + pytest.main([__file__]) diff --git a/backend/core/urls.py b/backend/core/urls.py index 04012d8..93c9b12 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -6,30 +6,39 @@ from core.views import ( UserRegisterView, VerifyEmailView, KnowledgeBaseViewSet, ChatRoomMessagesView, MeView, GenerateAPIKeyView, - IntegrationViewSet, WidgetView, LoadAvailableConfigurationView, AppNotificationUpdateView + IntegrationViewSet, AppIntegrationViewSet, WidgetView, LoadAvailableConfigurationView, AppNotificationUpdateView ) +from core.views.change_password import ChangePasswordView from core.views.resend_verification import ResendVerificationView from core.views.custom_auth import CustomAuthToken -from core.views.application import ApplicationViewSet, ApplicationChatRoomsPreviewView +from core.views.application import ApplicationViewSet, ApplicationChatRoomsPreviewView, UserChatRoomsView from core.views.chatroom import ChatRoomDetailView from core.views.configure_app import ConfigureAppIntegrationView, LoadAppConfigurationView from core.views.integration import supported_integrations from core.views.llm_model import LLMModelViewSet from core.views.message import SendMessageView +from core.views.human_agent import HumanAgentInfoView from core.views.ingestion import IngestApplicationKBView from core.views.notification_profile import NotificationProfileViewSet from core.views.app_model import ConfigureAppModelsView from core.views.forgot_password import ForgotPasswordView from core.views.reset_password import ResetPasswordView, ResetPasswordVerifyView +from core.views.ai_provider import AIProviderViewSet +from core.views.app_ai_provider import AppAIProviderViewSet +from core.views.github_ingestion import GitHubIngestionViewSet router = DefaultRouter() router.register(r'applications', ApplicationViewSet, basename='applications') +router.register(r'ai-providers', AIProviderViewSet, basename='ai-provider') router.register(r'notification-profiles', NotificationProfileViewSet, basename='notificationprofile') router.register(r'models', LLMModelViewSet, basename='model'), -router.register(r'integrations', IntegrationViewSet, basename='integration'), +router.register(r'integrations', IntegrationViewSet, basename='integration') +router.register(r'app-integrations', AppIntegrationViewSet, basename='app-integration') +router.register(r'github-ingestion', GitHubIngestionViewSet, basename='github-ingestion'), nested_router = NestedDefaultRouter(router, r'applications', lookup='application') nested_router.register(r'knowledge-bases', KnowledgeBaseViewSet, basename='application-knowledge-bases') +nested_router.register(r'ai-providers', AppAIProviderViewSet, basename='application-ai-providers') urlpatterns = [ path('login/', CustomAuthToken.as_view(), name='api_login'), @@ -55,6 +64,10 @@ path('applications//chatrooms/', ApplicationChatRoomsPreviewView.as_view(), name='application-chatroom-previews'), + path('applications//user-chatrooms/', UserChatRoomsView.as_view(), + name='user-chatrooms'), + path('applications//human-agent/', HumanAgentInfoView.as_view(), + name='human-agent-info'), path('chatrooms//', ChatRoomDetailView.as_view(), name='chatroom-detail'), path('applications//ingests/', IngestApplicationKBView.as_view(), name='application-ingest'), @@ -94,4 +107,5 @@ path('forgot-password/', ForgotPasswordView.as_view(), name='forgot-password'), path('reset-password/', ResetPasswordView.as_view(), name='reset-password'), # remove path('reset-password//', ResetPasswordView.as_view(), name='reset-password'), + path('change-password/', ChangePasswordView.as_view(), name='change-password'), ] diff --git a/backend/core/utils.py b/backend/core/utils.py index 15087c7..9c19ce0 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -1,5 +1,6 @@ import json import re +from typing import Any, Callable, Dict, List, Optional, Union def parse_llm_response(content: str) -> dict: try: @@ -12,3 +13,43 @@ def parse_llm_response(content: str) -> dict: except json.JSONDecodeError as e: print("Failed to parse LLM response as JSON:", e) raise + + +def extract_and_merge_fields( + validated_data: Dict[str, Any], + field_selector: Union[List[str], Callable[[str], bool]], + existing_data: Optional[Dict[str, Any]] = None, + merge: bool = True +) -> Dict[str, Any]: + extracted_data = existing_data.copy() if existing_data and merge else {} + + if isinstance(field_selector, list): + main_fields = set(field_selector) + fields_to_extract = [field for field in validated_data.keys() if field not in main_fields] + elif callable(field_selector): + fields_to_extract = [field for field in validated_data.keys() if field_selector(field)] + else: + raise ValueError("field_selector must be a list of field names or a callable") + + for field in fields_to_extract: + value = validated_data.pop(field) + extracted_data[field] = str(value).strip() if value is not None else '' + + return extracted_data + + +def normalize_model_name_by_provider(model: str, provider: str) -> str: + """ + Normalize model name based on provider. + + Args: + model: The model name to normalize + provider: The AI provider name + + Returns: + Normalized model name + """ + if provider.lower() == 'gemini': + if model.startswith('model/'): + return model[6:] + return model diff --git a/backend/core/utils/config_manager.py b/backend/core/utils/config_manager.py new file mode 100644 index 0000000..a1539b5 --- /dev/null +++ b/backend/core/utils/config_manager.py @@ -0,0 +1,174 @@ +import os +from typing import Any, Dict, Optional, Union +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class DatabaseConfig: + """Database configuration""" + name: str + user: str + password: str + host: str + port: int + test_name: Optional[str] = None + test_user: Optional[str] = None + test_password: Optional[str] = None + test_host: Optional[str] = None + test_port: Optional[int] = None + + +@dataclass +class QdrantConfig: + """Qdrant vector database configuration""" + local_host: str = "localhost" + local_port: int = 6333 + cloud_host: Optional[str] = None + cloud_port: Optional[int] = None + cloud_api_key: Optional[str] = None + connect_to_local: bool = False + + +@dataclass +class AIProviderConfig: + """AI provider configuration""" + gemini_api_key: Optional[str] = None + openai_api_key: Optional[str] = None + + +@dataclass +class AppConfig: + """Application configuration""" + secret_key: str + debug: bool = False + allowed_hosts: list = None + closed_alpha_signups: list = None + require_account_approval: bool = False + + def __post_init__(self): + if self.allowed_hosts is None: + self.allowed_hosts = [] + if self.closed_alpha_signups is None: + self.closed_alpha_signups = [] + + +class ConfigManager: + """ + Centralized configuration management with environment variable support + """ + + def __init__(self, base_dir: Optional[Path] = None): + self.base_dir = base_dir or Path(__file__).parent.parent.parent.parent + self._env_loaded = False + + def _ensure_env_loaded(self): + """Ensure environment variables are loaded""" + if not self._env_loaded: + try: + from dotenv import load_dotenv + load_dotenv(self.base_dir / '.env') + self._env_loaded = True + except ImportError: + self._env_loaded = True + + def get_env_var( + self, + key: str, + default: Any = None, + var_type: type = str + ) -> Any: + """ + Get environment variable with type conversion + + Args: + key: Environment variable key + default: Default value if not found + var_type: Type to convert to + + Returns: + Converted environment variable or default + """ + self._ensure_env_loaded() + + value = os.environ.get(key) + if value is None: + return default + + if var_type == bool: + return value.lower() in ('true', '1', 'yes', 'on') + elif var_type == int: + return int(value) + elif var_type == float: + return float(value) + elif var_type == list: + return [item.strip() for item in value.split(',') if item.strip()] + else: + return value + + def get_database_config(self) -> DatabaseConfig: + """Get database configuration""" + return DatabaseConfig( + name=self.get_env_var('DB_NAME', 'chatterbox'), + user=self.get_env_var('DB_USER', 'postgres'), + password=self.get_env_var('PASSWORD', 'postgres'), + host=self.get_env_var('DB_HOST', 'localhost'), + port=self.get_env_var('PORT', 5432, int), + test_name=self.get_env_var('TEST_DB_NAME'), + test_user=self.get_env_var('TEST_DB_USER'), + test_password=self.get_env_var('TEST_DB_PASSWORD'), + test_host=self.get_env_var('TEST_DB_HOST'), + test_port=self.get_env_var('TEST_DB_PORT', None, int) + ) + + def get_qdrant_config(self) -> QdrantConfig: + """Get Qdrant configuration""" + return QdrantConfig( + local_host=self.get_env_var('QDRANT_LOCAL_HOST', 'localhost'), + local_port=self.get_env_var('QDRANT_LOCAL_PORT', 6333, int), + cloud_host=self.get_env_var('QDRANT_CLOUD_HOST'), + cloud_port=self.get_env_var('QDRANT_CLOUD_PORT', None, int), + cloud_api_key=self.get_env_var('QDRANT_CLOUD_API_KEY'), + connect_to_local=self.get_env_var('CONNECT_TO_LOCAL_VECTOR_DB', 'False', bool) + ) + + def get_ai_provider_config(self) -> AIProviderConfig: + """Get AI provider configuration""" + return AIProviderConfig( + gemini_api_key=self.get_env_var('GEMINI_API_KEY'), + openai_api_key=self.get_env_var('OPENAI_API_KEY') + ) + + def get_app_config(self) -> AppConfig: + """Get application configuration""" + return AppConfig( + secret_key=self.get_env_var('APP_SECRET_KEY', ''), + debug=self.get_env_var('DEBUG', 'False', bool), + allowed_hosts=self.get_env_var('ALLOWED_HOSTS', [], list), + closed_alpha_signups=self.get_env_var('CLOSED_ALPHA_SIGN_UPS', [], list), + require_account_approval=self.get_env_var('REQUIRE_ACCOUNT_APPROVAL', 'False', bool) + ) + + def get_url_config(self) -> Dict[str, str]: + """Get URL configuration""" + return { + 'api_base_url': self.get_env_var('API_BASE_URL', 'http://localhost:8000/api'), + 'frontend_url': self.get_env_var('FRONTEND_URL', 'http://localhost:3000'), + 'widget_url': self.get_env_var('WIDGET_URL', 'https://widget.ch8r.com') + } + + def get_email_config(self) -> Dict[str, str]: + """Get email configuration""" + return { + 'mailersend_api_key': self.get_env_var('MAILERSEND_API_KEY', ''), + 'default_from_email': self.get_env_var('DEFAULT_FROM_EMAIL', ''), + 'discord_signup_webhook_url': self.get_env_var('DISCORD_SIGNUP_WEBHOOK_URL', '') + } + + def get_security_config(self) -> Dict[str, str]: + """Get security configuration""" + return { + 'secret_encryption_key': self.get_env_var('SECRET_ENCRYPTION_KEY', ''), + } + +config_manager = ConfigManager() diff --git a/backend/core/utils/error_handling.py b/backend/core/utils/error_handling.py new file mode 100644 index 0000000..d984535 --- /dev/null +++ b/backend/core/utils/error_handling.py @@ -0,0 +1,190 @@ +import logging +import traceback +from typing import Optional, Dict, Any, Callable +from functools import wraps +from django.core.exceptions import ValidationError +from rest_framework.exceptions import APIException + +logger = logging.getLogger(__name__) + + +class GitHubIngestionError(Exception): + """Custom exception for GitHub ingestion errors""" + pass + + +class AIProviderError(Exception): + """Custom exception for AI provider errors""" + pass + + +class ValidationError(Exception): + """Custom validation error""" + pass + + +def handle_service_errors( + error_class: type = Exception, + default_message: str = "An error occurred", + log_level: str = "error" +): + """ + Decorator for consistent error handling in services + + Args: + error_class: Exception class to raise + default_message: Default error message + log_level: Logging level ('error', 'warning', 'info') + """ + def decorator(func: Callable): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + log_method = getattr(logger, log_level) + + log_method( + f"Error in {func.__name__}: {str(e)}\n" + f"Traceback: {traceback.format_exc()}" + ) + + raise error_class(default_message) from e + + return wrapper + return decorator + + +def safe_execute( + func: Callable, + default_value: Any = None, + error_message: Optional[str] = None +) -> Any: + """ + Safely execute a function and return default value on error + + Args: + func: Function to execute + default_value: Value to return on error + error_message: Optional error message to log + + Returns: + Function result or default value + """ + try: + return func() + except Exception as e: + if error_message: + logger.warning(f"{error_message}: {str(e)}") + else: + logger.warning(f"Error in {func.__name__}: {str(e)}") + return default_value + + +def validate_required_fields(data: Dict[str, Any], required_fields: list) -> None: + """ + Validate that required fields are present in data + + Args: + data: Dictionary to validate + required_fields: List of required field names + + Raises: + ValidationError: If required fields are missing + """ + missing_fields = [field for field in required_fields if field not in data or not data[field]] + if missing_fields: + raise ValidationError(f"Missing required fields: {', '.join(missing_fields)}") + + +def log_api_call( + endpoint: str, + method: str = "GET", + params: Optional[Dict[str, Any]] = None, + response_status: Optional[int] = None, + error: Optional[str] = None +): + """ + Log API call details for debugging and monitoring + + Args: + endpoint: API endpoint + method: HTTP method + params: Request parameters + response_status: Response status code + error: Error message if any + """ + log_data = { + 'endpoint': endpoint, + 'method': method, + 'params': params or {} + } + + if response_status: + log_data['status'] = response_status + + if error: + log_data['error'] = error + logger.error(f"API call failed: {log_data}") + else: + logger.info(f"API call: {log_data}") + + +class ErrorContext: + """Context manager for error handling and logging""" + + def __init__( + self, + operation: str, + reraise: bool = True, + default_return: Any = None + ): + self.operation = operation + self.reraise = reraise + self.default_return = default_return + + def __enter__(self): + logger.info(f"Starting operation: {self.operation}") + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is None: + logger.info(f"Completed operation: {self.operation}") + return True + + logger.error( + f"Failed operation: {self.operation}\\n" + f"Error: {exc_val}\\n" + f"Traceback: {traceback.format_exception(exc_type, exc_val, exc_tb)}" + ) + + if self.reraise: + return False + + return True # Suppress the exception + + +def format_error_response( + error: Exception, + user_friendly_message: Optional[str] = None +) -> Dict[str, Any]: + """ + Format error response for API endpoints + + Args: + error: Exception that occurred + user_friendly_message: Optional user-friendly message + + Returns: + Formatted error response + """ + response = { + 'error': user_friendly_message or 'An unexpected error occurred', + 'type': error.__class__.__name__ + } + + if logger.isEnabledFor(logging.DEBUG): + response['details'] = str(error) + response['traceback'] = traceback.format_exc() + + return response diff --git a/backend/core/utils/github_migration_helper.py b/backend/core/utils/github_migration_helper.py new file mode 100644 index 0000000..f3e986b --- /dev/null +++ b/backend/core/utils/github_migration_helper.py @@ -0,0 +1,318 @@ +import logging +import time +from typing import Dict, List, Optional, Any +from core.services.github_client import GitHubAPIClient +from core.services.github_graphql_client import GitHubGraphQLClient +from core.models import AppIntegration + +logger = logging.getLogger(__name__) + + +class GitHubMigrationHelper: + def __init__(self, app_integration: AppIntegration): + self.app_integration = app_integration + self.rest_client = None + self.graphql_client = None + + def _get_rest_client(self) -> GitHubAPIClient: + if not self.rest_client: + token = self.app_integration.integration.config.get('token') + if not token: + raise ValueError("GitHub token not found in integration config") + self.rest_client = GitHubAPIClient(token) + return self.rest_client + + def _get_graphql_client(self) -> GitHubGraphQLClient: + if not self.graphql_client: + token = self.app_integration.integration.config.get('token') + if not token: + raise ValueError("GitHub token not found in integration config") + self.graphql_client = GitHubGraphQLClient(token) + return self.graphql_client + + def compare_issue_ingestion_performance(self, owner: str, repo: str, limit: int = 50) -> Dict[str, Any]: + results = { + 'repository': f"{owner}/{repo}", + 'limit': limit, + 'rest_api': {}, + 'graphql': {}, + 'improvement': {} + } + + logger.info(f"Starting performance comparison for {owner}/{repo}") + + try: + logger.info("Testing REST API performance...") + rest_start = time.time() + + rest_client = self._get_rest_client() + issues = rest_client.get_issues(owner, repo, state='all') + + issues_to_test = issues[:limit] + + rest_api_calls = 1 + total_comments = 0 + + for issue in issues_to_test: + comments = rest_client.get_issue_comments(owner, repo, issue['number']) + total_comments += len(comments) + rest_api_calls += 1 + + rest_end = time.time() + rest_duration = rest_end - rest_start + + results['rest_api'] = { + 'duration_seconds': round(rest_duration, 2), + 'api_calls': rest_api_calls, + 'issues_processed': len(issues_to_test), + 'total_comments': total_comments, + 'avg_time_per_issue': round(rest_duration / len(issues_to_test), 3) if issues_to_test else 0 + } + + logger.info(f"REST API: {rest_api_calls} calls, {rest_duration:.2f}s for {len(issues_to_test)} issues") + + except Exception as e: + logger.error(f"REST API test failed: {e}") + results['rest_api']['error'] = str(e) + + try: + logger.info("Testing GraphQL performance...") + graphql_start = time.time() + + graphql_client = self._get_graphql_client() + issues_with_comments = graphql_client.get_all_issues_with_comments( + owner, repo, states=['OPEN', 'CLOSED'] + ) + + issues_to_test = issues_with_comments[:limit] + + graphql_end = time.time() + graphql_duration = graphql_end - graphql_start + + total_comments = sum( + len(issue.get('comments', {}).get('edges', [])) + for issue in issues_to_test + ) + + results['graphql'] = { + 'duration_seconds': round(graphql_duration, 2), + 'api_calls': 1, + 'issues_processed': len(issues_to_test), + 'total_comments': total_comments, + 'avg_time_per_issue': round(graphql_duration / len(issues_to_test), 3) if issues_to_test else 0 + } + + logger.info(f"GraphQL: 1 call, {graphql_duration:.2f}s for {len(issues_to_test)} issues") + + except Exception as e: + logger.error(f"GraphQL test failed: {e}") + results['graphql']['error'] = str(e) + + if 'duration_seconds' in results['rest_api'] and 'duration_seconds' in results['graphql']: + rest_time = results['rest_api']['duration_seconds'] + graphql_time = results['graphql']['duration_seconds'] + + if rest_time > 0: + time_improvement = ((rest_time - graphql_time) / rest_time) * 100 + results['improvement'] = { + 'time_reduction_percent': round(time_improvement, 1), + 'api_call_reduction': results['rest_api']['api_calls'] - results['graphql']['api_calls'], + 'speed_multiplier': round(rest_time / graphql_time, 1) if graphql_time > 0 else float('inf') + } + + return results + + def compare_pr_ingestion_performance(self, owner: str, repo: str, limit: int = 50) -> Dict[str, Any]: + results = { + 'repository': f"{owner}/{repo}", + 'limit': limit, + 'rest_api': {}, + 'graphql': {}, + 'improvement': {} + } + + logger.info(f"Starting PR performance comparison for {owner}/{repo}") + + try: + logger.info("Testing REST API PR performance...") + rest_start = time.time() + + rest_client = self._get_rest_client() + prs = rest_client.get_pull_requests(owner, repo, state='all') + + prs_to_test = prs[:limit] + + rest_api_calls = 1 + total_comments = 0 + + for pr in prs_to_test: + comments = rest_client.get_pull_request_comments(owner, repo, pr['number']) + total_comments += len(comments) + rest_api_calls += 1 + + rest_end = time.time() + rest_duration = rest_end - rest_start + + results['rest_api'] = { + 'duration_seconds': round(rest_duration, 2), + 'api_calls': rest_api_calls, + 'prs_processed': len(prs_to_test), + 'total_comments': total_comments, + 'avg_time_per_pr': round(rest_duration / len(prs_to_test), 3) if prs_to_test else 0 + } + + logger.info(f"REST API PRs: {rest_api_calls} calls, {rest_duration:.2f}s for {len(prs_to_test)} PRs") + + except Exception as e: + logger.error(f"REST API PR test failed: {e}") + results['rest_api']['error'] = str(e) + + try: + logger.info("Testing GraphQL PR performance...") + graphql_start = time.time() + + graphql_client = self._get_graphql_client() + prs_with_comments = graphql_client.get_all_pull_requests_with_comments( + owner, repo, states=['OPEN', 'CLOSED', 'MERGED'] + ) + + prs_to_test = prs_with_comments[:limit] + + graphql_end = time.time() + graphql_duration = graphql_end - graphql_start + + total_comments = sum( + len(pr.get('comments', {}).get('edges', [])) + for pr in prs_to_test + ) + + results['graphql'] = { + 'duration_seconds': round(graphql_duration, 2), + 'api_calls': 1, + 'prs_processed': len(prs_to_test), + 'total_comments': total_comments, + 'avg_time_per_pr': round(graphql_duration / len(prs_to_test), 3) if prs_to_test else 0 + } + + logger.info(f"GraphQL PRs: 1 call, {graphql_duration:.2f}s for {len(prs_to_test)} PRs") + + except Exception as e: + logger.error(f"GraphQL PR test failed: {e}") + results['graphql']['error'] = str(e) + + if 'duration_seconds' in results['rest_api'] and 'duration_seconds' in results['graphql']: + rest_time = results['rest_api']['duration_seconds'] + graphql_time = results['graphql']['duration_seconds'] + + if rest_time > 0: + time_improvement = ((rest_time - graphql_time) / rest_time) * 100 + results['improvement'] = { + 'time_reduction_percent': round(time_improvement, 1), + 'api_call_reduction': results['rest_api']['api_calls'] - results['graphql']['api_calls'], + 'speed_multiplier': round(rest_time / graphql_time, 1) if graphql_time > 0 else float('inf') + } + + return results + + def generate_migration_report(self, owner: str, repo: str) -> str: + report = [] + report.append("# GitHub API Migration Report") + report.append(f"Repository: {owner}/{repo}") + report.append(f"Generated: {time.strftime('%Y-%m-%d %H:%M:%S')}") + report.append("") + + issue_results = self.compare_issue_ingestion_performance(owner, repo, limit=30) + report.append("## Issues Performance Comparison") + + if 'error' not in issue_results['rest_api'] and 'error' not in issue_results['graphql']: + report.append(f"### REST API") + report.append(f"- Duration: {issue_results['rest_api']['duration_seconds']}s") + report.append(f"- API Calls: {issue_results['rest_api']['api_calls']}") + report.append(f"- Issues: {issue_results['rest_api']['issues_processed']}") + report.append(f"- Comments: {issue_results['rest_api']['total_comments']}") + report.append("") + + report.append(f"### GraphQL") + report.append(f"- Duration: {issue_results['graphql']['duration_seconds']}s") + report.append(f"- API Calls: {issue_results['graphql']['api_calls']}") + report.append(f"- Issues: {issue_results['graphql']['issues_processed']}") + report.append(f"- Comments: {issue_results['graphql']['total_comments']}") + report.append("") + + if issue_results.get('improvement'): + imp = issue_results['improvement'] + report.append(f"### Improvements") + report.append(f"- Time Reduction: {imp['time_reduction_percent']}%") + report.append(f"- API Call Reduction: {imp['api_call_reduction']}") + report.append(f"- Speed Multiplier: {imp['speed_multiplier']}x faster") + report.append("") + else: + report.append("Error in performance comparison. Check logs for details.") + report.append("") + + pr_results = self.compare_pr_ingestion_performance(owner, repo, limit=30) + report.append("## Pull Requests Performance Comparison") + + if 'error' not in pr_results['rest_api'] and 'error' not in pr_results['graphql']: + report.append(f"### REST API") + report.append(f"- Duration: {pr_results['rest_api']['duration_seconds']}s") + report.append(f"- API Calls: {pr_results['rest_api']['api_calls']}") + report.append(f"- PRs: {pr_results['rest_api']['prs_processed']}") + report.append(f"- Comments: {pr_results['rest_api']['total_comments']}") + report.append("") + + report.append(f"### GraphQL") + report.append(f"- Duration: {pr_results['graphql']['duration_seconds']}s") + report.append(f"- API Calls: {pr_results['graphql']['api_calls']}") + report.append(f"- PRs: {pr_results['graphql']['prs_processed']}") + report.append(f"- Comments: {pr_results['graphql']['total_comments']}") + report.append("") + + if pr_results.get('improvement'): + imp = pr_results['improvement'] + report.append(f"### Improvements") + report.append(f"- Time Reduction: {imp['time_reduction_percent']}%") + report.append(f"- API Call Reduction: {imp['api_call_reduction']}") + report.append(f"- Speed Multiplier: {imp['speed_multiplier']}x faster") + report.append("") + else: + report.append("Error in PR performance comparison. Check logs for details.") + report.append("") + + report.append("## Migration Recommendations") + report.append("### Benefits of GraphQL:") + report.append("- ✅ Single API call for issues with comments") + report.append("- ✅ Single API call for PRs with comments") + report.append("- ✅ Reduced rate limiting risk") + report.append("- ✅ Better performance for large repositories") + report.append("- ✅ Flexible data selection") + report.append("") + + report.append("### Migration Steps:") + report.append("1. Install GraphQL client: `pip install gql==3.5.0`") + report.append("2. Update ingestion service to use GraphQL:") + report.append(" ```python") + report.append(" from core.services.github_ingestion import GitHubDataIngestionService") + report.append(" ") + report.append(" # Enable GraphQL (default)") + report.append(" service = GitHubDataIngestionService(app_integration, use_graphql=True)") + report.append(" ") + report.append(" # Or disable to fall back to REST") + report.append(" service = GitHubDataIngestionService(app_integration, use_graphql=False)") + report.append(" ```") + report.append("") + + report.append("### Notes:") + report.append("- GraphQL is used for issues, PRs, and comments") + report.append("- REST API is still used for PR files (not available in GraphQL)") + report.append("- Backward compatibility is maintained") + report.append("- Rate limits are significantly reduced") + report.append("- Performance improvement is especially noticeable for large repos") + + return "\n".join(report) + + def close(self): + if self.rest_client: + self.rest_client.close() + if self.graphql_client: + self.graphql_client.close() diff --git a/backend/core/views/__init__.py b/backend/core/views/__init__.py index 1cbdc71..294d2d8 100644 --- a/backend/core/views/__init__.py +++ b/backend/core/views/__init__.py @@ -14,4 +14,6 @@ from .custom_auth import CustomAuthToken from .app_model import ConfigureAppModelsView from .reset_password import ResetPasswordView, ResetPasswordVerifyView -from .forgot_password import ForgotPasswordView \ No newline at end of file +from .forgot_password import ForgotPasswordView +from .ai_provider import AIProviderViewSet +from .integration import AppIntegrationViewSet diff --git a/backend/core/views/ai_provider.py b/backend/core/views/ai_provider.py new file mode 100644 index 0000000..c48cf98 --- /dev/null +++ b/backend/core/views/ai_provider.py @@ -0,0 +1,183 @@ +from rest_framework import status, viewsets, permissions +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination +from rest_framework.decorators import action +from django.db import models + +from core.serializers.ai_provider import AIProviderCreateSerializer, AIProviderSerializer +from core.models import AIProvider, AIProviderModels +from core.consts import SUPPORTED_AI_PROVIDERS +from core.services.ai_client_service import AIClientService + + +class AIProviderViewSet(viewsets.ModelViewSet): + + permission_classes = [permissions.IsAuthenticated] + lookup_field = 'uuid' + http_method_names = ['get', 'post', 'put', 'patch', 'delete'] + pagination_class = PageNumberPagination + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ai_service = AIClientService() + + def get_queryset(self): + """Get queryset filtered by user""" + user = self.request.user + return AIProvider.objects.filter( + models.Q(creator=user) | models.Q(is_builtin=True) + ) + + def get_serializer_class(self): + """Get appropriate serializer based on action""" + if self.action in ['create', 'update', 'partial_update']: + return AIProviderCreateSerializer + return AIProviderSerializer + + def create(self, request, *args, **kwargs): + """Create a new AI provider with validation""" + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + result = self._create_ai_provider(serializer.validated_data, request.user) + return Response(result, status=status.HTTP_201_CREATED) + + except Exception as e: + return self._handle_validation_error(e) + + def _create_ai_provider(self, validated_data, user): + """Create AI provider with validation""" + is_valid, provider_models = self.ai_service.validate_ai_provider(validated_data) + + if not is_valid: + raise ValueError('Failed to validate AI provider connection') + + ai_provider = self.get_serializer().create(validated_data) + ai_provider.creator = user + ai_provider.save() + + self._store_provider_models(ai_provider, provider_models, user) + + return self._format_creation_response(ai_provider, provider_models) + + def _store_provider_models(self, ai_provider, provider_models, user): + """Store provider models in database""" + AIProviderModels.objects.update_or_create( + ai_provider=ai_provider, + defaults={ + 'models_data': provider_models, + 'creator': user + } + ) + + def _format_creation_response(self, ai_provider, provider_models): + """Format creation response""" + response_serializer = AIProviderSerializer(ai_provider) + return { + 'ai_provider': response_serializer.data, + 'validation': { + 'is_valid': True, + 'models': provider_models + } + } + + def list(self, request, *args, **kwargs): + """List AI providers with supported providers info""" + response = super().list(request, *args, **kwargs) + return self._format_list_response(response) + + def _format_list_response(self, response): + """Format list response with supported providers""" + if isinstance(response.data, dict): + response.data['supported_ai_providers'] = SUPPORTED_AI_PROVIDERS + else: + response.data = { + 'results': response.data, + 'supported_ai_providers': SUPPORTED_AI_PROVIDERS + } + return response + + def update(self, request, *args, **kwargs): + """Update AI provider with validation""" + partial = kwargs.pop('partial', False) + instance = self.get_object() + serializer = self.get_serializer(instance, data=request.data, partial=partial) + serializer.is_valid(raise_exception=True) + + try: + result = self._update_ai_provider(serializer, instance, request.user) + return Response(result) + + except Exception as e: + return self._handle_validation_error(e) + + def _update_ai_provider(self, serializer, instance, user): + validated_data = serializer.validated_data + + api_key_to_validate = self._get_api_key_to_validate(validated_data, instance) + + if api_key_to_validate and api_key_to_validate.strip(): + is_valid, provider_models = self.ai_service.validate_ai_provider( + validated_data, instance + ) + + if not is_valid: + raise ValueError('Failed to validate AI provider connection') + + self._store_provider_models(instance, provider_models, user) + + updated_instance = serializer.save() + + return self._format_update_response(updated_instance) + + def _get_api_key_to_validate(self, validated_data, instance): + """Get API key that needs validation""" + return validated_data.get('provider_api_key') or instance.provider_api_key + + def _format_update_response(self, instance): + """Format update response""" + response_serializer = AIProviderSerializer(instance) + return { + 'ai_provider': response_serializer.data, + 'message': 'AI provider updated successfully' + } + + def _handle_validation_error(self, error): + """Handle validation errors consistently""" + return Response( + { + 'error': 'Failed to validate AI provider connection', + 'details': str(error) + }, + status=status.HTTP_400_BAD_REQUEST + ) + + @action(detail=True, methods=['post']) + def test_connection(self, request, uuid=None): + """Test AI provider connection""" + instance = self.get_object() + + try: + test_data = { + 'provider': instance.provider, + 'provider_api_key': instance.provider_api_key, + **(instance.metadata or {}) + } + + is_valid, provider_models = self.ai_service.validate_ai_provider(test_data) + + return Response({ + 'is_valid': is_valid, + 'models': provider_models if is_valid else None, + 'message': 'Connection successful' if is_valid else 'Connection failed' + }) + + except Exception as e: + return Response( + { + 'is_valid': False, + 'error': str(e) + }, + status=status.HTTP_400_BAD_REQUEST + ) diff --git a/backend/core/views/app_ai_provider.py b/backend/core/views/app_ai_provider.py new file mode 100644 index 0000000..6e3662a --- /dev/null +++ b/backend/core/views/app_ai_provider.py @@ -0,0 +1,63 @@ +from rest_framework import viewsets, permissions, status +from rest_framework.response import Response +from django.shortcuts import get_object_or_404 + +from core.models.app_ai_provider import AppAIProvider +from core.models.application import Application +from core.serializers.app_ai_provider import ( + AppAIProviderSerializer, + AppAIProviderCreateSerializer, + AppAIProviderUpdateSerializer +) + +class AppAIProviderViewSet(viewsets.ModelViewSet): + lookup_field = 'uuid' + permission_classes = [permissions.IsAuthenticated] + http_method_names = ['get', 'post', 'put', 'patch', 'delete'] + + def get_queryset(self): + application = get_object_or_404( + Application, + uuid=self.kwargs['application_uuid'], + owner=self.request.user + ) + + queryset = AppAIProvider.objects.filter(application=application) + + context = self.request.query_params.get('context') + capability = self.request.query_params.get('capability') + + if context: + queryset = queryset.filter(context=context) + if capability: + queryset = queryset.filter(capability=capability) + + return queryset + + def get_serializer_class(self): + if self.action == 'create': + return AppAIProviderCreateSerializer + elif self.action in ['update', 'partial_update']: + return AppAIProviderUpdateSerializer + return AppAIProviderSerializer + + def get_serializer_context(self): + context = super().get_serializer_context() + application = get_object_or_404( + Application, + uuid=self.kwargs['application_uuid'], + owner=self.request.user + ) + context['application'] = application + return context + + def perform_create(self, serializer): + serializer.save() + + def destroy(self, request, *args, **kwargs): + instance = self.get_object() + self.perform_destroy(instance) + return Response( + {"detail": "deleted"}, + status=status.HTTP_200_OK + ) \ No newline at end of file diff --git a/backend/core/views/application.py b/backend/core/views/application.py index 16fb354..f7c4539 100644 --- a/backend/core/views/application.py +++ b/backend/core/views/application.py @@ -5,14 +5,16 @@ from rest_framework.response import Response from rest_framework.views import APIView +from core.consts import DASHBOARD_USER_ID_PREFIX from core.models import Application, ChatRoom, Message, LLMModel, AppModel +from core.models.chatroom_participant import ChatroomParticipant from core.permissions import HasAPIKeyPermission from core.serializers import ApplicationCreateSerializer, ApplicationViewSerializer from core.serializers.chatroom import ChatRoomPreviewSerializer from core.services.kb_utils import parse_kb_from_request from core.services.kb_utils import create_kb_records from core.tasks import process_kb -from core.widget_auth import WidgetTokenAuthentication +from core.widget_auth import WidgetTokenAuthentication, IsAuthenticatedOrWidget class ApplicationViewSet(viewsets.ModelViewSet): @@ -33,8 +35,6 @@ def create(self, request, *args, **kwargs): create_serializer.is_valid(raise_exception=True) app_instance = create_serializer.save(owner=request.user) - # TODO: may be we need to handle proper log and error messages if default - # TODO: models are not configured yet. AppModel.configure_defaults(app_instance) parsed_kb_items = parse_kb_from_request(request) @@ -81,5 +81,62 @@ def get(self, request, application_uuid): last_message_time=Subquery(last_message_time_subquery, output_field=DateTimeField()) ).order_by('-last_message_time', '-created_at').prefetch_related('messages') - serializer = ChatRoomPreviewSerializer(chatrooms, many=True) + serializer = ChatRoomPreviewSerializer(chatrooms, many=True, context={'user_identifier': f"{DASHBOARD_USER_ID_PREFIX}_{request.user.id}"}) + return Response({'chatrooms': serializer.data}) + + +class UserChatRoomsView(APIView): + """ + Returns chatrooms for a specific widget user (sender_identifier). + Supports optional ?type=human|ai filter based on participant roles. + """ + authentication_classes = [WidgetTokenAuthentication, SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticatedOrWidget | HasAPIKeyPermission] + + def get(self, request, application_uuid): + if request.user and request.user.is_authenticated: + app = Application.objects.filter(uuid=application_uuid, owner=request.user).first() + else: + app = getattr(request, 'application', None) + if not app or str(app.uuid) != str(application_uuid): + return Response({'detail': 'Invalid or unauthorized widget token'}, status=403) + + if not app: + return Response({'detail': 'Application not found.'}, status=status.HTTP_404_NOT_FOUND) + + sender_identifier = request.query_params.get('sender_identifier') + if not sender_identifier: + return Response({'detail': 'sender_identifier is required.'}, status=status.HTTP_400_BAD_REQUEST) + + chat_type = request.query_params.get('type') + + chatroom_ids = ChatroomParticipant.objects.filter( + user_identifier=sender_identifier, + chatroom__application=app, + ).values_list('chatroom_id', flat=True) + + chatrooms = ChatRoom.objects.filter(id__in=chatroom_ids) + + if chat_type == 'human': + human_chatroom_ids = ChatroomParticipant.objects.filter( + chatroom_id__in=chatroom_ids, + role='human_agent', + ).values_list('chatroom_id', flat=True) + chatrooms = chatrooms.filter(id__in=human_chatroom_ids) + elif chat_type == 'ai': + human_chatroom_ids = ChatroomParticipant.objects.filter( + chatroom_id__in=chatroom_ids, + role='human_agent', + ).values_list('chatroom_id', flat=True) + chatrooms = chatrooms.exclude(id__in=human_chatroom_ids) + + last_message_time_subquery = Message.objects.filter( + chatroom=OuterRef('pk') + ).order_by('-created_at').values('created_at')[:1] + + chatrooms = chatrooms.annotate( + last_message_time=Subquery(last_message_time_subquery, output_field=DateTimeField()) + ).order_by('-last_message_time', '-created_at').prefetch_related('messages') + + serializer = ChatRoomPreviewSerializer(chatrooms, many=True, context={'user_identifier': sender_identifier}) return Response({'chatrooms': serializer.data}) diff --git a/backend/core/views/change_password.py b/backend/core/views/change_password.py new file mode 100644 index 0000000..19d5356 --- /dev/null +++ b/backend/core/views/change_password.py @@ -0,0 +1,32 @@ +from rest_framework import status, permissions +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.views import APIView +from django.contrib.auth import update_session_auth_hash +from rest_framework.authtoken.models import Token + +from core.serializers.change_password import ChangePasswordSerializer + + +class ChangePasswordView(APIView): + permission_classes = [IsAuthenticated] + + def post(self, request): + serializer = ChangePasswordSerializer(data=request.data, context={'request': request}) + + if serializer.is_valid(): + user = serializer.save() + + update_session_auth_hash(request, user) + + try: + token = Token.objects.get(user=user) + except Token.DoesNotExist: + token = Token.objects.create(user=user) + + return Response({ + 'message': 'Password changed successfully', + 'token': token.key + }, status=status.HTTP_200_OK) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) diff --git a/backend/core/views/chatroom.py b/backend/core/views/chatroom.py index 2755d09..5ed6193 100644 --- a/backend/core/views/chatroom.py +++ b/backend/core/views/chatroom.py @@ -1,23 +1,51 @@ from rest_framework import status, permissions +from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.views import APIView from rest_framework.response import Response from django.shortcuts import get_object_or_404 from core.permissions import HasAPIKeyPermission from core.models.application import Application from core.models.chatroom import ChatRoom +from core.models.chatroom_participant import ChatroomParticipant +from core.widget_auth import WidgetTokenAuthentication, IsAuthenticatedOrWidget +from core.consts import DASHBOARD_USER_ID_PREFIX +from core.services.unread import mark_read_for_participant, broadcast_unread_update from core.serializers.chatroom import ChatRoomWithMessagesSerializer, ChatRoomDetailSerializer class ChatRoomMessagesView(APIView): - permission_classes = [permissions.IsAuthenticated | HasAPIKeyPermission] + authentication_classes = [WidgetTokenAuthentication, SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticatedOrWidget | HasAPIKeyPermission] def get(self, request, application_uuid, chatroom_uuid): - application = get_object_or_404(Application, uuid=application_uuid, owner=request.user) + if request.user and request.user.is_authenticated: + application = get_object_or_404(Application, uuid=application_uuid, owner=request.user) + else: + application = getattr(request, 'application', None) + if not application or str(application.uuid) != str(application_uuid): + return Response({'detail': 'Invalid or unauthorized widget token'}, status=403) + chatroom = get_object_or_404(ChatRoom, uuid=chatroom_uuid, application=application) - serializer = ChatRoomWithMessagesSerializer(chatroom) + if request.user and request.user.is_authenticated: + user_identifier = f"{DASHBOARD_USER_ID_PREFIX}_{request.user.id}" + else: + user_identifier = ( + request.data.get('sender_identifier') + or request.query_params.get('sender_identifier') + ) + + if user_identifier: + mark_read_for_participant(chatroom, user_identifier) + broadcast_unread_update(user_identifier, str(chatroom.uuid), False, user_identifier) + + if request.user and request.user.is_authenticated: + messages_qs = chatroom.messages.all().order_by('created_at') + else: + messages_qs = chatroom.messages.filter(is_internal=False).order_by('created_at') + serializer = ChatRoomWithMessagesSerializer(chatroom, context={'messages_qs': messages_qs}) return Response(serializer.data) diff --git a/backend/core/views/dummy_view.py b/backend/core/views/dummy_view.py index 79f206d..35a113e 100644 --- a/backend/core/views/dummy_view.py +++ b/backend/core/views/dummy_view.py @@ -8,11 +8,6 @@ class DummyView(APIView): -# # authentication_classes = [APIKeyAuthentication] -# permission_classes = [HasAPIAccessPermission] -# api_action = 'widget_chat' -# -# authentication_classes = [APIKeyAuthentication] permission_classes = [IsAuthenticated | HasAPIKeyPermission] def get(self, request, application_uuid): return Response({ diff --git a/backend/core/views/generate_api_key.py b/backend/core/views/generate_api_key.py index 9a7ac1e..f14f12f 100644 --- a/backend/core/views/generate_api_key.py +++ b/backend/core/views/generate_api_key.py @@ -18,7 +18,8 @@ def post(self, request, *args, **kwargs): "name": api_key_instance.name, "application": api_key_instance.application.id, "permissions": api_key_instance.permissions, - "created": api_key_instance.created + "created": api_key_instance.created, + "owner": api_key_instance.owner.id }, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) @@ -35,7 +36,7 @@ def delete(self, request, api_key_id=None, *args, **kwargs): return Response({"detail": "You do not have permission to delete this API key."}, status=status.HTTP_403_FORBIDDEN) api_key_instance.delete() - return Response({"detail": "API key deleted successfully."}, status=status.HTTP_204_NO_CONTENT) + return Response({"detail": "deleted"}, status=status.HTTP_200_OK) def get(self, request, application_uuid=None): if application_uuid is None: diff --git a/backend/core/views/github_ingestion.py b/backend/core/views/github_ingestion.py new file mode 100644 index 0000000..e10cc32 --- /dev/null +++ b/backend/core/views/github_ingestion.py @@ -0,0 +1,315 @@ +import logging + +from django.shortcuts import get_object_or_404 +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response + +from core.models import AppIntegration +from core.models.github_data import GitHubRepository +from core.serializers.github_serializers import ( + GitHubIngestionRequestSerializer, + GitHubIssueSerializer, + GitHubPullRequestSerializer, + GitHubRepositoryDetailSerializer, + GitHubRepositorySerializer, +) +from core.tasks.github_tasks import ingest_github_repository_task + +logger = logging.getLogger(__name__) + + +def _get_repository_for_user(pk, user): + """Fetch a GitHubRepository and verify ownership in one place.""" + repo = get_object_or_404( + GitHubRepository.objects.select_related('app_integration__application__owner'), + pk=pk, + ) + if repo.app_integration.application.owner != user: + return None, Response( + {'error': 'You do not have access to this repository'}, + status=status.HTTP_403_FORBIDDEN, + ) + return repo, None + + +class GitHubIngestionViewSet(viewsets.ViewSet): + """API endpoints for GitHub data ingestion and retrieval.""" + + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + operation_description="Queue async ingestion of a GitHub repository.", + request_body=GitHubIngestionRequestSerializer, + responses={ + 202: openapi.Response(description="Ingestion queued"), + 400: openapi.Response(description="Invalid parameters"), + 403: openapi.Response(description="Access denied"), + }, + ) + @action(detail=False, methods=['post'], url_path='ingest-repository') + def ingest_repository(self, request): + serializer = GitHubIngestionRequestSerializer( + data=request.data, context={'request': request} + ) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + data = serializer.validated_data + owner = data['owner'] + repo = data['repo'] + since = data.get('since') + app_integration_id = data['app_integration_id'] + full_name = f'{owner}/{repo}' + + app_integration = AppIntegration.objects.select_related('application').get( + id=app_integration_id + ) + + existing_repo = GitHubRepository.objects.filter( + full_name=full_name, + app_integration_id=app_integration_id, + ingestion_status='running', + ).first() + + if existing_repo: + import datetime + from django.utils import timezone + + if existing_repo.updated_at and (timezone.now() - existing_repo.updated_at).total_seconds() > 1800: + logger.warning(f"Found stuck repository ingestion for {full_name}, marking as failed") + existing_repo.ingestion_status = 'failed' + existing_repo.save() + else: + return Response( + { + 'error': 'Ingestion already in progress for this repository.', + 'repository_id': existing_repo.id, + 'status': existing_repo.ingestion_status, + 'last_updated': existing_repo.updated_at.isoformat() if existing_repo.updated_at else None + }, + status=status.HTTP_409_CONFLICT, + ) + + from core.models import KnowledgeBase + from core.tasks.kb import send_kb_update + kb, _ = KnowledgeBase.objects.get_or_create( + application=app_integration.application, + source_type='github', + path=full_name, + defaults={ + 'status': 'pending', + 'metadata': {'source': 'github', 'repository': full_name, 'content': ''}, + } + ) + send_kb_update(kb, kb.status) + + since_str = since.isoformat() if since else None + ingest_github_repository_task.delay(app_integration_id, owner, repo, since_str) + + return Response( + { + 'status': 'queued', + 'repository': full_name, + 'message': 'Repository ingestion started.', + 'kb_uuid': str(kb.uuid), + }, + status=status.HTTP_202_ACCEPTED, + ) + + @swagger_auto_schema( + operation_description="List all ingested repositories for an app integration.", + manual_parameters=[ + openapi.Parameter( + 'app_integration_id', openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, required=True, + ) + ], + responses={200: GitHubRepositorySerializer(many=True)}, + ) + @action(detail=False, methods=['get'], url_path='repositories') + def list_repositories(self, request): + app_integration_id = request.query_params.get('app_integration_id') + if not app_integration_id: + return Response( + {'error': 'app_integration_id is required'}, + status=status.HTTP_400_BAD_REQUEST, + ) + + app_integration = get_object_or_404( + AppIntegration.objects.select_related('application__owner'), + id=app_integration_id, + ) + if app_integration.application.owner != request.user: + return Response( + {'error': 'You do not have access to this integration'}, + status=status.HTTP_403_FORBIDDEN, + ) + + repositories = GitHubRepository.objects.filter( + app_integration=app_integration + ).order_by('-created_at') + return Response(GitHubRepositorySerializer(repositories, many=True).data) + + @swagger_auto_schema( + operation_description="Get full details of a single ingested repository.", + responses={200: GitHubRepositoryDetailSerializer()}, + ) + @action(detail=True, methods=['get'], url_path='detail') + def retrieve_repository(self, request, pk=None): + repository, err = _get_repository_for_user(pk, request.user) + if err: + return err + return Response(GitHubRepositoryDetailSerializer(repository).data) + + @swagger_auto_schema( + operation_description="List issues for a repository.", + manual_parameters=[ + openapi.Parameter( + 'state', openapi.IN_QUERY, + type=openapi.TYPE_STRING, + enum=['open', 'closed', 'all'], default='all', + ) + ], + responses={200: GitHubIssueSerializer(many=True)}, + ) + @action(detail=True, methods=['get'], url_path='issues') + def list_issues(self, request, pk=None): + repository, err = _get_repository_for_user(pk, request.user) + if err: + return err + + state_filter = request.query_params.get('state', 'all') + issues = repository.issues.prefetch_related('comments').all() + if state_filter in ('open', 'closed'): + issues = issues.filter(state=state_filter) + + return Response(GitHubIssueSerializer(issues, many=True).data) + + @swagger_auto_schema( + operation_description="List pull requests for a repository.", + manual_parameters=[ + openapi.Parameter( + 'state', openapi.IN_QUERY, + type=openapi.TYPE_STRING, + enum=['open', 'closed', 'merged', 'all'], default='all', + ) + ], + responses={200: GitHubPullRequestSerializer(many=True)}, + ) + @action(detail=True, methods=['get'], url_path='pull-requests') + def list_pull_requests(self, request, pk=None): + repository, err = _get_repository_for_user(pk, request.user) + if err: + return err + + state_filter = request.query_params.get('state', 'all') + prs = repository.pull_requests.prefetch_related('comments', 'files').all() + if state_filter in ('open', 'closed', 'merged'): + prs = prs.filter(state=state_filter) + + return Response(GitHubPullRequestSerializer(prs, many=True).data) + + + + + @swagger_auto_schema( + operation_description="Queue re-ingestion of an already-ingested repository.", + request_body=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + 'since': openapi.Schema( + type=openapi.TYPE_STRING, + description='ISO datetime — only ingest data after this point (optional)', + ) + }, + ), + responses={ + 202: openapi.Response(description="Re-ingestion queued"), + 403: openapi.Response(description="Access denied"), + 409: openapi.Response(description="Ingestion already running"), + }, + ) + @action(detail=True, methods=['post'], url_path='re-ingest') + def re_ingest_repository(self, request, pk=None): + repository, err = _get_repository_for_user(pk, request.user) + if err: + return err + + if repository.ingestion_status == 'running': + import datetime + from django.utils import timezone + + if repository.updated_at and (timezone.now() - repository.updated_at).total_seconds() > 1800: + logger.warning(f"Found stuck repository ingestion for {repository.full_name}, marking as failed") + repository.ingestion_status = 'failed' + repository.save() + else: + return Response( + { + 'error': 'Ingestion already in progress for this repository.', + 'repository_id': repository.id, + 'status': repository.ingestion_status, + 'last_updated': repository.updated_at.isoformat() if repository.updated_at else None + }, + status=status.HTTP_409_CONFLICT, + ) + + since = request.data.get('since') or request.query_params.get('since') + owner, repo = repository.full_name.split('/', 1) + ingest_github_repository_task.delay( + repository.app_integration_id, owner, repo, since + ) + + return Response( + { + 'status': 'queued', + 'repository': repository.full_name, + 'message': 'Repository re-ingestion started.', + }, + status=status.HTTP_202_ACCEPTED, + ) + + @swagger_auto_schema( + operation_description="Delete a repository and all its ingested data, including vectors.", + responses={ + 204: openapi.Response(description="Deleted"), + 403: openapi.Response(description="Access denied"), + }, + ) + @action(detail=True, methods=['delete'], url_path='delete') + def delete_repository(self, request, pk=None): + repository, err = _get_repository_for_user(pk, request.user) + if err: + return err + + full_name = repository.full_name + app = repository.app_integration.application + + try: + from core.models import KnowledgeBase, IngestedChunk + from core.services.ingestion import delete_vectors_from_qdrant + + kb = KnowledgeBase.objects.filter( + application=app, + source_type='github', + path=full_name, + ).first() + + if kb: + chunks = IngestedChunk.objects.filter(knowledge_base=kb) + qdrant_ids = [str(c.uuid) for c in chunks] + chunks.delete() + delete_vectors_from_qdrant(qdrant_ids) + kb.delete() + logger.info(f"Deleted KnowledgeBase and {len(qdrant_ids)} vectors for {full_name}") + except Exception as e: + logger.warning(f"Failed to clean up KB/vectors for {full_name}: {e}") + + repository.delete() + logger.info(f"Deleted GitHub repository record: {full_name}") + + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/backend/core/views/human_agent.py b/backend/core/views/human_agent.py new file mode 100644 index 0000000..b698ee4 --- /dev/null +++ b/backend/core/views/human_agent.py @@ -0,0 +1,34 @@ +from rest_framework.views import APIView +from rest_framework.response import Response +from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from django.shortcuts import get_object_or_404 + +from core.models.application import Application +from core.widget_auth import WidgetTokenAuthentication, IsAuthenticatedOrWidget +from core.permissions import HasAPIKeyPermission +from core.consts import DASHBOARD_USER_ID_PREFIX + + +class HumanAgentInfoView(APIView): + """ + Returns the human agent (app owner) info for the widget's Agent tab. + The owner's user_identifier is `reg:{user.id}` — stable and unique per user. + """ + authentication_classes = [WidgetTokenAuthentication, SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticatedOrWidget | HasAPIKeyPermission] + + def get(self, request, application_uuid): + if request.user and request.user.is_authenticated: + app = get_object_or_404(Application, uuid=application_uuid, owner=request.user) + else: + app = getattr(request, 'application', None) + if not app or str(app.uuid) != str(application_uuid): + return Response({'detail': 'Invalid or unauthorized widget token'}, status=403) + + owner = app.owner + return Response({ + 'user_identifier': f"{DASHBOARD_USER_ID_PREFIX}:{owner.id}", + 'name': owner.get_full_name() or owner.username, + 'email': owner.email, + 'is_online': True, # always online for now; can be extended with presence tracking + }) diff --git a/backend/core/views/ingestion.py b/backend/core/views/ingestion.py index 78ec66a..7ab83a5 100644 --- a/backend/core/views/ingestion.py +++ b/backend/core/views/ingestion.py @@ -7,14 +7,12 @@ from core.permissions import HasAPIKeyPermission from core.tasks import process_kb -# Not yet used - Use it when we allow users to re-process knowledge base class IngestApplicationKBView(APIView): permission_classes = [permissions.IsAuthenticated | HasAPIKeyPermission] def post(self, request, application_uuid): app = get_object_or_404(Application, uuid=application_uuid, owner=request.user) kbs = app.knowledge_bases.filter(status='pending') - - # TODO: Check if text & embedding models are configured + process_kb.delay([kb.id for kb in kbs]) return Response({"message": "Ingestion completed."}) diff --git a/backend/core/views/integration.py b/backend/core/views/integration.py index f049f47..3dfd344 100644 --- a/backend/core/views/integration.py +++ b/backend/core/views/integration.py @@ -5,8 +5,8 @@ from core.integrations.registry import SUPPORTED_INTEGRATIONS, INTEGRATION_TOOLS, \ SUPPORTED_PROVIDERS -from core.models import Integration -from core.serializers import IntegrationCreateSerializer, IntegrationViewSerializer +from core.models import Integration, AppIntegration +from core.serializers import IntegrationCreateSerializer, IntegrationViewSerializer, AppIntegrationViewSerializer class IntegrationViewSet(viewsets.ModelViewSet): queryset = Integration.objects.none() @@ -39,6 +39,31 @@ def destroy(self, request, *args, **kwargs): status=status.HTTP_200_OK ) +class AppIntegrationViewSet(viewsets.ModelViewSet): + queryset = AppIntegration.objects.none() + permission_classes = [IsAuthenticated] + lookup_field = 'uuid' + http_method_names = ['get', 'post', 'delete'] + + def get_queryset(self): + return AppIntegration.objects.filter(application__owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return AppIntegrationCreateSerializer + return AppIntegrationViewSerializer + + def perform_create(self, serializer): + serializer.save() + + def destroy(self, request, *args, **kwargs): + instance = self.get_object() + self.perform_destroy(instance) + return Response( + {"detail": "Deleted"}, + status=status.HTTP_200_OK + ) + @api_view(['GET']) @permission_classes([IsAuthenticated]) def supported_integrations(request): diff --git a/backend/core/views/knowledge_base.py b/backend/core/views/knowledge_base.py index ce12615..de78a7a 100644 --- a/backend/core/views/knowledge_base.py +++ b/backend/core/views/knowledge_base.py @@ -4,11 +4,12 @@ from rest_framework.generics import get_object_or_404 from rest_framework.parsers import MultiPartParser, FormParser, JSONParser from rest_framework.response import Response -from core.models import KnowledgeBase, Application, IngestedChunk, LLMModel +from core.models import KnowledgeBase, Application, IngestedChunk from core.serializers import KnowledgeBaseCreateSerializer, KnowledgeBaseViewSerializer, ApplicationViewSerializer from core.permissions import HasAPIKeyPermission from core.services.ingestion import delete_vectors_from_qdrant from core.services.kb_utils import create_kb_records, parse_kb_from_request, format_text_uri +from core.services.ai_client_service import AIClientService from core.tasks import process_kb import logging @@ -58,18 +59,19 @@ def create(self, request, *args, **kwargs): created_kbs = create_kb_records(application, items) - # TODO: May be we need a better error handling approach here - # TODO: Serializer with validator might work + ai_client_service = AIClientService() + _, embedding_model = ai_client_service.get_client_and_model( + app=application, context='response', capability='embedding' + ) + _, text_model = ai_client_service.get_client_and_model( + app=application, context='response', capability='text' + ) + errors = {} - text_model = application.get_model_by_type(LLMModel.ModelType.TEXT) if not text_model: - errors[ - "text_model"] = f"Text model not found for application {application.name}. Please configure a TEXT model." - - embedding_model = application.get_model_by_type(LLMModel.ModelType.EMBEDDING) + errors["text_model"] = f"Text model not found for application {application.name}. Please configure a TEXT model." if not embedding_model: - errors[ - "embedding_model"] = f"Embedding model not found for application {application.name}. Please configure an EMBEDDING model." + errors["embedding_model"] = f"Embedding model not found for application {application.name}. Please configure an EMBEDDING model." if errors: return Response({"errors": errors}, status=status.HTTP_400_BAD_REQUEST) @@ -100,8 +102,7 @@ def partial_update(self, request, *args, **kwargs): fields_to_update.append("path") kb.save(update_fields=fields_to_update) - - # TODO: We need to check whether text & embedding models are configured here as well + process_kb.delay([kb.id]) return Response(KnowledgeBaseViewSerializer(kb).data) @@ -126,4 +127,4 @@ def destroy(self, request, *args, **kwargs): logger.warning(f"Failed to delete file for KB {kb.uuid}: {e}") kb.delete() - return Response(status=status.HTTP_204_NO_CONTENT) \ No newline at end of file + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/backend/core/views/message.py b/backend/core/views/message.py index cbc04d6..76d4bc6 100644 --- a/backend/core/views/message.py +++ b/backend/core/views/message.py @@ -10,10 +10,13 @@ import logging from core.consts import LIVE_UPDATES_PREFIX +from core.services.unread import mark_unread_for_participants, broadcast_unread_update from core.models.application import Application from core.models.chatroom import ChatRoom from core.models.chatroom_participant import ChatroomParticipant from core.models.message import Message +from core.models.ai_provider import AIProvider +from core.utils import normalize_model_name_by_provider from core.serializers.message import CreateMessageSerializer, ViewMessageSerializer from core.tasks import generate_bot_response @@ -29,8 +32,8 @@ def generate_chatroom_name(a, b): class SendMessageView(APIView): authentication_classes = [WidgetTokenAuthentication, SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticatedOrWidget | HasAPIKeyPermission] + def post(self, request, application_uuid): - # Refactor if request.user and request.user.is_authenticated: app = get_object_or_404(Application, uuid=application_uuid, owner=request.user) else: @@ -38,7 +41,9 @@ def post(self, request, application_uuid): if not app or str(app.uuid) != str(application_uuid): return Response({'detail': 'Invalid or unauthorized widget token'}, status=403) - serializer = CreateMessageSerializer(data=request.data) + platform = 'dashboard' if (request.user and request.user.is_authenticated) else 'widget' + + serializer = CreateMessageSerializer(data=request.data, app_owner=app.owner) serializer.is_valid(raise_exception=True) data = serializer.validated_data @@ -47,8 +52,17 @@ def post(self, request, application_uuid): message_text = data['message'] metadata = data.get('metadata', {}) - send_to_user = data['send_to_user'] - metadata['send_to_user'] = send_to_user + is_internal = data.get('is_internal', False) if (request.user and request.user.is_authenticated) else False + + if platform == 'dashboard' and not is_internal and data.get('ai_mode', False): + return Response( + {'detail': 'AI mode is not permitted on external messages (is_internal=false).'}, + status=status.HTTP_400_BAD_REQUEST + ) + + ai_mode = data.get('ai_mode', False) + ai_provider_id = data.get('ai_provider') + model = data.get('model') if not chatroom_uuid and not sender_id: return Response( @@ -59,8 +73,27 @@ def post(self, request, application_uuid): if chatroom_uuid != 'new_chat': chatroom = get_object_or_404(ChatRoom, uuid=chatroom_uuid, application=app) - if sender_id and not ChatroomParticipant.objects.filter(chatroom=chatroom, - user_identifier=sender_id).exists(): + updated = False + if ai_provider_id is not None and ai_provider_id != chatroom.ai_provider_id: + chatroom.ai_provider_id = ai_provider_id + updated = True + if model is not None and model != chatroom.model: + provider_name = '' + if ai_provider_id: + try: + ai_provider = AIProvider.objects.only('provider').get(id=ai_provider_id) + provider_name = ai_provider.provider + except AIProvider.DoesNotExist: + pass + model = normalize_model_name_by_provider(model, provider_name) + chatroom.model = model + updated = True + if updated: + chatroom.save() + + if sender_id and not ChatroomParticipant.objects.filter( + chatroom=chatroom, user_identifier=sender_id + ).exists(): ChatroomParticipant.objects.create( chatroom=chatroom, user_identifier=sender_id, @@ -73,30 +106,56 @@ def post(self, request, application_uuid): with transaction.atomic(): chatroom = ChatRoom.objects.create( application=app, - name=chatroom_name + name=chatroom_name, + ai_provider_id=ai_provider_id, + model=model, ) + agent_identifier = metadata.get('human_agent_identifier', AGENT_IDENTIFIER) + ChatroomParticipant.objects.bulk_create([ ChatroomParticipant(chatroom=chatroom, user_identifier=sender_id, role='user'), - ChatroomParticipant(chatroom=chatroom, user_identifier=AGENT_IDENTIFIER, role='agent'), + ChatroomParticipant(chatroom=chatroom, user_identifier=agent_identifier, role='agent'), ]) message = Message.objects.create( chatroom=chatroom, sender_identifier=sender_id, message=message_text, - metadata=metadata + metadata=metadata, + is_internal=is_internal, + platform=platform, + ai_mode=ai_mode, + ai_provider_id=ai_provider_id, + model=model, ) - if send_to_user: - channel_layer = get_channel_layer() - participants = list( + unread_identifiers = mark_unread_for_participants(chatroom, sender_id, is_internal=is_internal) + for user_identifier in unread_identifiers: + broadcast_unread_update(user_identifier, str(chatroom.uuid), True, sender_id) + + channel_layer = get_channel_layer() + + def get_widget_participants(): + return list( message.chatroom.participants.filter( - Q(role='user') & Q(user_identifier__startswith='anon_') + user_identifier__startswith='widget_' + ).exclude( + user_identifier=sender_id ).values_list('user_identifier', flat=True) ) - for participant_id in participants: + def get_dashboard_participants(): + return list( + message.chatroom.participants.filter( + Q(user_identifier__startswith='dashboard_') | Q(role='human_agent') + ).exclude( + user_identifier=sender_id + ).values_list('user_identifier', flat=True) + ) + + def broadcast_to(participant_ids): + for participant_id in participant_ids: group_name = f"{LIVE_UPDATES_PREFIX}_{participant_id}" try: async_to_sync(channel_layer.group_send)( @@ -109,11 +168,24 @@ def post(self, request, application_uuid): except Exception as e: logger.warning(f"Failed to send message to {group_name}: {str(e)}") + if platform == 'widget': + if ai_mode: + broadcast_to(get_widget_participants() + get_dashboard_participants()) + generate_bot_response.delay(message.id, app.uuid, ai_provider_id, model) + else: + broadcast_to(get_dashboard_participants()) + else: - generate_bot_response.delay(message.id, app.uuid) + if is_internal: + if ai_mode: + broadcast_to(get_dashboard_participants()) + generate_bot_response.delay(message.id, app.uuid, ai_provider_id, model) + else: + pass + else: + broadcast_to(get_widget_participants()) response_data = ViewMessageSerializer(message).data response_data['message_status'] = 'message_sent' - response_data['llm_processing'] = True - response_data['chatroom_identifier'] = chatroom.uuid - return Response(ViewMessageSerializer(message).data, status=status.HTTP_200_OK) + response_data['chatroom_identifier'] = str(chatroom.uuid) + return Response(response_data, status=status.HTTP_200_OK) diff --git a/backend/github_data/__init__.py b/backend/github_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/github_data/admin.py b/backend/github_data/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/backend/github_data/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/backend/github_data/apps.py b/backend/github_data/apps.py new file mode 100644 index 0000000..0548126 --- /dev/null +++ b/backend/github_data/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class GithubDataConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'github_data' diff --git a/backend/github_data/migrations/0001_initial.py b/backend/github_data/migrations/0001_initial.py new file mode 100644 index 0000000..9dfd781 --- /dev/null +++ b/backend/github_data/migrations/0001_initial.py @@ -0,0 +1,12 @@ +# Generated by Django 5.2.3 on 2026-03-13 18:43 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + ] diff --git a/backend/github_data/migrations/__init__.py b/backend/github_data/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/github_data/models.py b/backend/github_data/models.py new file mode 100644 index 0000000..724d092 --- /dev/null +++ b/backend/github_data/models.py @@ -0,0 +1,11 @@ +from core.models.github_data import ( + GitHubRepository, GitHubIssue, GitHubIssueComment, GitHubPullRequest, + GitHubPRComment, GitHubPRFile, GitHubDiscussion, GitHubDiscussionComment, + GitHubWikiPage, GitHubRepositoryFile +) + +__all__ = [ + 'GitHubRepository', 'GitHubIssue', 'GitHubIssueComment', 'GitHubPullRequest', + 'GitHubPRComment', 'GitHubPRFile', 'GitHubDiscussion', 'GitHubDiscussionComment', + 'GitHubWikiPage', 'GitHubRepositoryFile' +] diff --git a/backend/core/tests.py b/backend/github_data/tests.py similarity index 100% rename from backend/core/tests.py rename to backend/github_data/tests.py diff --git a/backend/github_data/views.py b/backend/github_data/views.py new file mode 100644 index 0000000..2800278 --- /dev/null +++ b/backend/github_data/views.py @@ -0,0 +1,2 @@ +from django.shortcuts import render + diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..a211eea --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,20 @@ +[tool:pytest] +DJANGO_SETTINGS_MODULE = config.test_settings +python_files = tests.py test_*.py *_tests.py +python_classes = Test* +python_functions = test_* +addopts = + --strict-markers + --strict-config + --disable-warnings + --tb=short + --cov=core + --cov-report=html + --cov-report=term-missing + --cov-fail-under=80 +testpaths = core/tests +markers = + unit: Unit tests + integration: Integration tests + api: API endpoint tests + slow: Slow running tests diff --git a/backend/requirements.txt b/backend/requirements.txt index 71dc9c8..feb01b9 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -107,7 +107,7 @@ python-docx==1.2.0 python-dotenv==1.1.1 pytz==2025.2 PyYAML==6.0.2 -qdrant-client==1.14.3 +qdrant-client==1.17.0 redis==6.2.0 regex==2024.11.6 requests==2.32.4 @@ -128,7 +128,7 @@ tenacity==8.5.0 threadpoolctl==3.6.0 tokenizers==0.21.2 tomli==2.2.1 -torch==2.7.1 +torch==2.2.2 tqdm==4.67.1 transformers==4.53.0 Twisted==25.5.0 @@ -144,3 +144,4 @@ wcwidth==0.2.13 websockets==15.0.1 zope.interface==7.2 zstandard==0.23.0 +gql==3.5.0 diff --git a/backend/templates/customer_support.j2 b/backend/templates/prompts/default.j2 similarity index 96% rename from backend/templates/customer_support.j2 rename to backend/templates/prompts/default.j2 index 2d5c247..fb5033e 100644 --- a/backend/templates/customer_support.j2 +++ b/backend/templates/prompts/default.j2 @@ -1,4 +1,4 @@ -You are a professional and fact-based customer support assistant for **{{ product_name }}**. +You are a professional and fact-based customer support specialist for **{{ product_name }}**. You can call tools when needed to fetch information. Always follow the schema. diff --git a/curl/get_issue_comments b/curl/get_issue_comments new file mode 100644 index 0000000..d66b3ca --- /dev/null +++ b/curl/get_issue_comments @@ -0,0 +1,18 @@ +#!/bin/bash + +# GitHub API - Get Issue Comments +# Usage: ./get_issue_comments.sh owner repo issue_number [page] [per_page] + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +ISSUE_NUMBER=${3:-"1"} +PAGE=${4:-"1"} +PER_PAGE=${5:-"100"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/repos/${OWNER}/${REPO}/issues/${ISSUE_NUMBER}/comments?page=${PAGE}&per_page=${PER_PAGE}" diff --git a/curl/get_issues b/curl/get_issues new file mode 100644 index 0000000..e5e807a --- /dev/null +++ b/curl/get_issues @@ -0,0 +1,25 @@ +#!/bin/bash + +# GitHub API - Get Issues +# Usage: ./get_issues.sh owner repo [state] [since] [page] [per_page] + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +STATE=${3:-"all"} +SINCE=${4:-""} +PAGE=${5:-"1"} +PER_PAGE=${6:-"100"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +URL="https://api.github.com/repos/${OWNER}/${REPO}/issues?page=${PAGE}&per_page=${PER_PAGE}&state=${STATE}&sort=created&direction=desc" + +if [ -n "$SINCE" ]; then + URL="${URL}&since=${SINCE}" +fi + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "${URL}" diff --git a/curl/get_pull_request_comments b/curl/get_pull_request_comments new file mode 100644 index 0000000..7cef03d --- /dev/null +++ b/curl/get_pull_request_comments @@ -0,0 +1,18 @@ +#!/bin/bash + +# GitHub API - Get Pull Request Comments +# Usage: ./get_pull_request_comments.sh owner repo pr_number [page] [per_page] + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +PR_NUMBER=${3:-"1"} +PAGE=${4:-"1"} +PER_PAGE=${5:-"100"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/repos/${OWNER}/${REPO}/pulls/${PR_NUMBER}/comments?page=${PAGE}&per_page=${PER_PAGE}" diff --git a/curl/get_pull_request_files b/curl/get_pull_request_files new file mode 100644 index 0000000..75fe92b --- /dev/null +++ b/curl/get_pull_request_files @@ -0,0 +1,18 @@ +#!/bin/bash + +# GitHub API - Get Pull Request Files +# Usage: ./get_pull_request_files.sh owner repo pr_number [page] [per_page] + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +PR_NUMBER=${3:-"1"} +PAGE=${4:-"1"} +PER_PAGE=${5:-"100"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/repos/${OWNER}/${REPO}/pulls/${PR_NUMBER}/files?page=${PAGE}&per_page=${PER_PAGE}" diff --git a/curl/get_pull_requests b/curl/get_pull_requests new file mode 100644 index 0000000..2d23cc1 --- /dev/null +++ b/curl/get_pull_requests @@ -0,0 +1,25 @@ +#!/bin/bash + +# GitHub API - Get Pull Requests +# Usage: ./get_pull_requests.sh owner repo [state] [since] [page] [per_page] + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +STATE=${3:-"all"} +SINCE=${4:-""} +PAGE=${5:-"1"} +PER_PAGE=${6:-"100"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +URL="https://api.github.com/repos/${OWNER}/${REPO}/pulls?page=${PAGE}&per_page=${PER_PAGE}&state=${STATE}&sort=created&direction=desc" + +if [ -n "$SINCE" ]; then + URL="${URL}&since=${SINCE}" +fi + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "${URL}" diff --git a/curl/get_rate_limit b/curl/get_rate_limit new file mode 100644 index 0000000..ac019c7 --- /dev/null +++ b/curl/get_rate_limit @@ -0,0 +1,13 @@ +#!/bin/bash + +# GitHub API - Get Rate Limit Status +# Usage: ./get_rate_limit.sh + +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/rate_limit" diff --git a/curl/get_repository_file b/curl/get_repository_file new file mode 100644 index 0000000..b3101a1 --- /dev/null +++ b/curl/get_repository_file @@ -0,0 +1,16 @@ +#!/bin/bash + +# GitHub API - Get Repository File +# Usage: ./get_repository_file.sh owner repo path + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +PATH=${3:-"README.md"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/repos/${OWNER}/${REPO}/contents/${PATH}" diff --git a/curl/get_repository_info b/curl/get_repository_info new file mode 100644 index 0000000..95aa70c --- /dev/null +++ b/curl/get_repository_info @@ -0,0 +1,15 @@ +#!/bin/bash + +# GitHub API - Get Repository Information +# Usage: ./get_repository_info.sh owner repo + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/repos/${OWNER}/${REPO}" diff --git a/curl/get_wiki_pages b/curl/get_wiki_pages new file mode 100644 index 0000000..29bff1e --- /dev/null +++ b/curl/get_wiki_pages @@ -0,0 +1,18 @@ +#!/bin/bash + +# GitHub API - Get Wiki Pages +# Usage: ./get_wiki_pages.sh owner repo + +OWNER=${1:-"owner"} +REPO=${2:-"repo"} +TOKEN=${GITHUB_TOKEN:-"your_github_token_here"} + +# Note: This would require GraphQL API for full wiki access +# This is a simplified version that gets basic wiki info + +curl -X GET \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -H "User-Agent: Ch8r-GitHub-Ingestion/1.0" \ + "https://api.github.com/repos/${OWNER}/${REPO}/pages" diff --git a/frontend/.env.example b/frontend/.env.example index 30da259..8f82244 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -1,3 +1,4 @@ DOMAIN=localhost:8000 BACKEND_BASE_URL=http://localhost:8000 API_BASE_URL=http://localhost:8000/api +NODE_ENV=production diff --git a/frontend/app.vue b/frontend/app.vue index e6e3ba4..afb99ab 100644 --- a/frontend/app.vue +++ b/frontend/app.vue @@ -1,8 +1,49 @@ + +const userStore = useUserStore() +useHead({ + script: [ + { + 'src': '/widget.js', + 'defer': true, + 'data-api-base-url': 'http://localhost:8000', + 'data-app-uuid': '2279ab6a-2bd2-4112-ad82-eddbfc561e2b', + 'data-token': 'widget_SHXjQG4WjiY5sco85YlMT_x5mVaXNs5xe3MQ9Zsxg20', + 'data-user-identifier': userStore.getUser.id, + 'data-app-name': 'ch8r support', + 'data-app-description': 'We\'re here to help', + 'data-app-logo-url': 'http://localhost:3000/favicon.ico', + 'data-offset-bottom': '64' + } + ] +}) + +// onMounted(() => { +// window.addEventListener('message', (event) => { +// if (event.data?.type === 'ch8r-resize') { +// const iframe = document.getElementById('ch8r-widget-iframe') +// if (iframe && event.data.height) { +// iframe.style.height = `${event.data.height}px` +// if (event.data.height > 100) { +// iframe.style.width = '360px' +// } else { +// iframe.style.width = '88px' +// } +// } +// } +// }) +// }) + diff --git a/frontend/components/AIProvider/NewAIProvider.vue b/frontend/components/AIProvider/NewAIProvider.vue new file mode 100644 index 0000000..c92ba16 --- /dev/null +++ b/frontend/components/AIProvider/NewAIProvider.vue @@ -0,0 +1,270 @@ + + + diff --git a/frontend/components/AIProvider/UpdateAIProvider.vue b/frontend/components/AIProvider/UpdateAIProvider.vue new file mode 100644 index 0000000..f197ff7 --- /dev/null +++ b/frontend/components/AIProvider/UpdateAIProvider.vue @@ -0,0 +1,217 @@ + + diff --git a/frontend/components/ApiKey/NewApiKey.vue b/frontend/components/ApiKey/NewApiKey.vue index ec13903..931cfe1 100644 --- a/frontend/components/ApiKey/NewApiKey.vue +++ b/frontend/components/ApiKey/NewApiKey.vue @@ -1,13 +1,26 @@ + diff --git a/frontend/components/App/AppAIModelsConfiguration.vue b/frontend/components/App/AppAIModelsConfiguration.vue new file mode 100644 index 0000000..9187b1c --- /dev/null +++ b/frontend/components/App/AppAIModelsConfiguration.vue @@ -0,0 +1,33 @@ + + + diff --git a/frontend/components/App/AppNotificationConfiguration.vue b/frontend/components/App/AppNotificationConfiguration.vue new file mode 100644 index 0000000..795e1dd --- /dev/null +++ b/frontend/components/App/AppNotificationConfiguration.vue @@ -0,0 +1,88 @@ + + + diff --git a/frontend/components/App/AppPMSConfiguration.vue b/frontend/components/App/AppPMSConfiguration.vue new file mode 100644 index 0000000..0e33506 --- /dev/null +++ b/frontend/components/App/AppPMSConfiguration.vue @@ -0,0 +1,94 @@ + + + diff --git a/frontend/components/App/ConfigureAIModels.vue b/frontend/components/App/ConfigureAIModels.vue new file mode 100644 index 0000000..85f052e --- /dev/null +++ b/frontend/components/App/ConfigureAIModels.vue @@ -0,0 +1,239 @@ + + + diff --git a/frontend/components/App/ConfigureApp.vue b/frontend/components/App/ConfigureApp.vue index dcf18c3..a0915ec 100644 --- a/frontend/components/App/ConfigureApp.vue +++ b/frontend/components/App/ConfigureApp.vue @@ -1,256 +1,79 @@ + diff --git a/frontend/components/App/ConfigureEmbeddingModels.vue b/frontend/components/App/ConfigureEmbeddingModels.vue new file mode 100644 index 0000000..2f7bcbd --- /dev/null +++ b/frontend/components/App/ConfigureEmbeddingModels.vue @@ -0,0 +1,17 @@ + + + diff --git a/frontend/components/App/ConfigureTextResponseModels.vue b/frontend/components/App/ConfigureTextResponseModels.vue new file mode 100644 index 0000000..d7e5cb8 --- /dev/null +++ b/frontend/components/App/ConfigureTextResponseModels.vue @@ -0,0 +1,17 @@ + + + diff --git a/frontend/components/AppSidebar.vue b/frontend/components/AppSidebar.vue index e9ecc7c..e44edce 100644 --- a/frontend/components/AppSidebar.vue +++ b/frontend/components/AppSidebar.vue @@ -14,9 +14,10 @@ import { ChevronDown, ChevronUp, Puzzle, + Lock, } from 'lucide-vue-next' import SlideOver from '~/components/SlideOver.vue' -import { ref, computed, onMounted, watch } from 'vue' +import { ref, computed, onMounted, onBeforeUnmount, watch } from 'vue' import { Sidebar, @@ -95,13 +96,13 @@ watch( } if ( - newPath.includes('/knowledge-base') && - !newPath.match(/\/knowledge-base\/[^/]+$/) + newPath.includes('/knowledge-base') + && !newPath.match(/\/knowledge-base\/[^/]+$/) ) { setActiveMenu('knowledge-base') } else if ( - newPath.includes('/api-keys-and-widget') && - !newPath.match(/\/api-keys-and-widget\/[^/]+$/) + newPath.includes('/api-keys-and-widget') + && !newPath.match(/\/api-keys-and-widget\/[^/]+$/) ) { setActiveMenu('api-keys') } @@ -112,6 +113,33 @@ watch( const { isMobile } = useSidebar() const applicationsStore = useApplicationsStore() const chatroomStore = useChatroomStore() +const liveUpdateStore = useLiveUpdateStore() + +// Subscribe to unread_update events from the WebSocket +const unsubscribeUnread = liveUpdateStore.subscribe((msg) => { + if (msg.type === 'unread_update') { + const { chatroom_uuid, has_unread } = msg as any + if (has_unread) { + chatroomStore.markUnread(chatroom_uuid) + } else { + chatroomStore.markRead(chatroom_uuid) + } + } + if (msg.type === 'message') { + const data = (msg as any).data + if (data?.chatroom_identifier) { + chatroomStore.updateLastMessage(data.chatroom_identifier, data) + } + } +}) +onBeforeUnmount(() => unsubscribeUnread()) + +// Re-fetch chatrooms on WebSocket reconnect to reconcile unread state +watch(() => liveUpdateStore.isConnected, (connected, wasConnected) => { + if (connected && wasConnected === false && selectedApplication.value?.uuid) { + chatroomStore.fetchChatrooms(selectedApplication.value.uuid) + } +}) const { selectAppAndNavigate } = useNavigation() const { ellipsis } = useTextUtils() @@ -168,7 +196,10 @@ async function initNewChat() { - +
@@ -201,10 +232,20 @@ async function initNewChat() { Settings - - + + -
+
@@ -258,6 +299,24 @@ async function initNewChat() { Notification Profile + + + + + Change Password + +
Knowledge Base
@@ -283,11 +342,11 @@ async function initNewChat() { ? 'bg-sidebar-accent text-sidebar-accent-foreground font-semibold' : '', ]" - @click="setActiveMenu('api-keys')" >
API Keys & Widget
@@ -311,40 +370,47 @@ async function initNewChat() { -
- Conversations +
+ Conversations
- - + + -
- {{ ellipsis(chatroom.name, 30) }} - - {{ $dayjs(chatroom.last_message?.created_at).fromNow() }} +
+
+ {{ ellipsis(chatroom.name, 30) }} + + {{ $dayjs(chatroom.last_message?.created_at).fromNow() }} + +
+ + {{ chatroom.last_message?.message }}
- - {{ chatroom.last_message?.message }} - + - + + + + {{ apiError.error }} + +

{{ apiError.details }}

+ + + + + diff --git a/frontend/components/C8Button.vue b/frontend/components/C8Button.vue index cb6b646..7ab8560 100644 --- a/frontend/components/C8Button.vue +++ b/frontend/components/C8Button.vue @@ -1,6 +1,7 @@ + + diff --git a/frontend/components/C8Dialog.vue b/frontend/components/C8Dialog.vue new file mode 100644 index 0000000..13f1c04 --- /dev/null +++ b/frontend/components/C8Dialog.vue @@ -0,0 +1,81 @@ + + + diff --git a/frontend/components/C8Item.vue b/frontend/components/C8Item.vue new file mode 100644 index 0000000..69e2662 --- /dev/null +++ b/frontend/components/C8Item.vue @@ -0,0 +1,69 @@ + + + diff --git a/frontend/components/C8Loader.vue b/frontend/components/C8Loader.vue new file mode 100644 index 0000000..6c80ace --- /dev/null +++ b/frontend/components/C8Loader.vue @@ -0,0 +1,20 @@ + + + diff --git a/frontend/components/C8Select.vue b/frontend/components/C8Select.vue index 1766493..7f0aaee 100644 --- a/frontend/components/C8Select.vue +++ b/frontend/components/C8Select.vue @@ -1,11 +1,21 @@