From 3983ee46f7163ba2246f1c9949d2d36581b76af9 Mon Sep 17 00:00:00 2001 From: TAP Date: Wed, 17 Dec 2025 22:52:28 +0530 Subject: [PATCH 01/14] feedback for plg --- rag_service/core/langchain_manager.py | 137 +++++++++++++++++- rag_service/handlers/feedback_handler.py | 27 +++- .../feedback_request/feedback_request.js | 31 +++- .../feedback_request/feedback_request.json | 51 ++++++- 4 files changed, 237 insertions(+), 9 deletions(-) diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py index 29b719d..c736c43 100644 --- a/rag_service/core/langchain_manager.py +++ b/rag_service/core/langchain_manager.py @@ -171,7 +171,7 @@ def get_default_response_format(self) -> Dict: "encouragement": "Encouraging message for the student" } - async def generate_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: + async def generate_feedback_universal(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: """Generate feedback using universal template approach""" try: print("\n=== Starting Universal Feedback Generation ===") @@ -262,6 +262,141 @@ async def generate_feedback(self, assignment_context: Dict, submission_url: str, # Return structured error response return self.create_error_feedback(assignment_context) + + + + async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, + plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: + """Generate feedback with plagiarism context""" + + result_status = "Pending" + + try: + # Check for plagiarism/AI-generated content first + if plagiarism_data: + is_plagiarized = plagiarism_data.get("is_plagiarized", False) + is_ai_generated = plagiarism_data.get("is_ai_generated", False) + match_type = plagiarism_data.get("match_type", "original") + plagiarism_source = plagiarism_data.get("plagiarism_source", "none") + + # Handle AI-generated submissions + if is_ai_generated: + result_status = "Success - Flagged" + feedback = self._create_ai_generated_feedback( + plagiarism_data, + assignment_context + ) + await self._update_result_status(feedback_request_id, result_status) + return feedback + + # Handle plagiarized submissions + if is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: + result_status = "Success - Flagged" + feedback = self._create_plagiarism_feedback( + plagiarism_data, + assignment_context + ) + await self._update_result_status(feedback_request_id, result_status) + return feedback + + # Continue with normal feedback generation for original work + result_status = "Success - Original" + feedback = await self.generate_feedback_universal(assignment_context, submission_url,submission_id) + await self._update_result_status(feedback_request_id, result_status) + return feedback + + except Exception as e: + result_status = "Failed" + await self._update_result_status(feedback_request_id, result_status, str(e)) + raise + + async def _update_result_status(self, feedback_request_id: str, status: str, error_message: str = None): + """Update Feedback Request result_status""" + if not feedback_request_id: + return + + update_data = {"result_status": status} + if error_message: + update_data["error_message"] = error_message[:500] # Truncate long errors + + frappe.db.set_value( + "Feedback Request", + feedback_request_id, + update_data, + update_modified=True + ) + frappe.db.commit() + + def _create_ai_generated_feedback(self, plagiarism_data: Dict, assignment_context: Dict) -> Dict: + """Create feedback for AI-generated submissions""" + + ai_source = plagiarism_data.get("ai_detection_source", "unknown") + ai_confidence = plagiarism_data.get("ai_confidence", 0.0) + + return { + "overall_feedback": f"""Your submission appears to be generated by an AI tool + (detected source: {ai_source}, confidence: {ai_confidence:.0%}). + + At MentorMe, we encourage original creative work that reflects your own learning + and artistic development. AI-generated images, while interesting, don't demonstrate + the skills and creativity we're looking to nurture. + + Please submit your own original artwork for this assignment.""", + + "strengths": ["N/A - AI-generated content detected"], + "areas_for_improvement": [ + "Submit original artwork created by you", + "Review assignment guidelines for creative direction" + ], + "learning_objectives_feedback": [ + "Unable to assess - submission flagged as AI-generated" + ], + "grade_recommendation": 0, + "encouragement": "We believe in your creative abilities!", + "plagiarism_flag": { + "is_flagged": True, + "flag_type": "ai_generated", + "confidence": ai_confidence + } + } + + def _create_plagiarism_feedback( + self, + plagiarism_data: Dict, + assignment_context: Dict + ) -> Dict: + """Create feedback for plagiarized submissions""" + + match_type = plagiarism_data.get("match_type") + plagiarism_source = plagiarism_data.get("plagiarism_source") + similarity_score = plagiarism_data.get("similarity_score", 0.0) + + return { + "overall_feedback": f"""Your submission has been flagged for similarity + (similarity: {similarity_score:.0%}, source: {plagiarism_source}). + + Academic integrity is fundamental to the learning process. Please ensure your + submissions represent your own original work.""", + + "strengths": ["N/A - Submission flagged for similarity"], + "areas_for_improvement": [ + "Create original artwork for this assignment", + "Review academic integrity guidelines" + ], + "learning_objectives_feedback": [ + "Unable to assess - submission flagged for similarity" + ], + "grade_recommendation": 0, + "encouragement": "Every artist develops their unique style through practice!", + "plagiarism_flag": { + "is_flagged": True, + "flag_type": plagiarism_source, + "similarity_score": similarity_score + } + } + + + def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: """Ensure feedback has all required fields with correct types""" # Ensure all expected fields are present diff --git a/rag_service/handlers/feedback_handler.py b/rag_service/handlers/feedback_handler.py index b4b2e79..98b5a2d 100644 --- a/rag_service/handlers/feedback_handler.py +++ b/rag_service/handlers/feedback_handler.py @@ -22,6 +22,23 @@ async def handle_submission(self, message_data: Dict) -> None: try: print("\n=== Processing New Submission ===") print(f"Submission ID: {message_data.get('submission_id')}") + + + submission_id = message_data.get("submission_id") + is_plagiarized = message_data.get("is_plagiarized", False) + match_type = message_data.get("match_type", "original") + similarity_score = message_data.get("similarity_score", 0.0) + plagiarism_source = message_data.get("plagiarism_source", "none") + similar_sources = message_data.get("similar_sources", []) + is_ai_generated = message_data.get("is_ai_generated", False) + ai_detection_source = message_data.get("ai_detection_source", "unknown") + ai_confidence = message_data.get("ai_confidence", 0.0) + + print(f"\nPlagiarism Check - ID: {submission_id}, \ + Plagiarized: {is_plagiarized}, Match Type: {match_type}, \ + Similarity Score: {similarity_score}, Source: {plagiarism_source}, \ + Similar Sources: {similar_sources}, \ + AI Generated: {is_ai_generated}, AI Confidence: {ai_confidence}") # Create or update feedback request request_id = await self.create_feedback_request(message_data) @@ -41,7 +58,10 @@ async def handle_submission(self, message_data: Dict) -> None: feedback = await self.langchain_manager.generate_feedback( assignment_context=assignment_context, submission_url=message_data["img_url"], - submission_id=request_id + submission_id=request_id, + plagiarism_data=message_data, + feedback_request_id=request_id + ) print("\nFeedback generated, processing feedback...") @@ -92,6 +112,11 @@ async def create_feedback_request(self, message_data: Dict) -> str: "assignment_id": message_data["assignment_id"], "submission_content": message_data["img_url"], "plagiarism_score": message_data.get("plagiarism_score", 0.0), + "is_plagiarized": message_data.get("is_plagiarized", False), + "plagiarism_source": message_data.get("plagiarism_source", "none"), + "match_type": message_data.get("match_type", "original"), + "is_ai_generated": message_data.get("is_ai_generated", False), + "ai_confidence": message_data.get("ai_confidence", 0.0), "similar_sources": json.dumps(message_data.get("similar_sources", [])), "status": "Processing", "created_at": datetime.now(), diff --git a/rag_service/rag_service/doctype/feedback_request/feedback_request.js b/rag_service/rag_service/doctype/feedback_request/feedback_request.js index 2def124..4724608 100644 --- a/rag_service/rag_service/doctype/feedback_request/feedback_request.js +++ b/rag_service/rag_service/doctype/feedback_request/feedback_request.js @@ -1,8 +1,27 @@ -// Copyright (c) 2024, TAP and contributors -// For license information, please see license.txt +frappe.listview_settings['Feedback Request'] = { + add_fields: ["result_status", "is_plagiarized", "is_ai_generated"], -// frappe.ui.form.on("Feedback Request", { -// refresh(frm) { + get_indicator: function(doc) { + const status_map = { + "Pending": ["orange", "Pending"], + "Success - Original": ["green", "Original"], + "Success - Flagged": ["red", "Flagged"], + "Failed": ["darkgrey", "Failed"] + }; -// }, -// }); + const [color, label] = status_map[doc.result_status] || ["grey", "Unknown"]; + return [__(label), color, `result_status,=,${doc.result_status}`]; + }, + + formatters: { + result_status: function(value) { + const badges = { + "Pending": 'Pending', + "Success - Original": '✓ Original', + "Success - Flagged": '⚠ Flagged', + "Failed": '✗ Failed' + }; + return badges[value] || value; + } + } +}; diff --git a/rag_service/rag_service/doctype/feedback_request/feedback_request.json b/rag_service/rag_service/doctype/feedback_request/feedback_request.json index 12aef79..b56599e 100644 --- a/rag_service/rag_service/doctype/feedback_request/feedback_request.json +++ b/rag_service/rag_service/doctype/feedback_request/feedback_request.json @@ -21,7 +21,14 @@ "model_used", "processing_attempts", "error_log", - "is_archived" + "is_archived", + "result_status", + "is_plagiarized", + "match_type", + "plagiarism_source", + "is_ai_generated", + "ai_detection_source", + "ai_confidence" ], "fields": [ { @@ -107,6 +114,48 @@ "fieldname": "is_archived", "fieldtype": "Check", "label": "Is Archived" + }, + { + "fieldname": "result_status", + "fieldtype": "Select", + "label": "Result Status", + "options": "Pending\nSuccess - Original\nSuccess - Flagged\nFailed", + "default": "Pending", + "in_list_view": 1, + "in_standard_filter": 1, + "description": "Overall feedback generation result status" + }, + { + "fieldname": "is_plagiarized", + "fieldtype": "Check", + "label": "Is Plagiarized" + }, + { + "fieldname": "match_type", + "fieldtype": "Select", + "label": "Match Type", + "options": "\noriginal\nexact_duplicate\nnear_duplicate\nsemantic_match\nai_generated\nresubmission_allowed" + }, + { + "fieldname": "plagiarism_source", + "fieldtype": "Select", + "label": "Plagiarism Source", + "options": "\nnone\npeer\npeer_collusion\nself_cross_assignment\nself_late_resubmission\nreference\nai_generated" + }, + { + "fieldname": "is_ai_generated", + "fieldtype": "Check", + "label": "Is AI Generated" + }, + { + "fieldname": "ai_detection_source", + "fieldtype": "Data", + "label": "AI Detection Source" + }, + { + "fieldname": "ai_confidence", + "fieldtype": "Float", + "label": "AI Confidence" } ], "index_web_pages_for_search": 1, From 0f908b9c9f8fbbe51904c023827ac9d0528086b0 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 17 Dec 2025 22:54:11 +0530 Subject: [PATCH 02/14] feedback for plg --- rag_service/handlers/feedback_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag_service/handlers/feedback_handler.py b/rag_service/handlers/feedback_handler.py index 98b5a2d..d5542bc 100644 --- a/rag_service/handlers/feedback_handler.py +++ b/rag_service/handlers/feedback_handler.py @@ -37,7 +37,7 @@ async def handle_submission(self, message_data: Dict) -> None: print(f"\nPlagiarism Check - ID: {submission_id}, \ Plagiarized: {is_plagiarized}, Match Type: {match_type}, \ Similarity Score: {similarity_score}, Source: {plagiarism_source}, \ - Similar Sources: {similar_sources}, \ + Similar Sources: {similar_sources}, Detection Source: {ai_detection_source}, \ AI Generated: {is_ai_generated}, AI Confidence: {ai_confidence}") # Create or update feedback request From d39118a354c5efa586854a7e4ee2d82209cc9d48 Mon Sep 17 00:00:00 2001 From: Manu Date: Fri, 19 Dec 2025 11:43:59 +0530 Subject: [PATCH 03/14] Feedback payload fixed --- rag_service/core/feedback_processor.py | 57 ++---- rag_service/core/langchain_manager.py | 184 ++++++++++-------- rag_service/handlers/feedback_handler.py | 8 +- .../feedback_request/feedback_request.json | 5 +- 4 files changed, 128 insertions(+), 126 deletions(-) diff --git a/rag_service/core/feedback_processor.py b/rag_service/core/feedback_processor.py index da459ec..d350efc 100644 --- a/rag_service/core/feedback_processor.py +++ b/rag_service/core/feedback_processor.py @@ -10,56 +10,23 @@ class FeedbackProcessor: def __init__(self): self.queue_manager = QueueManager() - async def process_feedback(self, request_id: str, feedback: Dict) -> None: + async def process_feedback(self, request_id: str, feedback: Dict, model_used: str, template_used: str) -> None: """Process and store feedback in Feedback Request DocType""" try: print(f"\n=== Processing Feedback for Request: {request_id} ===") - + # Get the feedback request document feedback_request = frappe.get_doc("Feedback Request", request_id) print(f"Found Feedback Request: {feedback_request.name}") - - # Format feedback for display - formatted_feedback = self.format_feedback_for_display(feedback) - + print("\nUpdating Feedback Request fields...") # Update document fields using db_set feedback_request.db_set('status', 'Completed', update_modified=True) feedback_request.db_set('generated_feedback', json.dumps(feedback, indent=2), update_modified=True) feedback_request.db_set('feedback_summary', formatted_feedback, update_modified=True) feedback_request.db_set('completed_at', datetime.now(), update_modified=True) - - # Get and set LLM model info - llm_settings = frappe.get_list( - "LLM Settings", - filters={"is_active": 1}, - limit=1 - ) - if llm_settings: - feedback_request.db_set('model_used', llm_settings[0].name, update_modified=True) - - # FIXED: Get universal template (no assignment_type filtering) - print("Getting universal template...") - try: - # Get any active template (same logic as langchain_manager.py) - templates = frappe.get_list( - "Prompt Template", - filters={"is_active": 1}, # Only filter by active status - order_by="version desc", - limit=1 - ) - - if templates: - feedback_request.db_set('template_used', templates[0].name, update_modified=True) - print(f"Using universal template: {templates[0].name}") - else: - print("No active template found - leaving template_used empty") - feedback_request.db_set('template_used', '', update_modified=True) - - except Exception as template_error: - print(f"Error getting template: {str(template_error)}") - # Don't fail the entire process for template tracking issues - feedback_request.db_set('template_used', '', update_modified=True) + feedback_request.db_set('model_used', model_used, update_modified=True) + feedback_request.db_set('template_used', template_used, update_modified=True) # Commit changes frappe.db.commit() @@ -70,6 +37,7 @@ async def process_feedback(self, request_id: str, feedback: Dict) -> None: print(f"Status: {updated_doc.status}") print(f"Has Generated Feedback: {bool(updated_doc.generated_feedback)}") print(f"Has Feedback Summary: {bool(updated_doc.feedback_summary)}") + print(f"Model Used: {updated_doc.model_used}") print(f"Template Used: {updated_doc.template_used}") # Prepare and send message to TAP LMS @@ -79,9 +47,18 @@ async def process_feedback(self, request_id: str, feedback: Dict) -> None: "assignment_id": feedback_request.assignment_id, "feedback": feedback, "summary": formatted_feedback, + + "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], + "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], + "match_type": feedback['plagiarism_output']['match_type'], + "plagiarism_source": feedback['plagiarism_output']['plagiarism_source'], + "similarity_score": feedback['plagiarism_output']['similarity_score'], + "ai_detection_source": feedback['plagiarism_output']['ai_detection_source'], + "ai_confidence": feedback['plagiarism_output']['ai_confidence'], + "generated_at": feedback_request.completed_at.isoformat() if feedback_request.completed_at else datetime.now().isoformat(), - "plagiarism_score": feedback_request.plagiarism_score, - "similar_sources": json.loads(feedback_request.similar_sources or '[]') + # "plagiarism_score": feedback_request.plagiarism_score, + # "similar_sources": json.loads(feedback_request.similar_sources or '[]') } # Send to TAP LMS queue diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py index c736c43..c510afe 100644 --- a/rag_service/core/langchain_manager.py +++ b/rag_service/core/langchain_manager.py @@ -25,10 +25,12 @@ def setup_llm(self): raise Exception("No active LLM configuration found") settings = frappe.get_doc("LLM Settings", llm_settings[0].name) + self.model_used = llm_settings[0].name print("\nUsing LLM Settings:") print(f"Provider: {settings.provider}") print(f"Model: {settings.model_name}") + # Create LLM provider based on settings self.llm_provider = create_llm_provider( provider=settings.provider, @@ -121,27 +123,27 @@ def __init__(self): self.template_name = "Built-in Universal Template" self.system_prompt = """You are an expert educational feedback assistant that provides constructive, age-appropriate feedback on student submissions across all subjects and assignment types. You adapt your evaluation criteria and language based on the assignment context provided. -CRITICAL: You must ALWAYS respond with valid JSON, never plain text.""" + CRITICAL: You must ALWAYS respond with valid JSON, never plain text.""" self.user_prompt = """Assignment Context: -Assignment Name: {assignment_name} -Subject Area: {course_vertical} -Assignment Type: {assignment_type} -Description: {assignment_description} + Assignment Name: {assignment_name} + Subject Area: {course_vertical} + Assignment Type: {assignment_type} + Description: {assignment_description} -Learning Objectives: -{learning_objectives} + Learning Objectives: + {learning_objectives} -Please analyze this student submission and provide feedback in the required JSON format.""" + Please analyze this student submission and provide feedback in the required JSON format.""" self.response_format = """{ - "overall_feedback": "Comprehensive feedback about the submission", - "strengths": ["Specific strength 1", "Specific strength 2", "Specific strength 3"], - "areas_for_improvement": ["Improvement area 1", "Improvement area 2"], - "learning_objectives_feedback": ["Feedback on learning objective 1"], - "grade_recommendation": 85, - "encouragement": "Encouraging message for the student" -}""" + "overall_feedback": "Comprehensive feedback about the submission", + "strengths": ["Specific strength 1", "Specific strength 2", "Specific strength 3"], + "areas_for_improvement": ["Improvement area 1", "Improvement area 2"], + "learning_objectives_feedback": ["Feedback on learning objective 1"], + "grade_recommendation": 85, + "encouragement": "Encouraging message for the student" + }""" return BuiltinTemplate() @@ -251,8 +253,32 @@ async def generate_feedback_universal(self, assignment_context: Dict, submission # Create structured fallback response feedback = self.create_fallback_feedback(assignment_context, expected_format) + # Attach default plagiarism/AI-detection metadata + plagiarism_output = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + feedback["plagiarism_output"] = plagiarism_output + + try: + if hasattr(template, 'name'): + template_used = template.name + else: + template_used = "Built-in Universal Template" + + except Exception as template_error: + print("Used Default Template:") + template_used = "Built-in Universal Template" + # Don't fail the entire process for template tracking issues + print("\n=== Feedback Generation Completed Successfully ===") - return feedback + return feedback, template_used except Exception as e: error_msg = f"Error generating feedback for submission {submission_id}: {str(e)}" @@ -260,7 +286,9 @@ async def generate_feedback_universal(self, assignment_context: Dict, submission frappe.log_error(message=error_msg, title="Feedback Generation Error") # Return structured error response - return self.create_error_feedback(assignment_context) + template_used = "Built-in Universal Template for Error" + return self.create_error_feedback(assignment_context), template_used + @@ -286,24 +314,24 @@ async def generate_feedback( self, assignment_context: Dict, submission_url: str plagiarism_data, assignment_context ) - await self._update_result_status(feedback_request_id, result_status) - return feedback + tempalate_used = "Feedback Template for AI Generated Submission" # Handle plagiarized submissions - if is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: + elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: result_status = "Success - Flagged" feedback = self._create_plagiarism_feedback( plagiarism_data, assignment_context ) - await self._update_result_status(feedback_request_id, result_status) - return feedback + tempalate_used = "Feedback Template for Plagiarized Submission" + + # Continue with normal feedback generation for original work + else: + result_status = "Success - Original" + feedback, tempalate_used = await self.generate_feedback_universal(assignment_context, submission_url,submission_id) - # Continue with normal feedback generation for original work - result_status = "Success - Original" - feedback = await self.generate_feedback_universal(assignment_context, submission_url,submission_id) await self._update_result_status(feedback_request_id, result_status) - return feedback + return feedback, self.model_used, tempalate_used except Exception as e: result_status = "Failed" @@ -332,69 +360,67 @@ def _create_ai_generated_feedback(self, plagiarism_data: Dict, assignment_contex ai_source = plagiarism_data.get("ai_detection_source", "unknown") ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - - return { - "overall_feedback": f"""Your submission appears to be generated by an AI tool - (detected source: {ai_source}, confidence: {ai_confidence:.0%}). - - At MentorMe, we encourage original creative work that reflects your own learning - and artistic development. AI-generated images, while interesting, don't demonstrate - the skills and creativity we're looking to nurture. - - Please submit your own original artwork for this assignment.""", - - "strengths": ["N/A - AI-generated content detected"], - "areas_for_improvement": [ - "Submit original artwork created by you", - "Review assignment guidelines for creative direction" - ], - "learning_objectives_feedback": [ - "Unable to assess - submission flagged as AI-generated" - ], - "grade_recommendation": 0, - "encouragement": "We believe in your creative abilities!", - "plagiarism_flag": { - "is_flagged": True, - "flag_type": "ai_generated", - "confidence": ai_confidence - } + response = { + "overall_feedback": f"Your submission appears to be generated by an \ + AI tool (detected source: {ai_source}, confidence: {ai_confidence:.0%}). \ + At MentorMe, we encourage original creative work that reflects your own learning \ + and artistic development. AI-generated images, while interesting, don't demonstrate \ + the skills and creativity we're looking to nurture. Please submit your own original \ + artwork for this assignment.", + "strengths": ["N/A - AI-generated content detected"], + "areas_for_improvement": ["Submit original artwork created by you", + "Review assignment guidelines for creative direction"], + "learning_objectives_feedback": ["Unable to assess - submission flagged as AI-generated"], + "grade_recommendation": 0, + "encouragement": "We believe in your creative abilities!", + "plagiarism_output": { + "is_plagiarized": False, + "is_ai_generated": True, + "match_type": "ai_generated", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": ai_source, + "ai_confidence": ai_confidence, + } } - def _create_plagiarism_feedback( - self, - plagiarism_data: Dict, - assignment_context: Dict - ) -> Dict: + return response + + + def _create_plagiarism_feedback( self, plagiarism_data: Dict, assignment_context: Dict) -> Dict: """Create feedback for plagiarized submissions""" match_type = plagiarism_data.get("match_type") plagiarism_source = plagiarism_data.get("plagiarism_source") similarity_score = plagiarism_data.get("similarity_score", 0.0) + ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - return { - "overall_feedback": f"""Your submission has been flagged for similarity - (similarity: {similarity_score:.0%}, source: {plagiarism_source}). - - Academic integrity is fundamental to the learning process. Please ensure your - submissions represent your own original work.""", - - "strengths": ["N/A - Submission flagged for similarity"], - "areas_for_improvement": [ - "Create original artwork for this assignment", - "Review academic integrity guidelines" - ], - "learning_objectives_feedback": [ - "Unable to assess - submission flagged for similarity" - ], - "grade_recommendation": 0, - "encouragement": "Every artist develops their unique style through practice!", - "plagiarism_flag": { - "is_flagged": True, - "flag_type": plagiarism_source, - "similarity_score": similarity_score - } + # respond with structured feedback + response = { + "overall_feedback": f"Your submission has been flagged for similarity \ + (similarity: {similarity_score:.0%}, source: {plagiarism_source}).\ + Academic integrity is fundamental to the learning process. Please ensure your \ + submissions represent your own original work.", + "strengths": ["N/A - Submission flagged for similarity"], + "areas_for_improvement": ["Create original artwork for this assignment", + "Review academic integrity guidelines"], + "learning_objectives_feedback": ["Unable to assess - submission flagged for similarity"], + "grade_recommendation": 0, + "encouragement": "Every artist develops their unique style through practice!", + "plagiarism_output": { + "is_plagiarized": True, + "is_ai_generated": False, + "match_type": match_type, + "plagiarism_source": plagiarism_source, + "similarity_score": similarity_score, + "ai_detection_source": "none", + "ai_confidence": ai_confidence, + } } + return response + + def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: diff --git a/rag_service/handlers/feedback_handler.py b/rag_service/handlers/feedback_handler.py index d5542bc..dc75f64 100644 --- a/rag_service/handlers/feedback_handler.py +++ b/rag_service/handlers/feedback_handler.py @@ -37,7 +37,7 @@ async def handle_submission(self, message_data: Dict) -> None: print(f"\nPlagiarism Check - ID: {submission_id}, \ Plagiarized: {is_plagiarized}, Match Type: {match_type}, \ Similarity Score: {similarity_score}, Source: {plagiarism_source}, \ - Similar Sources: {similar_sources}, Detection Source: {ai_detection_source}, \ + Similar Sources: {similar_sources}, \ AI Generated: {is_ai_generated}, AI Confidence: {ai_confidence}") # Create or update feedback request @@ -55,18 +55,17 @@ async def handle_submission(self, message_data: Dict) -> None: print("\nGenerating feedback...") # Generate feedback - feedback = await self.langchain_manager.generate_feedback( + feedback, model_used, template_used = await self.langchain_manager.generate_feedback( assignment_context=assignment_context, submission_url=message_data["img_url"], submission_id=request_id, plagiarism_data=message_data, feedback_request_id=request_id - ) print("\nFeedback generated, processing feedback...") # Process and deliver feedback - await self.feedback_processor.process_feedback(request_id, feedback) + await self.feedback_processor.process_feedback(request_id, feedback, model_used, template_used) print("\nFeedback processing completed") except Exception as e: @@ -118,6 +117,7 @@ async def create_feedback_request(self, message_data: Dict) -> str: "is_ai_generated": message_data.get("is_ai_generated", False), "ai_confidence": message_data.get("ai_confidence", 0.0), "similar_sources": json.dumps(message_data.get("similar_sources", [])), + "ai_detection_source": message_data.get("ai_detection_source", "unknown"), "status": "Processing", "created_at": datetime.now(), "processing_attempts": 1 diff --git a/rag_service/rag_service/doctype/feedback_request/feedback_request.json b/rag_service/rag_service/doctype/feedback_request/feedback_request.json index b56599e..2805452 100644 --- a/rag_service/rag_service/doctype/feedback_request/feedback_request.json +++ b/rag_service/rag_service/doctype/feedback_request/feedback_request.json @@ -89,9 +89,8 @@ }, { "fieldname": "template_used", - "fieldtype": "Link", - "label": "Template Used", - "options": "Prompt Template" + "fieldtype": "Text", + "label": "Template Used" }, { "fieldname": "model_used", From 51fc03588a7f742931d34d3caa30e1a58d623aeb Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Mon, 19 Jan 2026 05:47:21 +0000 Subject: [PATCH 04/14] rubric evaluation --- .gitignore | 3 +- .../core/assignment_context_manager.py | 106 +-- rag_service/core/feedback_processor.py | 7 +- rag_service/core/langchain_manager.py | 151 +++-- rag_service/handlers/feedback_handler.py | 22 - rag_service/scripts/test_feedback_prompt.py | 110 +++ rag_service/scripts/test_langchain_manager.py | 628 ++++++++++++++++++ 7 files changed, 867 insertions(+), 160 deletions(-) create mode 100644 rag_service/scripts/test_feedback_prompt.py create mode 100644 rag_service/scripts/test_langchain_manager.py diff --git a/.gitignore b/.gitignore index ba04025..83e74cd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ *.swp tags node_modules -__pycache__ \ No newline at end of file +__pycache__ +*scratch* \ No newline at end of file diff --git a/rag_service/core/assignment_context_manager.py b/rag_service/core/assignment_context_manager.py index c5cc8d7..b470f69 100644 --- a/rag_service/core/assignment_context_manager.py +++ b/rag_service/core/assignment_context_manager.py @@ -37,7 +37,14 @@ async def get_assignment_context(self, assignment_id: str) -> Dict: if cached_context: print("Found cached context") - return await self._format_cached_context(cached_context[0].name) + cached_context = frappe.get_doc("Assignment Context", cached_context[0].name).as_dict() + # remove fields where value is a datatime object + for key in list(cached_context.keys()): + if isinstance(cached_context[key], datetime): + cached_context.pop(key, None) + cached_context = {"assignment": cached_context,} + return cached_context + # return await self._format_cached_context(cached_context[0].name) # 2. If not in cache or caching disabled, fetch from API print("Fetching context from API...") @@ -48,9 +55,12 @@ async def get_assignment_context(self, assignment_id: str) -> Dict: print("Saving to cache...") await self._save_to_cache(assignment_id, context) - # 4. Format and return - return self._format_context_for_llm(context) - + if context["assignment"]["rubrics"] is None: + print("Rubrics not found in assignment context.") + raise Exception("Rubrics missing in assignment context") + + return context + except Exception as e: error_msg = f"Error getting assignment context: {str(e)}" print(f"\nError: {error_msg}") @@ -62,16 +72,11 @@ async def _fetch_from_api(self, assignment_id: str) -> Dict: try: # Construct API URL properly api_url = f"{self.settings.base_url.rstrip('/')}/{self.settings.assignment_context_endpoint.lstrip('/')}" - print(f"\nMaking API request to: {api_url}") payload = { "assignment_id": assignment_id } - print("\nRequest Details:") - print(f"Headers: {json.dumps({k: v if k != 'Authorization' else '[REDACTED]' for k, v in self.headers.items()}, indent=2)}") - print(f"Payload: {json.dumps(payload, indent=2)}") - response = requests.post( api_url, headers=self.headers, @@ -79,7 +84,6 @@ async def _fetch_from_api(self, assignment_id: str) -> Dict: timeout=30 ) - print(f"\nResponse Status: {response.status_code}") if response.status_code != 200: error_msg = f"API request failed with status {response.status_code}: {response.text}" @@ -191,88 +195,6 @@ async def _save_to_cache(self, assignment_id: str, context: Dict) -> None: frappe.db.rollback() # Rollback on error raise Exception(error_msg) - async def _format_cached_context(self, context_name: str) -> Dict: - """Format cached context for LLM""" - try: - context = frappe.get_doc("Assignment Context", context_name) - - # Parse learning objectives safely - learning_objectives = [] - try: - if context.learning_objectives: - learning_objectives = json.loads(context.learning_objectives) - except (json.JSONDecodeError, TypeError) as e: - print(f"Error parsing learning objectives: {str(e)}") - # Create an empty default if parsing fails - learning_objectives = [] - - return { - "assignment": { - "id": context.assignment_id, - "name": context.assignment_name, - "type": context.assignment_type, - "description": context.description, - "max_score": context.max_score, - "reference_image": context.reference_image - }, - "learning_objectives": learning_objectives, - "course_vertical": context.course_vertical, - "difficulty_level": context.difficulty_level - } - - except Exception as e: - error_msg = f"Error formatting cached context: {str(e)}" - print(f"\nError: {error_msg}") - raise Exception(error_msg) - - def _format_context_for_llm(self, api_context: Dict) -> Dict: - """Format API context for LLM""" - try: - assignment = api_context["assignment"] - - # Determine course vertical - course_vertical = "General" - if "subject" in assignment and assignment["subject"]: - subject_parts = assignment["subject"].split("-") - if len(subject_parts) > 1: - course_vertical = subject_parts[-1].strip() - - # Parse assignment type correctly - assignment_type = assignment.get("type", "Practical") - valid_types = ["Written", "Practical", "Performance", "Collaborative"] - if not assignment_type or assignment_type not in valid_types: - assignment_type = "Practical" - - # Format learning objectives - learning_objectives = [] - if "learning_objectives" in api_context and api_context["learning_objectives"]: - learning_objectives = [ - { - "objective_id": obj.get("objective", "Unknown"), - "description": obj.get("description", "").strip() - } - for obj in api_context["learning_objectives"] - ] - - return { - "assignment": { - "id": assignment.get("name", ""), # Using name as ID - "name": assignment.get("name", ""), - "type": assignment_type, - "description": assignment.get("description", ""), - "max_score": assignment.get("max_score", "100"), - "reference_image": assignment.get("reference_image", "") - }, - "learning_objectives": learning_objectives, - "course_vertical": course_vertical, - "difficulty_level": "Medium" # Default value - } - - except Exception as e: - error_msg = f"Error formatting API context: {str(e)}" - print(f"\nError: {error_msg}") - raise Exception(error_msg) - async def refresh_cache(self, assignment_id: str) -> None: """Manually refresh cache for an assignment""" try: diff --git a/rag_service/core/feedback_processor.py b/rag_service/core/feedback_processor.py index d350efc..d7c71e9 100644 --- a/rag_service/core/feedback_processor.py +++ b/rag_service/core/feedback_processor.py @@ -23,7 +23,7 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st # Update document fields using db_set feedback_request.db_set('status', 'Completed', update_modified=True) feedback_request.db_set('generated_feedback', json.dumps(feedback, indent=2), update_modified=True) - feedback_request.db_set('feedback_summary', formatted_feedback, update_modified=True) + feedback_request.db_set('feedback_summary', feedback['overall_feedback'], update_modified=True) feedback_request.db_set('completed_at', datetime.now(), update_modified=True) feedback_request.db_set('model_used', model_used, update_modified=True) feedback_request.db_set('template_used', template_used, update_modified=True) @@ -46,7 +46,7 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st "student_id": feedback_request.student_id, "assignment_id": feedback_request.assignment_id, "feedback": feedback, - "summary": formatted_feedback, + "summary": feedback['overall_feedback'], "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], @@ -65,6 +65,9 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st self.queue_manager.send_feedback_to_tap(message) print(f"\nFeedback processed and sent for request: {request_id}") + + print("Payload sent to TAP LMS queue:") + print(json.dumps(message, indent=2)) except Exception as e: error_msg = f"Error processing feedback: {str(e)}" diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py index c510afe..0879cc4 100644 --- a/rag_service/core/langchain_manager.py +++ b/rag_service/core/langchain_manager.py @@ -121,29 +121,74 @@ def get_builtin_template(self): class BuiltinTemplate: def __init__(self): self.template_name = "Built-in Universal Template" - self.system_prompt = """You are an expert educational feedback assistant that provides constructive, age-appropriate feedback on student submissions across all subjects and assignment types. You adapt your evaluation criteria and language based on the assignment context provided. + self.system_prompt = """You are an encouraging, knowledgeable educational assistant that provides constructive feedback on student submissions using a structured rubric-based evaluation. + EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. + CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. - CRITICAL: You must ALWAYS respond with valid JSON, never plain text.""" + Always provide feedback that is: + - Encouraging and positive while being constructive + - Age-appropriate and specific to observations + - Directly aligned with rubric criteria + - Clear about achievement gaps and growth areas - self.user_prompt = """Assignment Context: - Assignment Name: {assignment_name} - Subject Area: {course_vertical} - Assignment Type: {assignment_type} - Description: {assignment_description} + Structure your response by: + 1. Evaluating the submission against each rubric skill + 2. Assigning a single grade for each skill in the rubric [] + 3. Providing specific, actionable feedback + 4. Ending with motivating encouragement - Learning Objectives: - {learning_objectives} + CRITICAL: You must respond with valid JSON format only.""" - Please analyze this student submission and provide feedback in the required JSON format.""" + self.user_prompt = """Assignment Context: + - Name: {assignment_name} + - Subject: {course_vertical} + - Type: {assignment_type} + - Description: {assignment_description} + + Learning Objectives: {learning_objectives} + + Rubric Criteria: {rubric_criteria} + + CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. + + Analyze this submission and respond ONLY in this JSON format: + + { + "rubric_evaluations": [ + { + "skill": "Skill Name", + "grade_value": 1-5, + "observation": "specific evidence from submission" + } + ], + "overall_feedback": "30-50 words of constructive, encouraging feedback or 'Submission does not match assignment requirements.'", + "strengths": ["specific strength 1", "specific strength 2"], + "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], + "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)", + "encouragement": "motivating closing statement" + }""" self.response_format = """{ - "overall_feedback": "Comprehensive feedback about the submission", - "strengths": ["Specific strength 1", "Specific strength 2", "Specific strength 3"], - "areas_for_improvement": ["Improvement area 1", "Improvement area 2"], - "learning_objectives_feedback": ["Feedback on learning objective 1"], - "grade_recommendation": 85, - "encouragement": "Encouraging message for the student" - }""" + "rubric_evaluations": [ + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + }, + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + } + ], + "overall_feedback": "Overall assessment of the submission", + "strengths": ["Strength 1", "Strength 2", "Strength 3"], + "areas_for_improvement": ["Area 1", "Area 2"], + "learning_objectives_feedback": ["Feedback on objective 1"], + "grade_recommendation": 75, + "encouragement": "Encouraging message for the student" + } + """ return BuiltinTemplate() @@ -162,9 +207,31 @@ def format_objectives(self, objectives: List[Dict]) -> str: return "\n".join(formatted) + def format_rubrics(self, rubrics: Dict) -> str: + prompt = "" + for criterion, grades_list in rubrics.items(): + prompt += f"\n{criterion}:\n" + for grade_item in grades_list: + prompt += f" Grade {grade_item['grade_value']}: {grade_item['grade_description']}\n" + + return prompt + + def get_default_response_format(self) -> Dict: """Get default response format""" return { + "rubric_evaluations": [ + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + }, + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + } + ], "overall_feedback": "Overall assessment of the submission", "strengths": ["Strength 1", "Strength 2", "Strength 3"], "areas_for_improvement": ["Area 1", "Area 2"], @@ -173,7 +240,7 @@ def get_default_response_format(self) -> Dict: "encouragement": "Encouraging message for the student" } - async def generate_feedback_universal(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: + async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: """Generate feedback using universal template approach""" try: print("\n=== Starting Universal Feedback Generation ===") @@ -196,17 +263,20 @@ async def generate_feedback_universal(self, assignment_context: Dict, submission # Format learning objectives learning_objectives = self.format_objectives(assignment_context.get("learning_objectives", [])) - - # SIMPLIFIED: Use template directly without complex modifications - # The universal template handles all subject types internally - + rubric_criteria = self.format_rubrics(assignment_context["assignment"].get("rubrics", {})) + + print("User Prompt Context Prepared:") + print(json.dumps(assignment_context, indent=2)) + + # Format user prompt with assignment context user_prompt_vars = { "assignment_name": assignment_context["assignment"].get("name", ""), "assignment_description": assignment_context["assignment"].get("description", ""), - "course_vertical": assignment_context.get("course_vertical", "General"), - "assignment_type": assignment_context["assignment"].get("type", "Practical"), - "learning_objectives": learning_objectives + "course_vertical": assignment_context.get("subject", "General"), + # "assignment_type": assignment_context["assignment"].get("type", "Practical"), + "learning_objectives": learning_objectives, + "rubric_criteria": rubric_criteria } # Format the user prompt with available variables @@ -225,10 +295,6 @@ async def generate_feedback_universal(self, assignment_context: Dict, submission user_prompt=formatted_user_prompt, image_url=submission_url ) - - print(f"\nAssignment: {assignment_context['assignment'].get('name', 'Unknown')}") - print(f"Subject: {assignment_context.get('course_vertical', 'General')}") - print(f"Type: {assignment_context['assignment'].get('type', 'Unknown')}") print("\nSending request to LLM...") # Generate feedback - SINGLE LLM CALL (no separate validation) @@ -238,8 +304,6 @@ async def generate_feedback_universal(self, assignment_context: Dict, submission try: # Clean up the response text cleaned_text = self.clean_json_response(raw_text) - print(f"\nCleaned Response Text: {cleaned_text}") - feedback = json.loads(cleaned_text) print("\nSuccessfully parsed JSON response") @@ -290,9 +354,6 @@ async def generate_feedback_universal(self, assignment_context: Dict, submission return self.create_error_feedback(assignment_context), template_used - - - async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: """Generate feedback with plagiarism context""" @@ -311,8 +372,7 @@ async def generate_feedback( self, assignment_context: Dict, submission_url: str if is_ai_generated: result_status = "Success - Flagged" feedback = self._create_ai_generated_feedback( - plagiarism_data, - assignment_context + plagiarism_data ) tempalate_used = "Feedback Template for AI Generated Submission" @@ -320,15 +380,14 @@ async def generate_feedback( self, assignment_context: Dict, submission_url: str elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: result_status = "Success - Flagged" feedback = self._create_plagiarism_feedback( - plagiarism_data, - assignment_context + plagiarism_data ) tempalate_used = "Feedback Template for Plagiarized Submission" # Continue with normal feedback generation for original work else: result_status = "Success - Original" - feedback, tempalate_used = await self.generate_feedback_universal(assignment_context, submission_url,submission_id) + feedback, tempalate_used = await self.generate_ai_evaluated_feedback(assignment_context, submission_url,submission_id) await self._update_result_status(feedback_request_id, result_status) return feedback, self.model_used, tempalate_used @@ -355,7 +414,7 @@ async def _update_result_status(self, feedback_request_id: str, status: str, err ) frappe.db.commit() - def _create_ai_generated_feedback(self, plagiarism_data: Dict, assignment_context: Dict) -> Dict: + def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: """Create feedback for AI-generated submissions""" ai_source = plagiarism_data.get("ai_detection_source", "unknown") @@ -387,7 +446,7 @@ def _create_ai_generated_feedback(self, plagiarism_data: Dict, assignment_contex return response - def _create_plagiarism_feedback( self, plagiarism_data: Dict, assignment_context: Dict) -> Dict: + def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: """Create feedback for plagiarized submissions""" match_type = plagiarism_data.get("match_type") @@ -421,8 +480,6 @@ def _create_plagiarism_feedback( self, plagiarism_data: Dict, assignment_context return response - - def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: """Ensure feedback has all required fields with correct types""" # Ensure all expected fields are present @@ -464,6 +521,14 @@ def create_fallback_feedback(self, assignment_context: Dict, expected_format: Di fallback[field] = f"I encountered a formatting issue while processing your submission for {assignment_name}. This appears to be a technical problem on our end. Please try resubmitting if this issue persists." elif field == "grade_recommendation": fallback[field] = 50 # Neutral grade for technical issues + elif field == "rubric_evaluations": + fallback[field] = [ + { + "criterion": "Content Knowledge", + "grade_value": 2, + "observation": "Neutral evaluation due to processing issue" + } + ] elif isinstance(default_value, list): if "strength" in field: fallback[field] = ["Your submission was received and processed"] diff --git a/rag_service/handlers/feedback_handler.py b/rag_service/handlers/feedback_handler.py index dc75f64..dcb81f6 100644 --- a/rag_service/handlers/feedback_handler.py +++ b/rag_service/handlers/feedback_handler.py @@ -20,32 +20,12 @@ async def handle_submission(self, message_data: Dict) -> None: """Handle a new submission from plagiarism queue""" request_id = None try: - print("\n=== Processing New Submission ===") - print(f"Submission ID: {message_data.get('submission_id')}") - - submission_id = message_data.get("submission_id") - is_plagiarized = message_data.get("is_plagiarized", False) - match_type = message_data.get("match_type", "original") - similarity_score = message_data.get("similarity_score", 0.0) - plagiarism_source = message_data.get("plagiarism_source", "none") - similar_sources = message_data.get("similar_sources", []) - is_ai_generated = message_data.get("is_ai_generated", False) - ai_detection_source = message_data.get("ai_detection_source", "unknown") - ai_confidence = message_data.get("ai_confidence", 0.0) - - print(f"\nPlagiarism Check - ID: {submission_id}, \ - Plagiarized: {is_plagiarized}, Match Type: {match_type}, \ - Similarity Score: {similarity_score}, Source: {plagiarism_source}, \ - Similar Sources: {similar_sources}, \ - AI Generated: {is_ai_generated}, AI Confidence: {ai_confidence}") - # Create or update feedback request request_id = await self.create_feedback_request(message_data) print(f"\nFeedback Request Created/Updated: {request_id}") # Get assignment context - print(f"\nFetching assignment context for: {message_data['assignment_id']}") assignment_context = await self.assignment_context_manager.get_assignment_context( message_data["assignment_id"] ) @@ -128,8 +108,6 @@ async def create_feedback_request(self, message_data: Dict) -> str: # Explicitly commit the transaction frappe.db.commit() - - print(f"Feedback Request Created/Updated Successfully: {request_id}") return request_id except Exception as e: diff --git a/rag_service/scripts/test_feedback_prompt.py b/rag_service/scripts/test_feedback_prompt.py new file mode 100644 index 0000000..40754a8 --- /dev/null +++ b/rag_service/scripts/test_feedback_prompt.py @@ -0,0 +1,110 @@ +message_data_plg = { + "submission_id": "IMSUB-2601150154", + "student_id": "2724532", + "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601150154_IMSUB-25121730259_image.jpg", + "created_at": "2026-01-15 18:26:12.316528", + "similar_sources": [ + { + "submission_id": "2e7ef3cb-aa66-4eb2-9235-b43698e9e92d", + "student_id": "2724533", + "assignment_id": "fun-faces-1313", + "image_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601010088_IMSUB-25121730259_image.jpg", + "similarity_score": 1.0, + "role": "peer_exact_match" + }, + { + "submission_id": "ca593d80-015a-4195-ab60-627b43969352", + "student_id": "2724533", + "assignment_id": "fun-faces-1313", + "image_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601090012_IMSUB-25121730259_image.jpg", + "similarity_score": 1.0, + "role": "peer_exact_match" + } + ], + "similarity_score": 1.0, + "is_plagiarized": True, + "match_type": "exact_duplicate", + "assignment_id": "SC_L4_CA1-Basic", + "is_ai_generated": False, + "ai_detection_source": "", + "ai_confidence": 0.0, + "plagiarism_source": "peer_collusion" +} + +message_data = { + "submission_id": "IMSUB-2601160155", + "student_id": "2724532", + "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601160155_20251105064001_C107277_F32580_M18081088.png", + "created_at": "2026-01-16 10:49:13.515436", + "similar_sources": "null", + "similarity_score": "null", + "is_plagiarized": False, + "match_type": "original", + "assignment_id": "SC_L4_CA1-Basic", + "is_ai_generated": False, + "ai_detection_source": "", + "ai_confidence": 0.0, + "plagiarism_source": "" +} + +message_data = {'submission_id': 'IMSUB-2601160156', 'student_id': '27245334', +'img_url': 'https://storage.googleapis.com/bucket_tap_1/uploads/11/AugProccess/20251104143002_C5095389_F32580_M18009048.png', + 'created_at': '2026-01-16 13:03:49.054548', 'similar_sources': None, 'similarity_score': None, 'is_plagiarized': False, 'match_type': 'original', 'assignment_id': 'SC_L4_CA1-Basic', + 'is_ai_generated': False, 'ai_detection_source': '', 'ai_confidence': 0.0, 'plagiarism_source': ''} + + + + +assignment_context_sc = { + 'assignment': { + 'name': 'SC_L4_CA1', + 'description': "Hello coders!\nHave you ever shared your basic details for sports or school events? Did you also think that if you put these details in a proper format, it would become so easy to share them?\n\nSo let’s solve this problem and complete a challenge.\nYou have to create a bio-data program with the help of the Pydroid 3 application, which prints your Name, Class, Age, Tap Course and Favourite Activity on the screen.\nYou can easily complete this challenge using simple print statements.\n\nI also created my bio-data program in the same way and printed all the details on the output screen!\nFirst, I displayed the Name – Riya Roy,\nthen I displayed the Class – 11th, Age – 16, Tap course – Python, and Favourite Activity – Playing Chess.\n\nYou too create your own bio-data in the coding area and take a screenshot of the output window. Then share your output screenshot with your Tap Buddy on WhatsApp.\nAnd don’t forget to attend the quiz questions after completing the activity.\nSo see you in the next activity! Till then, keep creating, keep innovating", + 'type': 'Practical', + 'subject': 'Coding', + 'submission_guidelines': None, + 'reference_images': [], + 'max_score': '100', + 'rubrics': { + 'Content Knowledge': [ + {'grade_value': 1, 'grade_description': "Blank screenshot, unrelated screenshot, or no visible code/output.\nNo project-related content. The screenshot does not show the code or output window."}, + {'grade_value': 2, 'grade_description': "Output is present but lacks necessary details or contains incorrect formatting/ code is present only a few lines but not complete. The program attempts to categorize grades but is incomplete, with missing conditions or incorrect output."}, + {'grade_value': 3, 'grade_description': "The program categorizes grades based on marks but may have minor issues in logic or output formatting.\nShows some logic using if-else conditional statements."}, + {'grade_value': 4, 'grade_description': "Code mostly correct and clear. \nThe program categorizes grades correctly based on marks (Grade A for 90-100, B for 75-89, etc.). Code logic is clear, and output is correct.\nFormatting is jumbled but correct"}, + {'grade_value': 5, 'grade_description': "The program categorizes grades correctly, handles invalid input (e.g., outside 0-100), and outputs accurate grades. The logic is well-organized, and the code is clean."} + ], + 'Problem Solving': [ + {'grade_value': 1, 'grade_description': "Blank screenshot, unrelated screenshot, or no visible code/output.\nNo project-related content. The screenshot does not show the code or output window."}, + {'grade_value': 2, 'grade_description': "Code / Output window is visible but with limited details. The program runs but shows incorrect logic or output, e.g., grades are not correctly assigned or displayed."}, + {'grade_value': 3, 'grade_description': "Output or code shows partial understanding. \nThe program runs and shows the correct grades for valid input, but output may not be formatted correctly or program is not formatted correctly."}, + {'grade_value': 4, 'grade_description': "The program correctly assigns grades based on marks input (e.g., A for 90-100, B for 75-89). Handles invalid input correctly."}, + {'grade_value': 5, 'grade_description': "The program works flawlessly, correctly categorizing grades for all valid inputs, showing the correct grade, and handling invalid input (outside 0-100) with appropriate feedback."} + ] + } + }, + 'learning_objectives': [] +} + +assignment_context = {'assignment': {'name': 'VA_L1_CA1', 'description': '"“Close your eyes and imagine a creature that no one has ever seen before. Maybe it has dragon wings, zebra stripes, or fish scales…”\n“Today, your challenge is to invent it! Your task is to combine lines, shapes, and patterns to create a new, magical, and imaginative creature.”\n“Choose at least 3 shapes and 3 patterns for your creature. Then add unique features like wings, horns, multiple eyes, or any magical detail. Your creature should show your imagination!”\n“Step 1: Choose shapes – circles, triangles, rectangles, or ovals – and use them to form body parts. Step 2: Add patterns – stripes, spirals, zigzags, or dots – use at least three different patterns. Step 3: Add creative twists – mix and match special features like an elephant trunk with dragon wings, or a fish tail with monster horns. Following these steps will make your creature alive and interesting.”\n“You can add extra creativity to your creature – more patterns, colors, or unusual features. Let your imagination run free; the more unique and wild, the more fun it will be!”\n“Here’s how to make your creature successful:” • Use at least 3 shapes • Use at least 3 patterns • Creature should look creative and imaginative • Artwork should be neat and complete • Features should be unique and interesting\n“Once your drawing is complete, click a photo and share it with TAP Buddy. We can’t wait to see your creatures come alive on paper!”\n“Well done! You did amazing today. Thank you for creating with us—see you in the next activity!”"', 'type': 'Practical', 'subject': 'Arts', 'submission_guidelines': None, 'reference_images': [], 'max_score': '100', 'rubrics': {'Content Knowledge': [{'grade_value': 1, 'grade_description': '- Invalid or no submission.\n- The art work is digital and not hand drawn.\n - No shapes or patterns at all.'}, {'grade_value': 2, 'grade_description': '- Fewer than 3 shapes OR fewer than 3 patterns. \n- Shapes/patterns are very messy or hard to see. \n\n- Example: Only 1-2 circles drawn, no patterns or just 1 stripe.'}, {'grade_value': 3, 'grade_description': '- Uses 3 shapes AND 3 patterns, but some are unclear or blended together. \n- Not easy to count or spot all. \n\n- Example: Circles, triangles, rectangles with stripes, dots, zigzags but lines overlap and mix up.'}, {'grade_value': 4, 'grade_description': '- Clearly shows at least 3 different shapes AND 3 different patterns. \n- Easy to see and count each one. \n\n- Example: Circle body, triangle ears, rectangle legs with clear stripes on tail, dots on wings, zigzags on back.'}, {'grade_value': 5, 'grade_description': '- Uses more than 3 shapes AND more than 3 patterns, all clear and well-placed. \n- Extra shapes/patterns make it even better. \n\n- Example: Circle head, oval body, triangle wings, rectangle tail with stripes, dots, spirals, zigzags, plus checkerboard on legs.'}], 'Creativity': [{'grade_value': 1, 'grade_description': '- Invalid or no submission. \n- The art work is digital and not hand drawn.\n- No creature drawn.'}, {'grade_value': 2, 'grade_description': '- Creature has only 1-2 simple features. \n- Drawing is messy (colors outside lines) or not finished. \n- Looks boring, not imaginative. \n\n- Example: Plain circle with 1 basic wing; colors spill over edges.'}, {'grade_value': 3, 'grade_description': '- Creature has 3+ features with some imagination. \n- Drawing mostly neat but has small messy spots or unfinished parts. \n- Features are okay but not very unique. \n\n- Example: Triangle body, 2 wings, horn with patterns; a few smudges or blank areas.'}, {'grade_value': 4, 'grade_description': '- Creature looks very creative and imaginative. \n- Drawing is neat, complete, all parts colored nicely. \n- Many unique and interesting features mixed well. \n\n- Example: Oval body, spiral tail, 3 eyes, wings in bright colors; clean lines, no mess.'}, {'grade_value': 5, 'grade_description': '- Creature is super original, wild, and full of imagination. \n- Drawing is very neat, colorful, finished like a real artist. \n- Complex unique features that stand out. \n\n- Example: Mixed shapes with magical glowing wings, curly horns, 5 eyes, special tail; bright colors, perfect details.'}]}}, 'learning_objectives': []} + +import asyncio +from rag_service.core.langchain_manager import LangChainManager + +async def main(): + langchain_manager = LangChainManager() + request_id = 123 + print("\nGenerating feedback...") + # Generate feedback + feedback, model_used, template_used = await langchain_manager.generate_feedback( + assignment_context=assignment_context, + submission_url=message_data["img_url"], + submission_id=request_id, + plagiarism_data=message_data, + feedback_request_id=request_id + ) + + print("\nFeedback Generated:\n", feedback) + print("\nModel Used:", model_used) + print("\nTemplate Used:", template_used) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/rag_service/scripts/test_langchain_manager.py b/rag_service/scripts/test_langchain_manager.py new file mode 100644 index 0000000..c391bdd --- /dev/null +++ b/rag_service/scripts/test_langchain_manager.py @@ -0,0 +1,628 @@ +# rag_service/rag_service/core/langchain_manager.py + +import frappe +import json +from typing import Dict, List, Optional, Union +from datetime import datetime +from ..core.llm_providers import create_llm_provider, OpenAIProvider + +class LangChainManager: + def __init__(self): + self.llm = None + self.llm_provider = None + self.setup_llm() + + def setup_llm(self): + """Initialize LLM based on settings""" + try: + # llm_settings = frappe.get_list( + # "LLM Settings", + # filters={"is_active": 1}, + # limit=1 + # ) + + # if not llm_settings: + # raise Exception("No active LLM configuration found") + + # settings = frappe.get_doc("LLM Settings", llm_settings[0].name) + # self.model_used = llm_settings[0].name + # print("\nUsing LLM Settings:") + # print(f"Provider: {settings.provider}") + # print(f"Model: {settings.model_name}") + + # # Create LLM provider based on settings + # self.llm_provider = create_llm_provider( + # provider=settings.provider, + # api_key=settings.get_password('api_secret'), + # model_name=settings.model_name, + # temperature=settings.temperature, + # max_tokens=settings.max_tokens + # ) + + settings = {"name":"hv7j6uitvg","owner":"Administrator","creation":"2025-12-17 15:32:25.986861", + "modified":"2025-12-17 15:32:25.986861","modified_by":"Administrator","docstatus":0,"idx":18, + "provider":"OpenAI","model_name":"gpt-4o","temperature":0.7,"max_tokens":1500,"is_active":1, + "is_default":0, + "api_secret":"*****","doctype":"LLM Settings","__last_sync_on":"2026-01-16T04:56:38.776Z"} + self.model_used = settings["model_name"] + self.llm_provider = create_llm_provider( + provider=settings["provider"], + api_key=settings["api_secret"], + model_name=settings["model_name"], + temperature=settings["temperature"], + max_tokens=settings["max_tokens"] + ) + + + # Keep the llm reference for backward compatibility with OpenAI + if isinstance(self.llm_provider, OpenAIProvider): + self.llm = self.llm_provider.llm + + except Exception as e: + error_msg = f"LLM Setup Error: {str(e)}" + print(f"\nError: {error_msg}") + frappe.log_error(error_msg, "LLM Setup Error") + raise + + def clean_json_response(self, response: str) -> str: + """Clean JSON response from various formats""" + try: + # Remove markdown code blocks if present + if "```json" in response: + response = response.split("```json")[1].split("```")[0].strip() + elif "```" in response: + code_blocks = response.split("```") + if len(code_blocks) >= 3: # At least one code block exists + response = code_blocks[1].strip() + # Check if the extracted content looks like JSON + if not (response.startswith('{') or response.startswith('[')): + # If not, try to find JSON in the original response + json_start = response.find('{') + if json_start >= 0: + response = response[json_start:] + + # Try to extract JSON if response starts with explanation + if not response.strip().startswith('{'): + json_start = response.find('{') + if json_start >= 0: + response = response[json_start:] + + # Check if the response ends properly + if not response.strip().endswith('}'): + json_end = response.rfind('}') + if json_end >= 0: + response = response[:json_end+1] + + return response.strip() + except Exception as e: + print(f"Error cleaning JSON: {str(e)}") + return response + + def get_universal_template(self) -> Dict: + """Get any active template - no assignment_type filtering""" + try: + print("\n=== Getting Universal Template ===") + + # REMOVED: assignment_type filtering - get ANY active template + templates = frappe.get_list( + "Prompt Template", + filters={"is_active": 1}, + order_by="version desc", + limit=1 + ) + + if templates: + template = frappe.get_doc("Prompt Template", templates[0].name) + print(f"Using universal template: {template.template_name}") + + # Update the last_used timestamp + template.db_set('last_used', datetime.now()) + frappe.db.commit() + + return template + else: + print("No active template found, using built-in default") + return self.get_builtin_template() + + except Exception as e: + error_msg = f"Template Error: {str(e)}" + print(f"\nError: {error_msg}") + frappe.log_error(error_msg, "Template Error") + return self.get_builtin_template() + + def get_builtin_template(self): + """Return built-in default template as fallback""" + class BuiltinTemplate: + def __init__(self): + self.template_name = "Built-in Universal Template" + self.system_prompt = """You are an encouraging, knowledgeable educational assistant that provides constructive feedback on student submissions using a structured rubric-based evaluation. + EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. + CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. + + Always provide feedback that is: + - Encouraging and positive while being constructive + - Age-appropriate and specific to observations + - Directly aligned with rubric criteria + - Clear about achievement gaps and growth areas + + Structure your response by: + 1. Evaluating each rubric criterion against the submission + 2. Assigning grades based on rubric descriptors + 3. Providing specific, actionable feedback + 4. Ending with motivating encouragement + + CRITICAL: You must respond with valid JSON format only.""" + + self.user_prompt = """Assignment Context: + - Name: {assignment_name} + - Subject: {course_vertical} + - Type: {assignment_type} + - Description: {assignment_description} + + Learning Objectives: {learning_objectives} + + Rubric Criteria: {rubric_criteria} + + CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. + + Analyze this submission and respond ONLY in this JSON format: + + { + "rubric_evaluations": [ + { + "criterion": "criterion_name", + "grade_value": 1-5, + "observation": "specific evidence from submission" + } + ], + "overall_feedback": "30-50 words of constructive, encouraging feedback or 'Submission does not match assignment requirements.'", + "strengths": ["specific strength 1", "specific strength 2"], + "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], + "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)", + "encouragement": "motivating closing statement" + }""" + + self.response_format = """{ + "rubric_evaluations": [ + { + "criterion": "criterion_name", + "grade_value": 2, + "observation": "specific evidence from submission" + }, + { + "criterion": "criterion_name", + "grade_value": 2, + "observation": "specific evidence from submission" + } + ], + "overall_feedback": "Overall assessment of the submission", + "strengths": ["Strength 1", "Strength 2", "Strength 3"], + "areas_for_improvement": ["Area 1", "Area 2"], + "learning_objectives_feedback": ["Feedback on objective 1"], + "grade_recommendation": 75, + "encouragement": "Encouraging message for the student" + } + """ + + return BuiltinTemplate() + + def format_objectives(self, objectives: List[Dict]) -> str: + """Format learning objectives for prompt""" + if not objectives: + return "No specific learning objectives provided for this assignment." + + formatted = [] + for i, obj in enumerate(objectives, 1): + if isinstance(obj, dict): + description = obj.get('description', obj.get('objective_id', 'Unknown objective')) + else: + description = str(obj) + formatted.append(f"{i}. {description}") + + return "\n".join(formatted) + + def format_rubrics(self, rubrics: Dict) -> str: + prompt = "" + for criterion, grades_list in rubrics.items(): + prompt += f"\n{criterion}:\n" + for grade_item in grades_list: + prompt += f" Grade {grade_item['grade_value']}: {grade_item['grade_description']}\n" + + return prompt + + + def get_default_response_format(self) -> Dict: + """Get default response format""" + return { + "rubric_evaluations": [ + { + "criterion": "criterion_name", + "grade_value": 2, + "observation": "specific evidence from submission" + }, + { + "criterion": "criterion_name", + "grade_value": 2, + "observation": "specific evidence from submission" + } + ], + "overall_feedback": "Overall assessment of the submission", + "strengths": ["Strength 1", "Strength 2", "Strength 3"], + "areas_for_improvement": ["Area 1", "Area 2"], + "learning_objectives_feedback": ["Feedback on objective 1"], + "grade_recommendation": 75, + "encouragement": "Encouraging message for the student" + } + + async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: + """Generate feedback using universal template approach""" + try: + print("\n=== Starting Universal Feedback Generation ===") + + # Get universal template (no assignment_type filtering) + # template = self.get_universal_template() + template = self.get_builtin_template() # FOR TESTING ONLY + print("Template loaded successfully") + + # Get expected response format from template or use default + try: + if hasattr(template, 'response_format') and template.response_format: + expected_format = json.loads(template.response_format) + print("Using template-defined response format") + else: + expected_format = self.get_default_response_format() + print("Using default response format") + except json.JSONDecodeError: + expected_format = self.get_default_response_format() + print("Failed to parse template response format, using default") + + # Format learning objectives + learning_objectives = self.format_objectives(assignment_context.get("learning_objectives", [])) + rubric_criteria = self.format_rubrics(assignment_context["assignment"].get("rubrics", {})) + + + # SIMPLIFIED: Use template directly without complex modifications + # The universal template handles all subject types internally + + # Format user prompt with assignment context + user_prompt_vars = { + "assignment_name": assignment_context["assignment"].get("name", ""), + "assignment_description": assignment_context["assignment"].get("description", ""), + "course_vertical": assignment_context.get("course_vertical", "General"), + # "assignment_type": assignment_context["assignment"].get("type", "Practical"), + "learning_objectives": learning_objectives, + "rubric_criteria": rubric_criteria + } + + # Format the user prompt with available variables + formatted_user_prompt = template.user_prompt + for key, value in user_prompt_vars.items(): + placeholder = "{" + key + "}" + if placeholder in formatted_user_prompt: + formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) + + # Use template system prompt as-is (it already handles JSON requirement) + system_prompt = template.system_prompt + + # Prepare messages for the LLM provider + messages = self.llm_provider.format_messages( + system_prompt=system_prompt, + user_prompt=formatted_user_prompt, + image_url=submission_url + ) + + print(f"\nAssignment: {assignment_context['assignment'].get('name', 'Unknown')}") + print(f"Subject: {assignment_context.get('course_vertical', 'General')}") + print(f"Type: {assignment_context['assignment'].get('type', 'Unknown')}") + print("\nSending request to LLM...") + + # Generate feedback - SINGLE LLM CALL (no separate validation) + raw_text = await self.llm_provider.generate_with_vision(messages) + print(f"\nRaw LLM Response: {raw_text}") + + try: + # Clean up the response text + cleaned_text = self.clean_json_response(raw_text) + print(f"\nCleaned Response Text: {cleaned_text}") + + feedback = json.loads(cleaned_text) + print("\nSuccessfully parsed JSON response") + + # Validate and ensure required fields + feedback = self.validate_feedback_structure(feedback, expected_format) + + except json.JSONDecodeError as e: + print(f"\nJSON Parse Error: {str(e)}") + print("Using fallback feedback format") + + # Create structured fallback response + feedback = self.create_fallback_feedback(assignment_context, expected_format) + + # Attach default plagiarism/AI-detection metadata + plagiarism_output = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + feedback["plagiarism_output"] = plagiarism_output + + try: + if hasattr(template, 'name'): + template_used = template.name + else: + template_used = "Built-in Universal Template" + + except Exception as template_error: + print("Used Default Template:") + template_used = "Built-in Universal Template" + # Don't fail the entire process for template tracking issues + + print("\n=== Feedback Generation Completed Successfully ===") + return feedback, template_used + + except Exception as e: + error_msg = f"Error generating feedback for submission {submission_id}: {str(e)}" + print(f"\nError: {error_msg}") + frappe.log_error(message=error_msg, title="Feedback Generation Error") + + # Return structured error response + template_used = "Built-in Universal Template for Error" + return self.create_error_feedback(assignment_context), template_used + + async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, + plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: + """Generate feedback with plagiarism context""" + + result_status = "Pending" + + try: + # Check for plagiarism/AI-generated content first + if plagiarism_data: + is_plagiarized = plagiarism_data.get("is_plagiarized", False) + is_ai_generated = plagiarism_data.get("is_ai_generated", False) + match_type = plagiarism_data.get("match_type", "original") + plagiarism_source = plagiarism_data.get("plagiarism_source", "none") + + # Handle AI-generated submissions + if is_ai_generated: + result_status = "Success - Flagged" + feedback = self._create_ai_generated_feedback( + plagiarism_data + ) + tempalate_used = "Feedback Template for AI Generated Submission" + + # Handle plagiarized submissions + elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: + result_status = "Success - Flagged" + feedback = self._create_plagiarism_feedback( + plagiarism_data + ) + tempalate_used = "Feedback Template for Plagiarized Submission" + + # Continue with normal feedback generation for original work + else: + result_status = "Success - Original" + feedback, tempalate_used = await self.generate_ai_evaluated_feedback(assignment_context, submission_url,submission_id) + + await self._update_result_status(feedback_request_id, result_status) + return feedback, self.model_used, tempalate_used + + except Exception as e: + result_status = "Failed" + await self._update_result_status(feedback_request_id, result_status, str(e)) + raise + + async def _update_result_status(self, feedback_request_id: str, status: str, error_message: str = None): + """Update Feedback Request result_status""" + if not feedback_request_id: + return + + update_data = {"result_status": status} + if error_message: + update_data["error_message"] = error_message[:500] # Truncate long errors + + # frappe.db.set_value( + # "Feedback Request", + # feedback_request_id, + # update_data, + # update_modified=True + # ) + # frappe.db.commit() + + def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: + """Create feedback for AI-generated submissions""" + + ai_source = plagiarism_data.get("ai_detection_source", "unknown") + ai_confidence = plagiarism_data.get("ai_confidence", 0.0) + response = { + "overall_feedback": f"Your submission appears to be generated by an \ + AI tool (detected source: {ai_source}, confidence: {ai_confidence:.0%}). \ + At MentorMe, we encourage original creative work that reflects your own learning \ + and artistic development. AI-generated images, while interesting, don't demonstrate \ + the skills and creativity we're looking to nurture. Please submit your own original \ + artwork for this assignment.", + "strengths": ["N/A - AI-generated content detected"], + "areas_for_improvement": ["Submit original artwork created by you", + "Review assignment guidelines for creative direction"], + "learning_objectives_feedback": ["Unable to assess - submission flagged as AI-generated"], + "grade_recommendation": 0, + "encouragement": "We believe in your creative abilities!", + "plagiarism_output": { + "is_plagiarized": False, + "is_ai_generated": True, + "match_type": "ai_generated", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": ai_source, + "ai_confidence": ai_confidence, + } + } + + return response + + + def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: + """Create feedback for plagiarized submissions""" + + match_type = plagiarism_data.get("match_type") + plagiarism_source = plagiarism_data.get("plagiarism_source") + similarity_score = plagiarism_data.get("similarity_score", 0.0) + ai_confidence = plagiarism_data.get("ai_confidence", 0.0) + + # respond with structured feedback + response = { + "overall_feedback": f"Your submission has been flagged for similarity \ + (similarity: {similarity_score:.0%}, source: {plagiarism_source}).\ + Academic integrity is fundamental to the learning process. Please ensure your \ + submissions represent your own original work.", + "strengths": ["N/A - Submission flagged for similarity"], + "areas_for_improvement": ["Create original artwork for this assignment", + "Review academic integrity guidelines"], + "learning_objectives_feedback": ["Unable to assess - submission flagged for similarity"], + "grade_recommendation": 0, + "encouragement": "Every artist develops their unique style through practice!", + "plagiarism_output": { + "is_plagiarized": True, + "is_ai_generated": False, + "match_type": match_type, + "plagiarism_source": plagiarism_source, + "similarity_score": similarity_score, + "ai_detection_source": "none", + "ai_confidence": ai_confidence, + } + } + + return response + + + def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: + """Ensure feedback has all required fields with correct types""" + # Ensure all expected fields are present + for field in expected_format: + if field not in feedback: + if isinstance(expected_format[field], list): + feedback[field] = ["No information provided"] + elif isinstance(expected_format[field], (int, float)): + feedback[field] = 0 + else: + feedback[field] = "No information provided" + + # Validate grade_recommendation format for TAP LMS compatibility + try: + grade = feedback.get("grade_recommendation", 0) + if isinstance(grade, str): + # Extract numeric part only + grade_clean = ''.join(c for c in grade if c.isdigit() or c == '.') + grade = float(grade_clean) if grade_clean else 0 + feedback["grade_recommendation"] = max(0, min(100, float(grade))) + except (ValueError, TypeError): + feedback["grade_recommendation"] = 0 + + # Ensure list fields are lists + list_fields = ["strengths", "areas_for_improvement", "learning_objectives_feedback"] + for field in list_fields: + if field in feedback and not isinstance(feedback[field], list): + feedback[field] = [str(feedback[field])] + + return feedback + + def create_fallback_feedback(self, assignment_context: Dict, expected_format: Dict) -> Dict: + """Create structured fallback when JSON parsing fails""" + assignment_name = assignment_context["assignment"].get("name", "this assignment") + + fallback = {} + for field, default_value in expected_format.items(): + if field == "overall_feedback": + fallback[field] = f"I encountered a formatting issue while processing your submission for {assignment_name}. This appears to be a technical problem on our end. Please try resubmitting if this issue persists." + elif field == "grade_recommendation": + fallback[field] = 50 # Neutral grade for technical issues + elif field == "rubric_evaluations": + fallback[field] = [ + { + "criterion": "Content Knowledge", + "grade_value": 2, + "observation": "Neutral evaluation due to processing issue" + } + ] + elif isinstance(default_value, list): + if "strength" in field: + fallback[field] = ["Your submission was received and processed"] + elif "improvement" in field: + fallback[field] = ["Please ensure your submission clearly shows your work"] + else: + fallback[field] = ["Unable to provide specific feedback due to processing issue"] + else: + if field == "encouragement": + fallback[field] = "Technical issues don't reflect your effort - please try resubmitting!" + else: + fallback[field] = "Processing issue - please resubmit for detailed feedback" + + return fallback + + def create_error_feedback(self, assignment_context: Dict) -> Dict: + """Create feedback for system errors""" + assignment_name = assignment_context["assignment"].get("name", "this assignment") + + return { + "overall_feedback": f"I encountered a system error while processing your submission for {assignment_name}. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", + "strengths": ["Your submission was received successfully"], + "areas_for_improvement": ["No issues identified with your submission - this appears to be a technical problem"], + "learning_objectives_feedback": ["Unable to evaluate due to system error - please resubmit"], + "grade_recommendation": 0, + "encouragement": "Technical issues don't reflect your effort or ability - please try again!" + } + + @staticmethod + def format_feedback_for_display(feedback: Dict) -> str: + """Format feedback for human-readable display""" + try: + formatted = [] + + if "overall_feedback" in feedback: + formatted.append("Overall Feedback:") + formatted.append(feedback["overall_feedback"]) + + if "strengths" in feedback: + formatted.append("\nStrengths:") + for strength in feedback["strengths"]: + formatted.append(f"- {strength}") + + if "areas_for_improvement" in feedback: + formatted.append("\nAreas for Improvement:") + for area in feedback["areas_for_improvement"]: + formatted.append(f"- {area}") + + if "learning_objectives_feedback" in feedback: + formatted.append("\nLearning Objectives Feedback:") + for obj in feedback["learning_objectives_feedback"]: + formatted.append(f"- {obj}") + + if "grade_recommendation" in feedback: + formatted.append(f"\nGrade Recommendation: {feedback['grade_recommendation']}") + + if "encouragement" in feedback: + formatted.append(f"\nEncouragement: {feedback['encouragement']}") + + return "\n".join(formatted) + + except Exception as e: + error_msg = f"Error formatting feedback: {str(e)}" + print(f"\nError: {error_msg}") + return "Error formatting feedback for display. Please check the JSON feedback data." + + def get_current_config(self) -> Dict: + """Get current LLM configuration""" + if not self.llm_provider: + return {"status": "not_configured"} + + return { + "provider": self.llm_provider.__class__.__name__, + "model": self.llm_provider.model_name, + "temperature": self.llm_provider.temperature, + "max_tokens": self.llm_provider.max_tokens + } From 5247664c44bb65a094653b46d4c71bfd7ec7d5d6 Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Mon, 19 Jan 2026 05:53:58 +0000 Subject: [PATCH 05/14] rubric evaluation --- rag_service/core/langchain_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py index 0879cc4..17d0b1c 100644 --- a/rag_service/core/langchain_manager.py +++ b/rag_service/core/langchain_manager.py @@ -524,7 +524,7 @@ def create_fallback_feedback(self, assignment_context: Dict, expected_format: Di elif field == "rubric_evaluations": fallback[field] = [ { - "criterion": "Content Knowledge", + "skill": "Content Knowledge", "grade_value": 2, "observation": "Neutral evaluation due to processing issue" } From 01580b50a51985a1f3010cf9b38daa19fd680dfb Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Tue, 20 Jan 2026 09:53:50 +0000 Subject: [PATCH 06/14] changed LMS payload --- rag_service/core/feedback_processor.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/rag_service/core/feedback_processor.py b/rag_service/core/feedback_processor.py index d7c71e9..8d7381a 100644 --- a/rag_service/core/feedback_processor.py +++ b/rag_service/core/feedback_processor.py @@ -33,21 +33,12 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st # Verify the update updated_doc = frappe.get_doc("Feedback Request", request_id) - print("\nVerification after update:") - print(f"Status: {updated_doc.status}") - print(f"Has Generated Feedback: {bool(updated_doc.generated_feedback)}") - print(f"Has Feedback Summary: {bool(updated_doc.feedback_summary)}") - print(f"Model Used: {updated_doc.model_used}") - print(f"Template Used: {updated_doc.template_used}") - # Prepare and send message to TAP LMS message = { "submission_id": feedback_request.submission_id, "student_id": feedback_request.student_id, "assignment_id": feedback_request.assignment_id, "feedback": feedback, - "summary": feedback['overall_feedback'], - "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], "match_type": feedback['plagiarism_output']['match_type'], @@ -57,8 +48,7 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st "ai_confidence": feedback['plagiarism_output']['ai_confidence'], "generated_at": feedback_request.completed_at.isoformat() if feedback_request.completed_at else datetime.now().isoformat(), - # "plagiarism_score": feedback_request.plagiarism_score, - # "similar_sources": json.loads(feedback_request.similar_sources or '[]') + } # Send to TAP LMS queue From c9c7d493b638f062f06ecb8f93516176ad7544fe Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Fri, 30 Jan 2026 06:32:09 +0000 Subject: [PATCH 07/14] feedback transalation --- .../core/assignment_context_manager.py | 120 ++++--- rag_service/core/context_fetcher.py | 208 ------------ rag_service/core/embedding_utils.py | 79 ----- rag_service/core/feedback_generator.py | 135 -------- rag_service/core/feedback_processor.py | 18 +- rag_service/core/langchain_manager.py | 297 +++++++++--------- rag_service/core/rag_utils.py | 94 ------ rag_service/core/vector_store.py | 86 ----- rag_service/feedback_generator.py | 54 ---- rag_service/handlers/feedback_handler.py | 36 +-- rag_service/rabbitmq_utils.py | 72 ----- .../assignment_context.json | 4 +- rag_service/scripts/console_consumer.py | 4 + rag_service/scripts/test_consumer_payload.py | 141 +++++++++ rag_service/utils/commands.py | 29 -- rag_service/utils/queue_manager.py | 2 +- rag_service/utils/rabbitmq_consumer.py | 2 - rag_service/utils/rabbitmq_manager.py | 258 --------------- rag_service/utils/setup_test_data.py | 89 ------ 19 files changed, 378 insertions(+), 1350 deletions(-) delete mode 100644 rag_service/core/context_fetcher.py delete mode 100644 rag_service/core/embedding_utils.py delete mode 100644 rag_service/core/feedback_generator.py delete mode 100644 rag_service/core/rag_utils.py delete mode 100644 rag_service/core/vector_store.py delete mode 100644 rag_service/feedback_generator.py delete mode 100644 rag_service/rabbitmq_utils.py create mode 100644 rag_service/scripts/console_consumer.py create mode 100644 rag_service/scripts/test_consumer_payload.py delete mode 100644 rag_service/utils/commands.py delete mode 100644 rag_service/utils/rabbitmq_manager.py delete mode 100644 rag_service/utils/setup_test_data.py diff --git a/rag_service/core/assignment_context_manager.py b/rag_service/core/assignment_context_manager.py index b470f69..4c8f29b 100644 --- a/rag_service/core/assignment_context_manager.py +++ b/rag_service/core/assignment_context_manager.py @@ -19,12 +19,12 @@ def __init__(self): print("\nInitialized AssignmentContextManager") print(f"Using API Endpoint: {self.settings.base_url.rstrip('/')}/{self.settings.assignment_context_endpoint.lstrip('/')}") - async def get_assignment_context(self, assignment_id: str) -> Dict: + async def get_assignment_context(self, assignment_id: str, student_id: str) -> Dict: """Get assignment context from cache or API""" try: print(f"\n=== Getting Assignment Context for: {assignment_id} ===") - # 1. Check cache if enabled + # Check cache if enabled if self.settings.enable_caching: cached_context = frappe.get_list( "Assignment Context", @@ -34,7 +34,8 @@ async def get_assignment_context(self, assignment_id: str) -> Dict: }, limit=1 ) - + + context = None if cached_context: print("Found cached context") cached_context = frappe.get_doc("Assignment Context", cached_context[0].name).as_dict() @@ -42,23 +43,35 @@ async def get_assignment_context(self, assignment_id: str) -> Dict: for key in list(cached_context.keys()): if isinstance(cached_context[key], datetime): cached_context.pop(key, None) - cached_context = {"assignment": cached_context,} - return cached_context - # return await self._format_cached_context(cached_context[0].name) - - # 2. If not in cache or caching disabled, fetch from API - print("Fetching context from API...") - context = await self._fetch_from_api(assignment_id) - - # 3. Save to cache if enabled - if self.settings.enable_caching: - print("Saving to cache...") - await self._save_to_cache(assignment_id, context) + context = {"assignment": cached_context,} + + if not context: + # If not in cache or caching disabled, fetch from API + print("Fetching context from API...") + context = await self._fetch_assignment_from_api(assignment_id) + + # Save to cache if enabled + if self.settings.enable_caching: + print("Saving to cache...") + await self._save_to_cache(assignment_id, context) + + if context["assignment"]["rubrics"] is None: + print("Rubrics not found in assignment context.") + raise Exception("Rubrics missing in assignment context") + + if student_id is None: + raise Exception("Student ID is required to fetch student context") - if context["assignment"]["rubrics"] is None: - print("Rubrics not found in assignment context.") - raise Exception("Rubrics missing in assignment context") + student_details = await self._fetch_student_from_api(student_id) + # student_details = { + # "student_id":"ST0001", + # "grade":"6", + # "level":"2", + # "language": "Hindi" + # } + context["student"] = {**student_details} + print("Assignment context",context) return context except Exception as e: @@ -67,7 +80,7 @@ async def get_assignment_context(self, assignment_id: str) -> Dict: frappe.log_error(error_msg, "Assignment Context Error") raise - async def _fetch_from_api(self, assignment_id: str) -> Dict: + async def _fetch_assignment_from_api(self, assignment_id: str) -> Dict: """Fetch assignment context from TAP LMS API""" try: # Construct API URL properly @@ -76,7 +89,6 @@ async def _fetch_from_api(self, assignment_id: str) -> Dict: payload = { "assignment_id": assignment_id } - response = requests.post( api_url, headers=self.headers, @@ -84,7 +96,6 @@ async def _fetch_from_api(self, assignment_id: str) -> Dict: timeout=30 ) - if response.status_code != 200: error_msg = f"API request failed with status {response.status_code}: {response.text}" print(f"Error: {error_msg}") @@ -102,6 +113,39 @@ async def _fetch_from_api(self, assignment_id: str) -> Dict: print(f"\nError: {error_msg}") raise Exception(error_msg) + async def _fetch_student_from_api(self, student_id: str) -> Dict: + """Fetch student details from TAP LMS API""" + try: + # Construct API URL properly + api_url = f"{self.settings.base_url.rstrip('/')}/{self.settings.student_context_endpoint.lstrip('/')}" + + payload = { + "student_id": student_id + } + response = requests.post( + api_url, + headers=self.headers, + json=payload, + timeout=30 + ) + + if response.status_code != 200: + error_msg = f"API request failed with status {response.status_code}: {response.text}" + print(f"Error: {error_msg}") + raise Exception(error_msg) + + data = response.json() + if "message" not in data: + raise Exception("Invalid API response format") + + print("Student API request successful") + return data["message"] + + except requests.RequestException as e: + error_msg = f"API request failed: {str(e)}" + print(f"\nError: {error_msg}") + raise Exception(error_msg) + async def _save_to_cache(self, assignment_id: str, context: Dict) -> None: """Save assignment context to cache""" try: @@ -161,7 +205,8 @@ async def _save_to_cache(self, assignment_id: str, context: Dict) -> None: "last_updated": now_datetime(), "cache_valid_till": cache_valid_till, "last_sync_status": "Success", - "version": (doc.version or 0) + 1 + "version": (doc.version or 0) + 1, + "rubrics": json.dumps(assignment.get("rubrics", {})) }) doc.save() print(f"Updated existing cache for assignment {assignment_id}") @@ -181,7 +226,8 @@ async def _save_to_cache(self, assignment_id: str, context: Dict) -> None: "last_updated": now_datetime(), "cache_valid_till": cache_valid_till, "last_sync_status": "Success", - "version": 1 + "version": 1, + "rubrics": json.dumps(assignment.get("rubrics", {})) }) doc.insert() print(f"Created new cache for assignment {assignment_id}") @@ -201,7 +247,7 @@ async def refresh_cache(self, assignment_id: str) -> None: print(f"\n=== Refreshing Cache for Assignment: {assignment_id} ===") # Force fetch from API - context = await self._fetch_from_api(assignment_id) + context = await self._fetch_assignment_from_api(assignment_id) # Save to cache await self._save_to_cache(assignment_id, context) @@ -212,29 +258,3 @@ async def refresh_cache(self, assignment_id: str) -> None: error_msg = f"Error refreshing cache: {str(e)}" print(f"\nError: {error_msg}") raise Exception(error_msg) - - def verify_settings(self) -> Dict: - """Verify RAG Settings configuration""" - try: - results = { - "base_url": bool(self.settings.base_url), - "api_key": bool(self.settings.api_key), - "api_secret": bool(self.settings.get_password('api_secret')), - "endpoints": bool(self.settings.assignment_context_endpoint), - "cache_config": bool(self.settings.cache_duration_days is not None) - } - - missing = [k for k, v in results.items() if not v] - - return { - "status": "Valid" if not missing else "Invalid", - "missing_settings": missing, - "cache_enabled": self.settings.enable_caching, - "cache_duration": self.settings.cache_duration_days - } - - except Exception as e: - return { - "status": "Error", - "error": str(e) - } diff --git a/rag_service/core/context_fetcher.py b/rag_service/core/context_fetcher.py deleted file mode 100644 index d2d6edb..0000000 --- a/rag_service/core/context_fetcher.py +++ /dev/null @@ -1,208 +0,0 @@ -# rag_service/rag_service/core/context_fetcher.py - -import frappe -import json -import httpx -from datetime import datetime, timedelta -from typing import Dict, Optional -from urllib.parse import urljoin - -class AssignmentContextFetcher: - def __init__(self): - self.settings = frappe.get_single("RAG Settings") - self.api_url = urljoin( - self.settings.base_url, - self.settings.assignment_context_endpoint - ) - self.cache_duration = timedelta(days=self.settings.cache_duration_days) - self.enable_caching = self.settings.enable_caching - self.max_retries = 3 - self.retry_delay = 1 # seconds - - def _get_headers(self) -> Dict[str, str]: - """Get request headers with authentication""" - api_key = self.settings.api_key - api_secret = self.settings.get_password('api_secret') - - return { - "Authorization": f"token {api_key}:{api_secret}", - "Content-Type": "application/json" - } - - async def get_assignment_context(self, assignment_id: str) -> Dict: - """ - Get assignment context - first check cache, then fetch from API if needed - """ - try: - # Check cache if enabled - if self.enable_caching: - cached_context = self._get_cached_context(assignment_id) - if cached_context: - frappe.logger().debug(f"Cache hit for assignment {assignment_id}") - return cached_context - - # If not in cache or caching disabled, fetch from API - context_data = await self._fetch_from_api(assignment_id) - - # Cache if enabled - if self.enable_caching: - self._cache_context(assignment_id, context_data) - frappe.logger().debug(f"Cached context for assignment {assignment_id}") - - return context_data - - except Exception as e: - frappe.log_error( - message=f"Error fetching assignment context: {str(e)}", - title="Assignment Context Error" - ) - raise - - async def _fetch_from_api(self, assignment_id: str) -> Dict: - """Fetch context from TAP LMS API with retries""" - last_error = None - headers = self._get_headers() - - for attempt in range(self.max_retries): - try: - async with httpx.AsyncClient() as client: - response = await client.post( - self.api_url, - json={"assignment_id": assignment_id}, - headers=headers, - timeout=30.0 - ) - - if response.status_code == 401: - raise ValueError("Authentication failed - check API key and secret") - - response.raise_for_status() - - context_data = response.json() - if not context_data.get("message"): - raise ValueError("Invalid response format from API") - - frappe.logger().debug(f"Successfully fetched context for assignment {assignment_id}") - return context_data["message"] - - except httpx.HTTPStatusError as e: - last_error = f"HTTP error {e.response.status_code}: {str(e)}" - if e.response.status_code in [401, 403, 404]: # Don't retry auth or not found errors - break - await self._wait_before_retry(attempt) - - except httpx.RequestError as e: - last_error = f"Request error: {str(e)}" - await self._wait_before_retry(attempt) - - except Exception as e: - last_error = str(e) - await self._wait_before_retry(attempt) - - frappe.log_error( - message=f"Failed to fetch context after {self.max_retries} attempts: {last_error}", - title="API Error" - ) - raise Exception(f"Failed to fetch assignment context: {last_error}") - - async def _wait_before_retry(self, attempt: int): - """Exponential backoff for retries""" - import asyncio - wait_time = self.retry_delay * (2 ** attempt) # exponential backoff - await asyncio.sleep(wait_time) - - def _get_cached_context(self, assignment_id: str) -> Optional[Dict]: - """Check if we have a valid cached context""" - try: - cached = frappe.get_list( - "Assignment Context", - filters={ - "assignment_id": assignment_id, - "cache_valid_till": [">", datetime.now()], - "last_sync_status": "Success" - }, - order_by="version desc", - limit=1 - ) - - if not cached: - return None - - context_doc = frappe.get_doc("Assignment Context", cached[0].name) - return { - "assignment": { - "name": context_doc.assignment_name, - "description": context_doc.description, - "type": context_doc.assignment_type, - "subject": context_doc.course_vertical, - "submission_guidelines": context_doc.submission_guidelines, - "reference_image": context_doc.reference_image, - "max_score": context_doc.max_score, - }, - "learning_objectives": json.loads(context_doc.learning_objectives) - } - - except Exception as e: - frappe.log_error( - message=f"Error retrieving cached context: {str(e)}", - title="Cache Retrieval Error" - ) - return None - - def _cache_context(self, assignment_id: str, context_data: Dict) -> None: - """Store context in cache""" - try: - assignment = context_data["assignment"] - cache_valid_till = datetime.now() + self.cache_duration - - doc = frappe.get_doc({ - "doctype": "Assignment Context", - "assignment_id": assignment_id, - "assignment_name": assignment["name"], - "course_vertical": assignment["subject"], - "description": assignment["description"], - "submission_guidelines": assignment["submission_guidelines"], - "reference_image": assignment["reference_image"], - "learning_objectives": json.dumps(context_data["learning_objectives"]), - "max_score": assignment["max_score"], - "last_updated": datetime.now(), - "cache_valid_till": cache_valid_till, - "last_sync_status": "Success", - "version": 1 - }) - - existing_docs = frappe.get_list( - "Assignment Context", - filters={"assignment_id": assignment_id} - ) - - if existing_docs: - doc.name = existing_docs[0].name - doc.version = frappe.get_value("Assignment Context", existing_docs[0].name, "version") + 1 - - doc.save() - frappe.db.commit() - - except Exception as e: - frappe.log_error( - message=f"Error caching context: {str(e)}", - title="Cache Error" - ) - raise - - def invalidate_cache(self, assignment_id: str) -> None: - """Manually invalidate cache for an assignment""" - try: - frappe.db.sql(""" - UPDATE `tabAssignment Context` - SET cache_valid_till = NOW() - WHERE assignment_id = %s - """, (assignment_id,)) - frappe.db.commit() - - except Exception as e: - frappe.log_error( - message=f"Error invalidating cache: {str(e)}", - title="Cache Error" - ) - raise diff --git a/rag_service/core/embedding_utils.py b/rag_service/core/embedding_utils.py deleted file mode 100644 index afe81ed..0000000 --- a/rag_service/core/embedding_utils.py +++ /dev/null @@ -1,79 +0,0 @@ -# File: ~/frappe-bench/apps/rag_service/rag_service/core/embedding_utils.py - -import frappe -from sentence_transformers import SentenceTransformer -import numpy as np -import os -from frappe.utils import now_datetime -import json - -class EmbeddingManager: - def __init__(self): - self.model = None - self.model_name = 'all-MiniLM-L6-v2' - self.embedding_dimension = 384 - - def get_model(self): - if self.model is None: - self.model = SentenceTransformer(self.model_name) - return self.model - - def generate_embedding(self, text): - """Generate embedding for given text""" - model = self.get_model() - embedding = model.encode(text) - return embedding - - def save_embedding(self, reference_id, content, content_type="Submission"): - """Save embedding to Vector Store""" - try: - # Generate embedding - embedding = self.generate_embedding(content) - - # Create a file path for the embedding - site_path = frappe.get_site_path() - embedding_dir = os.path.join(site_path, 'private', 'files', 'embeddings') - os.makedirs(embedding_dir, exist_ok=True) - - file_path = os.path.join(embedding_dir, f"{content_type}_{reference_id}_{now_datetime().strftime('%Y%m%d_%H%M%S')}.npy") - - # Save the embedding to file - np.save(file_path, embedding) - - # Create Vector Store entry - vector_store = frappe.get_doc({ - "doctype": "Vector Store", - "content_type": content_type, - "reference_id": reference_id, - "content": content, - "embedding_file": os.path.relpath(file_path, site_path), - "created_at": now_datetime() - }) - - vector_store.insert() - frappe.db.commit() - - return vector_store.name - - except Exception as e: - frappe.log_error(f"Error saving embedding: {str(e)}") - raise - - def load_embedding(self, vector_store_name): - """Load embedding from Vector Store""" - try: - vector_store = frappe.get_doc("Vector Store", vector_store_name) - embedding_path = os.path.join(frappe.get_site_path(), vector_store.embedding_file) - - if not os.path.exists(embedding_path): - frappe.throw(f"Embedding file not found: {embedding_path}") - - embedding = np.load(embedding_path) - return embedding - - except Exception as e: - frappe.log_error(f"Error loading embedding: {str(e)}") - raise - -# Create a singleton instance -embedding_manager = EmbeddingManager() diff --git a/rag_service/core/feedback_generator.py b/rag_service/core/feedback_generator.py deleted file mode 100644 index 663db13..0000000 --- a/rag_service/core/feedback_generator.py +++ /dev/null @@ -1,135 +0,0 @@ -# File: ~/frappe-bench/apps/rag_service/rag_service/core/feedback_generator.py - -import frappe -from .embedding_utils import embedding_manager -from .vector_store import faiss_manager -import json -from datetime import datetime - -class FeedbackGenerator: - def __init__(self): - self.feedback_templates = { - "general": """ -Based on the submission content and similar examples, here's the feedback: - -Strengths: -{strengths} - -Areas for Improvement: -{improvements} - -Suggestions: -{suggestions} - -Overall Assessment: -{assessment} - """.strip(), - - "plagiarism_alert": """ -⚠️ Plagiarism Concern: -The submission shows significant similarity ({similarity_score:.2f}%) with existing content. -Please review and ensure original work. - -Similar Content Found: -{similar_content} - -Recommendation: -{recommendation} - """.strip() - } - - def generate_structured_feedback(self, submission_content, similar_contents, plagiarism_score=None): - """Generate structured feedback using RAG approach""" - try: - # Analyze strengths - strengths = self._analyze_strengths(submission_content) - - # Analyze areas for improvement - improvements = self._analyze_improvements(submission_content, similar_contents) - - # Generate suggestions - suggestions = self._generate_suggestions(submission_content, similar_contents) - - # Create feedback - feedback = self.feedback_templates["general"].format( - strengths="\n".join(f"- {s}" for s in strengths), - improvements="\n".join(f"- {i}" for i in improvements), - suggestions="\n".join(f"- {s}" for s in suggestions), - assessment=self._create_overall_assessment( - submission_content, - strengths, - improvements - ) - ) - - # Add plagiarism warning if score is high - if plagiarism_score and plagiarism_score > 0.8: - feedback += "\n\n" + self.feedback_templates["plagiarism_alert"].format( - similarity_score=plagiarism_score * 100, - similar_content=self._format_similar_content(similar_contents[:1]), - recommendation="Please revise your submission to ensure originality." - ) - - return { - "feedback": feedback, - "metadata": { - "strengths_count": len(strengths), - "improvements_count": len(improvements), - "suggestions_count": len(suggestions), - "has_plagiarism_warning": plagiarism_score > 0.8 if plagiarism_score else False, - "generated_at": str(datetime.now()) - } - } - - except Exception as e: - frappe.log_error(f"Error generating feedback: {str(e)}") - raise - - def _analyze_strengths(self, content): - """Analyze submission strengths""" - # Placeholder - Implement actual strength analysis - strengths = [ - "Clear presentation of concepts", - "Good structure and organization", - "Effective use of examples" - ] - return strengths - - def _analyze_improvements(self, content, similar_contents): - """Analyze areas for improvement""" - # Placeholder - Implement actual improvement analysis - improvements = [ - "Consider adding more detailed explanations", - "Include more specific examples", - "Expand on key concepts" - ] - return improvements - - def _generate_suggestions(self, content, similar_contents): - """Generate specific suggestions""" - # Placeholder - Implement actual suggestion generation - suggestions = [ - "Reference related topics to strengthen understanding", - "Include practical applications of concepts", - "Add visual representations where applicable" - ] - return suggestions - - def _create_overall_assessment(self, content, strengths, improvements): - """Create overall assessment""" - # Placeholder - Implement actual assessment logic - return "The submission demonstrates good understanding of the concepts while having room for enhancement in specific areas." - - def _format_similar_content(self, similar_contents): - """Format similar content for feedback""" - if not similar_contents: - return "No similar content found" - - formatted = [] - for content in similar_contents: - formatted.append(f"- Content: {content['content']}\n Similarity: {content['similarity_score']:.2%}") - - return "\n".join(formatted) - -# Create singleton instance -feedback_generator = FeedbackGenerator() diff --git a/rag_service/core/feedback_processor.py b/rag_service/core/feedback_processor.py index 8d7381a..2db85d9 100644 --- a/rag_service/core/feedback_processor.py +++ b/rag_service/core/feedback_processor.py @@ -22,7 +22,7 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st print("\nUpdating Feedback Request fields...") # Update document fields using db_set feedback_request.db_set('status', 'Completed', update_modified=True) - feedback_request.db_set('generated_feedback', json.dumps(feedback, indent=2), update_modified=True) + feedback_request.db_set('generated_feedback', json.dumps(feedback, indent=2, ensure_ascii=False), update_modified=True) feedback_request.db_set('feedback_summary', feedback['overall_feedback'], update_modified=True) feedback_request.db_set('completed_at', datetime.now(), update_modified=True) feedback_request.db_set('model_used', model_used, update_modified=True) @@ -39,13 +39,13 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st "student_id": feedback_request.student_id, "assignment_id": feedback_request.assignment_id, "feedback": feedback, - "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], - "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], - "match_type": feedback['plagiarism_output']['match_type'], - "plagiarism_source": feedback['plagiarism_output']['plagiarism_source'], - "similarity_score": feedback['plagiarism_output']['similarity_score'], - "ai_detection_source": feedback['plagiarism_output']['ai_detection_source'], - "ai_confidence": feedback['plagiarism_output']['ai_confidence'], + # "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], + # "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], + # "match_type": feedback['plagiarism_output']['match_type'], + # "plagiarism_source": feedback['plagiarism_output']['plagiarism_source'], + # "similarity_score": feedback['plagiarism_output']['similarity_score'], + # "ai_detection_source": feedback['plagiarism_output']['ai_detection_source'], + # "ai_confidence": feedback['plagiarism_output']['ai_confidence'], "generated_at": feedback_request.completed_at.isoformat() if feedback_request.completed_at else datetime.now().isoformat(), @@ -57,7 +57,7 @@ async def process_feedback(self, request_id: str, feedback: Dict, model_used: st print(f"\nFeedback processed and sent for request: {request_id}") print("Payload sent to TAP LMS queue:") - print(json.dumps(message, indent=2)) + print(json.dumps(message, indent=2, ensure_ascii=False)) except Exception as e: error_msg = f"Error processing feedback: {str(e)}" diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py index 17d0b1c..87f072a 100644 --- a/rag_service/core/langchain_manager.py +++ b/rag_service/core/langchain_manager.py @@ -87,7 +87,7 @@ def clean_json_response(self, response: str) -> str: def get_universal_template(self) -> Dict: """Get any active template - no assignment_type filtering""" try: - print("\n=== Getting Universal Template ===") + print("\n=== Getting Prompt Template ===") # REMOVED: assignment_type filtering - get ANY active template templates = frappe.get_list( @@ -122,51 +122,68 @@ class BuiltinTemplate: def __init__(self): self.template_name = "Built-in Universal Template" self.system_prompt = """You are an encouraging, knowledgeable educational assistant that provides constructive feedback on student submissions using a structured rubric-based evaluation. - EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. - CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. - - Always provide feedback that is: - - Encouraging and positive while being constructive - - Age-appropriate and specific to observations - - Directly aligned with rubric criteria - - Clear about achievement gaps and growth areas - - Structure your response by: - 1. Evaluating the submission against each rubric skill - 2. Assigning a single grade for each skill in the rubric [] - 3. Providing specific, actionable feedback - 4. Ending with motivating encouragement - - CRITICAL: You must respond with valid JSON format only.""" + EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. Prioritize growth recognition over perfection. + + GRADING PHILOSOPHY: + - Credit partial mastery: A student showing 60% competency deserves acknowledgment of that progress. Grades need not be binary (good/bad). + - Growth mindset framing: Every submission represents learning in progress. Frame gaps as natural and achievable, not deficiencies. + - Reserve lower grades (1-2) only for minimal engagement or complete absence of skill demonstration + - Lean toward higher grades when effort and authenticity are evident + + + Always provide feedback that is: + - Encouraging and positive while being constructive + - Age-appropriate and specific to observations + - Directly aligned with rubric criteria + - Clear about achievement gaps and growth areas + + Structure your response by: + 1. Opening with what the student did well (be specific) + 2. Evaluating the submission against each rubric skill + 3. Assigning a single grade for each skill in the rubric criteria + 4. Evaluating only the skills mentioned in the rubric criteria + 5. Providing specific, actionable feedback for growth + 6. Ending with motivating encouragement + 7. Translate the overall_feedback. Translation rules: + - Formal but friendly tone (customer communication). + - Natural, conversational phrasing. Not literal translation. + - Use native script for the language. + + CRITICAL: You must respond with valid JSON format only. + """ self.user_prompt = """Assignment Context: - - Name: {assignment_name} - - Subject: {course_vertical} - - Type: {assignment_type} - - Description: {assignment_description} + - Name: {assignment_name} + - Subject: {course_vertical} + - Type: {assignment_type} + - Description: {assignment_description} - Learning Objectives: {learning_objectives} + Learning Objectives: {learning_objectives} - Rubric Criteria: {rubric_criteria} + Rubric Criteria: {rubric_criteria} - CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. + CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. - Analyze this submission and respond ONLY in this JSON format: + Analyze this submission and respond ONLY in this JSON format: + { + "rubric_evaluations": [ { - "rubric_evaluations": [ - { - "skill": "Skill Name", - "grade_value": 1-5, - "observation": "specific evidence from submission" - } - ], - "overall_feedback": "30-50 words of constructive, encouraging feedback or 'Submission does not match assignment requirements.'", - "strengths": ["specific strength 1", "specific strength 2"], - "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], - "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)", - "encouragement": "motivating closing statement" - }""" + "Skill": "Skill Name", + "grade_value": 1-5, + "observation": "specific evidence from submission" + } + ], + "strengths": ["specific strength 1", "specific strength 2"], + "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], + "encouragement": "motivating closing statement", + "overall_feedback": "30-50 words of encouraging feedback addressing the student in a friendly tone that summarises strengths and improvement potential. Or 'Submission does not match assignment requirements.'", + "overall_feedback_translated": "Translation of overall_feedback in {Language} for a Grade {Grade_Level} student as per translation rules.", + "learning_objectives_feedback": ["Feedback on objective 1",], + "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)" + + } + """ self.response_format = """{ "rubric_evaluations": [ @@ -181,12 +198,13 @@ def __init__(self): "observation": "specific evidence from submission" } ], - "overall_feedback": "Overall assessment of the submission", "strengths": ["Strength 1", "Strength 2", "Strength 3"], "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": 75, - "encouragement": "Encouraging message for the student" + "encouragement": "Encouraging message for the student", + "overall_feedback": "Overall assessment of the submission", + "overall_feedback_translated": "Translation of overall_feedback.", + "learning_objectives_feedback": ["Feedback on objective 1",], + "final_grade": 75, } """ @@ -207,8 +225,10 @@ def format_objectives(self, objectives: List[Dict]) -> str: return "\n".join(formatted) - def format_rubrics(self, rubrics: Dict) -> str: + def format_rubrics(self, rubrics) -> str: prompt = "" + if isinstance(rubrics, str): + rubrics = json.loads(rubrics) for criterion, grades_list in rubrics.items(): prompt += f"\n{criterion}:\n" for grade_item in grades_list: @@ -216,34 +236,34 @@ def format_rubrics(self, rubrics: Dict) -> str: return prompt - def get_default_response_format(self) -> Dict: """Get default response format""" return { - "rubric_evaluations": [ - { - "skill": "Skill Name", - "grade_value": 2, - "observation": "specific evidence from submission" - }, - { - "skill": "Skill Name", - "grade_value": 2, - "observation": "specific evidence from submission" + "rubric_evaluations": [ + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + }, + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + } + ], + "strengths": ["Strength 1", "Strength 2", "Strength 3"], + "areas_for_improvement": ["Area 1", "Area 2"], + "encouragement": "Encouraging message for the student", + "overall_feedback": "Overall assessment of the submission", + "overall_feedback_translated": "Translation of overall_feedback.", + "learning_objectives_feedback": ["Feedback on objective 1"], + "final_grade": 75 } - ], - "overall_feedback": "Overall assessment of the submission", - "strengths": ["Strength 1", "Strength 2", "Strength 3"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": 75, - "encouragement": "Encouraging message for the student" - } - + async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: """Generate feedback using universal template approach""" try: - print("\n=== Starting Universal Feedback Generation ===") + print("\n=== Starting AI Feedback Generation ===") # Get universal template (no assignment_type filtering) template = self.get_universal_template() @@ -263,20 +283,18 @@ async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submiss # Format learning objectives learning_objectives = self.format_objectives(assignment_context.get("learning_objectives", [])) - rubric_criteria = self.format_rubrics(assignment_context["assignment"].get("rubrics", {})) - - print("User Prompt Context Prepared:") - print(json.dumps(assignment_context, indent=2)) - + rubric_criteria = self.format_rubrics(assignment_context["assignment"].get("rubrics", "")) # Format user prompt with assignment context user_prompt_vars = { "assignment_name": assignment_context["assignment"].get("name", ""), "assignment_description": assignment_context["assignment"].get("description", ""), "course_vertical": assignment_context.get("subject", "General"), - # "assignment_type": assignment_context["assignment"].get("type", "Practical"), + "assignment_type": assignment_context["assignment"].get("type", "Practical"), "learning_objectives": learning_objectives, - "rubric_criteria": rubric_criteria + "rubric_criteria": rubric_criteria, + "Language": assignment_context["student"].get("language", "English"), + "Grade_Level": assignment_context["student"].get("grade", "1") } # Format the user prompt with available variables @@ -289,6 +307,9 @@ async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submiss # Use template system prompt as-is (it already handles JSON requirement) system_prompt = template.system_prompt + print("User Prompt Prepared:") + print(formatted_user_prompt) + # Prepare messages for the LLM provider messages = self.llm_provider.format_messages( system_prompt=system_prompt, @@ -297,11 +318,10 @@ async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submiss ) print("\nSending request to LLM...") - # Generate feedback - SINGLE LLM CALL (no separate validation) - raw_text = await self.llm_provider.generate_with_vision(messages) - print(f"\nRaw LLM Response: {raw_text}") - try: + # Generate feedback - SINGLE LLM CALL (no separate validation) + raw_text = await self.llm_provider.generate_with_vision(messages) + print(f"\nRaw LLM Response: {raw_text}") # Clean up the response text cleaned_text = self.clean_json_response(raw_text) feedback = json.loads(cleaned_text) @@ -315,7 +335,7 @@ async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submiss print("Using fallback feedback format") # Create structured fallback response - feedback = self.create_fallback_feedback(assignment_context, expected_format) + feedback = self.create_fallback_feedback(expected_format) # Attach default plagiarism/AI-detection metadata plagiarism_output = { @@ -351,8 +371,7 @@ async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submiss # Return structured error response template_used = "Built-in Universal Template for Error" - return self.create_error_feedback(assignment_context), template_used - + return self.create_error_feedback(), template_used async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: @@ -366,7 +385,6 @@ async def generate_feedback( self, assignment_context: Dict, submission_url: str is_plagiarized = plagiarism_data.get("is_plagiarized", False) is_ai_generated = plagiarism_data.get("is_ai_generated", False) match_type = plagiarism_data.get("match_type", "original") - plagiarism_source = plagiarism_data.get("plagiarism_source", "none") # Handle AI-generated submissions if is_ai_generated: @@ -388,7 +406,8 @@ async def generate_feedback( self, assignment_context: Dict, submission_url: str else: result_status = "Success - Original" feedback, tempalate_used = await self.generate_ai_evaluated_feedback(assignment_context, submission_url,submission_id) - + + feedback["translation_language"] = assignment_context["student"].get("language", "English") await self._update_result_status(feedback_request_id, result_status) return feedback, self.model_used, tempalate_used @@ -420,18 +439,27 @@ def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: ai_source = plagiarism_data.get("ai_detection_source", "unknown") ai_confidence = plagiarism_data.get("ai_confidence", 0.0) response = { - "overall_feedback": f"Your submission appears to be generated by an \ - AI tool (detected source: {ai_source}, confidence: {ai_confidence:.0%}). \ + "overall_feedback": "Your submission appears to be generated by an AI tool. \ At MentorMe, we encourage original creative work that reflects your own learning \ and artistic development. AI-generated images, while interesting, don't demonstrate \ the skills and creativity we're looking to nurture. Please submit your own original \ artwork for this assignment.", - "strengths": ["N/A - AI-generated content detected"], + "overall_feedback_translated": "Your submission appears to be generated by an AI tool. \ + At MentorMe, we encourage original creative work that reflects your own learning \ + and artistic development. AI-generated images, while interesting, don't demonstrate \ + the skills and creativity we're looking to nurture. Please submit your own original \ + artwork for this assignment.", + "strengths": ["N/A - AI-generated content detected."], "areas_for_improvement": ["Submit original artwork created by you", "Review assignment guidelines for creative direction"], - "learning_objectives_feedback": ["Unable to assess - submission flagged as AI-generated"], + "learning_objectives_feedback": ["N/A - AI-generated content detected."], "grade_recommendation": 0, "encouragement": "We believe in your creative abilities!", + "rubric_evaluations": [{ + "Skill": "Content Knowledge", + "grade_value": 0, + "observation": "N/A - AI-generated content detected." + }], "plagiarism_output": { "is_plagiarized": False, "is_ai_generated": True, @@ -445,7 +473,6 @@ def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: return response - def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: """Create feedback for plagiarized submissions""" @@ -456,16 +483,23 @@ def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: # respond with structured feedback response = { - "overall_feedback": f"Your submission has been flagged for similarity \ - (similarity: {similarity_score:.0%}, source: {plagiarism_source}).\ + "overall_feedback": "Your submission has been flagged for similarity. \ + Academic integrity is fundamental to the learning process. Please ensure your \ + submissions represent your own original work.", + "overall_feedback_translated": "Your submission has been flagged for similarity. \ Academic integrity is fundamental to the learning process. Please ensure your \ submissions represent your own original work.", "strengths": ["N/A - Submission flagged for similarity"], "areas_for_improvement": ["Create original artwork for this assignment", "Review academic integrity guidelines"], - "learning_objectives_feedback": ["Unable to assess - submission flagged for similarity"], + "learning_objectives_feedback": ["N/A - Submission flagged for similarity"], "grade_recommendation": 0, "encouragement": "Every artist develops their unique style through practice!", + "rubric_evaluations": [{ + "Skill": "Content Knowledge", + "grade_value": 0, + "observation": "N/A - Submission flagged for similarity." + }], "plagiarism_output": { "is_plagiarized": True, "is_ai_generated": False, @@ -479,7 +513,6 @@ def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: return response - def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: """Ensure feedback has all required fields with correct types""" # Ensure all expected fields are present @@ -511,14 +544,15 @@ def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> return feedback - def create_fallback_feedback(self, assignment_context: Dict, expected_format: Dict) -> Dict: + def create_fallback_feedback(self, expected_format: Dict) -> Dict: """Create structured fallback when JSON parsing fails""" - assignment_name = assignment_context["assignment"].get("name", "this assignment") fallback = {} for field, default_value in expected_format.items(): if field == "overall_feedback": - fallback[field] = f"I encountered a formatting issue while processing your submission for {assignment_name}. This appears to be a technical problem on our end. Please try resubmitting if this issue persists." + fallback[field] = "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor." + elif field == "overall_feedback_translated": + fallback[field] = "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor." elif field == "grade_recommendation": fallback[field] = 50 # Neutral grade for technical issues elif field == "rubric_evaluations": @@ -544,65 +578,34 @@ def create_fallback_feedback(self, assignment_context: Dict, expected_format: Di return fallback - def create_error_feedback(self, assignment_context: Dict) -> Dict: + def create_error_feedback(self) -> Dict: """Create feedback for system errors""" - assignment_name = assignment_context["assignment"].get("name", "this assignment") - - return { - "overall_feedback": f"I encountered a system error while processing your submission for {assignment_name}. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", + + feedback = { + "overall_feedback": "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", + "overall_feedback_translated": "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", "strengths": ["Your submission was received successfully"], "areas_for_improvement": ["No issues identified with your submission - this appears to be a technical problem"], "learning_objectives_feedback": ["Unable to evaluate due to system error - please resubmit"], "grade_recommendation": 0, - "encouragement": "Technical issues don't reflect your effort or ability - please try again!" + "encouragement": "Technical issues don't reflect your effort or ability - please try again!", + "rubric_evaluations": [{ + "Skill": "Content Knowledge", + "grade_value": 2, + "observation": "Neutral evaluation due to processing issue" + }], } + plagiarism_output = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + feedback["plagiarism_output"] = plagiarism_output + + return feedback - @staticmethod - def format_feedback_for_display(feedback: Dict) -> str: - """Format feedback for human-readable display""" - try: - formatted = [] - - if "overall_feedback" in feedback: - formatted.append("Overall Feedback:") - formatted.append(feedback["overall_feedback"]) - - if "strengths" in feedback: - formatted.append("\nStrengths:") - for strength in feedback["strengths"]: - formatted.append(f"- {strength}") - - if "areas_for_improvement" in feedback: - formatted.append("\nAreas for Improvement:") - for area in feedback["areas_for_improvement"]: - formatted.append(f"- {area}") - - if "learning_objectives_feedback" in feedback: - formatted.append("\nLearning Objectives Feedback:") - for obj in feedback["learning_objectives_feedback"]: - formatted.append(f"- {obj}") - - if "grade_recommendation" in feedback: - formatted.append(f"\nGrade Recommendation: {feedback['grade_recommendation']}") - - if "encouragement" in feedback: - formatted.append(f"\nEncouragement: {feedback['encouragement']}") - - return "\n".join(formatted) - - except Exception as e: - error_msg = f"Error formatting feedback: {str(e)}" - print(f"\nError: {error_msg}") - return "Error formatting feedback for display. Please check the JSON feedback data." - - def get_current_config(self) -> Dict: - """Get current LLM configuration""" - if not self.llm_provider: - return {"status": "not_configured"} - - return { - "provider": self.llm_provider.__class__.__name__, - "model": self.llm_provider.model_name, - "temperature": self.llm_provider.temperature, - "max_tokens": self.llm_provider.max_tokens - } diff --git a/rag_service/core/rag_utils.py b/rag_service/core/rag_utils.py deleted file mode 100644 index d4fb093..0000000 --- a/rag_service/core/rag_utils.py +++ /dev/null @@ -1,94 +0,0 @@ -# File: ~/frappe-bench/apps/rag_service/rag_service/core/rag_utils.py - -import frappe -import json -from .embedding_utils import embedding_manager -from .vector_store import faiss_manager -from .feedback_generator import feedback_generator - -def process_submission(submission_id, content): - """Process a new submission through the RAG pipeline""" - try: - # Generate and store embedding - vector_store_name = embedding_manager.save_embedding( - reference_id=submission_id, - content=content, - content_type="submission" - ) - - # Add to FAISS index - faiss_manager.add_vector(vector_store_name) - - # Find similar submissions - embedding = embedding_manager.load_embedding(vector_store_name) - similar_submissions = faiss_manager.search_similar(embedding) - - return { - "vector_store_name": vector_store_name, - "similar_submissions": similar_submissions, - "plagiarism_score": min([s['distance'] for s in similar_submissions]) if similar_submissions else 0 - } - - except Exception as e: - frappe.log_error(f"Error processing submission: {str(e)}") - raise - -def find_similar_content(query_text, k=5): - """Find similar content for given text""" - try: - # Generate embedding for query - query_embedding = embedding_manager.generate_embedding(query_text) - - # Search for similar content - similar_content = faiss_manager.search_similar(query_embedding, k) - - # Get full content for results - results = [] - for item in similar_content: - vector_store = frappe.get_doc("Vector Store", item["vector_store"]) - results.append({ - "content": vector_store.content, - "content_type": vector_store.content_type, - "reference_id": vector_store.reference_id, - "similarity_score": item["distance"] - }) - - return results - - except Exception as e: - frappe.log_error(f"Error finding similar content: {str(e)}") - raise - -def generate_feedback(submission_id, content): - """Generate feedback for a submission""" - try: - # Process submission and get similar content - process_result = process_submission(submission_id, content) - - # Get similar contents with their full details - similar_contents = find_similar_content(content) - - # Generate feedback - feedback_result = feedback_generator.generate_structured_feedback( - content, - similar_contents, - plagiarism_score=process_result.get("plagiarism_score", None) - ) - - # Store feedback in Vector Store - feedback_store_name = embedding_manager.save_embedding( - reference_id=f"feedback_{submission_id}", - content=feedback_result["feedback"], - content_type="feedback" - ) - - return { - "feedback": feedback_result["feedback"], - "metadata": feedback_result["metadata"], - "similar_contents": similar_contents[:3], # Top 3 similar contents - "feedback_id": feedback_store_name - } - - except Exception as e: - frappe.log_error(f"Error generating feedback: {str(e)}") - raise diff --git a/rag_service/core/vector_store.py b/rag_service/core/vector_store.py deleted file mode 100644 index 5729d29..0000000 --- a/rag_service/core/vector_store.py +++ /dev/null @@ -1,86 +0,0 @@ -# File: rag_service/rag_service/core/vector_store.py - -import frappe -import faiss -import numpy as np -import os -from .embedding_utils import embedding_manager - -class FAISSManager: - def __init__(self): - self.index = None - self.dimension = embedding_manager.embedding_dimension - self.vector_ids = [] # To maintain mapping between FAISS and Vector Store - - def initialize_index(self): - """Initialize FAISS index""" - if self.index is None: - self.index = faiss.IndexFlatL2(self.dimension) - self._load_existing_vectors() - - def _load_existing_vectors(self): - """Load existing vectors from Vector Store into FAISS index""" - try: - vector_stores = frappe.get_all( - "Vector Store", - fields=["name", "embedding_file"] - ) - - vectors = [] - for vs in vector_stores: - try: - embedding = embedding_manager.load_embedding(vs.name) - vectors.append(embedding) - self.vector_ids.append(vs.name) - except Exception as e: - frappe.log_error(f"Error loading vector {vs.name}: {str(e)}") - - if vectors: - vectors_array = np.array(vectors).astype('float32') - self.index.add(vectors_array) - - except Exception as e: - frappe.log_error(f"Error loading existing vectors: {str(e)}") - - def add_vector(self, vector_store_name): - """Add a new vector to the index""" - try: - self.initialize_index() - - embedding = embedding_manager.load_embedding(vector_store_name) - self.index.add(embedding.reshape(1, -1).astype('float32')) - self.vector_ids.append(vector_store_name) - - except Exception as e: - frappe.log_error(f"Error adding vector to FAISS: {str(e)}") - raise - - def search_similar(self, query_vector, k=5): - """Search for similar vectors""" - try: - self.initialize_index() - - distances, indices = self.index.search( - query_vector.reshape(1, -1).astype('float32'), - k - ) - - results = [] - for i, idx in enumerate(indices[0]): - if idx < len(self.vector_ids): - vector_store = frappe.get_doc("Vector Store", self.vector_ids[idx]) - results.append({ - "vector_store": vector_store.name, - "reference_doctype": vector_store.reference_doctype, - "reference_name": vector_store.reference_name, - "distance": float(distances[0][i]) - }) - - return results - - except Exception as e: - frappe.log_error(f"Error searching similar vectors: {str(e)}") - raise - -# Create a singleton instance -faiss_manager = FAISSManager() diff --git a/rag_service/feedback_generator.py b/rag_service/feedback_generator.py deleted file mode 100644 index b66c55c..0000000 --- a/rag_service/feedback_generator.py +++ /dev/null @@ -1,54 +0,0 @@ -import frappe -import json - -def get_student_context(student_id): - try: - return frappe.get_doc("Student Context", {"student_id": student_id}) - except frappe.DoesNotExistError: - frappe.log_error(f"Student Context not found for student_id: {student_id}") - return None - -def get_assignment_context(assignment_id): - try: - return frappe.get_doc("Assignment Context", {"assignment_id": assignment_id}) - except frappe.DoesNotExistError: - frappe.log_error(f"Assignment Context not found for assignment_id: {assignment_id}") - return None - -def generate_feedback(submission_data): - """ - Initial basic feedback generation - """ - try: - # Log the received data - frappe.logger().info(f"Generating feedback for submission: {submission_data}") - - # Extract basic information - student_id = submission_data.get("student_id") - assignment_id = submission_data.get("assignment_id") - plagiarism_score = submission_data.get("plagiarism_score", 0) - - # Get contexts - student_context = get_student_context(student_id) - assignment_context = get_assignment_context(assignment_id) - - # Generate basic feedback - feedback = { - "status": "completed", - "plagiarism_assessment": { - "score": plagiarism_score, - "flag": "high" if plagiarism_score > 0.8 else "low" - }, - "feedback_text": "Submission received and processed.", - "timestamp": frappe.utils.now_datetime() - } - - return feedback - - except Exception as e: - frappe.log_error(frappe.get_traceback(), "Error in generate_feedback") - return { - "status": "error", - "error_message": str(e), - "timestamp": frappe.utils.now_datetime() - } diff --git a/rag_service/handlers/feedback_handler.py b/rag_service/handlers/feedback_handler.py index dcb81f6..8c83f24 100644 --- a/rag_service/handlers/feedback_handler.py +++ b/rag_service/handlers/feedback_handler.py @@ -27,7 +27,7 @@ async def handle_submission(self, message_data: Dict) -> None: # Get assignment context assignment_context = await self.assignment_context_manager.get_assignment_context( - message_data["assignment_id"] + message_data["assignment_id"], message_data["student_id"] ) if not assignment_context: @@ -175,40 +175,6 @@ async def get_request_status(self, request_id: str) -> Dict: "status": "Unknown" } - async def retry_failed_request(self, request_id: str) -> None: - """Retry a failed feedback request""" - try: - print(f"\n=== Retrying Failed Request: {request_id} ===") - - feedback_request = frappe.get_doc("Feedback Request", request_id) - - if feedback_request.status != "Failed": - raise ValueError(f"Request {request_id} is not in failed state") - - if feedback_request.processing_attempts >= 3: - raise ValueError(f"Maximum retry attempts reached for request {request_id}") - - # Prepare message data for reprocessing - message_data = { - "submission_id": feedback_request.submission_id, - "student_id": feedback_request.student_id, - "assignment_id": feedback_request.assignment_id, - "img_url": feedback_request.submission_content, - "plagiarism_score": feedback_request.plagiarism_score, - "similar_sources": json.loads(feedback_request.similar_sources or '[]') - } - - # Process the request again - await self.handle_submission(message_data) - - print(f"Request {request_id} retried successfully") - - except Exception as e: - error_msg = f"Error retrying request: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "Request Retry Error") - raise - async def cleanup_old_requests(self, days: int = 30) -> None: """Clean up old completed requests""" try: diff --git a/rag_service/rabbitmq_utils.py b/rag_service/rabbitmq_utils.py deleted file mode 100644 index a7f026c..0000000 --- a/rag_service/rabbitmq_utils.py +++ /dev/null @@ -1,72 +0,0 @@ -# File: rag_service/rag_service/rabbitmq_utils.py - -import frappe -import pika -import json -from .core.rag_utils import process_submission, find_similar_content - -def process_message(ch, method, properties, body): - try: - message_data = json.loads(body) - - # Process the submission - result = process_submission( - message_data.get("submission_id"), - message_data.get("content") - ) - - # Create feedback request - feedback_request = frappe.get_doc({ - "doctype": "Feedback Request", - "request_id": message_data.get("submission_id"), - "student_id": message_data.get("student_id"), - "assignment_id": message_data.get("assignment_id"), - "submission_content": message_data.get("content"), - "plagiarism_score": message_data.get("plagiarism_score"), - "status": "Processing", - "similar_submissions": json.dumps(result.get("similar_submissions")) - }) - - feedback_request.insert() - frappe.db.commit() - - except Exception as e: - frappe.log_error(frappe.get_traceback(), "Error processing RabbitMQ message") - - -def start_consuming(): - try: - settings = get_rabbitmq_settings() - - # Log the connection attempt - frappe.logger().info(f"Connecting to RabbitMQ at {settings.host}:{settings.port}") - - # Create connection - credentials = pika.PlainCredentials(settings.username, settings.password) - parameters = pika.ConnectionParameters( - host=settings.host, - port=settings.port, - virtual_host=settings.virtual_host, - credentials=credentials - ) - - connection = pika.BlockingConnection(parameters) - channel = connection.channel() - - # Ensure queue exists - channel.queue_declare(queue=settings.plagiarism_results_queue, durable=True) - - # Set up consumer - channel.basic_consume( - queue=settings.plagiarism_results_queue, - on_message_callback=process_message, - auto_ack=True - ) - - frappe.logger().info("Started consuming messages...") - channel.start_consuming() - - except Exception as e: - frappe.log_error(frappe.get_traceback(), "Error in RabbitMQ consumer") - raise - diff --git a/rag_service/rag_service/doctype/assignment_context/assignment_context.json b/rag_service/rag_service/doctype/assignment_context/assignment_context.json index b7dceba..a4882a3 100644 --- a/rag_service/rag_service/doctype/assignment_context/assignment_context.json +++ b/rag_service/rag_service/doctype/assignment_context/assignment_context.json @@ -9,7 +9,7 @@ "assignment_name", "course_vertical", "description", - "rubric", + "rubrics", "learning_objectives", "difficulty_level", "last_updated", @@ -37,7 +37,7 @@ "label": "Course Vertical" }, { - "fieldname": "rubric", + "fieldname": "rubrics", "fieldtype": "Text", "label": "Rubric" }, diff --git a/rag_service/scripts/console_consumer.py b/rag_service/scripts/console_consumer.py new file mode 100644 index 0000000..4316092 --- /dev/null +++ b/rag_service/scripts/console_consumer.py @@ -0,0 +1,4 @@ +from rag_service.utils.rabbitmq_consumer import RabbitMQConsumer +consumer = RabbitMQConsumer(debug=True) +if consumer.test_connection(): + consumer.start_consuming() \ No newline at end of file diff --git a/rag_service/scripts/test_consumer_payload.py b/rag_service/scripts/test_consumer_payload.py new file mode 100644 index 0000000..e9d1aa4 --- /dev/null +++ b/rag_service/scripts/test_consumer_payload.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Test script to send payloads directly to the RabbitMQ consumer for testing. +Simply modify the payload dictionary below and run: python test_consumer_payload.py +""" + +import sys +import json +import pika +from datetime import datetime +from pathlib import Path + +# Add the apps directory to the path +sys.path.insert(0, str(Path(__file__).parent.parent)) + + + +# ============================================================================ +# EDIT THE PAYLOAD BELOW TO TEST DIFFERENT DATA +# ============================================================================ +PAYLOAD = { + "submission_id": "IMSUB-2601280188", + "student_id": "ST00000182", + "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601280188_20251105154700_C5099524_F32580_M18137454.png", + "created_at": "2026-01-28 16:58:27.423261", + "similar_sources": None, + "similarity_score": None, + "is_plagiarized": False, + "match_type": "original", + "assignment_id": "VA_L1_CA1-Basic", + "is_ai_generated": False, + "ai_detection_source": "", + "ai_confidence": 0.0, + "plagiarism_source": "" +} +# ============================================================================ + + +def get_rabbitmq_settings(): + """Get RabbitMQ settings from Frappe""" + try: + return { + "host": "rabbit-01.lmq.cloudamqp.com", + "port": "5672", + "username": "aoafhbrm", + "password": "****", + "virtual_host": "aoafhbrm", + "queue": "plg_result_q_local", + } + except Exception as e: + print(f"Error fetching RabbitMQ settings: {e}") + return None + + +def connect_to_rabbitmq(settings): + """Establish connection to RabbitMQ""" + try: + credentials = pika.PlainCredentials(settings["username"], settings["password"]) + parameters = pika.ConnectionParameters( + host=settings["host"], + port=settings["port"], + virtual_host=settings["virtual_host"], + credentials=credentials, + heartbeat=600, + blocked_connection_timeout=300, + ) + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + print(f"\n✓ Connected to RabbitMQ at {settings['host']}:{settings['port']}") + return connection, channel + except Exception as e: + print(f"\n✗ RabbitMQ Connection Error: {e}") + return None, None + + +def send_payload(connection, channel, queue_name, payload): + """Send a payload to RabbitMQ queue""" + try: + # Declare queue to ensure it exists + channel.queue_declare(queue=queue_name, durable=True) + + # Send message + channel.basic_publish( + exchange="", + routing_key=queue_name, + body=json.dumps(payload, ensure_ascii=False), + properties=pika.BasicProperties( + delivery_mode=2, # persistent + content_type="application/json", + ), + ) + print(f"\n✓ Payload sent to queue '{queue_name}'") + print(f" Submission ID: {payload.get('submission_id')}") + print(f" Student ID: {payload.get('student_id')}") + print(f" Assignment ID: {payload.get('assignment_id')}") + return True + except Exception as e: + print(f"\n✗ Error sending payload: {e}") + return False + finally: + if connection and not connection.is_closed: + connection.close() + + +def main(): + # Get RabbitMQ settings + settings = get_rabbitmq_settings() + if not settings: + print("\n✗ Unable to retrieve RabbitMQ settings") + return + + print(f"\n=== RabbitMQ Consumer Test Script ===") + print(f"Host: {settings['host']}") + print(f"Port: {settings['port']}") + print(f"Queue: {settings['queue']}") + + # Connect to RabbitMQ + connection, channel = connect_to_rabbitmq(settings) + if not connection or not channel: + return + + try: + # Validate required fields + required_fields = ["submission_id", "student_id", "assignment_id", "img_url"] + missing = [f for f in required_fields if f not in PAYLOAD] + if missing: + print(f"\n✗ Missing required fields: {', '.join(missing)}") + return + + print(f"\n--- Sending Payload ---") + print(f"Payload:\n{json.dumps(PAYLOAD, indent=2)}") + send_payload(connection, channel, settings["queue"], PAYLOAD) + + finally: + if connection and not connection.is_closed: + connection.close() + print("\n✓ Disconnected from RabbitMQ") + + +if __name__ == "__main__": + main() diff --git a/rag_service/utils/commands.py b/rag_service/utils/commands.py deleted file mode 100644 index 25d9ce0..0000000 --- a/rag_service/utils/commands.py +++ /dev/null @@ -1,29 +0,0 @@ -# apps/rag_service/rag_service/utils/commands.py - -import frappe -from frappe.commands import pass_context -import click - -@click.command('rag-consumer-start') -@pass_context -def start_consumer(context): - """Start the RAG Service consumer""" - from rag_service.rag_service.utils.rabbitmq_consumer import RabbitMQConsumer - - site = context.sites[0] - frappe.init(site=site) - frappe.connect() - - try: - click.echo(f"Starting RAG consumer for site: {site}") - consumer = RabbitMQConsumer() - consumer.start_consuming() - except Exception as e: - click.echo(f"Error starting consumer: {str(e)}") - finally: - frappe.destroy() - -# And update hooks.py: -commands = [ - "rag_service.utils.commands.start_consumer" -] diff --git a/rag_service/utils/queue_manager.py b/rag_service/utils/queue_manager.py index cd8bc51..ff917cf 100644 --- a/rag_service/utils/queue_manager.py +++ b/rag_service/utils/queue_manager.py @@ -77,7 +77,7 @@ def send_feedback_to_tap(self, feedback_data: Dict) -> None: self.channel.basic_publish( exchange='', routing_key=self.settings.feedback_results_queue, - body=json.dumps(message), + body=json.dumps(message, ensure_ascii=False), properties=pika.BasicProperties( delivery_mode=2, # make message persistent content_type='application/json' diff --git a/rag_service/utils/rabbitmq_consumer.py b/rag_service/utils/rabbitmq_consumer.py index fbe177d..05f59f1 100644 --- a/rag_service/utils/rabbitmq_consumer.py +++ b/rag_service/utils/rabbitmq_consumer.py @@ -3,9 +3,7 @@ import frappe import pika import json -import time import asyncio -from datetime import datetime from typing import Dict, Optional from ..handlers.feedback_handler import FeedbackHandler from .queue_manager import QueueManager diff --git a/rag_service/utils/rabbitmq_manager.py b/rag_service/utils/rabbitmq_manager.py deleted file mode 100644 index 9a43ba2..0000000 --- a/rag_service/utils/rabbitmq_manager.py +++ /dev/null @@ -1,258 +0,0 @@ -import frappe -import pika -import json -import time -from datetime import datetime -from typing import Dict, Optional, Union - -class RabbitMQManager: - def __init__(self, debug=True): - self.settings = frappe.get_single("RabbitMQ Settings") - self.debug = debug - self.connection = None - self.channel = None - self.processed_count = 0 - - def connect(self) -> None: - """Establish RabbitMQ connection""" - try: - if self.debug: - print(f"\nConnecting to RabbitMQ at {self.settings.host}...") - - credentials = pika.PlainCredentials( - self.settings.username, - self.settings.password - ) - - parameters = pika.ConnectionParameters( - host=self.settings.host, - port=int(self.settings.port), - virtual_host=self.settings.virtual_host, - credentials=credentials, - heartbeat=600, - blocked_connection_timeout=300 - ) - - self.connection = pika.BlockingConnection(parameters) - self.channel = self.connection.channel() - - if self.debug: - print("Connection established successfully!") - - except Exception as e: - error_msg = f"RabbitMQ Connection Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "RabbitMQ Connection Error") - raise - - def close(self) -> None: - """Close RabbitMQ connection""" - if self.connection and not self.connection.is_closed: - self.connection.close() - self.connection = None - self.channel = None - - def test_connection(self) -> bool: - """Test RabbitMQ connection""" - try: - self.connect() - print("Connection test successful!") - return True - except Exception as e: - print(f"Connection test failed: {str(e)}") - return False - finally: - self.close() - - def get_queue_info(self, queue_name: str = None) -> Dict: - """Get information about a specific queue or all queues""" - try: - self.connect() - - if queue_name is None: - queue_name = self.settings.plagiarism_results_queue - - queue_info = self.channel.queue_declare( - queue=queue_name, - durable=True, - passive=True - ) - - return { - 'queue': queue_name, - 'message_count': queue_info.method.message_count, - 'consumer_count': queue_info.method.consumer_count - } - - except Exception as e: - print(f"Error getting queue info: {str(e)}") - return {} - finally: - self.close() - - def peek_message(self, queue_name: str = None) -> Optional[dict]: - """Peek at the next message in the queue without consuming it""" - try: - self.connect() - - if queue_name is None: - queue_name = self.settings.plagiarism_results_queue - - # Get message without consuming - method_frame, header_frame, body = self.channel.basic_get( - queue=queue_name, - auto_ack=False - ) - - if not method_frame: - print("\nNo messages in queue") - return None - - try: - message = json.loads(body) - result = { - 'delivery_tag': method_frame.delivery_tag, - 'content': message, - 'raw_body': body.decode('utf-8') - } - except json.JSONDecodeError as e: - result = { - 'delivery_tag': method_frame.delivery_tag, - 'error': f"Invalid JSON: {str(e)}", - 'raw_body': body.decode('utf-8') - } - - # Return message to queue - self.channel.basic_nack( - delivery_tag=method_frame.delivery_tag, - requeue=True - ) - - return result - - except Exception as e: - print(f"Error peeking message: {str(e)}") - return None - finally: - self.close() - - def delete_message(self, delivery_tag: int, queue_name: str = None) -> bool: - """Delete a specific message using its delivery tag""" - try: - self.connect() - - if queue_name is None: - queue_name = self.settings.plagiarism_results_queue - - # Get and reject the message - method_frame, _, body = self.channel.basic_get( - queue=queue_name, - auto_ack=False - ) - - if not method_frame: - print("\nNo messages in queue") - return False - - if method_frame.delivery_tag == delivery_tag: - self.channel.basic_reject( - delivery_tag=delivery_tag, - requeue=False - ) - print(f"\nSuccessfully deleted message with delivery tag: {delivery_tag}") - return True - else: - print(f"\nMessage with delivery tag {delivery_tag} not found at queue head") - # Return message to queue - self.channel.basic_nack( - delivery_tag=method_frame.delivery_tag, - requeue=True - ) - return False - - except Exception as e: - error_msg = f"Error deleting message: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "Message Deletion Error") - return False - finally: - self.close() - - def peek_and_delete(self, queue_name: str = None) -> dict: - """Peek at the next message and optionally delete it""" - result = { - 'success': False, - 'message': None, - 'action_taken': None - } - - # First peek at the message - message_info = self.peek_message(queue_name) - - if not message_info: - result['message'] = "No messages in queue" - return result - - # Show the message content - print("\nNext message in queue:") - print(f"Delivery Tag: {message_info['delivery_tag']}") - print(f"Raw content: {message_info['raw_body']}") - - if 'error' in message_info: - print(f"Parse error: {message_info['error']}") - else: - print(f"Parsed content: {json.dumps(message_info['content'], indent=2)}") - - # Ask for confirmation - confirm = input("\nDo you want to delete this message? (y/n): ").lower() - - if confirm == 'y': - success = self.delete_message(message_info['delivery_tag'], queue_name) - result.update({ - 'success': success, - 'message': "Message deleted successfully" if success else "Failed to delete message", - 'action_taken': 'deleted' - }) - else: - result.update({ - 'success': True, - 'message': "Message left in queue", - 'action_taken': 'skipped' - }) - - return result - - def purge_queue(self, queue_name: str = None) -> bool: - """Purge all messages from a queue""" - try: - self.connect() - - if queue_name is None: - queue_name = self.settings.plagiarism_results_queue - - # Get message count before purging - queue_info = self.channel.queue_declare( - queue=queue_name, - durable=True, - passive=True - ) - message_count = queue_info.method.message_count - - # Confirm purge - confirm = input(f"\nAre you sure you want to purge {message_count} messages from queue '{queue_name}'? (y/n): ").lower() - - if confirm == 'y': - self.channel.queue_purge(queue=queue_name) - print(f"\nSuccessfully purged {message_count} messages from queue: {queue_name}") - return True - else: - print("\nPurge cancelled") - return False - - except Exception as e: - error_msg = f"Error purging queue: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "Queue Purge Error") - return False - finally: - self.close() diff --git a/rag_service/utils/setup_test_data.py b/rag_service/utils/setup_test_data.py deleted file mode 100644 index 06c1d52..0000000 --- a/rag_service/utils/setup_test_data.py +++ /dev/null @@ -1,89 +0,0 @@ -# File: ~/frappe-bench/apps/rag_service/rag_service/utils/setup_test_data.py - -import frappe -from frappe.utils import now_datetime -from ..core.embedding_utils import embedding_manager - -def create_test_data(): - """Create test data for RAG system""" - test_contents = [ - { - "content": "Python is a high-level programming language known for its simplicity and readability.", - "reference_id": "test_content_1", - "content_type": "reference" # Changed to lowercase - }, - { - "content": "Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience.", - "reference_id": "test_content_2", - "content_type": "reference" - }, - { - "content": "Natural Language Processing (NLP) helps computers understand and process human language.", - "reference_id": "test_content_3", - "content_type": "reference" - }, - { - "content": "This is a sample student submission about programming concepts.", - "reference_id": "submission_1", - "content_type": "submission" - }, - { - "content": "Great work on explaining the concepts. Consider adding more examples.", - "reference_id": "feedback_1", - "content_type": "feedback" - } - ] - - created_docs = [] - - try: - for content in test_contents: - # Check if already exists - existing = frappe.db.exists( - "Vector Store", - {"reference_id": content["reference_id"]} - ) - - if not existing: - # Create embedding and save - vector_store_name = embedding_manager.save_embedding( - reference_id=content["reference_id"], - content=content["content"], - content_type=content["content_type"] - ) - created_docs.append(vector_store_name) - print(f"Created: {vector_store_name} - {content['content_type']}") - - frappe.db.commit() - return f"Created {len(created_docs)} test documents: {', '.join(created_docs)}" - - except Exception as e: - frappe.log_error(frappe.get_traceback(), "Error creating test data") - return f"Error creating test data: {str(e)}" - -def verify_test_data(): - """Verify the created test data""" - try: - vector_stores = frappe.get_all( - "Vector Store", - fields=["name", "content_type", "reference_id", "content", "embedding_file"], - order_by="creation desc" - ) - - print(f"\nFound {len(vector_stores)} Vector Store entries:") - for vs in vector_stores: - print("\nVector Store:", vs.name) - print("Type:", vs.content_type) - print("Reference:", vs.reference_id) - print("Content:", vs.content) - print("Embedding File:", vs.embedding_file) - - return vector_stores - - except Exception as e: - print(f"Error verifying test data: {str(e)}") - return None - -# You can run this from bench console -if __name__ == "__main__": - create_test_data() From 3e28c941d5ce6669e60e517b59ddac698782a629 Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Fri, 30 Jan 2026 11:12:05 +0000 Subject: [PATCH 08/14] feedback translation --- rag_service/core/assignment_context_manager.py | 5 +++-- rag_service/core/langchain_manager.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rag_service/core/assignment_context_manager.py b/rag_service/core/assignment_context_manager.py index 4c8f29b..047b793 100644 --- a/rag_service/core/assignment_context_manager.py +++ b/rag_service/core/assignment_context_manager.py @@ -23,6 +23,8 @@ async def get_assignment_context(self, assignment_id: str, student_id: str) -> D """Get assignment context from cache or API""" try: print(f"\n=== Getting Assignment Context for: {assignment_id} ===") + + context = None # Check cache if enabled if self.settings.enable_caching: @@ -34,8 +36,7 @@ async def get_assignment_context(self, assignment_id: str, student_id: str) -> D }, limit=1 ) - - context = None + if cached_context: print("Found cached context") cached_context = frappe.get_doc("Assignment Context", cached_context[0].name).as_dict() diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py index 87f072a..244c20e 100644 --- a/rag_service/core/langchain_manager.py +++ b/rag_service/core/langchain_manager.py @@ -483,7 +483,7 @@ def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: # respond with structured feedback response = { - "overall_feedback": "Your submission has been flagged for similarity. \ + "overall_feedback": "Your submission has been flagged for similarity with another submission. \ Academic integrity is fundamental to the learning process. Please ensure your \ submissions represent your own original work.", "overall_feedback_translated": "Your submission has been flagged for similarity. \ From 3f201327c92c69f1fb508cb009ba46601975d17a Mon Sep 17 00:00:00 2001 From: Manu Date: Mon, 9 Feb 2026 12:20:34 +0530 Subject: [PATCH 09/14] video analysis --- .gitignore | 3 +- rag_service/api/test_api.py | 33 - rag_service/config/__init__.py | 0 .../{handlers => core}/feedback_handler.py | 12 +- rag_service/core/feedback_processor.py | 129 --- rag_service/core/feedback_service.py | 390 +++++++++ rag_service/core/langchain_manager.py | 611 ------------- rag_service/core/langchain_manager.py.bak | 466 ---------- rag_service/core/llm_providers.py | 152 +++- .../feedback_utils/evaluation_generation.py | 358 ++++++++ .../feedback_utils/image_evaluation.py | 69 ++ .../feedback_utils/video_evaluation.py | 90 ++ rag_service/handlers/__init__.py | 0 .../doctype/gemini_settings/__init__.py | 2 + .../gemini_settings/gemini_settings.js | 4 + .../gemini_settings/gemini_settings.json | 95 ++ .../gemini_settings/gemini_settings.py | 8 + .../gemini_settings/test_gemini_settings.py | 9 + .../prompt_template/prompt_template.json | 11 +- rag_service/scripts/assignment | 1 + rag_service/scripts/check_active_llms.py | 66 -- .../service_account_key.json} | 0 rag_service/scripts/test_feedback_prompt.py | 6 +- rag_service/scripts/test_langchain_manager.py | 628 -------------- rag_service/scripts/vertexai_client.py | 220 +++++ rag_service/scripts/video_analysis.py | 820 ++++++++++++++++++ rag_service/templates/__init__.py | 0 rag_service/templates/pages/__init__.py | 0 rag_service/utils/rabbitmq_consumer.py | 2 +- requirements.txt | 4 + 30 files changed, 2240 insertions(+), 1949 deletions(-) delete mode 100644 rag_service/api/test_api.py delete mode 100644 rag_service/config/__init__.py rename rag_service/{handlers => core}/feedback_handler.py (94%) delete mode 100644 rag_service/core/feedback_processor.py create mode 100644 rag_service/core/feedback_service.py delete mode 100644 rag_service/core/langchain_manager.py delete mode 100644 rag_service/core/langchain_manager.py.bak create mode 100644 rag_service/feedback_utils/evaluation_generation.py create mode 100644 rag_service/feedback_utils/image_evaluation.py create mode 100644 rag_service/feedback_utils/video_evaluation.py delete mode 100644 rag_service/handlers/__init__.py create mode 100644 rag_service/rag_service/doctype/gemini_settings/__init__.py create mode 100644 rag_service/rag_service/doctype/gemini_settings/gemini_settings.js create mode 100644 rag_service/rag_service/doctype/gemini_settings/gemini_settings.json create mode 100644 rag_service/rag_service/doctype/gemini_settings/gemini_settings.py create mode 100644 rag_service/rag_service/doctype/gemini_settings/test_gemini_settings.py create mode 100644 rag_service/scripts/assignment delete mode 100644 rag_service/scripts/check_active_llms.py rename rag_service/{api/__init__.py => scripts/service_account_key.json} (100%) delete mode 100644 rag_service/scripts/test_langchain_manager.py create mode 100644 rag_service/scripts/vertexai_client.py create mode 100644 rag_service/scripts/video_analysis.py delete mode 100644 rag_service/templates/__init__.py delete mode 100644 rag_service/templates/pages/__init__.py diff --git a/.gitignore b/.gitignore index 83e74cd..3b032c2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ tags node_modules __pycache__ -*scratch* \ No newline at end of file +*scratch* +*.vscode \ No newline at end of file diff --git a/rag_service/api/test_api.py b/rag_service/api/test_api.py deleted file mode 100644 index 18bf7f3..0000000 --- a/rag_service/api/test_api.py +++ /dev/null @@ -1,33 +0,0 @@ -# File: ~/frappe-bench/apps/rag_service/rag_service/api/test_api.py - -@frappe.whitelist(allow_guest=False) -def generate_submission_feedback(): - """Generate feedback for a submission""" - try: - data = frappe.request.get_json() - - if not data: - frappe.throw("No data provided") - - content = data.get("content") - submission_id = data.get("submission_id") or f"submission_{now()}" - - if not content: - frappe.throw("Content is required") - - result = generate_feedback(submission_id, content) - - return { - "status": "success", - "submission_id": submission_id, - "feedback": result["feedback"], - "metadata": result["metadata"], - "similar_contents": result["similar_contents"] - } - - except Exception as e: - frappe.log_error(frappe.get_traceback(), "Feedback Generation Error") - return { - "status": "error", - "message": str(e) - } diff --git a/rag_service/config/__init__.py b/rag_service/config/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rag_service/handlers/feedback_handler.py b/rag_service/core/feedback_handler.py similarity index 94% rename from rag_service/handlers/feedback_handler.py rename to rag_service/core/feedback_handler.py index 8c83f24..d7981cf 100644 --- a/rag_service/handlers/feedback_handler.py +++ b/rag_service/core/feedback_handler.py @@ -1,18 +1,16 @@ -# rag_service/rag_service/handlers/feedback_handler.py +# rag_service/rag_service/core/feedback_handler.py import frappe import json from datetime import datetime from typing import Dict, Optional -from ..core.langchain_manager import LangChainManager -from ..core.feedback_processor import FeedbackProcessor +from ..core.feedback_service import FeedbackService from ..core.assignment_context_manager import AssignmentContextManager from ..utils.queue_manager import QueueManager class FeedbackHandler: def __init__(self): - self.langchain_manager = LangChainManager() - self.feedback_processor = FeedbackProcessor() + self.feedback_service = FeedbackService() self.queue_manager = QueueManager() self.assignment_context_manager = AssignmentContextManager() @@ -35,7 +33,7 @@ async def handle_submission(self, message_data: Dict) -> None: print("\nGenerating feedback...") # Generate feedback - feedback, model_used, template_used = await self.langchain_manager.generate_feedback( + feedback, model_used, template_used = await self.feedback_service.generate_feedback( assignment_context=assignment_context, submission_url=message_data["img_url"], submission_id=request_id, @@ -45,7 +43,7 @@ async def handle_submission(self, message_data: Dict) -> None: print("\nFeedback generated, processing feedback...") # Process and deliver feedback - await self.feedback_processor.process_feedback(request_id, feedback, model_used, template_used) + await self.feedback_service.process_feedback(request_id, feedback, model_used, template_used) print("\nFeedback processing completed") except Exception as e: diff --git a/rag_service/core/feedback_processor.py b/rag_service/core/feedback_processor.py deleted file mode 100644 index 2db85d9..0000000 --- a/rag_service/core/feedback_processor.py +++ /dev/null @@ -1,129 +0,0 @@ -# rag_service/rag_service/core/feedback_processor.py - -import frappe -import json -from datetime import datetime -from typing import Dict, Optional -from ..utils.queue_manager import QueueManager - -class FeedbackProcessor: - def __init__(self): - self.queue_manager = QueueManager() - - async def process_feedback(self, request_id: str, feedback: Dict, model_used: str, template_used: str) -> None: - """Process and store feedback in Feedback Request DocType""" - try: - print(f"\n=== Processing Feedback for Request: {request_id} ===") - - # Get the feedback request document - feedback_request = frappe.get_doc("Feedback Request", request_id) - print(f"Found Feedback Request: {feedback_request.name}") - - print("\nUpdating Feedback Request fields...") - # Update document fields using db_set - feedback_request.db_set('status', 'Completed', update_modified=True) - feedback_request.db_set('generated_feedback', json.dumps(feedback, indent=2, ensure_ascii=False), update_modified=True) - feedback_request.db_set('feedback_summary', feedback['overall_feedback'], update_modified=True) - feedback_request.db_set('completed_at', datetime.now(), update_modified=True) - feedback_request.db_set('model_used', model_used, update_modified=True) - feedback_request.db_set('template_used', template_used, update_modified=True) - - # Commit changes - frappe.db.commit() - - # Verify the update - updated_doc = frappe.get_doc("Feedback Request", request_id) - # Prepare and send message to TAP LMS - message = { - "submission_id": feedback_request.submission_id, - "student_id": feedback_request.student_id, - "assignment_id": feedback_request.assignment_id, - "feedback": feedback, - # "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], - # "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], - # "match_type": feedback['plagiarism_output']['match_type'], - # "plagiarism_source": feedback['plagiarism_output']['plagiarism_source'], - # "similarity_score": feedback['plagiarism_output']['similarity_score'], - # "ai_detection_source": feedback['plagiarism_output']['ai_detection_source'], - # "ai_confidence": feedback['plagiarism_output']['ai_confidence'], - - "generated_at": feedback_request.completed_at.isoformat() if feedback_request.completed_at else datetime.now().isoformat(), - - } - - # Send to TAP LMS queue - self.queue_manager.send_feedback_to_tap(message) - - print(f"\nFeedback processed and sent for request: {request_id}") - - print("Payload sent to TAP LMS queue:") - print(json.dumps(message, indent=2, ensure_ascii=False)) - - except Exception as e: - error_msg = f"Error processing feedback: {str(e)}" - print(f"\nError: {error_msg}") - - try: - if 'feedback_request' in locals() and feedback_request: - feedback_request.db_set('status', 'Failed', update_modified=True) - feedback_request.db_set('error_log', error_msg, update_modified=True) - frappe.db.commit() - print(f"Request {request_id} marked as failed") - except Exception as save_error: - print(f"Error saving failure status: {str(save_error)}") - - frappe.log_error(error_msg, "Feedback Processing Error") - raise - - def format_feedback_for_display(self, feedback: Dict) -> str: - """Format feedback for human-readable display""" - try: - formatted = [] - - # Standard fields - if "overall_feedback" in feedback: - formatted.append("Overall Feedback:") - formatted.append(feedback["overall_feedback"]) - - if "strengths" in feedback: - formatted.append("\nStrengths:") - for strength in feedback["strengths"]: - formatted.append(f"- {strength}") - - if "areas_for_improvement" in feedback: - formatted.append("\nAreas for Improvement:") - for area in feedback["areas_for_improvement"]: - formatted.append(f"- {area}") - - if "learning_objectives_feedback" in feedback: - formatted.append("\nLearning Objectives Feedback:") - for obj in feedback["learning_objectives_feedback"]: - formatted.append(f"- {obj}") - - if "grade_recommendation" in feedback: - formatted.append(f"\nGrade Recommendation: {feedback['grade_recommendation']}") - - if "encouragement" in feedback: - formatted.append(f"\nEncouragement: {feedback['encouragement']}") - - # Include any additional fields not in the standard format - standard_fields = ["overall_feedback", "strengths", "areas_for_improvement", - "learning_objectives_feedback", "grade_recommendation", - "encouragement", "detected_type", "error"] - - # Process any custom fields in the feedback - for key, value in feedback.items(): - if key not in standard_fields: - formatted.append(f"\n{key.replace('_', ' ').title()}:") - if isinstance(value, list): - for item in value: - formatted.append(f"- {item}") - else: - formatted.append(str(value)) - - return "\n".join(formatted) - - except Exception as e: - error_msg = f"Error formatting feedback: {str(e)}" - print(f"\nError: {error_msg}") - return "Error formatting feedback for display. Please check the JSON feedback data." diff --git a/rag_service/core/feedback_service.py b/rag_service/core/feedback_service.py new file mode 100644 index 0000000..070c186 --- /dev/null +++ b/rag_service/core/feedback_service.py @@ -0,0 +1,390 @@ +# rag_service/rag_service/core/feedback_service.py + +import frappe +import json +from datetime import datetime +from typing import Dict +from ..feedback_utils.evaluation_generation import EvaluationGenerator +from ..utils.queue_manager import QueueManager + +class FeedbackService: + def __init__(self): + self.evaluation_generator = EvaluationGenerator(self) + self.queue_manager = QueueManager() + self.model_used = None + + async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, + plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: + """Generate feedback with plagiarism context""" + + result_status = "Pending" + model_used = "N/A" + tempalate_used = "N/A" + + try: + # Check for plagiarism/AI-generated content first + if plagiarism_data: + is_plagiarized = plagiarism_data.get("is_plagiarized", False) + is_ai_generated = plagiarism_data.get("is_ai_generated", False) + match_type = plagiarism_data.get("match_type", "original") + + # Handle AI-generated submissions + if is_ai_generated: + result_status = "Success - Flagged" + feedback = self._create_ai_generated_feedback( + plagiarism_data + ) + tempalate_used = "Feedback Template for AI Generated Submission" + + # Handle plagiarized submissions + elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: + result_status = "Success - Flagged" + feedback = self._create_plagiarism_feedback( + plagiarism_data + ) + tempalate_used = "Feedback Template for Plagiarized Submission" + + # Continue with normal feedback generation for original work + else: + result_status = "Success - Original" + feedback, model_used, tempalate_used = await self.evaluation_generator.generate_ai_feedback( + assignment_context, submission_url, submission_id + ) + + feedback["translation_language"] = assignment_context["student"].get("language", "English") + await self._update_result_status(feedback_request_id, result_status) + return feedback, model_used, tempalate_used + + except Exception as e: + result_status = "Failed" + await self._update_result_status(feedback_request_id, result_status, str(e)) + raise + + async def process_feedback( + self, request_id: str, feedback: Dict, model_used: str, template_used: str + ) -> None: + """Process and store feedback in Feedback Request DocType.""" + try: + print(f"\n=== Processing Feedback for Request: {request_id} ===") + + # Get the feedback request document + feedback_request = frappe.get_doc("Feedback Request", request_id) + print(f"Found Feedback Request: {feedback_request.name}") + + print("\nUpdating Feedback Request fields...") + # Update document fields using db_set + feedback_request.db_set("status", "Completed", update_modified=True) + feedback_request.db_set( + "generated_feedback", + json.dumps(feedback, indent=2, ensure_ascii=False), + update_modified=True, + ) + feedback_request.db_set( + "feedback_summary", feedback["overall_feedback"], update_modified=True + ) + feedback_request.db_set("completed_at", datetime.now(), update_modified=True) + feedback_request.db_set("model_used", model_used, update_modified=True) + feedback_request.db_set("template_used", template_used, update_modified=True) + + # Commit changes + frappe.db.commit() + + # Verify the update + updated_doc = frappe.get_doc("Feedback Request", request_id) + # Prepare and send message to TAP LMS + message = { + "submission_id": feedback_request.submission_id, + "student_id": feedback_request.student_id, + "assignment_id": feedback_request.assignment_id, + "feedback": feedback, + # "is_plagiarized": feedback['plagiarism_output']['is_plagiarized'], + # "is_ai_generated": feedback['plagiarism_output']['is_ai_generated'], + # "match_type": feedback['plagiarism_output']['match_type'], + # "plagiarism_source": feedback['plagiarism_output']['plagiarism_source'], + # "similarity_score": feedback['plagiarism_output']['similarity_score'], + # "ai_detection_source": feedback['plagiarism_output']['ai_detection_source'], + # "ai_confidence": feedback['plagiarism_output']['ai_confidence'], + + "generated_at": feedback_request.completed_at.isoformat() + if feedback_request.completed_at + else datetime.now().isoformat(), + } + + # Send to TAP LMS queue + self.queue_manager.send_feedback_to_tap(message) + + print(f"\nFeedback processed and sent for request: {request_id}") + + print("Payload sent to TAP LMS queue:") + print(json.dumps(message, indent=2, ensure_ascii=False)) + + except Exception as e: + error_msg = f"Error processing feedback: {str(e)}" + print(f"\nError: {error_msg}") + + try: + if "feedback_request" in locals() and feedback_request: + feedback_request.db_set("status", "Failed", update_modified=True) + feedback_request.db_set("error_log", error_msg, update_modified=True) + frappe.db.commit() + print(f"Request {request_id} marked as failed") + except Exception as save_error: + print(f"Error saving failure status: {str(save_error)}") + + frappe.log_error(error_msg, "Feedback Processing Error") + raise + + def format_feedback_for_display(self, feedback: Dict) -> str: + """Format feedback for human-readable display.""" + try: + formatted = [] + + # Standard fields + if "overall_feedback" in feedback: + formatted.append("Overall Feedback:") + formatted.append(feedback["overall_feedback"]) + + if "strengths" in feedback: + formatted.append("\nStrengths:") + for strength in feedback["strengths"]: + formatted.append(f"- {strength}") + + if "areas_for_improvement" in feedback: + formatted.append("\nAreas for Improvement:") + for area in feedback["areas_for_improvement"]: + formatted.append(f"- {area}") + + if "learning_objectives_feedback" in feedback: + formatted.append("\nLearning Objectives Feedback:") + for obj in feedback["learning_objectives_feedback"]: + formatted.append(f"- {obj}") + + if "grade_recommendation" in feedback: + formatted.append( + f"\nGrade Recommendation: {feedback['grade_recommendation']}" + ) + + if "encouragement" in feedback: + formatted.append(f"\nEncouragement: {feedback['encouragement']}") + + # Include any additional fields not in the standard format + standard_fields = [ + "overall_feedback", + "strengths", + "areas_for_improvement", + "learning_objectives_feedback", + "grade_recommendation", + "encouragement", + "detected_type", + "error", + ] + + # Process any custom fields in the feedback + for key, value in feedback.items(): + if key not in standard_fields: + formatted.append(f"\n{key.replace('_', ' ').title()}:") + if isinstance(value, list): + for item in value: + formatted.append(f"- {item}") + else: + formatted.append(str(value)) + + return "\n".join(formatted) + + except Exception as e: + error_msg = f"Error formatting feedback: {str(e)}" + print(f"\nError: {error_msg}") + return "Error formatting feedback for display. Please check the JSON feedback data." + + async def _update_result_status(self, feedback_request_id: str, status: str, error_message: str = None): + """Update Feedback Request result_status""" + if not feedback_request_id: + return + + update_data = {"result_status": status} + if error_message: + update_data["error_message"] = error_message[:500] # Truncate long errors + + frappe.db.set_value( + "Feedback Request", + feedback_request_id, + update_data, + update_modified=True + ) + frappe.db.commit() + + def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: + """Create feedback for AI-generated submissions""" + + ai_source = plagiarism_data.get("ai_detection_source", "unknown") + ai_confidence = plagiarism_data.get("ai_confidence", 0.0) + response = { + "overall_feedback": "Your submission appears to be generated by an AI tool. \ + At MentorMe, we encourage original creative work that reflects your own learning \ + and artistic development. AI-generated images, while interesting, don't demonstrate \ + the skills and creativity we're looking to nurture. Please submit your own original \ + artwork for this assignment.", + "overall_feedback_translated": "Your submission appears to be generated by an AI tool. \ + At MentorMe, we encourage original creative work that reflects your own learning \ + and artistic development. AI-generated images, while interesting, don't demonstrate \ + the skills and creativity we're looking to nurture. Please submit your own original \ + artwork for this assignment.", + "strengths": ["N/A - AI-generated content detected."], + "areas_for_improvement": ["Submit original artwork created by you", + "Review assignment guidelines for creative direction"], + "learning_objectives_feedback": ["N/A - AI-generated content detected."], + "grade_recommendation": 0, + "encouragement": "We believe in your creative abilities!", + "rubric_evaluations": [{ + "Skill": "Content Knowledge", + "grade_value": 0, + "observation": "N/A - AI-generated content detected." + }], + "plagiarism_output": { + "is_plagiarized": False, + "is_ai_generated": True, + "match_type": "ai_generated", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": ai_source, + "ai_confidence": ai_confidence, + } + } + + return response + + def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: + """Create feedback for plagiarized submissions""" + + match_type = plagiarism_data.get("match_type") + plagiarism_source = plagiarism_data.get("plagiarism_source") + similarity_score = plagiarism_data.get("similarity_score", 0.0) + ai_confidence = plagiarism_data.get("ai_confidence", 0.0) + + # respond with structured feedback + response = { + "overall_feedback": "Your submission has been flagged for similarity with another submission. \ + Academic integrity is fundamental to the learning process. Please ensure your \ + submissions represent your own original work.", + "overall_feedback_translated": "Your submission has been flagged for similarity. \ + Academic integrity is fundamental to the learning process. Please ensure your \ + submissions represent your own original work.", + "strengths": ["N/A - Submission flagged for similarity"], + "areas_for_improvement": ["Create original artwork for this assignment", + "Review academic integrity guidelines"], + "learning_objectives_feedback": ["N/A - Submission flagged for similarity"], + "grade_recommendation": 0, + "encouragement": "Every artist develops their unique style through practice!", + "rubric_evaluations": [{ + "Skill": "Content Knowledge", + "grade_value": 0, + "observation": "N/A - Submission flagged for similarity." + }], + "plagiarism_output": { + "is_plagiarized": True, + "is_ai_generated": False, + "match_type": match_type, + "plagiarism_source": plagiarism_source, + "similarity_score": similarity_score, + "ai_detection_source": "none", + "ai_confidence": ai_confidence, + } + } + + return response + + def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: + """Ensure feedback has all required fields with correct types""" + # Ensure all expected fields are present + for field in expected_format: + if field not in feedback: + if isinstance(expected_format[field], list): + feedback[field] = ["No information provided"] + elif isinstance(expected_format[field], (int, float)): + feedback[field] = 0 + else: + feedback[field] = "No information provided" + + # Validate grade_recommendation format for TAP LMS compatibility + try: + grade = feedback.get("grade_recommendation", 0) + if isinstance(grade, str): + # Extract numeric part only + grade_clean = ''.join(c for c in grade if c.isdigit() or c == '.') + grade = float(grade_clean) if grade_clean else 0 + feedback["grade_recommendation"] = max(0, min(100, float(grade))) + except (ValueError, TypeError): + feedback["grade_recommendation"] = 0 + + # Ensure list fields are lists + list_fields = ["strengths", "areas_for_improvement", "learning_objectives_feedback"] + for field in list_fields: + if field in feedback and not isinstance(feedback[field], list): + feedback[field] = [str(feedback[field])] + + return feedback + + def create_fallback_feedback(self, expected_format: Dict) -> Dict: + """Create structured fallback when JSON parsing fails""" + + fallback = {} + for field, default_value in expected_format.items(): + if field == "overall_feedback": + fallback[field] = "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor." + elif field == "overall_feedback_translated": + fallback[field] = "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor." + elif field == "grade_recommendation": + fallback[field] = 50 # Neutral grade for technical issues + elif field == "rubric_evaluations": + fallback[field] = [ + { + "skill": "Content Knowledge", + "grade_value": 2, + "observation": "Neutral evaluation due to processing issue" + } + ] + elif isinstance(default_value, list): + if "strength" in field: + fallback[field] = ["Your submission was received and processed"] + elif "improvement" in field: + fallback[field] = ["Please ensure your submission clearly shows your work"] + else: + fallback[field] = ["Unable to provide specific feedback due to processing issue"] + else: + if field == "encouragement": + fallback[field] = "Technical issues don't reflect your effort - please try resubmitting!" + else: + fallback[field] = "Processing issue - please resubmit for detailed feedback" + + return fallback + + def create_error_feedback(self) -> Dict: + """Create feedback for system errors""" + + feedback = { + "overall_feedback": "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", + "overall_feedback_translated": "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", + "strengths": ["Your submission was received successfully"], + "areas_for_improvement": ["No issues identified with your submission - this appears to be a technical problem"], + "learning_objectives_feedback": ["Unable to evaluate due to system error - please resubmit"], + "grade_recommendation": 0, + "encouragement": "Technical issues don't reflect your effort or ability - please try again!", + "rubric_evaluations": [{ + "Skill": "Content Knowledge", + "grade_value": 2, + "observation": "Neutral evaluation due to processing issue" + }], + } + plagiarism_output = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + feedback["plagiarism_output"] = plagiarism_output + + return feedback diff --git a/rag_service/core/langchain_manager.py b/rag_service/core/langchain_manager.py deleted file mode 100644 index 244c20e..0000000 --- a/rag_service/core/langchain_manager.py +++ /dev/null @@ -1,611 +0,0 @@ -# rag_service/rag_service/core/langchain_manager.py - -import frappe -import json -from typing import Dict, List, Optional, Union -from datetime import datetime -from .llm_providers import create_llm_provider, OpenAIProvider - -class LangChainManager: - def __init__(self): - self.llm = None - self.llm_provider = None - self.setup_llm() - - def setup_llm(self): - """Initialize LLM based on settings""" - try: - llm_settings = frappe.get_list( - "LLM Settings", - filters={"is_active": 1}, - limit=1 - ) - - if not llm_settings: - raise Exception("No active LLM configuration found") - - settings = frappe.get_doc("LLM Settings", llm_settings[0].name) - self.model_used = llm_settings[0].name - print("\nUsing LLM Settings:") - print(f"Provider: {settings.provider}") - print(f"Model: {settings.model_name}") - - - # Create LLM provider based on settings - self.llm_provider = create_llm_provider( - provider=settings.provider, - api_key=settings.get_password('api_secret'), - model_name=settings.model_name, - temperature=settings.temperature, - max_tokens=settings.max_tokens - ) - - # Keep the llm reference for backward compatibility with OpenAI - if isinstance(self.llm_provider, OpenAIProvider): - self.llm = self.llm_provider.llm - - except Exception as e: - error_msg = f"LLM Setup Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "LLM Setup Error") - raise - - def clean_json_response(self, response: str) -> str: - """Clean JSON response from various formats""" - try: - # Remove markdown code blocks if present - if "```json" in response: - response = response.split("```json")[1].split("```")[0].strip() - elif "```" in response: - code_blocks = response.split("```") - if len(code_blocks) >= 3: # At least one code block exists - response = code_blocks[1].strip() - # Check if the extracted content looks like JSON - if not (response.startswith('{') or response.startswith('[')): - # If not, try to find JSON in the original response - json_start = response.find('{') - if json_start >= 0: - response = response[json_start:] - - # Try to extract JSON if response starts with explanation - if not response.strip().startswith('{'): - json_start = response.find('{') - if json_start >= 0: - response = response[json_start:] - - # Check if the response ends properly - if not response.strip().endswith('}'): - json_end = response.rfind('}') - if json_end >= 0: - response = response[:json_end+1] - - return response.strip() - except Exception as e: - print(f"Error cleaning JSON: {str(e)}") - return response - - def get_universal_template(self) -> Dict: - """Get any active template - no assignment_type filtering""" - try: - print("\n=== Getting Prompt Template ===") - - # REMOVED: assignment_type filtering - get ANY active template - templates = frappe.get_list( - "Prompt Template", - filters={"is_active": 1}, - order_by="version desc", - limit=1 - ) - - if templates: - template = frappe.get_doc("Prompt Template", templates[0].name) - print(f"Using universal template: {template.template_name}") - - # Update the last_used timestamp - template.db_set('last_used', datetime.now()) - frappe.db.commit() - - return template - else: - print("No active template found, using built-in default") - return self.get_builtin_template() - - except Exception as e: - error_msg = f"Template Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "Template Error") - return self.get_builtin_template() - - def get_builtin_template(self): - """Return built-in default template as fallback""" - class BuiltinTemplate: - def __init__(self): - self.template_name = "Built-in Universal Template" - self.system_prompt = """You are an encouraging, knowledgeable educational assistant that provides constructive feedback on student submissions using a structured rubric-based evaluation. - EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. Prioritize growth recognition over perfection. - - GRADING PHILOSOPHY: - - Credit partial mastery: A student showing 60% competency deserves acknowledgment of that progress. Grades need not be binary (good/bad). - - Growth mindset framing: Every submission represents learning in progress. Frame gaps as natural and achievable, not deficiencies. - - Reserve lower grades (1-2) only for minimal engagement or complete absence of skill demonstration - - Lean toward higher grades when effort and authenticity are evident - - - Always provide feedback that is: - - Encouraging and positive while being constructive - - Age-appropriate and specific to observations - - Directly aligned with rubric criteria - - Clear about achievement gaps and growth areas - - Structure your response by: - 1. Opening with what the student did well (be specific) - 2. Evaluating the submission against each rubric skill - 3. Assigning a single grade for each skill in the rubric criteria - 4. Evaluating only the skills mentioned in the rubric criteria - 5. Providing specific, actionable feedback for growth - 6. Ending with motivating encouragement - 7. Translate the overall_feedback. Translation rules: - - Formal but friendly tone (customer communication). - - Natural, conversational phrasing. Not literal translation. - - Use native script for the language. - - CRITICAL: You must respond with valid JSON format only. - """ - - self.user_prompt = """Assignment Context: - - Name: {assignment_name} - - Subject: {course_vertical} - - Type: {assignment_type} - - Description: {assignment_description} - - Learning Objectives: {learning_objectives} - - Rubric Criteria: {rubric_criteria} - - CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. - - Analyze this submission and respond ONLY in this JSON format: - - { - "rubric_evaluations": [ - { - "Skill": "Skill Name", - "grade_value": 1-5, - "observation": "specific evidence from submission" - } - ], - "strengths": ["specific strength 1", "specific strength 2"], - "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], - "encouragement": "motivating closing statement", - "overall_feedback": "30-50 words of encouraging feedback addressing the student in a friendly tone that summarises strengths and improvement potential. Or 'Submission does not match assignment requirements.'", - "overall_feedback_translated": "Translation of overall_feedback in {Language} for a Grade {Grade_Level} student as per translation rules.", - "learning_objectives_feedback": ["Feedback on objective 1",], - "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)" - - } - """ - - self.response_format = """{ - "rubric_evaluations": [ - { - "skill": "Skill Name", - "grade_value": 2, - "observation": "specific evidence from submission" - }, - { - "skill": "Skill Name", - "grade_value": 2, - "observation": "specific evidence from submission" - } - ], - "strengths": ["Strength 1", "Strength 2", "Strength 3"], - "areas_for_improvement": ["Area 1", "Area 2"], - "encouragement": "Encouraging message for the student", - "overall_feedback": "Overall assessment of the submission", - "overall_feedback_translated": "Translation of overall_feedback.", - "learning_objectives_feedback": ["Feedback on objective 1",], - "final_grade": 75, - } - """ - - return BuiltinTemplate() - - def format_objectives(self, objectives: List[Dict]) -> str: - """Format learning objectives for prompt""" - if not objectives: - return "No specific learning objectives provided for this assignment." - - formatted = [] - for i, obj in enumerate(objectives, 1): - if isinstance(obj, dict): - description = obj.get('description', obj.get('objective_id', 'Unknown objective')) - else: - description = str(obj) - formatted.append(f"{i}. {description}") - - return "\n".join(formatted) - - def format_rubrics(self, rubrics) -> str: - prompt = "" - if isinstance(rubrics, str): - rubrics = json.loads(rubrics) - for criterion, grades_list in rubrics.items(): - prompt += f"\n{criterion}:\n" - for grade_item in grades_list: - prompt += f" Grade {grade_item['grade_value']}: {grade_item['grade_description']}\n" - - return prompt - - def get_default_response_format(self) -> Dict: - """Get default response format""" - return { - "rubric_evaluations": [ - { - "skill": "Skill Name", - "grade_value": 2, - "observation": "specific evidence from submission" - }, - { - "skill": "Skill Name", - "grade_value": 2, - "observation": "specific evidence from submission" - } - ], - "strengths": ["Strength 1", "Strength 2", "Strength 3"], - "areas_for_improvement": ["Area 1", "Area 2"], - "encouragement": "Encouraging message for the student", - "overall_feedback": "Overall assessment of the submission", - "overall_feedback_translated": "Translation of overall_feedback.", - "learning_objectives_feedback": ["Feedback on objective 1"], - "final_grade": 75 - } - - async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: - """Generate feedback using universal template approach""" - try: - print("\n=== Starting AI Feedback Generation ===") - - # Get universal template (no assignment_type filtering) - template = self.get_universal_template() - print("Template loaded successfully") - - # Get expected response format from template or use default - try: - if hasattr(template, 'response_format') and template.response_format: - expected_format = json.loads(template.response_format) - print("Using template-defined response format") - else: - expected_format = self.get_default_response_format() - print("Using default response format") - except json.JSONDecodeError: - expected_format = self.get_default_response_format() - print("Failed to parse template response format, using default") - - # Format learning objectives - learning_objectives = self.format_objectives(assignment_context.get("learning_objectives", [])) - rubric_criteria = self.format_rubrics(assignment_context["assignment"].get("rubrics", "")) - - # Format user prompt with assignment context - user_prompt_vars = { - "assignment_name": assignment_context["assignment"].get("name", ""), - "assignment_description": assignment_context["assignment"].get("description", ""), - "course_vertical": assignment_context.get("subject", "General"), - "assignment_type": assignment_context["assignment"].get("type", "Practical"), - "learning_objectives": learning_objectives, - "rubric_criteria": rubric_criteria, - "Language": assignment_context["student"].get("language", "English"), - "Grade_Level": assignment_context["student"].get("grade", "1") - } - - # Format the user prompt with available variables - formatted_user_prompt = template.user_prompt - for key, value in user_prompt_vars.items(): - placeholder = "{" + key + "}" - if placeholder in formatted_user_prompt: - formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) - - # Use template system prompt as-is (it already handles JSON requirement) - system_prompt = template.system_prompt - - print("User Prompt Prepared:") - print(formatted_user_prompt) - - # Prepare messages for the LLM provider - messages = self.llm_provider.format_messages( - system_prompt=system_prompt, - user_prompt=formatted_user_prompt, - image_url=submission_url - ) - print("\nSending request to LLM...") - - try: - # Generate feedback - SINGLE LLM CALL (no separate validation) - raw_text = await self.llm_provider.generate_with_vision(messages) - print(f"\nRaw LLM Response: {raw_text}") - # Clean up the response text - cleaned_text = self.clean_json_response(raw_text) - feedback = json.loads(cleaned_text) - print("\nSuccessfully parsed JSON response") - - # Validate and ensure required fields - feedback = self.validate_feedback_structure(feedback, expected_format) - - except json.JSONDecodeError as e: - print(f"\nJSON Parse Error: {str(e)}") - print("Using fallback feedback format") - - # Create structured fallback response - feedback = self.create_fallback_feedback(expected_format) - - # Attach default plagiarism/AI-detection metadata - plagiarism_output = { - "is_plagiarized": False, - "is_ai_generated": False, - "match_type": "original", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": "none", - "ai_confidence": 0.0, - "similar_sources": [] - } - feedback["plagiarism_output"] = plagiarism_output - - try: - if hasattr(template, 'name'): - template_used = template.name - else: - template_used = "Built-in Universal Template" - - except Exception as template_error: - print("Used Default Template:") - template_used = "Built-in Universal Template" - # Don't fail the entire process for template tracking issues - - print("\n=== Feedback Generation Completed Successfully ===") - return feedback, template_used - - except Exception as e: - error_msg = f"Error generating feedback for submission {submission_id}: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(message=error_msg, title="Feedback Generation Error") - - # Return structured error response - template_used = "Built-in Universal Template for Error" - return self.create_error_feedback(), template_used - - async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, - plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: - """Generate feedback with plagiarism context""" - - result_status = "Pending" - - try: - # Check for plagiarism/AI-generated content first - if plagiarism_data: - is_plagiarized = plagiarism_data.get("is_plagiarized", False) - is_ai_generated = plagiarism_data.get("is_ai_generated", False) - match_type = plagiarism_data.get("match_type", "original") - - # Handle AI-generated submissions - if is_ai_generated: - result_status = "Success - Flagged" - feedback = self._create_ai_generated_feedback( - plagiarism_data - ) - tempalate_used = "Feedback Template for AI Generated Submission" - - # Handle plagiarized submissions - elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: - result_status = "Success - Flagged" - feedback = self._create_plagiarism_feedback( - plagiarism_data - ) - tempalate_used = "Feedback Template for Plagiarized Submission" - - # Continue with normal feedback generation for original work - else: - result_status = "Success - Original" - feedback, tempalate_used = await self.generate_ai_evaluated_feedback(assignment_context, submission_url,submission_id) - - feedback["translation_language"] = assignment_context["student"].get("language", "English") - await self._update_result_status(feedback_request_id, result_status) - return feedback, self.model_used, tempalate_used - - except Exception as e: - result_status = "Failed" - await self._update_result_status(feedback_request_id, result_status, str(e)) - raise - - async def _update_result_status(self, feedback_request_id: str, status: str, error_message: str = None): - """Update Feedback Request result_status""" - if not feedback_request_id: - return - - update_data = {"result_status": status} - if error_message: - update_data["error_message"] = error_message[:500] # Truncate long errors - - frappe.db.set_value( - "Feedback Request", - feedback_request_id, - update_data, - update_modified=True - ) - frappe.db.commit() - - def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: - """Create feedback for AI-generated submissions""" - - ai_source = plagiarism_data.get("ai_detection_source", "unknown") - ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - response = { - "overall_feedback": "Your submission appears to be generated by an AI tool. \ - At MentorMe, we encourage original creative work that reflects your own learning \ - and artistic development. AI-generated images, while interesting, don't demonstrate \ - the skills and creativity we're looking to nurture. Please submit your own original \ - artwork for this assignment.", - "overall_feedback_translated": "Your submission appears to be generated by an AI tool. \ - At MentorMe, we encourage original creative work that reflects your own learning \ - and artistic development. AI-generated images, while interesting, don't demonstrate \ - the skills and creativity we're looking to nurture. Please submit your own original \ - artwork for this assignment.", - "strengths": ["N/A - AI-generated content detected."], - "areas_for_improvement": ["Submit original artwork created by you", - "Review assignment guidelines for creative direction"], - "learning_objectives_feedback": ["N/A - AI-generated content detected."], - "grade_recommendation": 0, - "encouragement": "We believe in your creative abilities!", - "rubric_evaluations": [{ - "Skill": "Content Knowledge", - "grade_value": 0, - "observation": "N/A - AI-generated content detected." - }], - "plagiarism_output": { - "is_plagiarized": False, - "is_ai_generated": True, - "match_type": "ai_generated", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": ai_source, - "ai_confidence": ai_confidence, - } - } - - return response - - def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: - """Create feedback for plagiarized submissions""" - - match_type = plagiarism_data.get("match_type") - plagiarism_source = plagiarism_data.get("plagiarism_source") - similarity_score = plagiarism_data.get("similarity_score", 0.0) - ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - - # respond with structured feedback - response = { - "overall_feedback": "Your submission has been flagged for similarity with another submission. \ - Academic integrity is fundamental to the learning process. Please ensure your \ - submissions represent your own original work.", - "overall_feedback_translated": "Your submission has been flagged for similarity. \ - Academic integrity is fundamental to the learning process. Please ensure your \ - submissions represent your own original work.", - "strengths": ["N/A - Submission flagged for similarity"], - "areas_for_improvement": ["Create original artwork for this assignment", - "Review academic integrity guidelines"], - "learning_objectives_feedback": ["N/A - Submission flagged for similarity"], - "grade_recommendation": 0, - "encouragement": "Every artist develops their unique style through practice!", - "rubric_evaluations": [{ - "Skill": "Content Knowledge", - "grade_value": 0, - "observation": "N/A - Submission flagged for similarity." - }], - "plagiarism_output": { - "is_plagiarized": True, - "is_ai_generated": False, - "match_type": match_type, - "plagiarism_source": plagiarism_source, - "similarity_score": similarity_score, - "ai_detection_source": "none", - "ai_confidence": ai_confidence, - } - } - - return response - - def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: - """Ensure feedback has all required fields with correct types""" - # Ensure all expected fields are present - for field in expected_format: - if field not in feedback: - if isinstance(expected_format[field], list): - feedback[field] = ["No information provided"] - elif isinstance(expected_format[field], (int, float)): - feedback[field] = 0 - else: - feedback[field] = "No information provided" - - # Validate grade_recommendation format for TAP LMS compatibility - try: - grade = feedback.get("grade_recommendation", 0) - if isinstance(grade, str): - # Extract numeric part only - grade_clean = ''.join(c for c in grade if c.isdigit() or c == '.') - grade = float(grade_clean) if grade_clean else 0 - feedback["grade_recommendation"] = max(0, min(100, float(grade))) - except (ValueError, TypeError): - feedback["grade_recommendation"] = 0 - - # Ensure list fields are lists - list_fields = ["strengths", "areas_for_improvement", "learning_objectives_feedback"] - for field in list_fields: - if field in feedback and not isinstance(feedback[field], list): - feedback[field] = [str(feedback[field])] - - return feedback - - def create_fallback_feedback(self, expected_format: Dict) -> Dict: - """Create structured fallback when JSON parsing fails""" - - fallback = {} - for field, default_value in expected_format.items(): - if field == "overall_feedback": - fallback[field] = "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor." - elif field == "overall_feedback_translated": - fallback[field] = "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor." - elif field == "grade_recommendation": - fallback[field] = 50 # Neutral grade for technical issues - elif field == "rubric_evaluations": - fallback[field] = [ - { - "skill": "Content Knowledge", - "grade_value": 2, - "observation": "Neutral evaluation due to processing issue" - } - ] - elif isinstance(default_value, list): - if "strength" in field: - fallback[field] = ["Your submission was received and processed"] - elif "improvement" in field: - fallback[field] = ["Please ensure your submission clearly shows your work"] - else: - fallback[field] = ["Unable to provide specific feedback due to processing issue"] - else: - if field == "encouragement": - fallback[field] = "Technical issues don't reflect your effort - please try resubmitting!" - else: - fallback[field] = "Processing issue - please resubmit for detailed feedback" - - return fallback - - def create_error_feedback(self) -> Dict: - """Create feedback for system errors""" - - feedback = { - "overall_feedback": "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", - "overall_feedback_translated": "I encountered a system error while processing your submission. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", - "strengths": ["Your submission was received successfully"], - "areas_for_improvement": ["No issues identified with your submission - this appears to be a technical problem"], - "learning_objectives_feedback": ["Unable to evaluate due to system error - please resubmit"], - "grade_recommendation": 0, - "encouragement": "Technical issues don't reflect your effort or ability - please try again!", - "rubric_evaluations": [{ - "Skill": "Content Knowledge", - "grade_value": 2, - "observation": "Neutral evaluation due to processing issue" - }], - } - plagiarism_output = { - "is_plagiarized": False, - "is_ai_generated": False, - "match_type": "original", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": "none", - "ai_confidence": 0.0, - "similar_sources": [] - } - feedback["plagiarism_output"] = plagiarism_output - - return feedback - diff --git a/rag_service/core/langchain_manager.py.bak b/rag_service/core/langchain_manager.py.bak deleted file mode 100644 index f51ce20..0000000 --- a/rag_service/core/langchain_manager.py.bak +++ /dev/null @@ -1,466 +0,0 @@ -# rag_service/rag_service/core/langchain_manager.py - -import frappe -from langchain_openai import ChatOpenAI -from langchain.schema import HumanMessage, SystemMessage -import json -from typing import Dict, List, Optional, Union -import httpx -from datetime import datetime - -class LangChainManager: - def __init__(self): - self.llm = None - self.setup_llm() - - def setup_llm(self): - """Initialize LLM based on settings""" - try: - llm_settings = frappe.get_list( - "LLM Settings", - filters={"is_active": 1}, - limit=1 - ) - - if not llm_settings: - raise Exception("No active LLM configuration found") - - settings = frappe.get_doc("LLM Settings", llm_settings[0].name) - print("\nUsing LLM Settings:") - print(f"Provider: {settings.provider}") - print(f"Model: {settings.model_name}") - - if settings.provider == "OpenAI": - self.llm = ChatOpenAI( - model_name=settings.model_name, - openai_api_key=settings.get_password('api_secret'), - temperature=settings.temperature, - max_tokens=settings.max_tokens - ) - elif settings.provider == "Anthropic": - # Add Anthropic model initialization if needed - raise Exception("Anthropic provider not yet implemented") - else: - raise Exception(f"Unsupported LLM provider: {settings.provider}") - - except Exception as e: - error_msg = f"LLM Setup Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "LLM Setup Error") - raise - - def clean_json_response(self, response: str) -> str: - """Clean JSON response from various formats""" - try: - # Remove markdown code blocks if present - if "```json" in response: - response = response.split("```json")[1].split("```")[0].strip() - elif "```" in response: - code_blocks = response.split("```") - if len(code_blocks) >= 3: # At least one code block exists - response = code_blocks[1].strip() - # Check if the extracted content looks like JSON - if not (response.startswith('{') or response.startswith('[')): - # If not, try to find JSON in the original response - json_start = response.find('{') - if json_start >= 0: - response = response[json_start:] - - # Try to extract JSON if response starts with explanation - if not response.strip().startswith('{'): - json_start = response.find('{') - if json_start >= 0: - response = response[json_start:] - - # Check if the response ends properly - if not response.strip().endswith('}'): - json_end = response.rfind('}') - if json_end >= 0: - response = response[:json_end+1] - - return response.strip() - except Exception as e: - print(f"Error cleaning JSON: {str(e)}") - return response - - async def validate_submission_image(self, image_url: str, assignment_type: str) -> Dict: - """Pre-validate if image appears to be appropriate for the assignment type""" - try: - print(f"\n=== Validating Submission Image ===") - print(f"URL: {image_url}") - print(f"Assignment Type: {assignment_type}") - - validation_prompt = f"""You are an artwork submission validator. -Analyze the image and determine if it is a valid submission for a {assignment_type} assignment. -You must respond ONLY with a JSON object containing these exact fields: -{{ - "is_valid": boolean, - "reason": "detailed explanation of why the image is valid or invalid", - "detected_type": "specific description of what type of image this appears to be" -}}""" - - messages = [ - SystemMessage(content=validation_prompt), - HumanMessage(content=[{ - "type": "image_url", - "image_url": {"url": image_url} - }]) - ] - - response = await self.llm.agenerate([messages]) - result = response.generations[0][0].text.strip() - print(f"\nRaw Validation Response: {result}") - - # Clean and parse the response - cleaned_result = self.clean_json_response(result) - print(f"\nCleaned Validation Response: {cleaned_result}") - - try: - validation_result = json.loads(cleaned_result) - print(f"\nParsed Validation Result: {json.dumps(validation_result, indent=2)}") - return validation_result - except json.JSONDecodeError as e: - print(f"JSON Decode Error: {str(e)}") - return { - "is_valid": True, # Default to True to avoid false negatives - "reason": "Failed to validate image format, proceeding with analysis", - "detected_type": "unvalidated_submission" - } - - except Exception as e: - error_msg = f"Image validation failed: {str(e)}" - print(f"\nError: {error_msg}") - return { - "is_valid": True, # Default to True to avoid false negatives - "reason": error_msg, - "detected_type": "error_during_validation" - } - - def format_objectives(self, objectives: List[Dict]) -> str: - """Format learning objectives for prompt""" - if not objectives: - return "No specific learning objectives provided for this assignment." - - return "\n".join([ - f"- {obj.get('description', obj.get('objective_id', 'Unknown objective'))}" - for obj in objectives - ]) - - async def get_image_content(self, image_url: str) -> Dict: - """Get image content in format required by GPT-4V""" - print(f"\nPreparing image content from URL: {image_url}") - return { - "type": "image_url", - "image_url": { - "url": image_url, - "detail": "high" - } - } - - def get_prompt_template(self, assignment_type: str) -> Dict: - """Get active prompt template for assignment type""" - try: - # First try to get an exact match for the assignment type - templates = frappe.get_list( - "Prompt Template", - filters={ - "assignment_type": assignment_type, - "is_active": 1 - }, - order_by="version desc", - limit=1 - ) - - # If no exact match found, try to get a generic template - if not templates: - print(f"\nNo specific template found for {assignment_type}, looking for generic template") - templates = frappe.get_list( - "Prompt Template", - filters={ - "is_active": 1 - }, - order_by="version desc", - limit=1 - ) - - if not templates: - raise Exception(f"No active prompt template found for {assignment_type}") - - template = frappe.get_doc("Prompt Template", templates[0].name) - print(f"\nUsing template: {template.template_name}") - - # Update the last_used timestamp - template.db_set('last_used', datetime.now()) - frappe.db.commit() - - return template - - except Exception as e: - error_msg = f"Prompt Template Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "Prompt Template Error") - raise - - def get_default_response_format(self, assignment_type: str) -> Dict: - """Get default response format if template doesn't define one""" - # Default format based on assignment type - default_formats = { - "Written": { - "overall_feedback": "Overall assessment of the written work", - "strengths": ["Strength 1", "Strength 2"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": "Numerical grade", - "encouragement": "Encouraging message for the student" - }, - "Practical": { - "overall_feedback": "Overall assessment of the practical work", - "strengths": ["Strength 1", "Strength 2"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": "Numerical grade", - "encouragement": "Encouraging message for the student" - }, - "Performance": { - "overall_feedback": "Overall assessment of the performance", - "strengths": ["Strength 1", "Strength 2"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": "Numerical grade", - "encouragement": "Encouraging message for the student" - }, - "Collaborative": { - "overall_feedback": "Overall assessment of the collaborative work", - "strengths": ["Strength 1", "Strength 2"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": "Numerical grade", - "encouragement": "Encouraging message for the student" - } - } - - # Return type-specific format or a generic one - return default_formats.get(assignment_type, default_formats["Practical"]) - - async def generate_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: - """Generate feedback using LangChain and GPT-4V""" - try: - print("\n=== Starting Feedback Generation ===") - - # Get assignment type from context - assignment_type = assignment_context["assignment"]["type"] - - # Get prompt template based on assignment type - template = self.get_prompt_template(assignment_type) - print("\nTemplate loaded successfully") - - # Get expected response format from template or use default - try: - if hasattr(template, 'response_format') and template.response_format: - expected_format = json.loads(template.response_format) - print("\nUsing template-defined response format") - else: - expected_format = self.get_default_response_format(assignment_type) - print("\nUsing default response format for assignment type:", assignment_type) - except json.JSONDecodeError: - expected_format = self.get_default_response_format(assignment_type) - print("\nFailed to parse template response format, using default") - - # Format expected format as JSON string - response_format_str = json.dumps(expected_format, indent=2) - - # Validate image first (keep this generic) - validation_result = await self.validate_submission_image( - submission_url, - assignment_type - ) - - # Process based on validation result - if validation_result.get("is_valid", False): - print("\nValid submission detected - generating feedback") - - # Format learning objectives if they exist - learning_objectives = "" - if assignment_context.get("learning_objectives") and len(assignment_context["learning_objectives"]) > 0: - learning_objectives = self.format_objectives(assignment_context["learning_objectives"]) - - # Prepare system prompt with expected format - enhanced_system_prompt = f""" - {template.system_prompt} - - IMPORTANT: You must ALWAYS respond with a valid JSON object matching this format exactly: - {response_format_str} - - If the image does not appear to be related to this assignment context, set the "overall_feedback" field to EXACTLY: - "Something went wrong—It looks like there's an issue from our end or your submission is incorrect! I am not able to provide feedback for your submission." - - Do not include any additional text, explanation, or markdown formatting outside the JSON object. - Return ONLY the JSON object, nothing else. - """ - - # Format variables for the user prompt - user_prompt_vars = { - "assignment_description": assignment_context["assignment"]["description"], - "learning_objectives": learning_objectives, - "assignment_type": assignment_type, - "assignment_name": assignment_context["assignment"]["name"] - } - - # Try to apply any additional variables from the template - if hasattr(template, 'variables') and template.variables: - for var in template.variables: - if var.variable_name in assignment_context: - user_prompt_vars[var.variable_name] = assignment_context[var.variable_name] - - # Format the user prompt with available variables - formatted_user_prompt = template.user_prompt - for key, value in user_prompt_vars.items(): - placeholder = "{" + key + "}" - if placeholder in formatted_user_prompt: - formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) - - # Prepare text content - text_content = { - "type": "text", - "text": formatted_user_prompt - } - - # Prepare image content - image_content = await self.get_image_content(submission_url) - - # Prepare messages - messages = [ - SystemMessage(content=enhanced_system_prompt), - HumanMessage(content=[text_content, image_content]) - ] - - print("\nSending request to LLM...") - - # Generate feedback - response = await self.llm.agenerate([messages]) - raw_text = response.generations[0][0].text.strip() - print("\nRaw LLM Response:") - print(raw_text) - - try: - # Clean up the response text - cleaned_text = self.clean_json_response(raw_text) - print("\nCleaned Response Text:") - print(cleaned_text) - - feedback = json.loads(cleaned_text) - print("\nSuccessfully parsed JSON response") - - # Verify that all expected fields are present - for field in expected_format: - if field not in feedback: - if isinstance(expected_format[field], list): - feedback[field] = ["No information provided"] - else: - feedback[field] = f"No information provided for {field}" - - except json.JSONDecodeError as e: - print(f"\nJSON Parse Error: {str(e)}") - print("Using fallback feedback format") - - # Create a fallback response matching the expected format - feedback = {} - for field in expected_format: - if isinstance(expected_format[field], list): - feedback[field] = ["Unable to generate proper feedback due to processing error"] - else: - feedback[field] = "Unable to generate proper feedback due to processing error" - - feedback["error"] = f"JSON parsing error: {str(e)}" - else: - print("\nInvalid submission detected - returning error feedback") - # Create an error feedback matching the expected format - feedback = {} - - # Set the special error message for overall_feedback - feedback["overall_feedback"] = "Something went wrong—It looks like there's an issue from our end or your submission is incorrect! I am not able to provide feedback for your submission." - - # Fill in other required fields - for field in expected_format: - if field != "overall_feedback": # Skip overall_feedback as we've already set it - if isinstance(expected_format[field], list): - feedback[field] = ["Please ensure your submission matches the assignment requirements"] - else: - feedback[field] = "Please ensure your submission matches the assignment requirements" - - # Add detected type information - feedback["detected_type"] = validation_result.get('detected_type', 'unknown') - feedback["reason"] = validation_result.get('reason', 'Unknown issue with submission') - - print("\n=== Feedback Generation Completed Successfully ===") - return feedback - - except Exception as e: - error_msg = f"Error generating feedback for submission {submission_id}: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(message=error_msg, title="Feedback Generation Error") - raise - - @staticmethod - def format_feedback_for_display(feedback: Dict) -> str: - """Format feedback for human-readable display""" - try: - formatted = [] - - if "overall_feedback" in feedback: - formatted.append("Overall Feedback:") - formatted.append(feedback["overall_feedback"]) - - if "strengths" in feedback: - formatted.append("\nStrengths:") - for strength in feedback["strengths"]: - formatted.append(f"- {strength}") - - if "areas_for_improvement" in feedback: - formatted.append("\nAreas for Improvement:") - for area in feedback["areas_for_improvement"]: - formatted.append(f"- {area}") - - if "learning_objectives_feedback" in feedback: - formatted.append("\nLearning Objectives Feedback:") - for obj in feedback["learning_objectives_feedback"]: - formatted.append(f"- {obj}") - - if "grade_recommendation" in feedback: - formatted.append(f"\nGrade Recommendation: {feedback['grade_recommendation']}") - - if "encouragement" in feedback: - formatted.append(f"\nEncouragement: {feedback['encouragement']}") - - # Include any additional fields not in the standard format - standard_fields = ["overall_feedback", "strengths", "areas_for_improvement", - "learning_objectives_feedback", "grade_recommendation", - "encouragement", "detected_type", "error"] - - for key, value in feedback.items(): - if key not in standard_fields: - formatted.append(f"\n{key.replace('_', ' ').title()}:") - if isinstance(value, list): - for item in value: - formatted.append(f"- {item}") - else: - formatted.append(str(value)) - - return "\n".join(formatted) - - except Exception as e: - error_msg = f"Error formatting feedback: {str(e)}" - print(f"\nError: {error_msg}") - return "Error formatting feedback" - - def get_current_config(self) -> Dict: - """Get current LLM configuration""" - if not self.llm: - return {"status": "not_configured"} - - return { - "provider": self.llm.__class__.__name__, - "model": self.llm.model_name, - "temperature": self.llm.temperature, - "max_tokens": self.llm.max_tokens - } diff --git a/rag_service/core/llm_providers.py b/rag_service/core/llm_providers.py index e409b9e..41c2a4f 100644 --- a/rag_service/core/llm_providers.py +++ b/rag_service/core/llm_providers.py @@ -2,10 +2,14 @@ import aiohttp import asyncio +import json from typing import List, Dict, Optional from langchain_openai import ChatOpenAI from langchain.schema import HumanMessage, SystemMessage from .llm_interface import BaseLLMInterface +import vertexai +from google.oauth2 import service_account +from vertexai.generative_models import GenerativeModel, Part class OpenAIProvider(BaseLLMInterface): """OpenAI provider using LangChain""" @@ -133,17 +137,159 @@ def format_messages(self, system_prompt: str, user_prompt: str, image_url: Optio return messages +class GeminiProvider(BaseLLMInterface): + """Gemini provider using Vertex AI.""" + + _vertex_init = {"key_data_id": None, "project_id": None, "location": None} + + def __init__( + self, + api_key: str, + model_name: str, + temperature: float = 0.7, + max_tokens: int = 2000, + key_data: Optional[Dict] = None, + location: str = "us-central1", + project_id: Optional[str] = None, + ): + super().__init__(api_key, model_name, temperature, max_tokens) + self.key_data = self._normalize_key_data(key_data) + self.location = location + self.project_id = project_id + self._ensure_vertex_init() + + def _normalize_key_data(self, key_data: Optional[Dict]) -> Optional[Dict]: + if not key_data: + return None + if isinstance(key_data, dict): + return key_data + if isinstance(key_data, str): + try: + return json.loads(key_data) + except json.JSONDecodeError: + return None + return None + + def _key_data_id(self) -> Optional[tuple]: + if not self.key_data: + return None + return ( + self.key_data.get("project_id"), + self.key_data.get("client_email"), + self.key_data.get("private_key_id"), + ) + + def _ensure_vertex_init(self) -> None: + init_state = GeminiProvider._vertex_init + if not self.key_data: + raise ValueError("Gemini service account key JSON is required") + + key_data_id = self._key_data_id() + resolved_project_id = self.project_id or self.key_data.get("project_id") + if ( + init_state["key_data_id"] == key_data_id + and init_state["project_id"] == resolved_project_id + and init_state["location"] == self.location + ): + return + + credentials = service_account.Credentials.from_service_account_info(self.key_data) + if not resolved_project_id: + raise ValueError("Project ID is required for Gemini provider initialization") + + vertexai.init(project=resolved_project_id, location=self.location, credentials=credentials) + GeminiProvider._vertex_init = { + "key_data_id": key_data_id, + "project_id": resolved_project_id, + "location": self.location, + } + + def _combine_messages(self, messages: List[Dict]) -> str: + parts = [] + for msg in messages: + content = msg.get("content", "") + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(item.get("text", "")) + else: + parts.append(str(content)) + return "\n\n".join([p for p in parts if p]) + + async def generate(self, messages: List[Dict]) -> str: + prompt = self._combine_messages(messages) + model = GenerativeModel(self.model_name) + response = model.generate_content( + prompt, + generation_config={ + "temperature": self.temperature, + }, + ) + return response.text or "" + + async def generate_with_vision(self, messages: List[Dict]) -> str: + prompt = self._combine_messages(messages) + image_url = None + for msg in messages: + content = msg.get("content", []) + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") == "image_url": + image_url = item.get("image_url", {}).get("url") + break + if image_url: + break + + if not image_url: + return await self.generate(messages) + + image_part = Part.from_uri(uri=image_url, mime_type="image/jpeg") + model = GenerativeModel(self.model_name) + response = model.generate_content( + [image_part, prompt], + generation_config={ + "temperature": self.temperature, + }, + ) + return response.text or "" + + async def generate_with_video(self, video_url: str, prompt: str) -> str: + video_uri = self._normalize_video_uri(video_url) + video_part = Part.from_uri(uri=video_uri, mime_type="video/mp4") + model = GenerativeModel(self.model_name) + response = model.generate_content( + [video_part, prompt], + generation_config={ + "temperature": self.temperature, + "response_mime_type": "application/json", + }, + ) + return response.text or "" + + def _normalize_video_uri(self, video_url: str) -> str: + if video_url.startswith("https://storage.googleapis.com/"): + return video_url.replace("https://storage.googleapis.com/", "gs://", 1) + return video_url + + # Factory function to create LLM providers -def create_llm_provider(provider: str, api_key: str, model_name: str, - temperature: float = 0.7, max_tokens: int = 2000) -> BaseLLMInterface: +def create_llm_provider( + provider: str, + api_key: str, + model_name: str, + temperature: float = 0.7, + max_tokens: int = 2000, + **kwargs, +) -> BaseLLMInterface: """Factory function to create LLM provider instances""" providers = { "OpenAI": OpenAIProvider, "Together AI": TogetherAIProvider, + "Gemini": GeminiProvider, } if provider not in providers: raise ValueError(f"Unsupported provider: {provider}") - return providers[provider](api_key, model_name, temperature, max_tokens) + return providers[provider](api_key, model_name, temperature, max_tokens, **kwargs) diff --git a/rag_service/feedback_utils/evaluation_generation.py b/rag_service/feedback_utils/evaluation_generation.py new file mode 100644 index 0000000..f717293 --- /dev/null +++ b/rag_service/feedback_utils/evaluation_generation.py @@ -0,0 +1,358 @@ +# rag_service/rag_service/feedback_utils/evaluation_generation.py + +import json +from datetime import datetime +from typing import Any, Dict, List, Tuple + +import frappe + +from .image_evaluation import ImageEvaluationGenerator +from .video_evaluation import VideoEvaluationGenerator + + +class BaseEvaluationGenerator: + """Shared evaluation generation utilities for different media types.""" + + IMAGE_EXTENSIONS = { + "jpg", + "jpeg", + "png", + "gif", + "webp", + "bmp", + "tiff", + "heic", + } + VIDEO_EXTENSIONS = { + "mp4", + "mov", + "webm", + "mkv", + "avi", + "mpeg", + "mpg", + } + + def __init__(self, feedback_service: Any): + self.feedback_service = feedback_service + + def detect_media_type(self, submission_url: str) -> str: + """Detect media type based on URL/extension.""" + if not submission_url: + return "image" + + url_without_query = submission_url.split("?", 1)[0].lower() + if "." in url_without_query: + ext = url_without_query.rsplit(".", 1)[-1] + if ext in self.IMAGE_EXTENSIONS: + return "image" + if ext in self.VIDEO_EXTENSIONS: + return "video" + + if "video" in url_without_query: + return "video" + return "image" + + def clean_json_response(self, response: str) -> str: + """Clean JSON response from various formats.""" + try: + # Remove markdown code blocks if present + if "```json" in response: + response = response.split("```json")[1].split("```")[0].strip() + elif "```" in response: + code_blocks = response.split("```") + if len(code_blocks) >= 3: # At least one code block exists + response = code_blocks[1].strip() + # Check if the extracted content looks like JSON + if not (response.startswith("{") or response.startswith("[")): + # If not, try to find JSON in the original response + json_start = response.find("{") + if json_start >= 0: + response = response[json_start:] + + # Try to extract JSON if response starts with explanation + if not response.strip().startswith("{"): + json_start = response.find("{") + if json_start >= 0: + response = response[json_start:] + + # Check if the response ends properly + if not response.strip().endswith("}"): + json_end = response.rfind("}") + if json_end >= 0: + response = response[: json_end + 1] + + return response.strip() + except Exception as e: + print(f"Error cleaning JSON: {str(e)}") + return response + + def format_objectives(self, objectives: List[Dict]) -> str: + """Format learning objectives for prompt.""" + if not objectives: + return "No specific learning objectives provided for this assignment." + + formatted = [] + for i, obj in enumerate(objectives, 1): + if isinstance(obj, dict): + description = obj.get("description", obj.get("objective_id", "Unknown objective")) + else: + description = str(obj) + formatted.append(f"{i}. {description}") + + return "\n".join(formatted) + + def format_rubrics(self, rubrics) -> str: + """Format rubric criteria for prompt.""" + prompt = "" + if isinstance(rubrics, str): + rubrics = json.loads(rubrics) + for criterion, grades_list in rubrics.items(): + prompt += f"\n{criterion}:\n" + for grade_item in grades_list: + prompt += ( + f" Grade {grade_item['grade_value']}: {grade_item['grade_description']}\n" + ) + + return prompt + + def get_default_response_format(self) -> Dict: + """Get default response format.""" + return { + "rubric_evaluations": [ + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission", + }, + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission", + }, + ], + "strengths": ["Strength 1", "Strength 2", "Strength 3"], + "areas_for_improvement": ["Area 1", "Area 2"], + "encouragement": "Encouraging message for the student", + "overall_feedback": "Overall assessment of the submission", + "overall_feedback_translated": "Translation of overall_feedback.", + "learning_objectives_feedback": ["Feedback on objective 1"], + "final_grade": 75, + } + + def get_builtin_template(self): + """Return built-in default template as fallback.""" + + class BuiltinTemplate: + def __init__(self): + self.template_name = "Built-in Universal Template" + self.system_prompt = """You are an encouraging, knowledgeable educational assistant that provides constructive feedback on student submissions using a structured rubric-based evaluation. + EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. Prioritize growth recognition over perfection. + + GRADING PHILOSOPHY: + - Credit partial mastery: A student showing 60% competency deserves acknowledgment of that progress. Grades need not be binary (good/bad). + - Growth mindset framing: Every submission represents learning in progress. Frame gaps as natural and achievable, not deficiencies. + - Reserve lower grades (1-2) only for minimal engagement or complete absence of skill demonstration + - Lean toward higher grades when effort and authenticity are evident + + + Always provide feedback that is: + - Encouraging and positive while being constructive + - Age-appropriate and specific to observations + - Directly aligned with rubric criteria + - Clear about achievement gaps and growth areas + + Structure your response by: + 1. Opening with what the student did well (be specific) + 2. Evaluating the submission against each rubric skill + 3. Assigning a single grade for each skill in the rubric criteria + 4. Evaluating only the skills mentioned in the rubric criteria + 5. Providing specific, actionable feedback for growth + 6. Ending with motivating encouragement + 7. Translate the overall_feedback. Translation rules: + - Formal but friendly tone (customer communication). + - Natural, conversational phrasing. Not literal translation. + - Use native script for the language. + + CRITICAL: You must respond with valid JSON format only. + """ + + self.user_prompt = """Assignment Context: + - Name: {assignment_name} + - Subject: {course_vertical} + - Type: {assignment_type} + - Description: {assignment_description} + + Learning Objectives: {learning_objectives} + + Rubric Criteria: {rubric_criteria} + + CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. + + Analyze this submission and respond ONLY in this JSON format: + + { + "rubric_evaluations": [ + { + "Skill": "Skill Name", + "grade_value": 1-5, + "observation": "specific evidence from submission" + } + ], + "strengths": ["specific strength 1", "specific strength 2"], + "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], + "encouragement": "motivating closing statement", + "overall_feedback": "30-50 words of encouraging feedback addressing the student in a friendly tone that summarises strengths and improvement potential. Or 'Submission does not match assignment requirements.'", + "overall_feedback_translated": "Translation of overall_feedback in {Language} for a Grade {Grade_Level} student as per translation rules.", + "learning_objectives_feedback": ["Feedback on objective 1",], + "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)" + + } + """ + + self.response_format = """{ + "rubric_evaluations": [ + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + }, + { + "skill": "Skill Name", + "grade_value": 2, + "observation": "specific evidence from submission" + } + ], + "strengths": ["Strength 1", "Strength 2", "Strength 3"], + "areas_for_improvement": ["Area 1", "Area 2"], + "encouragement": "Encouraging message for the student", + "overall_feedback": "Overall assessment of the submission", + "overall_feedback_translated": "Translation of overall_feedback.", + "learning_objectives_feedback": ["Feedback on objective 1",], + "final_grade": 75, + } + """ + + return BuiltinTemplate() + + def get_prompt_template(self, media_type: str): + """Get active template for the given media type.""" + try: + print("\n=== Getting Prompt Template ===") + + templates = frappe.get_list( + "Prompt Template", + filters={"is_active": 1, "media_type": media_type}, + order_by="version desc", + limit=1, + ) + + if templates: + template = frappe.get_doc("Prompt Template", templates[0].name) + print(f"Using {media_type} template: {template.template_name}") + + template.db_set("last_used", datetime.now()) + frappe.db.commit() + return template + + print("No active template found, using built-in default") + return self.get_builtin_template() + + except Exception as e: + error_msg = f"Template Error: {str(e)}" + print(f"\nError: {error_msg}") + frappe.log_error(error_msg, "Template Error") + return self.get_builtin_template() + + def _get_expected_format(self, template: Any) -> Dict: + try: + if hasattr(template, "response_format") and template.response_format: + return json.loads(template.response_format) + except json.JSONDecodeError: + pass + return self.get_default_response_format() + + def _format_user_prompt(self, template: Any, assignment_context: Dict, media_type: str) -> str: + learning_objectives = self.format_objectives( + assignment_context.get("learning_objectives", []) + ) + rubric_criteria = self.format_rubrics( + assignment_context.get("assignment", {}).get("rubrics", "") + ) + + user_prompt_vars = { + "assignment_name": assignment_context.get("assignment", {}).get("name", ""), + "assignment_description": assignment_context.get("assignment", {}).get("description", ""), + "course_vertical": assignment_context.get("subject", "General"), + "assignment_type": assignment_context.get("assignment", {}).get("type", "Practical"), + "learning_objectives": learning_objectives, + "rubric_criteria": rubric_criteria, + "Language": assignment_context.get("student", {}).get("language", "English"), + "Grade_Level": assignment_context.get("student", {}).get("grade", "1"), + } + + formatted_user_prompt = template.user_prompt + for key, value in user_prompt_vars.items(): + placeholder = "{" + key + "}" + if placeholder in formatted_user_prompt: + formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) + + if media_type == "video": + formatted_user_prompt += ( + "\n\nThis submission is a VIDEO. Evaluate the student's work shown in the video. " + "Ignore cinematography or editing. If the work is unclear, request resubmission." + ) + + return formatted_user_prompt + + def _parse_feedback(self, raw_text: str, expected_format: Dict) -> Dict: + cleaned_text = self.clean_json_response(raw_text or "") + try: + feedback = json.loads(cleaned_text) + feedback = self.feedback_service.validate_feedback_structure(feedback, expected_format) + except json.JSONDecodeError: + feedback = self.feedback_service.create_fallback_feedback(expected_format) + + return feedback + + def _attach_plagiarism_defaults(self, feedback: Dict) -> Dict: + feedback["plagiarism_output"] = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [], + } + return feedback + + def _template_used_name(self, template: Any) -> str: + try: + if hasattr(template, "name"): + return template.name + except Exception: + pass + return "Built-in Universal Template" + + +class EvaluationGenerator(BaseEvaluationGenerator): + """Generate AI feedback for different media types.""" + + def __init__(self, feedback_service: Any): + super().__init__(feedback_service) + self.image_generator = ImageEvaluationGenerator(feedback_service) + self.video_generator = VideoEvaluationGenerator(feedback_service) + + async def generate_ai_feedback( + self, assignment_context: Dict, submission_url: str, submission_id: str + ) -> Tuple[Dict, str, str]: + media_type = self.detect_media_type(submission_url) + if media_type == "video": + return await self.video_generator.generate_feedback( + assignment_context, submission_url, submission_id + ) + return await self.image_generator.generate_feedback( + assignment_context, submission_url, submission_id + ) diff --git a/rag_service/feedback_utils/image_evaluation.py b/rag_service/feedback_utils/image_evaluation.py new file mode 100644 index 0000000..9c22d0c --- /dev/null +++ b/rag_service/feedback_utils/image_evaluation.py @@ -0,0 +1,69 @@ +# rag_service/rag_service/feedback_utils/image_evaluation.py + +from typing import Any, Dict, Tuple + +import frappe + +from .evaluation_generation import BaseEvaluationGenerator +from .llm_providers import create_llm_provider + + +class ImageEvaluationGenerator(BaseEvaluationGenerator): + """Generate AI feedback for image submissions.""" + + def _create_llm_provider(self) -> Tuple[Any, str]: + llm_settings = frappe.get_list("LLM Settings", filters={"is_active": 1}, limit=1) + if not llm_settings: + raise Exception("No active LLM configuration found") + + settings = frappe.get_doc("LLM Settings", llm_settings[0].name) + model_used = llm_settings[0].name + + llm_provider = create_llm_provider( + provider=settings.provider, + api_key=settings.get_password("api_secret"), + model_name=settings.model_name, + temperature=settings.temperature, + max_tokens=settings.max_tokens, + ) + + return llm_provider, model_used + + async def generate_feedback( + self, assignment_context: Dict, submission_url: str, submission_id: str + ) -> Tuple[Dict, str, str]: + try: + print("\n=== Starting AI Feedback Generation (Image) ===") + + template = self.get_prompt_template("image") + expected_format = self._get_expected_format(template) + + formatted_user_prompt = self._format_user_prompt(template, assignment_context, "image") + system_prompt = template.system_prompt + + llm_provider, model_used = self._create_llm_provider() + + messages = llm_provider.format_messages( + system_prompt=system_prompt, + user_prompt=formatted_user_prompt, + image_url=submission_url, + ) + + raw_text = await llm_provider.generate_with_vision(messages) + feedback = self._parse_feedback(raw_text, expected_format) + feedback = self._attach_plagiarism_defaults(feedback) + + template_used = self._template_used_name(template) + + print("\n=== Image Feedback Generation Completed ===") + return feedback, model_used, template_used + + except Exception as e: + error_msg = f"Error generating image feedback for submission {submission_id}: {str(e)}" + print(f"\nError: {error_msg}") + frappe.log_error(message=error_msg, title="Image Feedback Generation Error") + + template_used = "Built-in Universal Template for Error" + error_feedback = self.feedback_service.create_error_feedback() + error_feedback = self._attach_plagiarism_defaults(error_feedback) + return error_feedback, "N/A", template_used diff --git a/rag_service/feedback_utils/video_evaluation.py b/rag_service/feedback_utils/video_evaluation.py new file mode 100644 index 0000000..ad89ad6 --- /dev/null +++ b/rag_service/feedback_utils/video_evaluation.py @@ -0,0 +1,90 @@ +# rag_service/rag_service/feedback_utils/video_evaluation.py + +import json +from typing import Any, Dict, Optional + +import frappe + +from .evaluation_generation import BaseEvaluationGenerator +from .llm_providers import create_llm_provider + + +class VideoEvaluationGenerator(BaseEvaluationGenerator): + """Generate AI feedback for video submissions.""" + + def _resolve_service_account_credentials(self, settings: Any) -> Optional[Dict]: + raw_key = settings.get("service_account_key_path") + + if isinstance(raw_key, dict): + return raw_key + if isinstance(raw_key, str): + raw_key = raw_key.strip() + if raw_key: + try: + return json.loads(raw_key) + except json.JSONDecodeError: + return None + + return None + + def _create_llm_provider(self) -> tuple[Any, str]: + gemini_settings = frappe.get_list("Gemini Settings", filters={"is_active": 1}, limit=1) + if not gemini_settings: + raise Exception("No active Gemini configuration found") + + settings = frappe.get_doc("Gemini Settings", gemini_settings[0].name) + model_used = gemini_settings[0].name + + key_data = self._resolve_service_account_credentials(settings) + if not key_data: + raise Exception("Gemini service account key JSON is required") + + llm_provider = create_llm_provider( + provider="Gemini", + api_key="", + model_name=settings.model_name, + temperature=settings.temperature or 0, + max_tokens=settings.max_tokens or 2000, + key_data=key_data, + location=settings.location or "us-central1", + project_id=settings.project_id or None, + ) + + return llm_provider, model_used + + async def generate_feedback( + self, assignment_context: Dict, submission_url: str, submission_id: str + ) -> Tuple[Dict, str, str]: + try: + print("\n=== Starting AI Feedback Generation (Video) ===") + + template = self.get_prompt_template("video") + expected_format = self._get_expected_format(template) + + formatted_user_prompt = self._format_user_prompt(template, assignment_context, "video") + system_prompt = template.system_prompt + combined_prompt = f"{system_prompt}\n\n{formatted_user_prompt}" + + llm_provider, model_used = self._create_llm_provider() + + if not hasattr(llm_provider, "generate_with_video"): + raise Exception("Gemini provider does not support video generation") + + raw_text = await llm_provider.generate_with_video(submission_url, combined_prompt) + feedback = self._parse_feedback(raw_text, expected_format) + feedback = self._attach_plagiarism_defaults(feedback) + + template_used = self._template_used_name(template) + + print("\n=== Video Feedback Generation Completed ===") + return feedback, model_used, template_used + + except Exception as e: + error_msg = f"Error generating video feedback for submission {submission_id}: {str(e)}" + print(f"\nError: {error_msg}") + frappe.log_error(message=error_msg, title="Video Feedback Generation Error") + + template_used = "Built-in Universal Template for Error" + error_feedback = self.feedback_service.create_error_feedback() + error_feedback = self._attach_plagiarism_defaults(error_feedback) + return error_feedback, "N/A", template_used diff --git a/rag_service/handlers/__init__.py b/rag_service/handlers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rag_service/rag_service/doctype/gemini_settings/__init__.py b/rag_service/rag_service/doctype/gemini_settings/__init__.py new file mode 100644 index 0000000..272296f --- /dev/null +++ b/rag_service/rag_service/doctype/gemini_settings/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026, TAP and contributors +# For license information, please see license.txt diff --git a/rag_service/rag_service/doctype/gemini_settings/gemini_settings.js b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.js new file mode 100644 index 0000000..8a42017 --- /dev/null +++ b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.js @@ -0,0 +1,4 @@ +// Copyright (c) 2026, TAP and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Gemini Settings", {}); diff --git a/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json new file mode 100644 index 0000000..6e67ed2 --- /dev/null +++ b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json @@ -0,0 +1,95 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2026-02-06 10:00:00.000000", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "model_name", + "location", + "project_id", + "service_account_key_path", + "temperature", + "max_tokens", + "is_active", + "is_default", + "description" + ], + "fields": [ + { + "fieldname": "model_name", + "fieldtype": "Data", + "label": "Model Name" + }, + { + "default": "us-central1", + "fieldname": "location", + "fieldtype": "Data", + "label": "Location" + }, + { + "fieldname": "project_id", + "fieldtype": "Data", + "label": "Project ID" + }, + { + "fieldname": "service_account_key_path", + "fieldtype": "Data", + "label": "Service Account Key Path", + "length": 300 + }, + { + "default": "0", + "fieldname": "temperature", + "fieldtype": "Float", + "label": "Temperature" + }, + { + "default": "2000", + "fieldname": "max_tokens", + "fieldtype": "Int", + "label": "Max Tokens" + }, + { + "default": "0", + "fieldname": "is_active", + "fieldtype": "Check", + "label": "Is Active" + }, + { + "default": "0", + "fieldname": "is_default", + "fieldtype": "Check", + "label": "Is Default" + }, + { + "fieldname": "description", + "fieldtype": "Small Text", + "label": "Description" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2026-02-06 10:00:00.000000", + "modified_by": "Administrator", + "module": "Rag Service", + "name": "Gemini Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/rag_service/rag_service/doctype/gemini_settings/gemini_settings.py b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.py new file mode 100644 index 0000000..086c765 --- /dev/null +++ b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.py @@ -0,0 +1,8 @@ +# Copyright (c) 2026, TAP and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class GeminiSettings(Document): + pass diff --git a/rag_service/rag_service/doctype/gemini_settings/test_gemini_settings.py b/rag_service/rag_service/doctype/gemini_settings/test_gemini_settings.py new file mode 100644 index 0000000..95d1e94 --- /dev/null +++ b/rag_service/rag_service/doctype/gemini_settings/test_gemini_settings.py @@ -0,0 +1,9 @@ +# Copyright (c) 2026, TAP and contributors +# For license information, please see license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestGeminiSettings(FrappeTestCase): + pass diff --git a/rag_service/rag_service/doctype/prompt_template/prompt_template.json b/rag_service/rag_service/doctype/prompt_template/prompt_template.json index 0176106..628eb68 100644 --- a/rag_service/rag_service/doctype/prompt_template/prompt_template.json +++ b/rag_service/rag_service/doctype/prompt_template/prompt_template.json @@ -7,6 +7,7 @@ "field_order": [ "template_name", "assignment_type", + "media_type", "system_prompt", "user_prompt", "response_format", @@ -33,6 +34,14 @@ "options": "Written\nPractical\nPerformance\nCollaborative", "reqd": 1 }, + { + "default": "image", + "fieldname": "media_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Media Type", + "options": "image\nvideo" + }, { "fieldname": "system_prompt", "fieldtype": "Text", @@ -106,4 +115,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} \ No newline at end of file +} diff --git a/rag_service/scripts/assignment b/rag_service/scripts/assignment new file mode 100644 index 0000000..86bcad3 --- /dev/null +++ b/rag_service/scripts/assignment @@ -0,0 +1 @@ +"assignment": {'name': 'VA_L2_CA1-Basic','description': '"Have you ever noticed how colorful walls, buses, shops, and even market stalls look? Things like movie posters, cricket match ads, or the signs on sweet shops are all in bright colors and bold designs. These ads grab your attention really quickly, right?\n\nThis idea was made popular by a famous 20th-century artist named Andy Warhol with something called Pop Art. He turned everyday things like bottles, soup cans, and pictures of famous people into bright and bold artworks using bright colors and repeating patterns.\n\nPop Art is when we take ordinary objects and everyday items and turn them into bold, fun, and eye-catching art!\n\nToday, we\'re going to be inspired by this idea and design a product in a colorful and vibrant Pop Art style. Just like Andy Warhol, we will advertise our product in a fun way. \n\nChoose a simple product — like a cup, a food item, or maybe something from your school supplies. Draw the same product repeatedly, fill it with bright and eye-catching colors, and use different lines and patterns for details.\n\nHere are the important steps to follow so your project is complete:\n- Your artwork should clearly show a product.\n- The colors in your artwork should immediately attract the audience.\n- The product should be drawn repeatedly, like Andy Warhol did.\n- There should be some details shown with lines and patterns.\n\nShare a picture of your final artwork with TAP Buddy. And don\'t forget to take the quiz!\n\nRemember, kids, art is not just a hobby — it\'s a way to express your ideas and imagination. So, explore your creativity and see how simple objects can become vibrant and exciting art! Keep creating and have fun with colors!"', 'assignment_type': 'Practical', 'subject': 'Arts', 'rubric_grades': [{'name': '1-Content Knowledge-Arts-Level 2-Arts-C0141', 'idx': 1, 'rubric_name': None, 'grade_value': 1, 'grade_name': 'Novice', 'grade_description': '- Invalid or no submission. \n- No product drawn', 'skill_name': 'Content Knowledge', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '2-Content Knowledge-Arts-Level 2-Arts-C0141', 'idx': 2, 'rubric_name': None, 'grade_value': 2, 'grade_name': 'Beginner', 'grade_description': '- Random product drawn; no repetition \n- Colored randomly without referring to color wheel', 'skill_name': 'Content Knowledge', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '3-Content Knowledge-Arts-Level 2-Arts-C0141', 'idx': 3, 'rubric_name': None, 'grade_value': 3, 'grade_name': 'Emerging', 'grade_description': '- A thoughtful product drawn; repeated a few times, but not andy warhol style \n- Colors are bold, but lacks contrast', 'skill_name': 'Content Knowledge', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '4-Content Knowledge-Arts-Level 2-Arts-C0141', 'idx': 4, 'rubric_name': None, 'grade_value': 4, 'grade_name': 'Proficient', 'grade_description': '- Product is repeated in a grid pattern, similar to Andy Warhol \n- Colors are bold, contrasting, and visually engaging.', 'skill_name': 'Content Knowledge', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '5-Content Knowledge-Arts-Level 2-Arts-C0141', 'idx': 5, 'rubric_name': None, 'grade_value': 5, 'grade_name': 'Expert', 'grade_description': '- Product is repeated in a grid pattern, similar to Andy Warhol \n- Choice of color matches the mood of the product, and has been enhanced with the use lines & patterns', 'skill_name': 'Content Knowledge', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '1-Creativity-Arts-Level 2-Arts-C0141', 'idx': 6, 'rubric_name': None, 'grade_value': 1, 'grade_name': 'Novice', 'grade_description': '- Invalid or no submission. \n- No product drawn', 'skill_name': 'Creativity', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '2-Creativity-Arts-Level 2-Arts-C0141', 'idx': 7, 'rubric_name': None, 'grade_value': 2, 'grade_name': 'Beginner', 'grade_description': '- Chosen product is basic & way too common. Eg: A cup, A pencil \n- Dull irrevelant colors are chosen. Eg: Orange & Pink for a pencil', 'skill_name': 'Creativity', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '3-Creativity-Arts-Level 2-Arts-C0141', 'idx': 8, 'rubric_name': None, 'grade_value': 3, 'grade_name': 'Emerging', 'grade_description': '- Chosen product is common, but has been presented differently than usual.\n- Product has been repeated, but is not cohesive. \n- Basic bright colors have been choosen', 'skill_name': 'Creativity', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '4-Creativity-Arts-Level 2-Arts-C0141', 'idx': 9, 'rubric_name': None, 'grade_value': 4, 'grade_name': 'Proficient', 'grade_description': '- Chosen product is an interesting pop art related object. \n- Product has been repeated in a grid pattern\n- Bright, complementary colors as per color wheel have been used \n\nExample: A shoe or a coffee cup, repeated in a grid filled with distinct patterns (e.g., stripes, dots)', 'skill_name': 'Creativity', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}, {'name': '5-Creativity-Arts-Level 2-Arts-C0141', 'idx': 10, 'rubric_name': None, 'grade_value': 5, 'grade_name': 'Expert', 'grade_description': '- Chosen product is imaginary & unique \n- Repeated in a grid pattern\n- Brightly colored detailed with multiple patterns and lines \n\nExample: An imaginative creature like an alien, monster, or fictional character with unique details, bold lines, and multiple creative patterns like swirls, zigzags, geometric shapes', 'skill_name': 'Creativity', 'course_vertical': 'Arts', 'course_level': 'Level 2-Arts-C0141', 'parent': 'VA_L2_CA1-Basic', 'parentfield': 'rubric_grades', 'parenttype': 'Assignment', 'doctype': 'Rubric Grade'}]} diff --git a/rag_service/scripts/check_active_llms.py b/rag_service/scripts/check_active_llms.py deleted file mode 100644 index 502b9e0..0000000 --- a/rag_service/scripts/check_active_llms.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python - -import sys -import os - -# Add frappe to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'apps')) - -def check_active_llms(): - """Check all active LLM settings and which one would be selected""" - - # Get all active settings - all_active = frappe.get_list( - "LLM Settings", - filters={"is_active": 1}, - fields=["name", "provider", "model_name", "modified", "creation"], - order_by="modified desc" - ) - - print(f"\n=== Active LLM Settings ({len(all_active)} found) ===\n") - - for i, setting in enumerate(all_active): - status = "✓ WOULD BE SELECTED" if i == 0 else " Would be ignored" - print(f"{status}: {setting.provider} - {setting.model_name}") - print(f" Name: {setting.name}") - print(f" Modified: {setting.modified}") - print(f" Created: {setting.creation}") - print() - - if len(all_active) > 1: - print("⚠️ WARNING: Multiple active LLM settings found!") - print(" Only the first one will be used.") - print(" Please deactivate all but one to avoid confusion.") - elif len(all_active) == 0: - print("❌ No active LLM settings found!") - print(" Please activate at least one LLM setting.") - - # Show what setup_llm would select - selected = frappe.get_list( - "LLM Settings", - filters={"is_active": 1}, - limit=1, - order_by="modified desc" - ) - - if selected: - settings = frappe.get_doc("LLM Settings", selected[0].name) - print(f"\n✓ The system will use: {settings.provider} - {settings.model_name}") - print(f" Name: {settings.name}") - -if __name__ == "__main__": - import frappe - - # Use the site name from command line or default - site_name = sys.argv[1] if len(sys.argv) > 1 else 'rag-dev.localhost' - - try: - frappe.init(site=site_name) - frappe.connect() - check_active_llms() - except Exception as e: - print(f"Error: {str(e)}") - print(f"Make sure the site '{site_name}' exists") - finally: - if frappe and frappe.db: - frappe.destroy() diff --git a/rag_service/api/__init__.py b/rag_service/scripts/service_account_key.json similarity index 100% rename from rag_service/api/__init__.py rename to rag_service/scripts/service_account_key.json diff --git a/rag_service/scripts/test_feedback_prompt.py b/rag_service/scripts/test_feedback_prompt.py index 40754a8..84dc8c7 100644 --- a/rag_service/scripts/test_feedback_prompt.py +++ b/rag_service/scripts/test_feedback_prompt.py @@ -87,14 +87,14 @@ assignment_context = {'assignment': {'name': 'VA_L1_CA1', 'description': '"“Close your eyes and imagine a creature that no one has ever seen before. Maybe it has dragon wings, zebra stripes, or fish scales…”\n“Today, your challenge is to invent it! Your task is to combine lines, shapes, and patterns to create a new, magical, and imaginative creature.”\n“Choose at least 3 shapes and 3 patterns for your creature. Then add unique features like wings, horns, multiple eyes, or any magical detail. Your creature should show your imagination!”\n“Step 1: Choose shapes – circles, triangles, rectangles, or ovals – and use them to form body parts. Step 2: Add patterns – stripes, spirals, zigzags, or dots – use at least three different patterns. Step 3: Add creative twists – mix and match special features like an elephant trunk with dragon wings, or a fish tail with monster horns. Following these steps will make your creature alive and interesting.”\n“You can add extra creativity to your creature – more patterns, colors, or unusual features. Let your imagination run free; the more unique and wild, the more fun it will be!”\n“Here’s how to make your creature successful:” • Use at least 3 shapes • Use at least 3 patterns • Creature should look creative and imaginative • Artwork should be neat and complete • Features should be unique and interesting\n“Once your drawing is complete, click a photo and share it with TAP Buddy. We can’t wait to see your creatures come alive on paper!”\n“Well done! You did amazing today. Thank you for creating with us—see you in the next activity!”"', 'type': 'Practical', 'subject': 'Arts', 'submission_guidelines': None, 'reference_images': [], 'max_score': '100', 'rubrics': {'Content Knowledge': [{'grade_value': 1, 'grade_description': '- Invalid or no submission.\n- The art work is digital and not hand drawn.\n - No shapes or patterns at all.'}, {'grade_value': 2, 'grade_description': '- Fewer than 3 shapes OR fewer than 3 patterns. \n- Shapes/patterns are very messy or hard to see. \n\n- Example: Only 1-2 circles drawn, no patterns or just 1 stripe.'}, {'grade_value': 3, 'grade_description': '- Uses 3 shapes AND 3 patterns, but some are unclear or blended together. \n- Not easy to count or spot all. \n\n- Example: Circles, triangles, rectangles with stripes, dots, zigzags but lines overlap and mix up.'}, {'grade_value': 4, 'grade_description': '- Clearly shows at least 3 different shapes AND 3 different patterns. \n- Easy to see and count each one. \n\n- Example: Circle body, triangle ears, rectangle legs with clear stripes on tail, dots on wings, zigzags on back.'}, {'grade_value': 5, 'grade_description': '- Uses more than 3 shapes AND more than 3 patterns, all clear and well-placed. \n- Extra shapes/patterns make it even better. \n\n- Example: Circle head, oval body, triangle wings, rectangle tail with stripes, dots, spirals, zigzags, plus checkerboard on legs.'}], 'Creativity': [{'grade_value': 1, 'grade_description': '- Invalid or no submission. \n- The art work is digital and not hand drawn.\n- No creature drawn.'}, {'grade_value': 2, 'grade_description': '- Creature has only 1-2 simple features. \n- Drawing is messy (colors outside lines) or not finished. \n- Looks boring, not imaginative. \n\n- Example: Plain circle with 1 basic wing; colors spill over edges.'}, {'grade_value': 3, 'grade_description': '- Creature has 3+ features with some imagination. \n- Drawing mostly neat but has small messy spots or unfinished parts. \n- Features are okay but not very unique. \n\n- Example: Triangle body, 2 wings, horn with patterns; a few smudges or blank areas.'}, {'grade_value': 4, 'grade_description': '- Creature looks very creative and imaginative. \n- Drawing is neat, complete, all parts colored nicely. \n- Many unique and interesting features mixed well. \n\n- Example: Oval body, spiral tail, 3 eyes, wings in bright colors; clean lines, no mess.'}, {'grade_value': 5, 'grade_description': '- Creature is super original, wild, and full of imagination. \n- Drawing is very neat, colorful, finished like a real artist. \n- Complex unique features that stand out. \n\n- Example: Mixed shapes with magical glowing wings, curly horns, 5 eyes, special tail; bright colors, perfect details.'}]}}, 'learning_objectives': []} import asyncio -from rag_service.core.langchain_manager import LangChainManager +from rag_service.core.feedback_service import FeedbackService async def main(): - langchain_manager = LangChainManager() + feedback_service = FeedbackService() request_id = 123 print("\nGenerating feedback...") # Generate feedback - feedback, model_used, template_used = await langchain_manager.generate_feedback( + feedback, model_used, template_used = await feedback_service.generate_feedback( assignment_context=assignment_context, submission_url=message_data["img_url"], submission_id=request_id, diff --git a/rag_service/scripts/test_langchain_manager.py b/rag_service/scripts/test_langchain_manager.py deleted file mode 100644 index c391bdd..0000000 --- a/rag_service/scripts/test_langchain_manager.py +++ /dev/null @@ -1,628 +0,0 @@ -# rag_service/rag_service/core/langchain_manager.py - -import frappe -import json -from typing import Dict, List, Optional, Union -from datetime import datetime -from ..core.llm_providers import create_llm_provider, OpenAIProvider - -class LangChainManager: - def __init__(self): - self.llm = None - self.llm_provider = None - self.setup_llm() - - def setup_llm(self): - """Initialize LLM based on settings""" - try: - # llm_settings = frappe.get_list( - # "LLM Settings", - # filters={"is_active": 1}, - # limit=1 - # ) - - # if not llm_settings: - # raise Exception("No active LLM configuration found") - - # settings = frappe.get_doc("LLM Settings", llm_settings[0].name) - # self.model_used = llm_settings[0].name - # print("\nUsing LLM Settings:") - # print(f"Provider: {settings.provider}") - # print(f"Model: {settings.model_name}") - - # # Create LLM provider based on settings - # self.llm_provider = create_llm_provider( - # provider=settings.provider, - # api_key=settings.get_password('api_secret'), - # model_name=settings.model_name, - # temperature=settings.temperature, - # max_tokens=settings.max_tokens - # ) - - settings = {"name":"hv7j6uitvg","owner":"Administrator","creation":"2025-12-17 15:32:25.986861", - "modified":"2025-12-17 15:32:25.986861","modified_by":"Administrator","docstatus":0,"idx":18, - "provider":"OpenAI","model_name":"gpt-4o","temperature":0.7,"max_tokens":1500,"is_active":1, - "is_default":0, - "api_secret":"*****","doctype":"LLM Settings","__last_sync_on":"2026-01-16T04:56:38.776Z"} - self.model_used = settings["model_name"] - self.llm_provider = create_llm_provider( - provider=settings["provider"], - api_key=settings["api_secret"], - model_name=settings["model_name"], - temperature=settings["temperature"], - max_tokens=settings["max_tokens"] - ) - - - # Keep the llm reference for backward compatibility with OpenAI - if isinstance(self.llm_provider, OpenAIProvider): - self.llm = self.llm_provider.llm - - except Exception as e: - error_msg = f"LLM Setup Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "LLM Setup Error") - raise - - def clean_json_response(self, response: str) -> str: - """Clean JSON response from various formats""" - try: - # Remove markdown code blocks if present - if "```json" in response: - response = response.split("```json")[1].split("```")[0].strip() - elif "```" in response: - code_blocks = response.split("```") - if len(code_blocks) >= 3: # At least one code block exists - response = code_blocks[1].strip() - # Check if the extracted content looks like JSON - if not (response.startswith('{') or response.startswith('[')): - # If not, try to find JSON in the original response - json_start = response.find('{') - if json_start >= 0: - response = response[json_start:] - - # Try to extract JSON if response starts with explanation - if not response.strip().startswith('{'): - json_start = response.find('{') - if json_start >= 0: - response = response[json_start:] - - # Check if the response ends properly - if not response.strip().endswith('}'): - json_end = response.rfind('}') - if json_end >= 0: - response = response[:json_end+1] - - return response.strip() - except Exception as e: - print(f"Error cleaning JSON: {str(e)}") - return response - - def get_universal_template(self) -> Dict: - """Get any active template - no assignment_type filtering""" - try: - print("\n=== Getting Universal Template ===") - - # REMOVED: assignment_type filtering - get ANY active template - templates = frappe.get_list( - "Prompt Template", - filters={"is_active": 1}, - order_by="version desc", - limit=1 - ) - - if templates: - template = frappe.get_doc("Prompt Template", templates[0].name) - print(f"Using universal template: {template.template_name}") - - # Update the last_used timestamp - template.db_set('last_used', datetime.now()) - frappe.db.commit() - - return template - else: - print("No active template found, using built-in default") - return self.get_builtin_template() - - except Exception as e: - error_msg = f"Template Error: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(error_msg, "Template Error") - return self.get_builtin_template() - - def get_builtin_template(self): - """Return built-in default template as fallback""" - class BuiltinTemplate: - def __init__(self): - self.template_name = "Built-in Universal Template" - self.system_prompt = """You are an encouraging, knowledgeable educational assistant that provides constructive feedback on student submissions using a structured rubric-based evaluation. - EVALUATION GUIDELINES: Assess submissions against the provided rubric criteria. For each criterion, determine the appropriate grade level (1-5 scale) based on the rubric descriptions provided. - CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. - - Always provide feedback that is: - - Encouraging and positive while being constructive - - Age-appropriate and specific to observations - - Directly aligned with rubric criteria - - Clear about achievement gaps and growth areas - - Structure your response by: - 1. Evaluating each rubric criterion against the submission - 2. Assigning grades based on rubric descriptors - 3. Providing specific, actionable feedback - 4. Ending with motivating encouragement - - CRITICAL: You must respond with valid JSON format only.""" - - self.user_prompt = """Assignment Context: - - Name: {assignment_name} - - Subject: {course_vertical} - - Type: {assignment_type} - - Description: {assignment_description} - - Learning Objectives: {learning_objectives} - - Rubric Criteria: {rubric_criteria} - - CRITICAL: It is crucial that the image looks like a photo clicked by a student using a mobile camera. It shouldn't be a digitally created image or one sourced from the internet. Grade it accordingly. - - Analyze this submission and respond ONLY in this JSON format: - - { - "rubric_evaluations": [ - { - "criterion": "criterion_name", - "grade_value": 1-5, - "observation": "specific evidence from submission" - } - ], - "overall_feedback": "30-50 words of constructive, encouraging feedback or 'Submission does not match assignment requirements.'", - "strengths": ["specific strength 1", "specific strength 2"], - "areas_for_improvement": ["actionable suggestion 1", "actionable suggestion 2"], - "final_grade": "average of all rubric grades (0-5 scale, converted to 0-100)", - "encouragement": "motivating closing statement" - }""" - - self.response_format = """{ - "rubric_evaluations": [ - { - "criterion": "criterion_name", - "grade_value": 2, - "observation": "specific evidence from submission" - }, - { - "criterion": "criterion_name", - "grade_value": 2, - "observation": "specific evidence from submission" - } - ], - "overall_feedback": "Overall assessment of the submission", - "strengths": ["Strength 1", "Strength 2", "Strength 3"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": 75, - "encouragement": "Encouraging message for the student" - } - """ - - return BuiltinTemplate() - - def format_objectives(self, objectives: List[Dict]) -> str: - """Format learning objectives for prompt""" - if not objectives: - return "No specific learning objectives provided for this assignment." - - formatted = [] - for i, obj in enumerate(objectives, 1): - if isinstance(obj, dict): - description = obj.get('description', obj.get('objective_id', 'Unknown objective')) - else: - description = str(obj) - formatted.append(f"{i}. {description}") - - return "\n".join(formatted) - - def format_rubrics(self, rubrics: Dict) -> str: - prompt = "" - for criterion, grades_list in rubrics.items(): - prompt += f"\n{criterion}:\n" - for grade_item in grades_list: - prompt += f" Grade {grade_item['grade_value']}: {grade_item['grade_description']}\n" - - return prompt - - - def get_default_response_format(self) -> Dict: - """Get default response format""" - return { - "rubric_evaluations": [ - { - "criterion": "criterion_name", - "grade_value": 2, - "observation": "specific evidence from submission" - }, - { - "criterion": "criterion_name", - "grade_value": 2, - "observation": "specific evidence from submission" - } - ], - "overall_feedback": "Overall assessment of the submission", - "strengths": ["Strength 1", "Strength 2", "Strength 3"], - "areas_for_improvement": ["Area 1", "Area 2"], - "learning_objectives_feedback": ["Feedback on objective 1"], - "grade_recommendation": 75, - "encouragement": "Encouraging message for the student" - } - - async def generate_ai_evaluated_feedback(self, assignment_context: Dict, submission_url: str, submission_id: str) -> Dict: - """Generate feedback using universal template approach""" - try: - print("\n=== Starting Universal Feedback Generation ===") - - # Get universal template (no assignment_type filtering) - # template = self.get_universal_template() - template = self.get_builtin_template() # FOR TESTING ONLY - print("Template loaded successfully") - - # Get expected response format from template or use default - try: - if hasattr(template, 'response_format') and template.response_format: - expected_format = json.loads(template.response_format) - print("Using template-defined response format") - else: - expected_format = self.get_default_response_format() - print("Using default response format") - except json.JSONDecodeError: - expected_format = self.get_default_response_format() - print("Failed to parse template response format, using default") - - # Format learning objectives - learning_objectives = self.format_objectives(assignment_context.get("learning_objectives", [])) - rubric_criteria = self.format_rubrics(assignment_context["assignment"].get("rubrics", {})) - - - # SIMPLIFIED: Use template directly without complex modifications - # The universal template handles all subject types internally - - # Format user prompt with assignment context - user_prompt_vars = { - "assignment_name": assignment_context["assignment"].get("name", ""), - "assignment_description": assignment_context["assignment"].get("description", ""), - "course_vertical": assignment_context.get("course_vertical", "General"), - # "assignment_type": assignment_context["assignment"].get("type", "Practical"), - "learning_objectives": learning_objectives, - "rubric_criteria": rubric_criteria - } - - # Format the user prompt with available variables - formatted_user_prompt = template.user_prompt - for key, value in user_prompt_vars.items(): - placeholder = "{" + key + "}" - if placeholder in formatted_user_prompt: - formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) - - # Use template system prompt as-is (it already handles JSON requirement) - system_prompt = template.system_prompt - - # Prepare messages for the LLM provider - messages = self.llm_provider.format_messages( - system_prompt=system_prompt, - user_prompt=formatted_user_prompt, - image_url=submission_url - ) - - print(f"\nAssignment: {assignment_context['assignment'].get('name', 'Unknown')}") - print(f"Subject: {assignment_context.get('course_vertical', 'General')}") - print(f"Type: {assignment_context['assignment'].get('type', 'Unknown')}") - print("\nSending request to LLM...") - - # Generate feedback - SINGLE LLM CALL (no separate validation) - raw_text = await self.llm_provider.generate_with_vision(messages) - print(f"\nRaw LLM Response: {raw_text}") - - try: - # Clean up the response text - cleaned_text = self.clean_json_response(raw_text) - print(f"\nCleaned Response Text: {cleaned_text}") - - feedback = json.loads(cleaned_text) - print("\nSuccessfully parsed JSON response") - - # Validate and ensure required fields - feedback = self.validate_feedback_structure(feedback, expected_format) - - except json.JSONDecodeError as e: - print(f"\nJSON Parse Error: {str(e)}") - print("Using fallback feedback format") - - # Create structured fallback response - feedback = self.create_fallback_feedback(assignment_context, expected_format) - - # Attach default plagiarism/AI-detection metadata - plagiarism_output = { - "is_plagiarized": False, - "is_ai_generated": False, - "match_type": "original", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": "none", - "ai_confidence": 0.0, - "similar_sources": [] - } - feedback["plagiarism_output"] = plagiarism_output - - try: - if hasattr(template, 'name'): - template_used = template.name - else: - template_used = "Built-in Universal Template" - - except Exception as template_error: - print("Used Default Template:") - template_used = "Built-in Universal Template" - # Don't fail the entire process for template tracking issues - - print("\n=== Feedback Generation Completed Successfully ===") - return feedback, template_used - - except Exception as e: - error_msg = f"Error generating feedback for submission {submission_id}: {str(e)}" - print(f"\nError: {error_msg}") - frappe.log_error(message=error_msg, title="Feedback Generation Error") - - # Return structured error response - template_used = "Built-in Universal Template for Error" - return self.create_error_feedback(assignment_context), template_used - - async def generate_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str, - plagiarism_data: Dict = None, feedback_request_id: str = None) -> Dict: - """Generate feedback with plagiarism context""" - - result_status = "Pending" - - try: - # Check for plagiarism/AI-generated content first - if plagiarism_data: - is_plagiarized = plagiarism_data.get("is_plagiarized", False) - is_ai_generated = plagiarism_data.get("is_ai_generated", False) - match_type = plagiarism_data.get("match_type", "original") - plagiarism_source = plagiarism_data.get("plagiarism_source", "none") - - # Handle AI-generated submissions - if is_ai_generated: - result_status = "Success - Flagged" - feedback = self._create_ai_generated_feedback( - plagiarism_data - ) - tempalate_used = "Feedback Template for AI Generated Submission" - - # Handle plagiarized submissions - elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: - result_status = "Success - Flagged" - feedback = self._create_plagiarism_feedback( - plagiarism_data - ) - tempalate_used = "Feedback Template for Plagiarized Submission" - - # Continue with normal feedback generation for original work - else: - result_status = "Success - Original" - feedback, tempalate_used = await self.generate_ai_evaluated_feedback(assignment_context, submission_url,submission_id) - - await self._update_result_status(feedback_request_id, result_status) - return feedback, self.model_used, tempalate_used - - except Exception as e: - result_status = "Failed" - await self._update_result_status(feedback_request_id, result_status, str(e)) - raise - - async def _update_result_status(self, feedback_request_id: str, status: str, error_message: str = None): - """Update Feedback Request result_status""" - if not feedback_request_id: - return - - update_data = {"result_status": status} - if error_message: - update_data["error_message"] = error_message[:500] # Truncate long errors - - # frappe.db.set_value( - # "Feedback Request", - # feedback_request_id, - # update_data, - # update_modified=True - # ) - # frappe.db.commit() - - def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: - """Create feedback for AI-generated submissions""" - - ai_source = plagiarism_data.get("ai_detection_source", "unknown") - ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - response = { - "overall_feedback": f"Your submission appears to be generated by an \ - AI tool (detected source: {ai_source}, confidence: {ai_confidence:.0%}). \ - At MentorMe, we encourage original creative work that reflects your own learning \ - and artistic development. AI-generated images, while interesting, don't demonstrate \ - the skills and creativity we're looking to nurture. Please submit your own original \ - artwork for this assignment.", - "strengths": ["N/A - AI-generated content detected"], - "areas_for_improvement": ["Submit original artwork created by you", - "Review assignment guidelines for creative direction"], - "learning_objectives_feedback": ["Unable to assess - submission flagged as AI-generated"], - "grade_recommendation": 0, - "encouragement": "We believe in your creative abilities!", - "plagiarism_output": { - "is_plagiarized": False, - "is_ai_generated": True, - "match_type": "ai_generated", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": ai_source, - "ai_confidence": ai_confidence, - } - } - - return response - - - def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: - """Create feedback for plagiarized submissions""" - - match_type = plagiarism_data.get("match_type") - plagiarism_source = plagiarism_data.get("plagiarism_source") - similarity_score = plagiarism_data.get("similarity_score", 0.0) - ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - - # respond with structured feedback - response = { - "overall_feedback": f"Your submission has been flagged for similarity \ - (similarity: {similarity_score:.0%}, source: {plagiarism_source}).\ - Academic integrity is fundamental to the learning process. Please ensure your \ - submissions represent your own original work.", - "strengths": ["N/A - Submission flagged for similarity"], - "areas_for_improvement": ["Create original artwork for this assignment", - "Review academic integrity guidelines"], - "learning_objectives_feedback": ["Unable to assess - submission flagged for similarity"], - "grade_recommendation": 0, - "encouragement": "Every artist develops their unique style through practice!", - "plagiarism_output": { - "is_plagiarized": True, - "is_ai_generated": False, - "match_type": match_type, - "plagiarism_source": plagiarism_source, - "similarity_score": similarity_score, - "ai_detection_source": "none", - "ai_confidence": ai_confidence, - } - } - - return response - - - def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: - """Ensure feedback has all required fields with correct types""" - # Ensure all expected fields are present - for field in expected_format: - if field not in feedback: - if isinstance(expected_format[field], list): - feedback[field] = ["No information provided"] - elif isinstance(expected_format[field], (int, float)): - feedback[field] = 0 - else: - feedback[field] = "No information provided" - - # Validate grade_recommendation format for TAP LMS compatibility - try: - grade = feedback.get("grade_recommendation", 0) - if isinstance(grade, str): - # Extract numeric part only - grade_clean = ''.join(c for c in grade if c.isdigit() or c == '.') - grade = float(grade_clean) if grade_clean else 0 - feedback["grade_recommendation"] = max(0, min(100, float(grade))) - except (ValueError, TypeError): - feedback["grade_recommendation"] = 0 - - # Ensure list fields are lists - list_fields = ["strengths", "areas_for_improvement", "learning_objectives_feedback"] - for field in list_fields: - if field in feedback and not isinstance(feedback[field], list): - feedback[field] = [str(feedback[field])] - - return feedback - - def create_fallback_feedback(self, assignment_context: Dict, expected_format: Dict) -> Dict: - """Create structured fallback when JSON parsing fails""" - assignment_name = assignment_context["assignment"].get("name", "this assignment") - - fallback = {} - for field, default_value in expected_format.items(): - if field == "overall_feedback": - fallback[field] = f"I encountered a formatting issue while processing your submission for {assignment_name}. This appears to be a technical problem on our end. Please try resubmitting if this issue persists." - elif field == "grade_recommendation": - fallback[field] = 50 # Neutral grade for technical issues - elif field == "rubric_evaluations": - fallback[field] = [ - { - "criterion": "Content Knowledge", - "grade_value": 2, - "observation": "Neutral evaluation due to processing issue" - } - ] - elif isinstance(default_value, list): - if "strength" in field: - fallback[field] = ["Your submission was received and processed"] - elif "improvement" in field: - fallback[field] = ["Please ensure your submission clearly shows your work"] - else: - fallback[field] = ["Unable to provide specific feedback due to processing issue"] - else: - if field == "encouragement": - fallback[field] = "Technical issues don't reflect your effort - please try resubmitting!" - else: - fallback[field] = "Processing issue - please resubmit for detailed feedback" - - return fallback - - def create_error_feedback(self, assignment_context: Dict) -> Dict: - """Create feedback for system errors""" - assignment_name = assignment_context["assignment"].get("name", "this assignment") - - return { - "overall_feedback": f"I encountered a system error while processing your submission for {assignment_name}. This appears to be a technical issue on our end. Please try resubmitting, and if the issue persists, contact your instructor.", - "strengths": ["Your submission was received successfully"], - "areas_for_improvement": ["No issues identified with your submission - this appears to be a technical problem"], - "learning_objectives_feedback": ["Unable to evaluate due to system error - please resubmit"], - "grade_recommendation": 0, - "encouragement": "Technical issues don't reflect your effort or ability - please try again!" - } - - @staticmethod - def format_feedback_for_display(feedback: Dict) -> str: - """Format feedback for human-readable display""" - try: - formatted = [] - - if "overall_feedback" in feedback: - formatted.append("Overall Feedback:") - formatted.append(feedback["overall_feedback"]) - - if "strengths" in feedback: - formatted.append("\nStrengths:") - for strength in feedback["strengths"]: - formatted.append(f"- {strength}") - - if "areas_for_improvement" in feedback: - formatted.append("\nAreas for Improvement:") - for area in feedback["areas_for_improvement"]: - formatted.append(f"- {area}") - - if "learning_objectives_feedback" in feedback: - formatted.append("\nLearning Objectives Feedback:") - for obj in feedback["learning_objectives_feedback"]: - formatted.append(f"- {obj}") - - if "grade_recommendation" in feedback: - formatted.append(f"\nGrade Recommendation: {feedback['grade_recommendation']}") - - if "encouragement" in feedback: - formatted.append(f"\nEncouragement: {feedback['encouragement']}") - - return "\n".join(formatted) - - except Exception as e: - error_msg = f"Error formatting feedback: {str(e)}" - print(f"\nError: {error_msg}") - return "Error formatting feedback for display. Please check the JSON feedback data." - - def get_current_config(self) -> Dict: - """Get current LLM configuration""" - if not self.llm_provider: - return {"status": "not_configured"} - - return { - "provider": self.llm_provider.__class__.__name__, - "model": self.llm_provider.model_name, - "temperature": self.llm_provider.temperature, - "max_tokens": self.llm_provider.max_tokens - } diff --git a/rag_service/scripts/vertexai_client.py b/rag_service/scripts/vertexai_client.py new file mode 100644 index 0000000..4836a2e --- /dev/null +++ b/rag_service/scripts/vertexai_client.py @@ -0,0 +1,220 @@ +import vertexai +from vertexai.generative_models import GenerativeModel, Part +from google.oauth2 import service_account +import json + +# ========================================== +# 0. COST ESTIMATION SETTINGS +# ========================================== + +# Prices are per 1M tokens unless noted otherwise. Update as pricing changes: +# https://cloud.google.com/vertex-ai/generative-ai/pricing +MODEL_PRICING_USD_PER_1M = { + "gemini-2.5-pro": {"input": 1.25, "output": 10.0, "audio_input": 1.25}, + "gemini-2.5-flash": {"input": 0.30, "output": 2.50, "audio_input": 1.0}, + "gemini-2.5-flash-lite": {"input": 0.10, "output": 0.40, "audio_input": 0.3}, +} + + +def _usage_get(usage_metadata, key_snake, key_camel=None): + if usage_metadata is None: + return None + if isinstance(usage_metadata, dict): + if key_snake in usage_metadata: + return usage_metadata[key_snake] + if key_camel and key_camel in usage_metadata: + return usage_metadata[key_camel] + return None + if hasattr(usage_metadata, key_snake): + return getattr(usage_metadata, key_snake) + if key_camel and hasattr(usage_metadata, key_camel): + return getattr(usage_metadata, key_camel) + return None + + +def _normalize_modality(modality): + if modality is None: + return None + if hasattr(modality, "name"): + modality = modality.name + elif hasattr(modality, "value"): + modality = modality.value + if isinstance(modality, str): + return modality.strip().lower() + return None + + +def _get_model_pricing(model_name): + if model_name in MODEL_PRICING_USD_PER_1M: + return MODEL_PRICING_USD_PER_1M[model_name] + for key, pricing in MODEL_PRICING_USD_PER_1M.items(): + if model_name.startswith(key): + return pricing + return None + + +def estimate_call_cost(usage_metadata, model_name): + pricing = _get_model_pricing(model_name) + if not pricing: + return None + + prompt_tokens = _usage_get(usage_metadata, "prompt_token_count", "promptTokenCount") or 0 + candidates_tokens = _usage_get( + usage_metadata, "candidates_token_count", "candidatesTokenCount" + ) or 0 + thoughts_tokens = _usage_get(usage_metadata, "thoughts_token_count", "thoughtsTokenCount") or 0 + total_tokens = _usage_get(usage_metadata, "total_token_count", "totalTokenCount") + + # Try to compute input cost by modality if prompt token breakdown is available. + prompt_details = _usage_get(usage_metadata, "prompt_tokens_details", "promptTokensDetails") + input_cost = 0.0 + if prompt_details: + for detail in prompt_details: + if isinstance(detail, dict): + modality = _normalize_modality(detail.get("modality")) + token_count = detail.get("token_count") or detail.get("tokenCount") or 0 + else: + modality = _normalize_modality(getattr(detail, "modality", None)) + token_count = getattr(detail, "token_count", 0) + if not token_count: + continue + if modality == "audio": + rate = pricing.get("audio_input", pricing["input"]) + else: + rate = pricing["input"] + input_cost += (token_count / 1_000_000) * rate + else: + input_cost = (prompt_tokens / 1_000_000) * pricing["input"] + + output_tokens = candidates_tokens + thoughts_tokens + output_cost = (output_tokens / 1_000_000) * pricing["output"] + estimated_cost = input_cost + output_cost + + return { + "prompt_tokens": prompt_tokens, + "candidates_tokens": candidates_tokens, + "thoughts_tokens": thoughts_tokens, + "total_tokens": total_tokens, + "estimated_cost_usd": estimated_cost, + } + +# ========================================== +# 1. SETUP & AUTHENTICATION +# ========================================== + +# Path to the JSON key file you created +key_path = '/Users/TAP/Documents/Git/rag_service/rag_service/core/service_account_key.json' + +# Load credentials from the JSON file +credentials = service_account.Credentials.from_service_account_file(key_path) + +# Extract project ID from the credentials file automatically +with open(key_path) as f: + project_id = json.load(f)['project_id'] + +# Initialize Vertex AI with the credentials +# location='us-central1' is standard, but change if your bucket is elsewhere +vertexai.init(project=project_id, location='us-central1', credentials=credentials) + +# ========================================== +# 2. DEFINE DATA & PROMPTS +# ========================================== + +# The Video URI (MUST be gs:// format for best performance) + + +# video_uri = "gs://your-bucket-name/student_art_submission.mp4" + +video_public = "https://storage.googleapis.com/bucket_tap_1/uploads/dance_l2_create/20251007140203_C455110_F32580_M16075921.mp4" +video_uri = video_public.replace("https://storage.googleapis.com/", "gs://") + +# The Assignment Context +assignment_description = """ +ASSIGNMENT: "Pendulum" +Task: Create a pendulumm from household items. +The student must record a video showing the work. +""" + +# The Grading Rubric +grading_rubric = """ +RUBRIC (Total 20 pts): +1. Pendulum Construction (5 pts): Is the pendulum properly constructed using household items? +2. Creativity (5 pts): Does the pendulum show creative use of materials? +3. Video Quality (5 pts): Is the video clear and shows the pendulum in motion? +4. Verbal Explanation (5 pts): Did the student clearly articulate their intent in the video? +""" + +# MODEL_NAME = "gemini-2.5-flash" +MODEL_NAME = "gemini-2.5-flash-lite" + +# ========================================== +# 3. ANALYSIS LOGIC +# ========================================== + +def analyze_student_video(): + print(f"Loading model and analyzing video: {video_uri}...") + + # Load the Model (Gemini 1.5 Flash is cost-efficient and fast for this) + + model = GenerativeModel(MODEL_NAME) + + # Create the Multimodal Prompt + # We combine the video file (referenced in Cloud Storage) with our text prompt + video_part = Part.from_uri( + uri=video_uri, + mime_type="video/mp4" + ) + + prompt = f""" + You are an expert science instructor. + + {assignment_description} + + Please analyze the attached video based strictly on the following rubric: + {grading_rubric} + + OUTPUT FORMAT: + Provide the output in JSON format with the following keys: + - total_score (integer) + - breakdown (object with score for each category) + - feedback_summary (string) + - specific_evidence (list of strings, citing timestamps from video if applicable) + """ + + # Generate the content + # temperature=0 ensures the grading is consistent and less "creative" + response = model.generate_content( + [video_part, prompt], + generation_config={"temperature": 0, "response_mime_type": "application/json"} + ) + + return response + +# ========================================== +# 4. EXECUTION +# ========================================== + +if __name__ == "__main__": + try: + response = analyze_student_video() + print("\n--- GRADING RESULTS ---") + print(response.text) + + print("\n--- RAW RESPONSE OBJECT ---") + print(response) + + cost_info = estimate_call_cost(response.usage_metadata, model_name=MODEL_NAME) + if cost_info: + print("\n--- ESTIMATED API COST ---") + print( + f"Prompt tokens: {cost_info['prompt_tokens']}, " + f"Output tokens: {cost_info['candidates_tokens']}, " + f"Thoughts tokens: {cost_info['thoughts_tokens']}, " + f"Total tokens: {cost_info['total_tokens']}" + ) + print(f"Estimated cost: ${cost_info['estimated_cost_usd']:.6f} USD") + else: + print("\n--- ESTIMATED API COST ---") + print("Pricing for this model is not configured yet.") + except Exception as e: + print(f"Error: {e}") diff --git a/rag_service/scripts/video_analysis.py b/rag_service/scripts/video_analysis.py new file mode 100644 index 0000000..b20b6db --- /dev/null +++ b/rag_service/scripts/video_analysis.py @@ -0,0 +1,820 @@ +from __future__ import annotations + +# rag_service/rag_service/core/video_analysis.py + +import os +import json +import base64 +import subprocess +import tempfile +import shutil +from typing import Any, Dict, List, Optional, Protocol, Tuple, runtime_checkable + +try: + import requests +except ImportError: + raise ImportError("requests library is required. Install with: pip install requests") + +try: + import cv2 + import numpy as np +except ImportError: + raise ImportError("opencv-python and numpy are required. Install with: pip install opencv-python numpy") + +@runtime_checkable +class LangChainManagerLike(Protocol): + """Minimal interface needed by VideoSubmissionAnalyzer. + + This keeps `video_analysis.py` importable for local testing without `frappe`. + """ + + llm_provider: Any + + def get_universal_template(self) -> Any: ... + def get_default_response_format(self) -> Dict: ... + def format_objectives(self, objectives: List[Dict]) -> str: ... + def format_rubrics(self, rubrics: Any) -> str: ... + def clean_json_response(self, response: str) -> str: ... + def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: ... + def create_error_feedback(self, assignment_context: Optional[Dict] = None) -> Dict: ... + + +class VideoSubmissionAnalyzer: + """Analyzes video submissions by extracting key frames and using vision LLM""" + + def __init__(self, langchain_manager: LangChainManagerLike): + """ + Initialize the video analyzer with a LangChainManager instance. + + Args: + langchain_manager: LangChainManager instance for LLM operations + """ + self.langchain_manager = langchain_manager + self.blur_threshold = 60 # Laplacian variance threshold for blur detection + + def frame_to_data_url(self, frame_path: str) -> str: + """ + Convert a frame image file to a base64 data URL. + + Args: + frame_path: Path to the frame image file + + Returns: + Data URL string in format: data:image/jpeg;base64,... + """ + try: + with open(frame_path, 'rb') as f: + image_bytes = f.read() + + base64_encoded = base64.b64encode(image_bytes).decode('utf-8') + return f"data:image/jpeg;base64,{base64_encoded}" + except Exception as e: + print(f"Error converting frame to data URL: {str(e)}") + raise + + def download_video(self, video_url: str, temp_dir: str) -> str: + """ + Download video from URL to a temporary file. + + Args: + video_url: URL of the video to download + temp_dir: Temporary directory to save the video + + Returns: + Path to the downloaded video file + + Raises: + Exception: If download fails + """ + try: + print(f"Downloading video from: {video_url}") + response = requests.get(video_url, stream=True, timeout=300) + response.raise_for_status() + + video_path = os.path.join(temp_dir, "video.mp4") + with open(video_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + print(f"Video downloaded to: {video_path}") + return video_path + except Exception as e: + error_msg = f"Error downloading video: {str(e)}" + print(f"Error: {error_msg}") + raise Exception(error_msg) + + def check_ffmpeg(self) -> bool: + """ + Check if ffmpeg is available in the system. + + Returns: + True if ffmpeg is available, False otherwise + """ + try: + subprocess.run( + ['ffmpeg', '-version'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True + ) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + return False + + def extract_evenly_spaced_frames(self, video_path: str, output_dir: str, num_frames: int = 10) -> List[str]: + """ + Extract evenly spaced frames from video using ffmpeg. + + Args: + video_path: Path to the video file + output_dir: Directory to save extracted frames + num_frames: Number of frames to extract (default: 10) + + Returns: + List of paths to extracted frame files + """ + try: + print(f"Extracting {num_frames} evenly spaced frames...") + + # Get video duration using ffprobe + try: + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', video_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + timeout=30 + ) + duration = float(result.stdout.strip()) + except (subprocess.CalledProcessError, FileNotFoundError, ValueError) as e: + print(f"Warning: Could not get video duration with ffprobe: {str(e)}") + print("Attempting to extract frames without duration info...") + # Fallback: extract frames at fixed intervals + duration = 10.0 # Assume 10 seconds if we can't determine + + # Calculate frame intervals + if num_frames <= 1: + intervals = [duration / 2] + else: + intervals = [duration * i / (num_frames - 1) for i in range(num_frames)] + + frame_paths = [] + for i, timestamp in enumerate(intervals): + frame_path = os.path.join(output_dir, f"frame_even_{i:03d}.jpg") + try: + subprocess.run( + ['ffmpeg', '-i', video_path, '-ss', str(timestamp), + '-vframes', '1', '-q:v', '2', '-y', frame_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + timeout=30 + ) + if os.path.exists(frame_path): + frame_paths.append(frame_path) + print(f"Extracted frame at {timestamp:.2f}s: {frame_path}") + except subprocess.TimeoutExpired: + print(f"Timeout extracting frame at {timestamp:.2f}s") + except subprocess.CalledProcessError as e: + print(f"Error extracting frame at {timestamp:.2f}s: {str(e)}") + + if not frame_paths: + raise Exception("Failed to extract any frames from video") + + return frame_paths + except subprocess.CalledProcessError as e: + error_msg = f"Error extracting frames with ffmpeg: {str(e)}" + print(f"Error: {error_msg}") + raise Exception(error_msg) + except Exception as e: + error_msg = f"Unexpected error extracting frames: {str(e)}" + print(f"Error: {error_msg}") + raise Exception(error_msg) + + def score_blur(self, frame_path: str) -> float: + """ + Calculate blur score for a frame using Laplacian variance. + + Args: + frame_path: Path to the frame image + + Returns: + Blur score (Laplacian variance). Higher values indicate sharper images. + """ + try: + image = cv2.imread(frame_path) + if image is None: + return 0.0 + + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + laplacian_var = cv2.Laplacian(gray, cv2.CV_64F).var() + return float(laplacian_var) + except Exception as e: + print(f"Error scoring blur for {frame_path}: {str(e)}") + return 0.0 + + def extract_sharp_frames(self, video_path: str, output_dir: str, num_frames: int = 3) -> List[Tuple[str, float]]: + """ + Extract frames with highest sharpness scores. + + Args: + video_path: Path to the video file + output_dir: Directory to save extracted frames + num_frames: Number of sharp frames to extract (default: 3) + + Returns: + List of tuples (frame_path, blur_score) sorted by score descending + """ + try: + print(f"Extracting {num_frames} sharpest frames...") + + # Extract more candidate frames for sharpness analysis + candidate_count = max(20, num_frames * 5) + temp_frames = self.extract_evenly_spaced_frames(video_path, output_dir, candidate_count) + + # Score all candidate frames + scored_frames = [] + for frame_path in temp_frames: + score = self.score_blur(frame_path) + scored_frames.append((frame_path, score)) + print(f"Frame {os.path.basename(frame_path)}: blur score = {score:.2f}") + + # Sort by score (highest first) and take top N + scored_frames.sort(key=lambda x: x[1], reverse=True) + top_frames = scored_frames[:num_frames] + + # Rename top frames to indicate they're sharp + sharp_frames = [] + for i, (frame_path, score) in enumerate(top_frames): + new_path = os.path.join(output_dir, f"frame_sharp_{i:03d}.jpg") + shutil.move(frame_path, new_path) + sharp_frames.append((new_path, score)) + + # Clean up remaining candidate frames + for frame_path, _ in scored_frames[num_frames:]: + if os.path.exists(frame_path): + os.remove(frame_path) + + return sharp_frames + except Exception as e: + error_msg = f"Error extracting sharp frames: {str(e)}" + print(f"Error: {error_msg}") + raise Exception(error_msg) + + def select_best_frames(self, evenly_spaced: List[str], sharp_frames: List[Tuple[str, float]], max_frames: int = 6) -> List[Tuple[str, float]]: + """ + Select best frames from evenly spaced and sharp frames. + + Args: + evenly_spaced: List of paths to evenly spaced frames + sharp_frames: List of tuples (frame_path, blur_score) for sharp frames + max_frames: Maximum number of frames to select (default: 6) + + Returns: + List of tuples (frame_path, blur_score) for selected frames + """ + try: + # Score evenly spaced frames + scored_evenly = [] + for frame_path in evenly_spaced: + score = self.score_blur(frame_path) + scored_evenly.append((frame_path, score)) + + # Combine and deduplicate (by path) + all_frames = {} + for frame_path, score in scored_evenly + sharp_frames: + all_frames[frame_path] = score + + # Sort by score and take top frames + sorted_frames = sorted(all_frames.items(), key=lambda x: x[1], reverse=True) + + # Mix: take some evenly spaced (for coverage) and some sharp (for quality) + selected = [] + + # Prioritize sharp frames, then fill with evenly spaced + for frame_path, score in sorted_frames: + if len(selected) >= max_frames: + break + + # Check if this is a sharp frame + is_sharp = any(frame_path == path for path, _ in sharp_frames) + if is_sharp and len(selected) < max_frames: + selected.append((frame_path, score)) + elif not is_sharp and len(selected) < max_frames: + # Add evenly spaced frame if we don't have enough + selected.append((frame_path, score)) + + # Ensure we have a good mix + if len(selected) < max_frames and len(sorted_frames) > len(selected): + # Add more from sorted list + for frame_path, score in sorted_frames: + if len(selected) >= max_frames: + break + if (frame_path, score) not in selected: + selected.append((frame_path, score)) + + print(f"Selected {len(selected)} frames for analysis") + return selected[:max_frames] + except Exception as e: + error_msg = f"Error selecting best frames: {str(e)}" + print(f"Error: {error_msg}") + raise Exception(error_msg) + + async def call_llm_on_frame(self, frame_data_url: str, system_prompt: str, user_prompt: str) -> Dict: + """ + Call LLM vision API on a single frame. + + Args: + frame_data_url: Data URL of the frame image + system_prompt: System prompt for the LLM + user_prompt: User prompt for the LLM + + Returns: + Parsed JSON feedback dictionary + """ + try: + # Format messages using the provider's format_messages method + messages = self.langchain_manager.llm_provider.format_messages( + system_prompt=system_prompt, + user_prompt=user_prompt, + image_url=frame_data_url + ) + + # Generate response + raw_text = await self.langchain_manager.llm_provider.generate_with_vision(messages) + + # Clean and parse JSON + cleaned_text = self.langchain_manager.clean_json_response(raw_text) + feedback = json.loads(cleaned_text) + + return feedback + except json.JSONDecodeError as e: + print(f"JSON parse error for frame: {str(e)}") + # Return empty dict, will be handled in aggregation + return {} + except Exception as e: + print(f"Error calling LLM on frame: {str(e)}") + return {} + + def aggregate_frame_results(self, frame_results: List[Dict], frame_info: List[Tuple[str, float]], expected_format: Dict) -> Dict: + """ + Aggregate feedback results from multiple frames. + + Args: + frame_results: List of feedback dictionaries from each frame + frame_info: List of tuples (frame_path, blur_score) for frames used + expected_format: Expected feedback format structure + + Returns: + Aggregated feedback dictionary + """ + try: + # Filter out empty results + valid_results = [r for r in frame_results if r and isinstance(r, dict)] + + if not valid_results: + return {} + + aggregated = {} + + # Aggregate scalar fields (average) + # Newer schemas may use `final_grade` instead of `grade_recommendation`. + scalar_fields = ["grade_recommendation", "final_grade"] + for field in scalar_fields: + if field in expected_format: + values = [] + for result in valid_results: + if field in result: + try: + val = float(result[field]) + values.append(val) + except (ValueError, TypeError): + pass + + if values: + avg_value = sum(values) / len(values) + # Clamp to 0-100 + aggregated[field] = max(0, min(100, avg_value)) + else: + aggregated[field] = expected_format.get(field, 0) + + # Aggregate text fields (choose longest non-fallback) + text_fields = ["overall_feedback", "overall_feedback_translated", "encouragement"] + for field in text_fields: + if field in expected_format: + candidates = [r.get(field, "") for r in valid_results if r.get(field)] + # Filter out fallback messages + non_fallback = [c for c in candidates if "technical" not in c.lower() and "error" not in c.lower() and "issue" not in c.lower()] + if non_fallback: + # Choose longest + aggregated[field] = max(non_fallback, key=len) + elif candidates: + aggregated[field] = candidates[0] + else: + aggregated[field] = expected_format.get(field, "") + + # Aggregate list fields (merge unique items) + list_fields = ["strengths", "areas_for_improvement", "learning_objectives_feedback"] + for field in list_fields: + if field in expected_format: + all_items = [] + for result in valid_results: + if field in result and isinstance(result[field], list): + all_items.extend(result[field]) + + # Deduplicate while preserving order + seen = set() + unique_items = [] + for item in all_items: + item_str = str(item).lower().strip() + if item_str and item_str not in seen: + seen.add(item_str) + unique_items.append(item) + + aggregated[field] = unique_items if unique_items else expected_format.get(field, []) + + # Aggregate rubric_evaluations (if schema expects it) + if "rubric_evaluations" in expected_format: + # Map skill -> {grade_values: [...], observations: [...]} + by_skill: Dict[str, Dict[str, Any]] = {} + for result in valid_results: + rubrics = result.get("rubric_evaluations") + if not isinstance(rubrics, list): + continue + for item in rubrics: + if not isinstance(item, dict): + continue + skill = item.get("skill") or item.get("Skill") or "Unknown" + skill_key = str(skill).strip() or "Unknown" + grade_val = item.get("grade_value") + obs = item.get("observation") + + bucket = by_skill.setdefault(skill_key, {"grade_values": [], "observations": []}) + try: + if grade_val is not None: + bucket["grade_values"].append(float(grade_val)) + except (ValueError, TypeError): + pass + if isinstance(obs, str) and obs.strip(): + bucket["observations"].append(obs.strip()) + + rubric_out: List[Dict[str, Any]] = [] + for skill_key, bucket in by_skill.items(): + grades = bucket["grade_values"] + avg_grade = round(sum(grades) / len(grades), 2) if grades else expected_format.get("rubric_evaluations", [{}])[0].get("grade_value", 0) + # Clamp 0-5 if that's the rubric scale, otherwise leave as-is + avg_grade = max(0.0, min(5.0, float(avg_grade))) + # Merge observations unique + seen_obs = set() + merged_obs: List[str] = [] + for o in bucket["observations"]: + key = o.lower() + if key not in seen_obs: + seen_obs.add(key) + merged_obs.append(o) + rubric_out.append( + { + "skill": skill_key, + "grade_value": avg_grade, + "observation": " | ".join(merged_obs) if merged_obs else "No specific evidence provided", + } + ) + + # Preserve expected key casing if provided (some templates use "Skill") + aggregated["rubric_evaluations"] = rubric_out if rubric_out else expected_format.get("rubric_evaluations", []) + + # Add video-specific metadata + blur_scores = [score for _, score in frame_info] + aggregated["video_evidence"] = [ + { + "frame": os.path.basename(path), + "frame_index": i, + "blur_score": score + } + for i, (path, score) in enumerate(frame_info) + ] + + aggregated["video_quality"] = { + "frames_extracted": len(frame_info), + "frames_used": len(frame_info), + "blur_scores": blur_scores, + "quality_flag": self._determine_quality_flag(blur_scores, len(frame_info)) + } + + # Calculate confidence + aggregated["confidence"] = self._calculate_confidence( + blur_scores, + aggregated.get("video_quality", {}).get("quality_flag", "ok"), + len(valid_results) + ) + + return aggregated + except Exception as e: + print(f"Error aggregating frame results: {str(e)}") + return {} + + def _determine_quality_flag(self, blur_scores: List[float], num_frames: int) -> str: + """ + Determine quality flag based on blur scores and frame count. + + Args: + blur_scores: List of blur scores + num_frames: Number of frames + + Returns: + Quality flag: "ok", "blurry", or "insufficient_frames" + """ + if num_frames < 3: + return "insufficient_frames" + + blurry_count = sum(1 for score in blur_scores if score < self.blur_threshold) + if blurry_count > len(blur_scores) / 2: + return "blurry" + + return "ok" + + def _calculate_confidence(self, blur_scores: List[float], quality_flag: str, valid_results: int) -> float: + """ + Calculate confidence score (0.0-1.0) based on quality metrics. + + Args: + blur_scores: List of blur scores + quality_flag: Quality flag from _determine_quality_flag + valid_results: Number of valid LLM results + + Returns: + Confidence score between 0.0 and 1.0 + """ + if quality_flag == "insufficient_frames": + return 0.2 + elif quality_flag == "blurry": + return 0.4 + + # Base confidence from frame count + frame_confidence = min(1.0, valid_results / 6.0) + + # Adjust based on blur scores + avg_blur = sum(blur_scores) / len(blur_scores) if blur_scores else 0 + blur_confidence = min(1.0, avg_blur / 100.0) # Normalize to 0-1 + + # Combine confidences + confidence = (frame_confidence * 0.6 + blur_confidence * 0.4) + return max(0.0, min(1.0, confidence)) + + def create_video_fallback_feedback(self, assignment_context: Dict, expected_format: Dict, reason: str = "unclear_video") -> Dict: + """ + Create fallback feedback for video submissions that cannot be graded reliably. + + Args: + assignment_context: Assignment context dictionary + expected_format: Expected feedback format + reason: Reason for fallback ("unclear_video", "insufficient_frames", etc.) + + Returns: + Fallback feedback dictionary + """ + assignment_name = assignment_context["assignment"].get("name", "this assignment") + + if reason == "insufficient_frames": + message = f"I was unable to extract enough clear frames from your video submission for {assignment_name}. Please ensure your video clearly shows your artwork and try resubmitting." + elif reason == "blurry": + message = f"The frames extracted from your video submission for {assignment_name} were too blurry to evaluate reliably. Please ensure your video is in focus and clearly shows your artwork, then resubmit." + else: + message = f"I encountered difficulty evaluating your video submission for {assignment_name}. The artwork shown in the video frames was unclear. Please resubmit with a clearer video that shows your work." + + fallback = {} + for field, default_value in expected_format.items(): + if field == "overall_feedback": + fallback[field] = message + elif field == "grade_recommendation": + fallback[field] = 0 + elif isinstance(default_value, list): + if "strength" in field: + fallback[field] = ["Unable to assess - video quality insufficient"] + elif "improvement" in field: + fallback[field] = ["Please resubmit with a clearer video"] + else: + fallback[field] = ["Unable to provide specific feedback due to video quality issues"] + else: + if field == "encouragement": + fallback[field] = "Please resubmit with a clearer video so we can properly evaluate your work!" + else: + fallback[field] = "Video quality insufficient - please resubmit" + + # Add video quality metadata + fallback["video_quality"] = { + "frames_extracted": 0, + "frames_used": 0, + "blur_scores": [], + "quality_flag": reason + } + fallback["confidence"] = 0.1 + fallback["video_evidence"] = [] + + return fallback + + async def analyze_video_submission(self, assignment_context: Dict, video_url: str, submission_id: str) -> Tuple[Dict, str]: + """ + Analyze a video submission by extracting frames and using vision LLM. + + Args: + assignment_context: Assignment context dictionary + video_url: URL of the video submission + submission_id: ID of the submission + + Returns: + Tuple of (feedback_dict, template_used_string) + """ + temp_dir = None + temp_video_path = None + + try: + print("\n=== Starting Video Submission Analysis ===") + + # Check ffmpeg availability + if not self.check_ffmpeg(): + raise Exception("ffmpeg is not available. Please install ffmpeg to process video submissions.") + + # Create temp directory for this submission + temp_dir = tempfile.mkdtemp(prefix=f"video_analysis_{submission_id}_") + print(f"Created temp directory: {temp_dir}") + + # Download video + temp_video_path = self.download_video(video_url, temp_dir) + + # Get template (same as image analysis) + template = self.langchain_manager.get_universal_template() + print("Template loaded successfully") + + # Get expected response format + try: + if hasattr(template, 'response_format') and template.response_format: + expected_format = json.loads(template.response_format) + print("Using template-defined response format") + else: + expected_format = self.langchain_manager.get_default_response_format() + print("Using default response format") + except json.JSONDecodeError: + expected_format = self.langchain_manager.get_default_response_format() + print("Failed to parse template response format, using default") + + # Format learning objectives + rubric criteria (if present in schema/context) + learning_objectives = self.langchain_manager.format_objectives( + assignment_context.get("learning_objectives", []) + ) + rubric_criteria = "" + try: + rubrics = assignment_context.get("assignment", {}).get("rubrics", "") + if rubrics and hasattr(self.langchain_manager, "format_rubrics"): + rubric_criteria = self.langchain_manager.format_rubrics(rubrics) + except Exception: + rubric_criteria = "" + + # Format user prompt (add video-specific instruction) + user_prompt_vars = { + "assignment_name": assignment_context["assignment"].get("name", ""), + "assignment_description": assignment_context["assignment"].get("description", ""), + # Newer contexts may use `subject` instead of `course_vertical` + "course_vertical": assignment_context.get("course_vertical", assignment_context.get("subject", "General")), + "assignment_type": assignment_context["assignment"].get("type", "Practical"), + "learning_objectives": learning_objectives, + "rubric_criteria": rubric_criteria, + "Language": assignment_context.get("student", {}).get("language", "English"), + "Grade_Level": assignment_context.get("student", {}).get("grade", "1"), + } + + formatted_user_prompt = template.user_prompt + for key, value in user_prompt_vars.items(): + placeholder = "{" + key + "}" + if placeholder in formatted_user_prompt: + formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) + + # Add video-specific instruction + video_instruction = "\n\nThis submission is a VIDEO. Evaluate the ARTWORK shown in the frames. Ignore cinematography. If the artwork is unclear in frames, return low confidence and request resubmission." + formatted_user_prompt += video_instruction + + system_prompt = template.system_prompt + + # Extract frames + print("\nExtracting frames from video...") + evenly_spaced = self.extract_evenly_spaced_frames(temp_video_path, temp_dir, num_frames=10) + sharp_frames_with_scores = self.extract_sharp_frames(temp_video_path, temp_dir, num_frames=3) + sharp_frames = [path for path, _ in sharp_frames_with_scores] + frames_extracted_total = len(set(evenly_spaced) | set(sharp_frames)) + + # Select best frames + selected_frames = self.select_best_frames(evenly_spaced, sharp_frames_with_scores, max_frames=6) + + # Quality gate check + blur_scores = [score for _, score in selected_frames] + quality_flag = self._determine_quality_flag(blur_scores, len(selected_frames)) + + if quality_flag != "ok" or len(selected_frames) < 3: + print(f"Video quality insufficient: {quality_flag}, frames: {len(selected_frames)}") + template_used = template.name if hasattr(template, 'name') else "Built-in Universal Template" + fallback = self.create_video_fallback_feedback(assignment_context, expected_format, quality_flag) + fallback["video_quality"] = { + "frames_extracted": frames_extracted_total, + "frames_used": 0, + "blur_scores": blur_scores, + "quality_flag": quality_flag, + } + fallback["confidence"] = self._calculate_confidence(blur_scores, quality_flag, 0) + fallback = self.langchain_manager.validate_feedback_structure(fallback, expected_format) + # Add plagiarism output + fallback["plagiarism_output"] = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + return fallback, template_used + + # Call LLM on each frame + print(f"\nCalling LLM on {len(selected_frames)} frames...") + frame_results = [] + for i, (frame_path, blur_score) in enumerate(selected_frames): + print(f"Processing frame {i+1}/{len(selected_frames)}: {os.path.basename(frame_path)} (blur: {blur_score:.2f})") + frame_data_url = self.frame_to_data_url(frame_path) + frame_feedback = await self.call_llm_on_frame(frame_data_url, system_prompt, formatted_user_prompt) + if frame_feedback: + frame_results.append(frame_feedback) + + # Aggregate results + print("\nAggregating results from frames...") + aggregated = self.aggregate_frame_results(frame_results, selected_frames, expected_format) + if isinstance(aggregated, dict): + aggregated.setdefault("video_quality", {}) + if isinstance(aggregated["video_quality"], dict): + aggregated["video_quality"]["frames_extracted"] = frames_extracted_total + + # Validate structure + aggregated = self.langchain_manager.validate_feedback_structure(aggregated, expected_format) + + # Add plagiarism output (same as image analysis) + aggregated["plagiarism_output"] = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + + # Get template name + try: + if hasattr(template, 'name'): + template_used = template.name + else: + template_used = "Built-in Universal Template" + except Exception: + template_used = "Built-in Universal Template" + + print("\n=== Video Analysis Completed Successfully ===") + return aggregated, template_used + + except Exception as e: + error_msg = f"Error analyzing video submission {submission_id}: {str(e)}" + print(f"\nError: {error_msg}") + + # Try to get template for error response + try: + template = self.langchain_manager.get_universal_template() + expected_format = json.loads(template.response_format) if hasattr(template, 'response_format') and template.response_format else self.langchain_manager.get_default_response_format() + except: + expected_format = self.langchain_manager.get_default_response_format() + + # LangChainManager signature differs across branches; support both. + try: + error_feedback = self.langchain_manager.create_error_feedback(assignment_context) + except TypeError: + error_feedback = self.langchain_manager.create_error_feedback() + error_feedback = self.langchain_manager.validate_feedback_structure(error_feedback, expected_format) + error_feedback["plagiarism_output"] = { + "is_plagiarized": False, + "is_ai_generated": False, + "match_type": "original", + "plagiarism_source": "none", + "similarity_score": 0.0, + "ai_detection_source": "none", + "ai_confidence": 0.0, + "similar_sources": [] + } + + template_used = "Built-in Universal Template for Error" + return error_feedback, template_used + + finally: + # Clean up temp files + if temp_dir and os.path.exists(temp_dir): + print(f"Cleaning up temp directory: {temp_dir}") + shutil.rmtree(temp_dir, ignore_errors=True) + diff --git a/rag_service/templates/__init__.py b/rag_service/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rag_service/templates/pages/__init__.py b/rag_service/templates/pages/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/rag_service/utils/rabbitmq_consumer.py b/rag_service/utils/rabbitmq_consumer.py index 05f59f1..583e04a 100644 --- a/rag_service/utils/rabbitmq_consumer.py +++ b/rag_service/utils/rabbitmq_consumer.py @@ -5,7 +5,7 @@ import json import asyncio from typing import Dict, Optional -from ..handlers.feedback_handler import FeedbackHandler +from ..core.feedback_handler import FeedbackHandler from .queue_manager import QueueManager class RabbitMQConsumer: diff --git a/requirements.txt b/requirements.txt index 5dd4609..69d6bd1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,6 @@ pikka langchain_openai +aiohttp +requests +numpy +opencv-python From 7c850ae5c74a1008bf40bf304af1f427e5668a57 Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Mon, 9 Feb 2026 08:00:35 +0000 Subject: [PATCH 10/14] video analysis --- .../core/assignment_context_manager.py | 3 ++- .../feedback_utils/evaluation_generation.py | 24 +++++++------------ .../feedback_utils/image_evaluation.py | 6 ++--- .../feedback_utils/video_evaluation.py | 10 ++++---- .../gemini_settings/gemini_settings.json | 12 ++++++---- rag_service/scripts/test_consumer_payload.py | 8 +++++-- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/rag_service/core/assignment_context_manager.py b/rag_service/core/assignment_context_manager.py index 047b793..ec6b481 100644 --- a/rag_service/core/assignment_context_manager.py +++ b/rag_service/core/assignment_context_manager.py @@ -72,7 +72,7 @@ async def get_assignment_context(self, assignment_id: str, student_id: str) -> D # } context["student"] = {**student_details} - print("Assignment context",context) + # print("Assignment context",context) return context except Exception as e: @@ -137,6 +137,7 @@ async def _fetch_student_from_api(self, student_id: str) -> Dict: data = response.json() if "message" not in data: + print(f"Error: Invalid API response format: {data}") raise Exception("Invalid API response format") print("Student API request successful") diff --git a/rag_service/feedback_utils/evaluation_generation.py b/rag_service/feedback_utils/evaluation_generation.py index f717293..e5be0b6 100644 --- a/rag_service/feedback_utils/evaluation_generation.py +++ b/rag_service/feedback_utils/evaluation_generation.py @@ -6,11 +6,7 @@ import frappe -from .image_evaluation import ImageEvaluationGenerator -from .video_evaluation import VideoEvaluationGenerator - - -class BaseEvaluationGenerator: +class EvaluationGenerator: """Shared evaluation generation utilities for different media types.""" IMAGE_EXTENSIONS = { @@ -336,23 +332,19 @@ def _template_used_name(self, template: Any) -> str: pass return "Built-in Universal Template" - -class EvaluationGenerator(BaseEvaluationGenerator): - """Generate AI feedback for different media types.""" - - def __init__(self, feedback_service: Any): - super().__init__(feedback_service) - self.image_generator = ImageEvaluationGenerator(feedback_service) - self.video_generator = VideoEvaluationGenerator(feedback_service) - async def generate_ai_feedback( self, assignment_context: Dict, submission_url: str, submission_id: str ) -> Tuple[Dict, str, str]: media_type = self.detect_media_type(submission_url) if media_type == "video": - return await self.video_generator.generate_feedback( + from .video_evaluation import VideoEvaluationGenerator + + return await VideoEvaluationGenerator(self.feedback_service).generate_feedback( assignment_context, submission_url, submission_id ) - return await self.image_generator.generate_feedback( + + from .image_evaluation import ImageEvaluationGenerator + + return await ImageEvaluationGenerator(self.feedback_service).generate_feedback( assignment_context, submission_url, submission_id ) diff --git a/rag_service/feedback_utils/image_evaluation.py b/rag_service/feedback_utils/image_evaluation.py index 9c22d0c..4bf271f 100644 --- a/rag_service/feedback_utils/image_evaluation.py +++ b/rag_service/feedback_utils/image_evaluation.py @@ -4,11 +4,11 @@ import frappe -from .evaluation_generation import BaseEvaluationGenerator -from .llm_providers import create_llm_provider +from .evaluation_generation import EvaluationGenerator +from ..core.llm_providers import create_llm_provider -class ImageEvaluationGenerator(BaseEvaluationGenerator): +class ImageEvaluationGenerator(EvaluationGenerator): """Generate AI feedback for image submissions.""" def _create_llm_provider(self) -> Tuple[Any, str]: diff --git a/rag_service/feedback_utils/video_evaluation.py b/rag_service/feedback_utils/video_evaluation.py index ad89ad6..cfab6a2 100644 --- a/rag_service/feedback_utils/video_evaluation.py +++ b/rag_service/feedback_utils/video_evaluation.py @@ -1,19 +1,19 @@ # rag_service/rag_service/feedback_utils/video_evaluation.py import json -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple import frappe -from .evaluation_generation import BaseEvaluationGenerator -from .llm_providers import create_llm_provider +from .evaluation_generation import EvaluationGenerator +from ..core.llm_providers import create_llm_provider -class VideoEvaluationGenerator(BaseEvaluationGenerator): +class VideoEvaluationGenerator(EvaluationGenerator): """Generate AI feedback for video submissions.""" def _resolve_service_account_credentials(self, settings: Any) -> Optional[Dict]: - raw_key = settings.get("service_account_key_path") + raw_key = settings.get("credentials_json") if isinstance(raw_key, dict): return raw_key diff --git a/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json index 6e67ed2..42c14df 100644 --- a/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json +++ b/rag_service/rag_service/doctype/gemini_settings/gemini_settings.json @@ -8,7 +8,7 @@ "model_name", "location", "project_id", - "service_account_key_path", + "credentials_json", "temperature", "max_tokens", "is_active", @@ -33,10 +33,12 @@ "label": "Project ID" }, { - "fieldname": "service_account_key_path", - "fieldtype": "Data", - "label": "Service Account Key Path", - "length": 300 + "fieldname": "credentials_json", + "fieldtype": "Code", + "in_list_view": 1, + "label": "Service Account Credentials (JSON)", + "options": "JSON", + "reqd": 1 }, { "default": "0", diff --git a/rag_service/scripts/test_consumer_payload.py b/rag_service/scripts/test_consumer_payload.py index e9d1aa4..63682f6 100644 --- a/rag_service/scripts/test_consumer_payload.py +++ b/rag_service/scripts/test_consumer_payload.py @@ -20,8 +20,11 @@ # ============================================================================ PAYLOAD = { "submission_id": "IMSUB-2601280188", - "student_id": "ST00000182", - "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601280188_20251105154700_C5099524_F32580_M18137454.png", + "student_id": "ST00000206", + +# "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601280188_20251105154700_C5099524_F32580_M18137454.png", + "img_url":"https://storage.googleapis.com/bucket_tap_1/uploads/11/AugProccess/20251105103501_C155227_F32580_M18105608.mp4", + "created_at": "2026-01-28 16:58:27.423261", "similar_sources": None, "similarity_score": None, @@ -36,6 +39,7 @@ # ============================================================================ + def get_rabbitmq_settings(): """Get RabbitMQ settings from Frappe""" try: From 6118388b847e9b317bf9c60e7eb96cd193003c39 Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Thu, 12 Feb 2026 11:06:00 +0000 Subject: [PATCH 11/14] stock messages --- rag_service/core/feedback_service.py | 26 ++++++++++--------- .../feedback_utils/image_evaluation.py | 2 +- .../feedback_utils/video_evaluation.py | 4 +-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/rag_service/core/feedback_service.py b/rag_service/core/feedback_service.py index 070c186..ab049a1 100644 --- a/rag_service/core/feedback_service.py +++ b/rag_service/core/feedback_service.py @@ -219,11 +219,7 @@ def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: ai_source = plagiarism_data.get("ai_detection_source", "unknown") ai_confidence = plagiarism_data.get("ai_confidence", 0.0) response = { - "overall_feedback": "Your submission appears to be generated by an AI tool. \ - At MentorMe, we encourage original creative work that reflects your own learning \ - and artistic development. AI-generated images, while interesting, don't demonstrate \ - the skills and creativity we're looking to nurture. Please submit your own original \ - artwork for this assignment.", + "overall_feedback": "Hi Champ, I found you have sent AI created work. Please send your work. I'm excited to see what you made.", "overall_feedback_translated": "Your submission appears to be generated by an AI tool. \ At MentorMe, we encourage original creative work that reflects your own learning \ and artistic development. AI-generated images, while interesting, don't demonstrate \ @@ -241,6 +237,7 @@ def _create_ai_generated_feedback(self, plagiarism_data: Dict) -> Dict: "observation": "N/A - AI-generated content detected." }], "plagiarism_output": { + "stock_audio_file": "invalid_submission_ai", "is_plagiarized": False, "is_ai_generated": True, "match_type": "ai_generated", @@ -260,15 +257,19 @@ def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: plagiarism_source = plagiarism_data.get("plagiarism_source") similarity_score = plagiarism_data.get("similarity_score", 0.0) ai_confidence = plagiarism_data.get("ai_confidence", 0.0) - + + if plagiarism_source =="peer": + feedback = "Hi Champ, I found you have sent another student's work. Please send your work. No cheating this time. Excited to see what you made." + elif plagiarism_source =="self": + feedback = "Hi Champ, I found you have sent the same work again. Please resend the final work. Excited to see what you made." + elif plagiarism_source == "reference": + feedback = "Hi Champ, I found you have sent work that closely matches reference materials. Please submit your own original work. I'm excited to see your unique creation!" + else: + feedback = "Hi Champ, I found you have sent work that closely matches existing content. Please submit your own original work. I'm excited to see your unique creation!" # respond with structured feedback response = { - "overall_feedback": "Your submission has been flagged for similarity with another submission. \ - Academic integrity is fundamental to the learning process. Please ensure your \ - submissions represent your own original work.", - "overall_feedback_translated": "Your submission has been flagged for similarity. \ - Academic integrity is fundamental to the learning process. Please ensure your \ - submissions represent your own original work.", + "overall_feedback": feedback, + "overall_feedback_translated": feedback, "strengths": ["N/A - Submission flagged for similarity"], "areas_for_improvement": ["Create original artwork for this assignment", "Review academic integrity guidelines"], @@ -281,6 +282,7 @@ def _create_plagiarism_feedback( self, plagiarism_data: Dict) -> Dict: "observation": "N/A - Submission flagged for similarity." }], "plagiarism_output": { + "stock_audio_file": "invalid_submission_ai", "is_plagiarized": True, "is_ai_generated": False, "match_type": match_type, diff --git a/rag_service/feedback_utils/image_evaluation.py b/rag_service/feedback_utils/image_evaluation.py index 4bf271f..823a5d9 100644 --- a/rag_service/feedback_utils/image_evaluation.py +++ b/rag_service/feedback_utils/image_evaluation.py @@ -5,7 +5,7 @@ import frappe from .evaluation_generation import EvaluationGenerator -from ..core.llm_providers import create_llm_provider +from .llm_providers import create_llm_provider class ImageEvaluationGenerator(EvaluationGenerator): diff --git a/rag_service/feedback_utils/video_evaluation.py b/rag_service/feedback_utils/video_evaluation.py index cfab6a2..226f263 100644 --- a/rag_service/feedback_utils/video_evaluation.py +++ b/rag_service/feedback_utils/video_evaluation.py @@ -6,14 +6,14 @@ import frappe from .evaluation_generation import EvaluationGenerator -from ..core.llm_providers import create_llm_provider +from .llm_providers import create_llm_provider class VideoEvaluationGenerator(EvaluationGenerator): """Generate AI feedback for video submissions.""" def _resolve_service_account_credentials(self, settings: Any) -> Optional[Dict]: - raw_key = settings.get("credentials_json") + raw_key = settings.get("service_account_key_path") if isinstance(raw_key, dict): return raw_key From 0dc59d1dbefedf46a058150155c44dabb2c8dcaa Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Thu, 12 Feb 2026 12:29:00 +0000 Subject: [PATCH 12/14] video analysis --- rag_service/feedback_utils/image_evaluation.py | 2 +- rag_service/feedback_utils/video_evaluation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rag_service/feedback_utils/image_evaluation.py b/rag_service/feedback_utils/image_evaluation.py index 823a5d9..4bf271f 100644 --- a/rag_service/feedback_utils/image_evaluation.py +++ b/rag_service/feedback_utils/image_evaluation.py @@ -5,7 +5,7 @@ import frappe from .evaluation_generation import EvaluationGenerator -from .llm_providers import create_llm_provider +from ..core.llm_providers import create_llm_provider class ImageEvaluationGenerator(EvaluationGenerator): diff --git a/rag_service/feedback_utils/video_evaluation.py b/rag_service/feedback_utils/video_evaluation.py index 226f263..a27dbaa 100644 --- a/rag_service/feedback_utils/video_evaluation.py +++ b/rag_service/feedback_utils/video_evaluation.py @@ -6,7 +6,7 @@ import frappe from .evaluation_generation import EvaluationGenerator -from .llm_providers import create_llm_provider +from ..core.llm_providers import create_llm_provider class VideoEvaluationGenerator(EvaluationGenerator): From 048476670fb5dcfb1c647cc3b4e19d508b185981 Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Tue, 17 Feb 2026 14:21:58 +0000 Subject: [PATCH 13/14] plg change --- rag_service/core/feedback_service.py | 2 +- rag_service/scripts/video_analysis.py | 820 -------------------------- 2 files changed, 1 insertion(+), 821 deletions(-) delete mode 100644 rag_service/scripts/video_analysis.py diff --git a/rag_service/core/feedback_service.py b/rag_service/core/feedback_service.py index ab049a1..d12b57f 100644 --- a/rag_service/core/feedback_service.py +++ b/rag_service/core/feedback_service.py @@ -37,7 +37,7 @@ async def generate_feedback( self, assignment_context: Dict, submission_url: str tempalate_used = "Feedback Template for AI Generated Submission" # Handle plagiarized submissions - elif is_plagiarized and match_type in ["exact_duplicate", "near_duplicate"]: + elif is_plagiarized: result_status = "Success - Flagged" feedback = self._create_plagiarism_feedback( plagiarism_data diff --git a/rag_service/scripts/video_analysis.py b/rag_service/scripts/video_analysis.py deleted file mode 100644 index b20b6db..0000000 --- a/rag_service/scripts/video_analysis.py +++ /dev/null @@ -1,820 +0,0 @@ -from __future__ import annotations - -# rag_service/rag_service/core/video_analysis.py - -import os -import json -import base64 -import subprocess -import tempfile -import shutil -from typing import Any, Dict, List, Optional, Protocol, Tuple, runtime_checkable - -try: - import requests -except ImportError: - raise ImportError("requests library is required. Install with: pip install requests") - -try: - import cv2 - import numpy as np -except ImportError: - raise ImportError("opencv-python and numpy are required. Install with: pip install opencv-python numpy") - -@runtime_checkable -class LangChainManagerLike(Protocol): - """Minimal interface needed by VideoSubmissionAnalyzer. - - This keeps `video_analysis.py` importable for local testing without `frappe`. - """ - - llm_provider: Any - - def get_universal_template(self) -> Any: ... - def get_default_response_format(self) -> Dict: ... - def format_objectives(self, objectives: List[Dict]) -> str: ... - def format_rubrics(self, rubrics: Any) -> str: ... - def clean_json_response(self, response: str) -> str: ... - def validate_feedback_structure(self, feedback: Dict, expected_format: Dict) -> Dict: ... - def create_error_feedback(self, assignment_context: Optional[Dict] = None) -> Dict: ... - - -class VideoSubmissionAnalyzer: - """Analyzes video submissions by extracting key frames and using vision LLM""" - - def __init__(self, langchain_manager: LangChainManagerLike): - """ - Initialize the video analyzer with a LangChainManager instance. - - Args: - langchain_manager: LangChainManager instance for LLM operations - """ - self.langchain_manager = langchain_manager - self.blur_threshold = 60 # Laplacian variance threshold for blur detection - - def frame_to_data_url(self, frame_path: str) -> str: - """ - Convert a frame image file to a base64 data URL. - - Args: - frame_path: Path to the frame image file - - Returns: - Data URL string in format: data:image/jpeg;base64,... - """ - try: - with open(frame_path, 'rb') as f: - image_bytes = f.read() - - base64_encoded = base64.b64encode(image_bytes).decode('utf-8') - return f"data:image/jpeg;base64,{base64_encoded}" - except Exception as e: - print(f"Error converting frame to data URL: {str(e)}") - raise - - def download_video(self, video_url: str, temp_dir: str) -> str: - """ - Download video from URL to a temporary file. - - Args: - video_url: URL of the video to download - temp_dir: Temporary directory to save the video - - Returns: - Path to the downloaded video file - - Raises: - Exception: If download fails - """ - try: - print(f"Downloading video from: {video_url}") - response = requests.get(video_url, stream=True, timeout=300) - response.raise_for_status() - - video_path = os.path.join(temp_dir, "video.mp4") - with open(video_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - - print(f"Video downloaded to: {video_path}") - return video_path - except Exception as e: - error_msg = f"Error downloading video: {str(e)}" - print(f"Error: {error_msg}") - raise Exception(error_msg) - - def check_ffmpeg(self) -> bool: - """ - Check if ffmpeg is available in the system. - - Returns: - True if ffmpeg is available, False otherwise - """ - try: - subprocess.run( - ['ffmpeg', '-version'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True - ) - return True - except (subprocess.CalledProcessError, FileNotFoundError): - return False - - def extract_evenly_spaced_frames(self, video_path: str, output_dir: str, num_frames: int = 10) -> List[str]: - """ - Extract evenly spaced frames from video using ffmpeg. - - Args: - video_path: Path to the video file - output_dir: Directory to save extracted frames - num_frames: Number of frames to extract (default: 10) - - Returns: - List of paths to extracted frame files - """ - try: - print(f"Extracting {num_frames} evenly spaced frames...") - - # Get video duration using ffprobe - try: - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', - '-of', 'default=noprint_wrappers=1:nokey=1', video_path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=True, - timeout=30 - ) - duration = float(result.stdout.strip()) - except (subprocess.CalledProcessError, FileNotFoundError, ValueError) as e: - print(f"Warning: Could not get video duration with ffprobe: {str(e)}") - print("Attempting to extract frames without duration info...") - # Fallback: extract frames at fixed intervals - duration = 10.0 # Assume 10 seconds if we can't determine - - # Calculate frame intervals - if num_frames <= 1: - intervals = [duration / 2] - else: - intervals = [duration * i / (num_frames - 1) for i in range(num_frames)] - - frame_paths = [] - for i, timestamp in enumerate(intervals): - frame_path = os.path.join(output_dir, f"frame_even_{i:03d}.jpg") - try: - subprocess.run( - ['ffmpeg', '-i', video_path, '-ss', str(timestamp), - '-vframes', '1', '-q:v', '2', '-y', frame_path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - timeout=30 - ) - if os.path.exists(frame_path): - frame_paths.append(frame_path) - print(f"Extracted frame at {timestamp:.2f}s: {frame_path}") - except subprocess.TimeoutExpired: - print(f"Timeout extracting frame at {timestamp:.2f}s") - except subprocess.CalledProcessError as e: - print(f"Error extracting frame at {timestamp:.2f}s: {str(e)}") - - if not frame_paths: - raise Exception("Failed to extract any frames from video") - - return frame_paths - except subprocess.CalledProcessError as e: - error_msg = f"Error extracting frames with ffmpeg: {str(e)}" - print(f"Error: {error_msg}") - raise Exception(error_msg) - except Exception as e: - error_msg = f"Unexpected error extracting frames: {str(e)}" - print(f"Error: {error_msg}") - raise Exception(error_msg) - - def score_blur(self, frame_path: str) -> float: - """ - Calculate blur score for a frame using Laplacian variance. - - Args: - frame_path: Path to the frame image - - Returns: - Blur score (Laplacian variance). Higher values indicate sharper images. - """ - try: - image = cv2.imread(frame_path) - if image is None: - return 0.0 - - gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - laplacian_var = cv2.Laplacian(gray, cv2.CV_64F).var() - return float(laplacian_var) - except Exception as e: - print(f"Error scoring blur for {frame_path}: {str(e)}") - return 0.0 - - def extract_sharp_frames(self, video_path: str, output_dir: str, num_frames: int = 3) -> List[Tuple[str, float]]: - """ - Extract frames with highest sharpness scores. - - Args: - video_path: Path to the video file - output_dir: Directory to save extracted frames - num_frames: Number of sharp frames to extract (default: 3) - - Returns: - List of tuples (frame_path, blur_score) sorted by score descending - """ - try: - print(f"Extracting {num_frames} sharpest frames...") - - # Extract more candidate frames for sharpness analysis - candidate_count = max(20, num_frames * 5) - temp_frames = self.extract_evenly_spaced_frames(video_path, output_dir, candidate_count) - - # Score all candidate frames - scored_frames = [] - for frame_path in temp_frames: - score = self.score_blur(frame_path) - scored_frames.append((frame_path, score)) - print(f"Frame {os.path.basename(frame_path)}: blur score = {score:.2f}") - - # Sort by score (highest first) and take top N - scored_frames.sort(key=lambda x: x[1], reverse=True) - top_frames = scored_frames[:num_frames] - - # Rename top frames to indicate they're sharp - sharp_frames = [] - for i, (frame_path, score) in enumerate(top_frames): - new_path = os.path.join(output_dir, f"frame_sharp_{i:03d}.jpg") - shutil.move(frame_path, new_path) - sharp_frames.append((new_path, score)) - - # Clean up remaining candidate frames - for frame_path, _ in scored_frames[num_frames:]: - if os.path.exists(frame_path): - os.remove(frame_path) - - return sharp_frames - except Exception as e: - error_msg = f"Error extracting sharp frames: {str(e)}" - print(f"Error: {error_msg}") - raise Exception(error_msg) - - def select_best_frames(self, evenly_spaced: List[str], sharp_frames: List[Tuple[str, float]], max_frames: int = 6) -> List[Tuple[str, float]]: - """ - Select best frames from evenly spaced and sharp frames. - - Args: - evenly_spaced: List of paths to evenly spaced frames - sharp_frames: List of tuples (frame_path, blur_score) for sharp frames - max_frames: Maximum number of frames to select (default: 6) - - Returns: - List of tuples (frame_path, blur_score) for selected frames - """ - try: - # Score evenly spaced frames - scored_evenly = [] - for frame_path in evenly_spaced: - score = self.score_blur(frame_path) - scored_evenly.append((frame_path, score)) - - # Combine and deduplicate (by path) - all_frames = {} - for frame_path, score in scored_evenly + sharp_frames: - all_frames[frame_path] = score - - # Sort by score and take top frames - sorted_frames = sorted(all_frames.items(), key=lambda x: x[1], reverse=True) - - # Mix: take some evenly spaced (for coverage) and some sharp (for quality) - selected = [] - - # Prioritize sharp frames, then fill with evenly spaced - for frame_path, score in sorted_frames: - if len(selected) >= max_frames: - break - - # Check if this is a sharp frame - is_sharp = any(frame_path == path for path, _ in sharp_frames) - if is_sharp and len(selected) < max_frames: - selected.append((frame_path, score)) - elif not is_sharp and len(selected) < max_frames: - # Add evenly spaced frame if we don't have enough - selected.append((frame_path, score)) - - # Ensure we have a good mix - if len(selected) < max_frames and len(sorted_frames) > len(selected): - # Add more from sorted list - for frame_path, score in sorted_frames: - if len(selected) >= max_frames: - break - if (frame_path, score) not in selected: - selected.append((frame_path, score)) - - print(f"Selected {len(selected)} frames for analysis") - return selected[:max_frames] - except Exception as e: - error_msg = f"Error selecting best frames: {str(e)}" - print(f"Error: {error_msg}") - raise Exception(error_msg) - - async def call_llm_on_frame(self, frame_data_url: str, system_prompt: str, user_prompt: str) -> Dict: - """ - Call LLM vision API on a single frame. - - Args: - frame_data_url: Data URL of the frame image - system_prompt: System prompt for the LLM - user_prompt: User prompt for the LLM - - Returns: - Parsed JSON feedback dictionary - """ - try: - # Format messages using the provider's format_messages method - messages = self.langchain_manager.llm_provider.format_messages( - system_prompt=system_prompt, - user_prompt=user_prompt, - image_url=frame_data_url - ) - - # Generate response - raw_text = await self.langchain_manager.llm_provider.generate_with_vision(messages) - - # Clean and parse JSON - cleaned_text = self.langchain_manager.clean_json_response(raw_text) - feedback = json.loads(cleaned_text) - - return feedback - except json.JSONDecodeError as e: - print(f"JSON parse error for frame: {str(e)}") - # Return empty dict, will be handled in aggregation - return {} - except Exception as e: - print(f"Error calling LLM on frame: {str(e)}") - return {} - - def aggregate_frame_results(self, frame_results: List[Dict], frame_info: List[Tuple[str, float]], expected_format: Dict) -> Dict: - """ - Aggregate feedback results from multiple frames. - - Args: - frame_results: List of feedback dictionaries from each frame - frame_info: List of tuples (frame_path, blur_score) for frames used - expected_format: Expected feedback format structure - - Returns: - Aggregated feedback dictionary - """ - try: - # Filter out empty results - valid_results = [r for r in frame_results if r and isinstance(r, dict)] - - if not valid_results: - return {} - - aggregated = {} - - # Aggregate scalar fields (average) - # Newer schemas may use `final_grade` instead of `grade_recommendation`. - scalar_fields = ["grade_recommendation", "final_grade"] - for field in scalar_fields: - if field in expected_format: - values = [] - for result in valid_results: - if field in result: - try: - val = float(result[field]) - values.append(val) - except (ValueError, TypeError): - pass - - if values: - avg_value = sum(values) / len(values) - # Clamp to 0-100 - aggregated[field] = max(0, min(100, avg_value)) - else: - aggregated[field] = expected_format.get(field, 0) - - # Aggregate text fields (choose longest non-fallback) - text_fields = ["overall_feedback", "overall_feedback_translated", "encouragement"] - for field in text_fields: - if field in expected_format: - candidates = [r.get(field, "") for r in valid_results if r.get(field)] - # Filter out fallback messages - non_fallback = [c for c in candidates if "technical" not in c.lower() and "error" not in c.lower() and "issue" not in c.lower()] - if non_fallback: - # Choose longest - aggregated[field] = max(non_fallback, key=len) - elif candidates: - aggregated[field] = candidates[0] - else: - aggregated[field] = expected_format.get(field, "") - - # Aggregate list fields (merge unique items) - list_fields = ["strengths", "areas_for_improvement", "learning_objectives_feedback"] - for field in list_fields: - if field in expected_format: - all_items = [] - for result in valid_results: - if field in result and isinstance(result[field], list): - all_items.extend(result[field]) - - # Deduplicate while preserving order - seen = set() - unique_items = [] - for item in all_items: - item_str = str(item).lower().strip() - if item_str and item_str not in seen: - seen.add(item_str) - unique_items.append(item) - - aggregated[field] = unique_items if unique_items else expected_format.get(field, []) - - # Aggregate rubric_evaluations (if schema expects it) - if "rubric_evaluations" in expected_format: - # Map skill -> {grade_values: [...], observations: [...]} - by_skill: Dict[str, Dict[str, Any]] = {} - for result in valid_results: - rubrics = result.get("rubric_evaluations") - if not isinstance(rubrics, list): - continue - for item in rubrics: - if not isinstance(item, dict): - continue - skill = item.get("skill") or item.get("Skill") or "Unknown" - skill_key = str(skill).strip() or "Unknown" - grade_val = item.get("grade_value") - obs = item.get("observation") - - bucket = by_skill.setdefault(skill_key, {"grade_values": [], "observations": []}) - try: - if grade_val is not None: - bucket["grade_values"].append(float(grade_val)) - except (ValueError, TypeError): - pass - if isinstance(obs, str) and obs.strip(): - bucket["observations"].append(obs.strip()) - - rubric_out: List[Dict[str, Any]] = [] - for skill_key, bucket in by_skill.items(): - grades = bucket["grade_values"] - avg_grade = round(sum(grades) / len(grades), 2) if grades else expected_format.get("rubric_evaluations", [{}])[0].get("grade_value", 0) - # Clamp 0-5 if that's the rubric scale, otherwise leave as-is - avg_grade = max(0.0, min(5.0, float(avg_grade))) - # Merge observations unique - seen_obs = set() - merged_obs: List[str] = [] - for o in bucket["observations"]: - key = o.lower() - if key not in seen_obs: - seen_obs.add(key) - merged_obs.append(o) - rubric_out.append( - { - "skill": skill_key, - "grade_value": avg_grade, - "observation": " | ".join(merged_obs) if merged_obs else "No specific evidence provided", - } - ) - - # Preserve expected key casing if provided (some templates use "Skill") - aggregated["rubric_evaluations"] = rubric_out if rubric_out else expected_format.get("rubric_evaluations", []) - - # Add video-specific metadata - blur_scores = [score for _, score in frame_info] - aggregated["video_evidence"] = [ - { - "frame": os.path.basename(path), - "frame_index": i, - "blur_score": score - } - for i, (path, score) in enumerate(frame_info) - ] - - aggregated["video_quality"] = { - "frames_extracted": len(frame_info), - "frames_used": len(frame_info), - "blur_scores": blur_scores, - "quality_flag": self._determine_quality_flag(blur_scores, len(frame_info)) - } - - # Calculate confidence - aggregated["confidence"] = self._calculate_confidence( - blur_scores, - aggregated.get("video_quality", {}).get("quality_flag", "ok"), - len(valid_results) - ) - - return aggregated - except Exception as e: - print(f"Error aggregating frame results: {str(e)}") - return {} - - def _determine_quality_flag(self, blur_scores: List[float], num_frames: int) -> str: - """ - Determine quality flag based on blur scores and frame count. - - Args: - blur_scores: List of blur scores - num_frames: Number of frames - - Returns: - Quality flag: "ok", "blurry", or "insufficient_frames" - """ - if num_frames < 3: - return "insufficient_frames" - - blurry_count = sum(1 for score in blur_scores if score < self.blur_threshold) - if blurry_count > len(blur_scores) / 2: - return "blurry" - - return "ok" - - def _calculate_confidence(self, blur_scores: List[float], quality_flag: str, valid_results: int) -> float: - """ - Calculate confidence score (0.0-1.0) based on quality metrics. - - Args: - blur_scores: List of blur scores - quality_flag: Quality flag from _determine_quality_flag - valid_results: Number of valid LLM results - - Returns: - Confidence score between 0.0 and 1.0 - """ - if quality_flag == "insufficient_frames": - return 0.2 - elif quality_flag == "blurry": - return 0.4 - - # Base confidence from frame count - frame_confidence = min(1.0, valid_results / 6.0) - - # Adjust based on blur scores - avg_blur = sum(blur_scores) / len(blur_scores) if blur_scores else 0 - blur_confidence = min(1.0, avg_blur / 100.0) # Normalize to 0-1 - - # Combine confidences - confidence = (frame_confidence * 0.6 + blur_confidence * 0.4) - return max(0.0, min(1.0, confidence)) - - def create_video_fallback_feedback(self, assignment_context: Dict, expected_format: Dict, reason: str = "unclear_video") -> Dict: - """ - Create fallback feedback for video submissions that cannot be graded reliably. - - Args: - assignment_context: Assignment context dictionary - expected_format: Expected feedback format - reason: Reason for fallback ("unclear_video", "insufficient_frames", etc.) - - Returns: - Fallback feedback dictionary - """ - assignment_name = assignment_context["assignment"].get("name", "this assignment") - - if reason == "insufficient_frames": - message = f"I was unable to extract enough clear frames from your video submission for {assignment_name}. Please ensure your video clearly shows your artwork and try resubmitting." - elif reason == "blurry": - message = f"The frames extracted from your video submission for {assignment_name} were too blurry to evaluate reliably. Please ensure your video is in focus and clearly shows your artwork, then resubmit." - else: - message = f"I encountered difficulty evaluating your video submission for {assignment_name}. The artwork shown in the video frames was unclear. Please resubmit with a clearer video that shows your work." - - fallback = {} - for field, default_value in expected_format.items(): - if field == "overall_feedback": - fallback[field] = message - elif field == "grade_recommendation": - fallback[field] = 0 - elif isinstance(default_value, list): - if "strength" in field: - fallback[field] = ["Unable to assess - video quality insufficient"] - elif "improvement" in field: - fallback[field] = ["Please resubmit with a clearer video"] - else: - fallback[field] = ["Unable to provide specific feedback due to video quality issues"] - else: - if field == "encouragement": - fallback[field] = "Please resubmit with a clearer video so we can properly evaluate your work!" - else: - fallback[field] = "Video quality insufficient - please resubmit" - - # Add video quality metadata - fallback["video_quality"] = { - "frames_extracted": 0, - "frames_used": 0, - "blur_scores": [], - "quality_flag": reason - } - fallback["confidence"] = 0.1 - fallback["video_evidence"] = [] - - return fallback - - async def analyze_video_submission(self, assignment_context: Dict, video_url: str, submission_id: str) -> Tuple[Dict, str]: - """ - Analyze a video submission by extracting frames and using vision LLM. - - Args: - assignment_context: Assignment context dictionary - video_url: URL of the video submission - submission_id: ID of the submission - - Returns: - Tuple of (feedback_dict, template_used_string) - """ - temp_dir = None - temp_video_path = None - - try: - print("\n=== Starting Video Submission Analysis ===") - - # Check ffmpeg availability - if not self.check_ffmpeg(): - raise Exception("ffmpeg is not available. Please install ffmpeg to process video submissions.") - - # Create temp directory for this submission - temp_dir = tempfile.mkdtemp(prefix=f"video_analysis_{submission_id}_") - print(f"Created temp directory: {temp_dir}") - - # Download video - temp_video_path = self.download_video(video_url, temp_dir) - - # Get template (same as image analysis) - template = self.langchain_manager.get_universal_template() - print("Template loaded successfully") - - # Get expected response format - try: - if hasattr(template, 'response_format') and template.response_format: - expected_format = json.loads(template.response_format) - print("Using template-defined response format") - else: - expected_format = self.langchain_manager.get_default_response_format() - print("Using default response format") - except json.JSONDecodeError: - expected_format = self.langchain_manager.get_default_response_format() - print("Failed to parse template response format, using default") - - # Format learning objectives + rubric criteria (if present in schema/context) - learning_objectives = self.langchain_manager.format_objectives( - assignment_context.get("learning_objectives", []) - ) - rubric_criteria = "" - try: - rubrics = assignment_context.get("assignment", {}).get("rubrics", "") - if rubrics and hasattr(self.langchain_manager, "format_rubrics"): - rubric_criteria = self.langchain_manager.format_rubrics(rubrics) - except Exception: - rubric_criteria = "" - - # Format user prompt (add video-specific instruction) - user_prompt_vars = { - "assignment_name": assignment_context["assignment"].get("name", ""), - "assignment_description": assignment_context["assignment"].get("description", ""), - # Newer contexts may use `subject` instead of `course_vertical` - "course_vertical": assignment_context.get("course_vertical", assignment_context.get("subject", "General")), - "assignment_type": assignment_context["assignment"].get("type", "Practical"), - "learning_objectives": learning_objectives, - "rubric_criteria": rubric_criteria, - "Language": assignment_context.get("student", {}).get("language", "English"), - "Grade_Level": assignment_context.get("student", {}).get("grade", "1"), - } - - formatted_user_prompt = template.user_prompt - for key, value in user_prompt_vars.items(): - placeholder = "{" + key + "}" - if placeholder in formatted_user_prompt: - formatted_user_prompt = formatted_user_prompt.replace(placeholder, str(value)) - - # Add video-specific instruction - video_instruction = "\n\nThis submission is a VIDEO. Evaluate the ARTWORK shown in the frames. Ignore cinematography. If the artwork is unclear in frames, return low confidence and request resubmission." - formatted_user_prompt += video_instruction - - system_prompt = template.system_prompt - - # Extract frames - print("\nExtracting frames from video...") - evenly_spaced = self.extract_evenly_spaced_frames(temp_video_path, temp_dir, num_frames=10) - sharp_frames_with_scores = self.extract_sharp_frames(temp_video_path, temp_dir, num_frames=3) - sharp_frames = [path for path, _ in sharp_frames_with_scores] - frames_extracted_total = len(set(evenly_spaced) | set(sharp_frames)) - - # Select best frames - selected_frames = self.select_best_frames(evenly_spaced, sharp_frames_with_scores, max_frames=6) - - # Quality gate check - blur_scores = [score for _, score in selected_frames] - quality_flag = self._determine_quality_flag(blur_scores, len(selected_frames)) - - if quality_flag != "ok" or len(selected_frames) < 3: - print(f"Video quality insufficient: {quality_flag}, frames: {len(selected_frames)}") - template_used = template.name if hasattr(template, 'name') else "Built-in Universal Template" - fallback = self.create_video_fallback_feedback(assignment_context, expected_format, quality_flag) - fallback["video_quality"] = { - "frames_extracted": frames_extracted_total, - "frames_used": 0, - "blur_scores": blur_scores, - "quality_flag": quality_flag, - } - fallback["confidence"] = self._calculate_confidence(blur_scores, quality_flag, 0) - fallback = self.langchain_manager.validate_feedback_structure(fallback, expected_format) - # Add plagiarism output - fallback["plagiarism_output"] = { - "is_plagiarized": False, - "is_ai_generated": False, - "match_type": "original", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": "none", - "ai_confidence": 0.0, - "similar_sources": [] - } - return fallback, template_used - - # Call LLM on each frame - print(f"\nCalling LLM on {len(selected_frames)} frames...") - frame_results = [] - for i, (frame_path, blur_score) in enumerate(selected_frames): - print(f"Processing frame {i+1}/{len(selected_frames)}: {os.path.basename(frame_path)} (blur: {blur_score:.2f})") - frame_data_url = self.frame_to_data_url(frame_path) - frame_feedback = await self.call_llm_on_frame(frame_data_url, system_prompt, formatted_user_prompt) - if frame_feedback: - frame_results.append(frame_feedback) - - # Aggregate results - print("\nAggregating results from frames...") - aggregated = self.aggregate_frame_results(frame_results, selected_frames, expected_format) - if isinstance(aggregated, dict): - aggregated.setdefault("video_quality", {}) - if isinstance(aggregated["video_quality"], dict): - aggregated["video_quality"]["frames_extracted"] = frames_extracted_total - - # Validate structure - aggregated = self.langchain_manager.validate_feedback_structure(aggregated, expected_format) - - # Add plagiarism output (same as image analysis) - aggregated["plagiarism_output"] = { - "is_plagiarized": False, - "is_ai_generated": False, - "match_type": "original", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": "none", - "ai_confidence": 0.0, - "similar_sources": [] - } - - # Get template name - try: - if hasattr(template, 'name'): - template_used = template.name - else: - template_used = "Built-in Universal Template" - except Exception: - template_used = "Built-in Universal Template" - - print("\n=== Video Analysis Completed Successfully ===") - return aggregated, template_used - - except Exception as e: - error_msg = f"Error analyzing video submission {submission_id}: {str(e)}" - print(f"\nError: {error_msg}") - - # Try to get template for error response - try: - template = self.langchain_manager.get_universal_template() - expected_format = json.loads(template.response_format) if hasattr(template, 'response_format') and template.response_format else self.langchain_manager.get_default_response_format() - except: - expected_format = self.langchain_manager.get_default_response_format() - - # LangChainManager signature differs across branches; support both. - try: - error_feedback = self.langchain_manager.create_error_feedback(assignment_context) - except TypeError: - error_feedback = self.langchain_manager.create_error_feedback() - error_feedback = self.langchain_manager.validate_feedback_structure(error_feedback, expected_format) - error_feedback["plagiarism_output"] = { - "is_plagiarized": False, - "is_ai_generated": False, - "match_type": "original", - "plagiarism_source": "none", - "similarity_score": 0.0, - "ai_detection_source": "none", - "ai_confidence": 0.0, - "similar_sources": [] - } - - template_used = "Built-in Universal Template for Error" - return error_feedback, template_used - - finally: - # Clean up temp files - if temp_dir and os.path.exists(temp_dir): - print(f"Cleaning up temp directory: {temp_dir}") - shutil.rmtree(temp_dir, ignore_errors=True) - From 0db4070b0296bcee81812985d734c34f87f8b8b1 Mon Sep 17 00:00:00 2001 From: Manu Agarwal Date: Tue, 17 Feb 2026 17:22:31 +0000 Subject: [PATCH 14/14] gemini settings --- rag_service/feedback_utils/video_evaluation.py | 2 +- rag_service/scripts/test_consumer_payload.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/rag_service/feedback_utils/video_evaluation.py b/rag_service/feedback_utils/video_evaluation.py index a27dbaa..cfab6a2 100644 --- a/rag_service/feedback_utils/video_evaluation.py +++ b/rag_service/feedback_utils/video_evaluation.py @@ -13,7 +13,7 @@ class VideoEvaluationGenerator(EvaluationGenerator): """Generate AI feedback for video submissions.""" def _resolve_service_account_credentials(self, settings: Any) -> Optional[Dict]: - raw_key = settings.get("service_account_key_path") + raw_key = settings.get("credentials_json") if isinstance(raw_key, dict): return raw_key diff --git a/rag_service/scripts/test_consumer_payload.py b/rag_service/scripts/test_consumer_payload.py index 63682f6..807fd60 100644 --- a/rag_service/scripts/test_consumer_payload.py +++ b/rag_service/scripts/test_consumer_payload.py @@ -19,22 +19,19 @@ # EDIT THE PAYLOAD BELOW TO TEST DIFFERENT DATA # ============================================================================ PAYLOAD = { - "submission_id": "IMSUB-2601280188", + "submission_id": "IMSUB-2602170459", "student_id": "ST00000206", - -# "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2601280188_20251105154700_C5099524_F32580_M18137454.png", - "img_url":"https://storage.googleapis.com/bucket_tap_1/uploads/11/AugProccess/20251105103501_C155227_F32580_M18105608.mp4", - - "created_at": "2026-01-28 16:58:27.423261", + "img_url": "https://storage.googleapis.com/tap-lms-submissions/submissions/IMSUB-2602170459_20251105103501_C155227_F32580_M18105608.mp4", + "created_at": "2026-02-17 22:32:44.472760", "similar_sources": None, - "similarity_score": None, + "similarity_score": 1.0, "is_plagiarized": False, "match_type": "original", - "assignment_id": "VA_L1_CA1-Basic", + "assignment_id": "VA_L2_CA1-Basic", "is_ai_generated": False, - "ai_detection_source": "", + "ai_detection_source": "None", "ai_confidence": 0.0, - "plagiarism_source": "" + "plagiarism_source": None } # ============================================================================