From f6e678d52f2a7b3f8f881c5b296907a5d179092e Mon Sep 17 00:00:00 2001 From: saikolasani Date: Thu, 27 Jun 2024 18:14:33 -0700 Subject: [PATCH 1/4] Fix Other Providers Issue --- ragulate/cli_commands/query.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ragulate/cli_commands/query.py b/ragulate/cli_commands/query.py index afb673c..7c8712e 100644 --- a/ragulate/cli_commands/query.py +++ b/ragulate/cli_commands/query.py @@ -101,8 +101,6 @@ def call_query( sample: float, seed: int, restart: bool, - provider: str, - model: str, **kwargs, ): if sample <= 0.0 or sample > 1.0: @@ -131,7 +129,5 @@ def call_query( sample_percent=sample, random_seed=seed, restart_pipeline=restart, - llm_provider=provider, - model_name=model, ) query_pipeline.query() From dc59a83f380af40dd31555c674286a71dea9aec2 Mon Sep 17 00:00:00 2001 From: saikolasani Date: Thu, 27 Jun 2024 18:21:20 -0700 Subject: [PATCH 2/4] Stores queries ran as a json file --- ragulate/pipelines/query_pipeline.py | 45 +++++++++------------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/ragulate/pipelines/query_pipeline.py b/ragulate/pipelines/query_pipeline.py index f290f7a..cc76b39 100644 --- a/ragulate/pipelines/query_pipeline.py +++ b/ragulate/pipelines/query_pipeline.py @@ -2,17 +2,11 @@ import signal import time from typing import Any, Dict, List, Optional +import pandas as pd from tqdm import tqdm from trulens_eval import Tru, TruChain -from trulens_eval.feedback.provider import ( - AzureOpenAI, - Bedrock, - Huggingface, - Langchain, - LiteLLM, - OpenAI, -) +from trulens_eval.feedback.provider import OpenAI from trulens_eval.feedback.provider.base import LLMProvider from trulens_eval.schema.feedback import FeedbackMode, FeedbackResultStatus @@ -56,8 +50,6 @@ def __init__( sample_percent: float = 1.0, random_seed: Optional[int] = None, restart_pipeline: Optional[bool] = False, - llm_provider: Optional[str] = "OpenAI", - model_name: Optional[str] = None, **kwargs, ): super().__init__( @@ -71,8 +63,6 @@ def __init__( self.sample_percent = sample_percent self.random_seed = random_seed self.restart_pipeline = restart_pipeline - self.llm_provider = llm_provider - self.model_name = model_name # Set up the signal handler for SIGINT (Ctrl-C) signal.signal(signal.SIGINT, self.signal_handler) @@ -116,6 +106,16 @@ def start_evaluation(self): self._tru.start_evaluator(disable_tqdm=True) self._evaluation_running = True + def export_results(self): + records = self._tru.get_records_and_feedbacks() + data = [record.__dict__ for record in records] + + # Convert to DataFrame + df = pd.DataFrame(data) + + # Export to JSON + df.to_json('results.json', orient='records') + def stop_evaluation(self, loc: str): if self._evaluation_running: try: @@ -123,6 +123,7 @@ def stop_evaluation(self, loc: str): self._tru.stop_evaluator() self._evaluation_running = False self._tru.delete_singleton() + self.export_results() except Exception as e: logger.error(f"issue stopping evaluator: {e}") finally: @@ -150,30 +151,12 @@ def update_progress(self, query_change: int = 0): self._finished_feedbacks = done - def get_provider(self) -> LLMProvider: - llm_provider = self.llm_provider.lower() - model_name = self.model_name - - if llm_provider == "openai": - return OpenAI(model_engine=model_name) - elif llm_provider == "azureopenai": - return AzureOpenAI(deployment_name=model_name) - elif llm_provider == "bedrock": - return Bedrock(model_id=model_name) - elif llm_provider == "litellm": - return LiteLLM(model_engine=model_name) - elif llm_provider == "Langchain": - return Langchain(model_engine=model_name) - elif llm_provider == "huggingface": - return Huggingface(name=model_name) - else: - raise ValueError(f"Unsupported provider: {llm_provider}") def query(self): query_method = self.get_method() pipeline = query_method(**self.ingredients) - llm_provider = self.get_provider() + llm_provider = OpenAI(model_engine="gpt-3.5-turbo") feedbacks = Feedbacks(llm_provider=llm_provider, pipeline=pipeline) From c0b7d14c6155264fa69212fe1f530966fa846983 Mon Sep 17 00:00:00 2001 From: saikolasani Date: Thu, 27 Jun 2024 23:06:09 -0700 Subject: [PATCH 3/4] Fix issue with providers --- ragulate/cli_commands/query.py | 20 ++++++++++++++++++++ ragulate/pipelines/query_pipeline.py | 23 ++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ragulate/cli_commands/query.py b/ragulate/cli_commands/query.py index 7c8712e..762dd66 100644 --- a/ragulate/cli_commands/query.py +++ b/ragulate/cli_commands/query.py @@ -88,6 +88,22 @@ def setup_query(subparsers): type=str, help="The name or id of the LLM model or deployment to use for Evaluation. Generally used in combination with the `--provider` param.", ) + query_parser.add_argument( + "--provider", + type=str, + help="The name of the LLM Provider to use for Evaluation.", + choices=[ + "OpenAI", + "AzureOpenAI", + "Huggingface", + ], + default="OpenAI", + ) + query_parser.add_argument( + "--model", + type=str, + help="The name or id of the LLM model or deployment to use for Evaluation. Generally used in combination with the `--provider` param.", + ) query_parser.set_defaults(func=lambda args: call_query(**vars(args))) def call_query( @@ -101,6 +117,8 @@ def call_query( sample: float, seed: int, restart: bool, + provider: str, + model: str, **kwargs, ): if sample <= 0.0 or sample > 1.0: @@ -129,5 +147,7 @@ def call_query( sample_percent=sample, random_seed=seed, restart_pipeline=restart, + llm_provider=provider, + model_name=model ) query_pipeline.query() diff --git a/ragulate/pipelines/query_pipeline.py b/ragulate/pipelines/query_pipeline.py index cc76b39..59fbf3b 100644 --- a/ragulate/pipelines/query_pipeline.py +++ b/ragulate/pipelines/query_pipeline.py @@ -6,6 +6,11 @@ from tqdm import tqdm from trulens_eval import Tru, TruChain +from trulens_eval.feedback.provider import ( + AzureOpenAI, + Huggingface, + OpenAI +) from trulens_eval.feedback.provider import OpenAI from trulens_eval.feedback.provider.base import LLMProvider from trulens_eval.schema.feedback import FeedbackMode, FeedbackResultStatus @@ -50,6 +55,8 @@ def __init__( sample_percent: float = 1.0, random_seed: Optional[int] = None, restart_pipeline: Optional[bool] = False, + llm_provider: Optional[str] = "OpenAI", + model_name: Optional[str] = None, **kwargs, ): super().__init__( @@ -63,6 +70,8 @@ def __init__( self.sample_percent = sample_percent self.random_seed = random_seed self.restart_pipeline = restart_pipeline + self.llm_provider = llm_provider + self.model_name = model_name # Set up the signal handler for SIGINT (Ctrl-C) signal.signal(signal.SIGINT, self.signal_handler) @@ -151,12 +160,24 @@ def update_progress(self, query_change: int = 0): self._finished_feedbacks = done + def get_provider(self) -> LLMProvider: + llm_provider = self.llm_provider.lower() + model_name = self.model_name + + if llm_provider == "openai": + return OpenAI(model_engine=model_name) + elif llm_provider == "azureopenai": + return AzureOpenAI(deployment_name=model_name) + elif llm_provider == "huggingface": + return Huggingface(name=model_name) + else: + raise ValueError(f"Unsupported provider: {llm_provider}") def query(self): query_method = self.get_method() pipeline = query_method(**self.ingredients) - llm_provider = OpenAI(model_engine="gpt-3.5-turbo") + llm_provider = self.get_provider() feedbacks = Feedbacks(llm_provider=llm_provider, pipeline=pipeline) From 74ff64b1cfb0b417244cbcfb109af61e3ad5d004 Mon Sep 17 00:00:00 2001 From: saikolasani Date: Thu, 27 Jun 2024 23:09:22 -0700 Subject: [PATCH 4/4] Fix formatting --- ragulate/cli_commands/query.py | 18 +----------------- ragulate/pipelines/query_pipeline.py | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/ragulate/cli_commands/query.py b/ragulate/cli_commands/query.py index 762dd66..afb673c 100644 --- a/ragulate/cli_commands/query.py +++ b/ragulate/cli_commands/query.py @@ -88,22 +88,6 @@ def setup_query(subparsers): type=str, help="The name or id of the LLM model or deployment to use for Evaluation. Generally used in combination with the `--provider` param.", ) - query_parser.add_argument( - "--provider", - type=str, - help="The name of the LLM Provider to use for Evaluation.", - choices=[ - "OpenAI", - "AzureOpenAI", - "Huggingface", - ], - default="OpenAI", - ) - query_parser.add_argument( - "--model", - type=str, - help="The name or id of the LLM model or deployment to use for Evaluation. Generally used in combination with the `--provider` param.", - ) query_parser.set_defaults(func=lambda args: call_query(**vars(args))) def call_query( @@ -148,6 +132,6 @@ def call_query( random_seed=seed, restart_pipeline=restart, llm_provider=provider, - model_name=model + model_name=model, ) query_pipeline.query() diff --git a/ragulate/pipelines/query_pipeline.py b/ragulate/pipelines/query_pipeline.py index 59fbf3b..25a1aaa 100644 --- a/ragulate/pipelines/query_pipeline.py +++ b/ragulate/pipelines/query_pipeline.py @@ -9,7 +9,7 @@ from trulens_eval.feedback.provider import ( AzureOpenAI, Huggingface, - OpenAI + OpenAI, ) from trulens_eval.feedback.provider import OpenAI from trulens_eval.feedback.provider.base import LLMProvider