From 3e7780f44494bd83a787a19ab2330c05bbb75fdf Mon Sep 17 00:00:00 2001 From: tatuke Date: Sun, 22 Jun 2025 14:04:25 +0800 Subject: [PATCH 1/5] new branch --- src/open_codex/resources/config.example.yaml | 157 +++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 src/open_codex/resources/config.example.yaml diff --git a/src/open_codex/resources/config.example.yaml b/src/open_codex/resources/config.example.yaml new file mode 100644 index 0000000..2d50ca8 --- /dev/null +++ b/src/open_codex/resources/config.example.yaml @@ -0,0 +1,157 @@ +type: llm +provider: litellm_llm +timeout: 120 +models: + - alias: default + model: gpt-4.1-nano-2025-04-14 + context_window_size: 1000000 + kwargs: + max_tokens: 4096 + n: 1 + seed: 0 + temperature: 0 + - model: gpt-4.1-mini-2025-04-14 + context_window_size: 1000000 + kwargs: + max_tokens: 4096 + n: 1 + seed: 0 + temperature: 0 + - model: gpt-4.1-2025-04-14 + context_window_size: 1000000 + kwargs: + max_tokens: 4096 + n: 1 + seed: 0 + temperature: 0 + +--- +type: embedder +provider: litellm_embedder +models: + - model: text-embedding-3-large + alias: default + timeout: 120 + +--- +type: engine +provider: wren_ui +endpoint: http://wren-ui:3000 + +--- +type: engine +provider: wren_ibis +endpoint: http://ibis-server:8000 + +--- +type: document_store +provider: qdrant +location: http://qdrant:6333 +embedding_model_dim: 3072 +timeout: 120 +recreate_index: true + +--- +type: pipeline +pipes: + - name: db_schema_indexing + embedder: litellm_embedder.default + document_store: qdrant + - name: historical_question_indexing + embedder: litellm_embedder.default + document_store: qdrant + - name: table_description_indexing + embedder: litellm_embedder.default + document_store: qdrant + - name: db_schema_retrieval + llm: litellm_llm.default + embedder: litellm_embedder.default + document_store: qdrant + - name: historical_question_retrieval + embedder: litellm_embedder.default + document_store: qdrant + - name: sql_generation + llm: litellm_llm.default + engine: wren_ui + - name: sql_correction + llm: litellm_llm.default + engine: wren_ui + - name: followup_sql_generation + llm: litellm_llm.default + engine: wren_ui + - name: sql_answer + llm: litellm_llm.default + - name: semantics_description + llm: litellm_llm.default + - name: relationship_recommendation + llm: litellm_llm.default + engine: wren_ui + - name: question_recommendation + llm: litellm_llm.default + - name: question_recommendation_db_schema_retrieval + llm: litellm_llm.default + embedder: litellm_embedder.default + document_store: qdrant + - name: question_recommendation_sql_generation + llm: litellm_llm.default + engine: wren_ui + - name: intent_classification + llm: litellm_llm.default + embedder: litellm_embedder.default + document_store: qdrant + - name: misleading_assistance + llm: litellm_llm.default + - name: data_assistance + llm: litellm_llm.default + - name: sql_pairs_indexing + document_store: qdrant + embedder: litellm_embedder.default + - name: sql_pairs_retrieval + document_store: qdrant + embedder: litellm_embedder.default + llm: litellm_llm.default + - name: preprocess_sql_data + llm: litellm_llm.default + - name: sql_executor + engine: wren_ui + - name: chart_generation + llm: litellm_llm.default + - name: chart_adjustment + llm: litellm_llm.default + - name: user_guide_assistance + llm: litellm_llm.default + - name: sql_question_generation + llm: litellm_llm.default + - name: sql_generation_reasoning + llm: litellm_llm.default + - name: followup_sql_generation_reasoning + llm: litellm_llm.default + - name: sql_regeneration + llm: litellm_llm.default + engine: wren_ui + - name: instructions_indexing + embedder: litellm_embedder.default + document_store: qdrant + - name: instructions_retrieval + embedder: litellm_embedder.default + document_store: qdrant + - name: sql_functions_retrieval + engine: wren_ibis + document_store: qdrant + - name: project_meta_indexing + document_store: qdrant + - name: sql_tables_extraction + llm: litellm_llm.default + +--- +settings: + + # langfuse_host: https://cloud.langfuse.com + # langfuse_enable: true + # logging_level: DEBUG + # development: false + # historical_question_retrieval_similarity_threshold: 0.9 + # sql_pairs_similarity_threshold: 0.7 + # sql_pairs_retrieval_max_size: 10 + # instructions_similarity_threshold: 0.7 + # instructions_top_k: 10 \ No newline at end of file From d42e925915438ccf00bc38997ba22237377704e8 Mon Sep 17 00:00:00 2001 From: tatuke Date: Sun, 22 Jun 2025 17:29:11 +0800 Subject: [PATCH 2/5] Initial commit on opencodex_ph4local --- src/open_codex/agent_builder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/open_codex/agent_builder.py b/src/open_codex/agent_builder.py index 6bddeae..96e586d 100644 --- a/src/open_codex/agent_builder.py +++ b/src/open_codex/agent_builder.py @@ -10,11 +10,11 @@ def get_system_prompt() -> str: .joinpath("prompt.txt") \ .read_text(encoding="utf-8") - @staticmethod - def get_phi_agent() -> LLMAgent: - from open_codex.agents.phi_4_mini_agent import Phi4MiniAgent - system_prompt = AgentBuilder.get_system_prompt() - return Phi4MiniAgent(system_prompt=system_prompt) + # @staticmethod + # def get_phi_agent() -> LLMAgent: + # from open_codex.agents.phi_4_mini_agent import Phi4MiniAgent + # system_prompt = AgentBuilder.get_system_prompt() + # return Phi4MiniAgent(system_prompt=system_prompt) @staticmethod def get_ollama_agent(model: str, host: str) -> LLMAgent: From fa5f34ca2cba3632bb08ff84a59c97cb1b7d4ef0 Mon Sep 17 00:00:00 2001 From: tatuke Date: Tue, 24 Jun 2025 00:40:47 +0800 Subject: [PATCH 3/5] add system info detection --- .gitignore | 3 +++ pyproject.toml | 1 + src/open_codex/agent_builder.py | 10 +++++----- src/open_codex/interfaces/llm_agent.py | 2 +- src/open_codex/main.py | 25 ++++++++++++++++++++----- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index aecef50..089e348 100644 --- a/.gitignore +++ b/.gitignore @@ -189,6 +189,8 @@ venv/ ENV/ env.bak/ venv.bak/ +config.example.yaml + # Spyder project settings .spyderproject @@ -238,3 +240,4 @@ pyrightconfig.json brew_formula/ .pypi_token +comm.py diff --git a/pyproject.toml b/pyproject.toml index 202000b..68a5236 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "prompt_toolkit", "pyinstaller>=6.13.0", "pyperclip>=1.9.0", + "distro=1.9.0", ] [project.scripts] diff --git a/src/open_codex/agent_builder.py b/src/open_codex/agent_builder.py index 96e586d..6bddeae 100644 --- a/src/open_codex/agent_builder.py +++ b/src/open_codex/agent_builder.py @@ -10,11 +10,11 @@ def get_system_prompt() -> str: .joinpath("prompt.txt") \ .read_text(encoding="utf-8") - # @staticmethod - # def get_phi_agent() -> LLMAgent: - # from open_codex.agents.phi_4_mini_agent import Phi4MiniAgent - # system_prompt = AgentBuilder.get_system_prompt() - # return Phi4MiniAgent(system_prompt=system_prompt) + @staticmethod + def get_phi_agent() -> LLMAgent: + from open_codex.agents.phi_4_mini_agent import Phi4MiniAgent + system_prompt = AgentBuilder.get_system_prompt() + return Phi4MiniAgent(system_prompt=system_prompt) @staticmethod def get_ollama_agent(model: str, host: str) -> LLMAgent: diff --git a/src/open_codex/interfaces/llm_agent.py b/src/open_codex/interfaces/llm_agent.py index 8b4a1a2..f201a46 100644 --- a/src/open_codex/interfaces/llm_agent.py +++ b/src/open_codex/interfaces/llm_agent.py @@ -2,6 +2,6 @@ class LLMAgent(ABC): @abstractmethod - def one_shot_mode(self, user_input: str) -> str: + def one_shot_mode(self, user_input: str, system_info: str) -> str: pass \ No newline at end of file diff --git a/src/open_codex/main.py b/src/open_codex/main.py index 0148002..6390a46 100644 --- a/src/open_codex/main.py +++ b/src/open_codex/main.py @@ -1,6 +1,7 @@ import sys import argparse import subprocess +import platform from open_codex.agent_builder import AgentBuilder from open_codex.interfaces.llm_agent import LLMAgent @@ -30,6 +31,20 @@ def get_keypress(): finally: termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) return key +# sys details +def get_system_info(): + system = platform.system() + release = platform.release() + version = platform.version() + info = f"OSsystem: {system} {release} {version}" + if system == "Linux": + try: + import distro + info += f"\nDistribution: {distro.name(pretty=True)}" + except ImportError: + pass + return info + def get_user_action(): print(f"{BLUE}What do you want to do with this command?{RESET}") @@ -65,9 +80,10 @@ def get_agent(args: argparse.Namespace) -> LLMAgent: print(f"{BLUE}Using model: phi-4-mini-instruct{RESET}") return AgentBuilder.get_phi_agent() -def run_one_shot(agent: LLMAgent, user_prompt: str) -> str: +def run_one_shot(agent: LLMAgent, user_prompt: str, system_info: str) -> str: + full_prompt = f"{user_prompt}\n\nSystem info: {system_info}" try: - return agent.one_shot_mode(user_prompt) + return agent.one_shot_mode(full_prompt) except ConnectionError: print(f"{RED}Could not connect to Model.{RESET}", file=sys.stderr) sys.exit(1) @@ -105,10 +121,9 @@ def parse_args() -> argparse.Namespace: def main(): args = parse_args() agent = get_agent(args) - - # join the prompt arguments into a single string + system_info = get_system_info() prompt = " ".join(args.prompt).strip() - response = run_one_shot(agent, prompt) + response = run_one_shot(agent, prompt, system_info) print_response(response) action = get_user_action() run_user_action(action, response) From 41aed7a57b068c19c3c13a0b07cf05c5fee889f4 Mon Sep 17 00:00:00 2001 From: tatuke Date: Tue, 24 Jun 2025 01:58:11 +0800 Subject: [PATCH 4/5] fix llm_agent illegal argument --- src/open_codex/interfaces/llm_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/open_codex/interfaces/llm_agent.py b/src/open_codex/interfaces/llm_agent.py index f201a46..8b4a1a2 100644 --- a/src/open_codex/interfaces/llm_agent.py +++ b/src/open_codex/interfaces/llm_agent.py @@ -2,6 +2,6 @@ class LLMAgent(ABC): @abstractmethod - def one_shot_mode(self, user_input: str, system_info: str) -> str: + def one_shot_mode(self, user_input: str) -> str: pass \ No newline at end of file From ca763250ac1c1e9f9cdbce4e26968d0efc87af44 Mon Sep 17 00:00:00 2001 From: tatuke Date: Tue, 24 Jun 2025 02:05:09 +0800 Subject: [PATCH 5/5] remove temp file yaml --- .gitignore | 1 + src/open_codex/resources/config.example.yaml | 157 ------------------- 2 files changed, 1 insertion(+), 157 deletions(-) delete mode 100644 src/open_codex/resources/config.example.yaml diff --git a/.gitignore b/.gitignore index 089e348..6e0f18a 100644 --- a/.gitignore +++ b/.gitignore @@ -189,6 +189,7 @@ venv/ ENV/ env.bak/ venv.bak/ + config.example.yaml diff --git a/src/open_codex/resources/config.example.yaml b/src/open_codex/resources/config.example.yaml deleted file mode 100644 index 2d50ca8..0000000 --- a/src/open_codex/resources/config.example.yaml +++ /dev/null @@ -1,157 +0,0 @@ -type: llm -provider: litellm_llm -timeout: 120 -models: - - alias: default - model: gpt-4.1-nano-2025-04-14 - context_window_size: 1000000 - kwargs: - max_tokens: 4096 - n: 1 - seed: 0 - temperature: 0 - - model: gpt-4.1-mini-2025-04-14 - context_window_size: 1000000 - kwargs: - max_tokens: 4096 - n: 1 - seed: 0 - temperature: 0 - - model: gpt-4.1-2025-04-14 - context_window_size: 1000000 - kwargs: - max_tokens: 4096 - n: 1 - seed: 0 - temperature: 0 - ---- -type: embedder -provider: litellm_embedder -models: - - model: text-embedding-3-large - alias: default - timeout: 120 - ---- -type: engine -provider: wren_ui -endpoint: http://wren-ui:3000 - ---- -type: engine -provider: wren_ibis -endpoint: http://ibis-server:8000 - ---- -type: document_store -provider: qdrant -location: http://qdrant:6333 -embedding_model_dim: 3072 -timeout: 120 -recreate_index: true - ---- -type: pipeline -pipes: - - name: db_schema_indexing - embedder: litellm_embedder.default - document_store: qdrant - - name: historical_question_indexing - embedder: litellm_embedder.default - document_store: qdrant - - name: table_description_indexing - embedder: litellm_embedder.default - document_store: qdrant - - name: db_schema_retrieval - llm: litellm_llm.default - embedder: litellm_embedder.default - document_store: qdrant - - name: historical_question_retrieval - embedder: litellm_embedder.default - document_store: qdrant - - name: sql_generation - llm: litellm_llm.default - engine: wren_ui - - name: sql_correction - llm: litellm_llm.default - engine: wren_ui - - name: followup_sql_generation - llm: litellm_llm.default - engine: wren_ui - - name: sql_answer - llm: litellm_llm.default - - name: semantics_description - llm: litellm_llm.default - - name: relationship_recommendation - llm: litellm_llm.default - engine: wren_ui - - name: question_recommendation - llm: litellm_llm.default - - name: question_recommendation_db_schema_retrieval - llm: litellm_llm.default - embedder: litellm_embedder.default - document_store: qdrant - - name: question_recommendation_sql_generation - llm: litellm_llm.default - engine: wren_ui - - name: intent_classification - llm: litellm_llm.default - embedder: litellm_embedder.default - document_store: qdrant - - name: misleading_assistance - llm: litellm_llm.default - - name: data_assistance - llm: litellm_llm.default - - name: sql_pairs_indexing - document_store: qdrant - embedder: litellm_embedder.default - - name: sql_pairs_retrieval - document_store: qdrant - embedder: litellm_embedder.default - llm: litellm_llm.default - - name: preprocess_sql_data - llm: litellm_llm.default - - name: sql_executor - engine: wren_ui - - name: chart_generation - llm: litellm_llm.default - - name: chart_adjustment - llm: litellm_llm.default - - name: user_guide_assistance - llm: litellm_llm.default - - name: sql_question_generation - llm: litellm_llm.default - - name: sql_generation_reasoning - llm: litellm_llm.default - - name: followup_sql_generation_reasoning - llm: litellm_llm.default - - name: sql_regeneration - llm: litellm_llm.default - engine: wren_ui - - name: instructions_indexing - embedder: litellm_embedder.default - document_store: qdrant - - name: instructions_retrieval - embedder: litellm_embedder.default - document_store: qdrant - - name: sql_functions_retrieval - engine: wren_ibis - document_store: qdrant - - name: project_meta_indexing - document_store: qdrant - - name: sql_tables_extraction - llm: litellm_llm.default - ---- -settings: - - # langfuse_host: https://cloud.langfuse.com - # langfuse_enable: true - # logging_level: DEBUG - # development: false - # historical_question_retrieval_similarity_threshold: 0.9 - # sql_pairs_similarity_threshold: 0.7 - # sql_pairs_retrieval_max_size: 10 - # instructions_similarity_threshold: 0.7 - # instructions_top_k: 10 \ No newline at end of file