From a4b8e9d82a62a894afeb2010006064de1c81a487 Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Tue, 13 May 2025 18:42:00 +0100 Subject: [PATCH 1/7] - Created PromptRules class to read saist.rules from the working directory - PROMPT_OVERRIDE to fully replace the prompt - PROMPT_PRE and PROMPT_POST to modify prompt before/after - Integrated apply_rules into analyze_single_file to adjust the prompt - Falls back to original prompt if saist.rules doesn't exist or is empty --- saist/main.py | 6 ++++-- saist/util/rules.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 saist/util/rules.py diff --git a/saist/main.py b/saist/main.py index 947393c..281340f 100644 --- a/saist/main.py +++ b/saist/main.py @@ -31,6 +31,8 @@ from util.output import print_banner, write_csv +from util.rules import PromptRules + prompts = prompts() load_dotenv(".env") @@ -42,9 +44,9 @@ async def analyze_single_file(scm: Scm, adapter: BaseLlmAdapter, filename, patch """ system_prompt = prompts.DETECT logger.debug(f"Processing {filename}") - prompt = ( + prompt = PromptRules.apply_rules( f"\n\nFile: {filename}\n{patch_text}\n" - ) + ) try: return (await adapter.prompt_structured(system_prompt, prompt, Findings, [scm.read_file_contents])).findings except Exception as e: diff --git a/saist/util/rules.py b/saist/util/rules.py new file mode 100644 index 0000000..e3890cd --- /dev/null +++ b/saist/util/rules.py @@ -0,0 +1,39 @@ +import os +import yaml + +class PromptRules: + RulesFile = "saist.rules" + + @staticmethod + def apply_rules(prompt): + rules = PromptRules.load_rules() + + override_prompt = rules.get("PROMPT_OVERRIDE") + if override_prompt: + #skip pre and post if override is present + return override_prompt + + pre = rules.get("PROMPT_PRE", "") + post = rules.get("PROMPT_POST", "") + + #add pre/post if they are present + return f"{pre}{prompt}{post}" + + @staticmethod + def load_rules(): + if not os.path.exists(PromptRules.RulesFile): + print("No saist.rules file found.") + return {} #return empty rules + + try: + with open(PromptRules.RulesFile, 'r') as file: + yaml_content = file.read() + rules = yaml.safe_load(yaml_content) #using safe_load to prevent exploits (thank you stack overflow) + return rules if rules is not None else {} + except Exception as ex: + print("Error reading saist.rules: " + str(ex)) + return {} + +#I first wrote this code in c# using dictionaries and a similar yaml parsing library for .net and then used a converter for python. +#I'm not too well versed with python yet but doing this excersise and seeing how things are converted, learning some syntax and hacking stuff together has been a massive help and has been really enjoyable! + From 5920f64df475447a0c6c0ec98e3bb9af94f49d3e Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Tue, 13 May 2025 19:50:52 +0100 Subject: [PATCH 2/7] - Applies rules to system_prompt instead of user prompt - PRE and POST no longer skipped when using OVERRIDE - Standardised logging - Added example rules file - Updated README --- README.md | 28 ++++++++++++++++++++++++++++ saist.rules | 8 ++++++++ saist/main.py | 4 ++-- saist/util/rules.py | 26 +++++++++++++++----------- 4 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 saist.rules diff --git a/README.md b/README.md index c0bba3a..d3670f1 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,34 @@ This setup will: --- +## 🧠 Custom Prompt Rules + +You can modify the system prompt used by creating a file named `saist.rules` in your working directory. + +This file supports the following YAML keys: + +PROMPT_OVERRIDE - Fully replaces the system prompt +PROMPT_PRE - Adds text before the existing prompt +PROMPT_POST - Adds text after the existing prompt + +If multiple keys are defined, `PRE` and `POST` will wrap around the override text. + +--- + +### 📝 `saist.rules.example` + +```yaml +# Example prompt modification + +# Use this to completely override the system prompt: +PROMPT_OVERRIDE: "Identify any potential vulnerabilites in the following code. Focus on input validation" + +# Use these to prepend or append context to the prompt +PROMPT_PRE: "You are reviewing critical code. Please be strict.\n\n" +PROMPT_POST: "\n\nPlease provide practical advice." + +--- + ## 🛣️ Roadmap - Ability to influence the prompts diff --git a/saist.rules b/saist.rules new file mode 100644 index 0000000..3e940f5 --- /dev/null +++ b/saist.rules @@ -0,0 +1,8 @@ +# Example prompt modification + +# Use this to completely override the system prompt: +# PROMPT_OVERRIDE: "Identify any potential vulnerabilites in the following code. Focus on input validation" + +# Use these to prepend or append context to the prompt +PROMPT_PRE: "You are reviewing critical code. Please be strict.\n\n" +PROMPT_POST: "\n\nPlease provide practical advice." diff --git a/saist/main.py b/saist/main.py index 281340f..47dc488 100644 --- a/saist/main.py +++ b/saist/main.py @@ -42,9 +42,9 @@ async def analyze_single_file(scm: Scm, adapter: BaseLlmAdapter, filename, patch """ Analyzes a SINGLE file diff with OpenAI, returning a Findings object or None on error. """ - system_prompt = prompts.DETECT + system_prompt = PromptRules.apply_rules(prompts.DETECT) logger.debug(f"Processing {filename}") - prompt = PromptRules.apply_rules( + prompt =( f"\n\nFile: {filename}\n{patch_text}\n" ) try: diff --git a/saist/util/rules.py b/saist/util/rules.py index e3890cd..83ddfb4 100644 --- a/saist/util/rules.py +++ b/saist/util/rules.py @@ -1,5 +1,8 @@ import os import yaml +import logging + +logger = logging.getLogger("saist") class PromptRules: RulesFile = "saist.rules" @@ -7,31 +10,32 @@ class PromptRules: @staticmethod def apply_rules(prompt): rules = PromptRules.load_rules() - - override_prompt = rules.get("PROMPT_OVERRIDE") - if override_prompt: - #skip pre and post if override is present - return override_prompt - + + override = rules.get("PROMPT_OVERRIDE", prompt) pre = rules.get("PROMPT_PRE", "") post = rules.get("PROMPT_POST", "") - - #add pre/post if they are present - return f"{pre}{prompt}{post}" + return f"{pre}{override}{post}" @staticmethod def load_rules(): if not os.path.exists(PromptRules.RulesFile): - print("No saist.rules file found.") + logger.warning("No saist.rules file found.") return {} #return empty rules try: with open(PromptRules.RulesFile, 'r') as file: yaml_content = file.read() rules = yaml.safe_load(yaml_content) #using safe_load to prevent exploits (thank you stack overflow) + + keys = [key for key in ["PROMPT_OVERRIDE", "PROMPT_PRE", "PROMPT_POST"] if key in rules] + if keys: + logger.debug(f"Loaded prompt rules: {', '.join(keys)}") + else: + logger.debug("No valid keys found.") + return rules if rules is not None else {} except Exception as ex: - print("Error reading saist.rules: " + str(ex)) + logger.error(f"Error reading saist.rules: {ex}") return {} #I first wrote this code in c# using dictionaries and a similar yaml parsing library for .net and then used a converter for python. From a90c0ed546c3cb857511260442e2b1d0103a9067 Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Tue, 13 May 2025 20:09:25 +0100 Subject: [PATCH 3/7] - Changed logger name to use a module-level logger --- saist/util/rules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saist/util/rules.py b/saist/util/rules.py index 83ddfb4..a92e670 100644 --- a/saist/util/rules.py +++ b/saist/util/rules.py @@ -2,7 +2,7 @@ import yaml import logging -logger = logging.getLogger("saist") +logger = logging.getLogger(__name__) class PromptRules: RulesFile = "saist.rules" @@ -32,7 +32,7 @@ def load_rules(): logger.debug(f"Loaded prompt rules: {', '.join(keys)}") else: logger.debug("No valid keys found.") - + return rules if rules is not None else {} except Exception as ex: logger.error(f"Error reading saist.rules: {ex}") From d56a9162054937d4fb53f166f8ae1f5f80bb769e Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Tue, 13 May 2025 20:25:05 +0100 Subject: [PATCH 4/7] -Closed yaml block on README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d3670f1..7443d4d 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,7 @@ If multiple keys are defined, `PRE` and `POST` will wrap around the override tex ### 📝 `saist.rules.example` + ```yaml # Example prompt modification @@ -180,6 +181,7 @@ PROMPT_OVERRIDE: "Identify any potential vulnerabilites in the following code. F # Use these to prepend or append context to the prompt PROMPT_PRE: "You are reviewing critical code. Please be strict.\n\n" PROMPT_POST: "\n\nPlease provide practical advice." +``` --- From a79d3799c8c67feb414db33e78bc5f17710e4e85 Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Tue, 13 May 2025 21:13:56 +0100 Subject: [PATCH 5/7] - Amended apply_rules to work properly with prompts - Changed system prompt definition within analyze_single_file to match the updated logic --- saist/main.py | 2 +- saist/util/rules.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/saist/main.py b/saist/main.py index 47dc488..a096cb6 100644 --- a/saist/main.py +++ b/saist/main.py @@ -42,7 +42,7 @@ async def analyze_single_file(scm: Scm, adapter: BaseLlmAdapter, filename, patch """ Analyzes a SINGLE file diff with OpenAI, returning a Findings object or None on error. """ - system_prompt = PromptRules.apply_rules(prompts.DETECT) + system_prompt = PromptRules.apply_rules(prompts.DETECT_PRE, prompts.DETECT_POST) logger.debug(f"Processing {filename}") prompt =( f"\n\nFile: {filename}\n{patch_text}\n" diff --git a/saist/util/rules.py b/saist/util/rules.py index a92e670..f5eb265 100644 --- a/saist/util/rules.py +++ b/saist/util/rules.py @@ -8,13 +8,16 @@ class PromptRules: RulesFile = "saist.rules" @staticmethod - def apply_rules(prompt): + def apply_rules(pre_prompt: str, post_prompt: str = "") -> str: rules = PromptRules.load_rules() - override = rules.get("PROMPT_OVERRIDE", prompt) + override = rules.get("PROMPT_OVERRIDE") pre = rules.get("PROMPT_PRE", "") post = rules.get("PROMPT_POST", "") - return f"{pre}{override}{post}" + + final_prompt = override if override else (pre_prompt + post_prompt) #if override isn't specified, this should just use the prefix and suffix with the orig prompt + return f"{pre}{final_prompt}{post}" + @staticmethod def load_rules(): From a136131f0cdefb502ce15eee74099c3f5606b09d Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Thu, 15 May 2025 07:34:34 +0100 Subject: [PATCH 6/7] -Ammended apply rules to work as intended -Ammened system prompt back to DIRECT for simplicity - Moved return out of else in load_rules --- saist/main.py | 2 +- saist/util/rules.py | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/saist/main.py b/saist/main.py index a096cb6..47dc488 100644 --- a/saist/main.py +++ b/saist/main.py @@ -42,7 +42,7 @@ async def analyze_single_file(scm: Scm, adapter: BaseLlmAdapter, filename, patch """ Analyzes a SINGLE file diff with OpenAI, returning a Findings object or None on error. """ - system_prompt = PromptRules.apply_rules(prompts.DETECT_PRE, prompts.DETECT_POST) + system_prompt = PromptRules.apply_rules(prompts.DETECT) logger.debug(f"Processing {filename}") prompt =( f"\n\nFile: {filename}\n{patch_text}\n" diff --git a/saist/util/rules.py b/saist/util/rules.py index f5eb265..72d5604 100644 --- a/saist/util/rules.py +++ b/saist/util/rules.py @@ -8,15 +8,22 @@ class PromptRules: RulesFile = "saist.rules" @staticmethod - def apply_rules(pre_prompt: str, post_prompt: str = "") -> str: + def apply_rules(prompt: str) -> str: rules = PromptRules.load_rules() - + override = rules.get("PROMPT_OVERRIDE") - pre = rules.get("PROMPT_PRE", "") - post = rules.get("PROMPT_POST", "") + pre = rules.get("PROMPT_PRE") + post = rules.get("PROMPT_POST") + + final_prompt = override if override else prompt + + if pre: + final_prompt = f"{pre}{final_prompt}" + if post: + final_prompt = f"{final_prompt}{post}" + + return final_prompt - final_prompt = override if override else (pre_prompt + post_prompt) #if override isn't specified, this should just use the prefix and suffix with the orig prompt - return f"{pre}{final_prompt}{post}" @staticmethod @@ -36,7 +43,7 @@ def load_rules(): else: logger.debug("No valid keys found.") - return rules if rules is not None else {} + return rules if rules is not None else {} except Exception as ex: logger.error(f"Error reading saist.rules: {ex}") return {} From b133e4f5c9e190a57f7fb6ed573410f6425b0e96 Mon Sep 17 00:00:00 2001 From: Matthew Fisk Date: Thu, 15 May 2025 08:59:23 +0100 Subject: [PATCH 7/7] -Ammended apply rules again, my last itteration was overwriting the original prompt with the new pre and post. I don't think this is the desired intention. -final_prompt should now contain pre and post (if they are present) wrapping either the original detect prompt or the overridden detect prompt --- saist/util/rules.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/saist/util/rules.py b/saist/util/rules.py index 72d5604..0b4c607 100644 --- a/saist/util/rules.py +++ b/saist/util/rules.py @@ -12,20 +12,16 @@ def apply_rules(prompt: str) -> str: rules = PromptRules.load_rules() override = rules.get("PROMPT_OVERRIDE") - pre = rules.get("PROMPT_PRE") - post = rules.get("PROMPT_POST") + pre = rules.get("PROMPT_PRE", "") + post = rules.get("PROMPT_POST", "") - final_prompt = override if override else prompt - - if pre: - final_prompt = f"{pre}{final_prompt}" - if post: - final_prompt = f"{final_prompt}{post}" + if override: + final_prompt = f"{pre}{override}{post}" + else: + final_prompt = f"{pre}{prompt}{post}" return final_prompt - - @staticmethod def load_rules(): if not os.path.exists(PromptRules.RulesFile):