diff --git a/CLIENT/add_message.py b/CLIENT/add_message.py new file mode 100644 index 0000000..0ade354 --- /dev/null +++ b/CLIENT/add_message.py @@ -0,0 +1,17 @@ +import requests +import time + +url = 'http://localhost:8000/add_message' + +user = input('User: ') +message = input('Enter your message: ') +timestamp = time.time() + +response = requests.post(url, params={ + 'message': message, + 'speaker': user, + 'timestamp': timestamp +}) + +print(response.status_code) +print(response.json()) diff --git a/CLIENT/maintain_tree.py b/CLIENT/maintain_tree.py new file mode 100644 index 0000000..f503944 --- /dev/null +++ b/CLIENT/maintain_tree.py @@ -0,0 +1,9 @@ +import requests +import time + +url = 'http://localhost:8000/maintain_tree' + +response = requests.post(url) + +print(response.status_code) +print(response.json()) diff --git a/CLIENT/rebuild_tree.py b/CLIENT/rebuild_tree.py new file mode 100644 index 0000000..b580617 --- /dev/null +++ b/CLIENT/rebuild_tree.py @@ -0,0 +1,9 @@ +import requests +import time + +url = 'http://localhost:8000/rebuild_tree' + +response = requests.post(url) + +print(response.status_code) +print(response.json()) \ No newline at end of file diff --git a/CLIENT/search.py b/CLIENT/search.py new file mode 100644 index 0000000..d5abc80 --- /dev/null +++ b/CLIENT/search.py @@ -0,0 +1,11 @@ +import requests +import time + +url = 'http://localhost:8000/search' + +query = input('search: ') + +response = requests.get(url, params={'query': query}) + +print(response.status_code) +print(response.json()) diff --git a/README.md b/README.md index 2d4053b..6ceed15 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,18 @@ To run REMO, you will need the following: - openai - PyYAML +## Installation + +Note: You may need to change tensorflow to tensowflow-macos in your requirements.txt file on certain OS X machines. + +1. Run `pip install -r requirements.txt` +2. Create key_openai.txt file and put your OpenAI API key inside. ## Usage 1. Start the FastAPI server: `uvicorn remo:app --reload` 2. Interact with the API using a REST client or web browser: `http://localhost:8000` - ## API Endpoints - **POST /add_message**: Add a new message to REMO. Speaker, timestamp, and content required. diff --git a/utils.py b/utils.py index de0be3a..194c247 100644 --- a/utils.py +++ b/utils.py @@ -10,30 +10,36 @@ import tensorflow_hub as hub -embedding_model = hub.load("https://tfhub.dev/google/universal-sentence-encoder-large/5") +embedding_model = hub.load( + "https://tfhub.dev/google/universal-sentence-encoder-large/5") def open_file(filepath): with open(filepath, 'r', encoding='utf-8', errors='ignore') as infile: return infile.read() + def save_file(filepath, content): with open(filepath, 'w', encoding='utf-8') as outfile: outfile.write(content) + def save_yaml(filepath, data): with open(filepath, 'w', encoding='utf-8') as file: yaml.dump(data, file, allow_unicode=True) + def load_yaml(filepath): with open(filepath, 'r', encoding='utf-8') as file: data = yaml.load(file, Loader=yaml.FullLoader) return data + def create_message(message: str, speaker: str, timestamp: float) -> Dict[str, Any]: # Create message dictionary return {"content": message, "speaker": speaker, "timestamp": timestamp} + def save_message(root_folder, message: Dict[str, Any]): timestamp, speaker = message['timestamp'], message['speaker'] filename = f"chat_{timestamp}_{speaker}.yaml" @@ -41,6 +47,7 @@ def save_message(root_folder, message: Dict[str, Any]): os.makedirs(os.path.dirname(filepath), exist_ok=True) save_yaml(filepath, message) + def search_tree(root_folder, query): # TODO add a "forks" parameter to allow for branching relevance # TODO add a "fuzziness" parameter that can generate a random vector to modify the search query @@ -55,13 +62,15 @@ def search_tree(root_folder, query): level -= 1 while level > 2: - level_files = [os.path.join(level_dir, f) for f in os.listdir(level_dir) if f.endswith(".yaml")] + level_files = [os.path.join(level_dir, f) for f in os.listdir( + level_dir) if f.endswith(".yaml")] max_similarity = -1 closest_file = None for file in level_files: data = load_yaml(file) - similarity = cosine_similarity(query_embedding, np.array(data["vector"]).reshape(1, -1))[0][0] + similarity = cosine_similarity( + query_embedding, np.array(data["vector"]).reshape(1, -1))[0][0] if similarity > max_similarity: max_similarity = similarity @@ -82,9 +91,9 @@ def search_tree(root_folder, query): def rebuild_tree(root_folder: str, max_cluster_size: int = 10): - # Delete all folders except L1_raw_logs, L2_message_pairs and .git + # Delete all folders except L1_raw_logs, L2_message_pairs, .git, CLIENT and REMO for folder_name in os.listdir(root_folder): - if folder_name not in {"L1_raw_logs", "L2_message_pairs", ".git"}: + if folder_name not in {"L1_raw_logs", "L2_message_pairs", ".git", "CLIENT", "REMO"}: folder_path = os.path.join(root_folder, folder_name) if os.path.isdir(folder_path): shutil.rmtree(folder_path) @@ -98,22 +107,26 @@ def rebuild_tree(root_folder: str, max_cluster_size: int = 10): process_missing_messages(root_folder) # Cluster L2 message pairs using cosine similarity, up to 10 per cluster - clusters = cluster_elements(root_folder, "L2_message_pairs", max_cluster_size) + clusters = cluster_elements( + root_folder, "L2_message_pairs", max_cluster_size) # Create summaries and save them in the next rank (L3_summaries) - create_summaries(root_folder, clusters, f"L3_summaries", "L2_message_pairs") + create_summaries(root_folder, clusters, + f"L3_summaries", "L2_message_pairs") # If top rank (e.g. L3_summaries) has > max_cluster_size files, repeat process, creating new taxonomical ranks current_rank = 3 while True: # calculate clusters at new rank - clusters = cluster_elements(root_folder, f"L{current_rank}_summaries", max_cluster_size) - + clusters = cluster_elements( + root_folder, f"L{current_rank}_summaries", max_cluster_size) + # summarize those clusters - create_summaries(root_folder, clusters, f"L{current_rank + 1}_summaries", f"L{current_rank}_summaries") + create_summaries(root_folder, clusters, + f"L{current_rank + 1}_summaries", f"L{current_rank}_summaries") current_rank += 1 - - # if clusters less than max cluster size, we are done :) + + # if clusters less than max cluster size, we are done :) if len(clusters) <= max_cluster_size: break @@ -157,9 +170,11 @@ def process_missing_messages(root_folder: str): } # Save message pair in L2_message_pairs folder - message_pair_path = os.path.join(message_pairs_dir, message_pair_filename) + message_pair_path = os.path.join( + message_pairs_dir, message_pair_filename) save_yaml(message_pair_path, message_pair_data) + def create_summaries(root_folder: str, clusters: List[List[str]], target_folder: str, source_folder: str): source_folder_path = os.path.join(root_folder, source_folder) target_folder_path = os.path.join(root_folder, target_folder) @@ -219,84 +234,94 @@ def cluster_elements(root_folder: str, target_folder: str, max_cluster_size: int return clusters + def maintain_tree(root_folder: str): l2_message_pairs_dir = os.path.join(root_folder, "L2_message_pairs") - + # Create L2 directory if it does not exist if not os.path.exists(l2_message_pairs_dir): os.makedirs(l2_message_pairs_dir) - + # Get list of files in L2 before processing missing messages l2_files_before = set(os.listdir(l2_message_pairs_dir)) - + # Process missing messages to generate new message pairs in L2 process_missing_messages(root_folder) - + # Get list of files in L2 after processing missing messages l2_files_after = set(os.listdir(l2_message_pairs_dir)) - + # Calculate the difference between the two lists to obtain the new message pairs new_message_pairs = l2_files_after - l2_files_before - #new_message_pairs = [os.path.join("L2_message_pairs", f) for f in l2_files_after - l2_files_before] - + # new_message_pairs = [os.path.join("L2_message_pairs", f) for f in l2_files_after - l2_files_before] + # Iterate through new files in L2 and check cosine similarity to files in L3 - integrate_new_elements(root_folder, "L3_summaries", new_message_pairs, 0.75) + integrate_new_elements(root_folder, "L3_summaries", + new_message_pairs, 0.75) + def integrate_new_elements(root_folder: str, target_folder: str, new_elements: List[str], threshold: float): target_dir = os.path.join(root_folder, target_folder) - + # Create target directory if it does not exist if not os.path.exists(target_dir): os.makedirs(target_dir) - + for new_element in new_elements: - new_element_path = os.path.join(root_folder, "L2_message_pairs", new_element) + new_element_path = os.path.join( + root_folder, "L2_message_pairs", new_element) new_element_data = load_yaml(new_element_path) - new_element_vector = np.array(new_element_data["vector"]).reshape(1, -1) - + new_element_vector = np.array( + new_element_data["vector"]).reshape(1, -1) + max_similarity = -1 closest_file = None - + for file in os.listdir(target_dir): file_path = os.path.join(target_dir, file) file_data = load_yaml(file_path) file_vector = np.array(file_data["vector"]).reshape(1, -1) - - similarity = cosine_similarity(new_element_vector, file_vector)[0][0] + + similarity = cosine_similarity( + new_element_vector, file_vector)[0][0] if similarity > max_similarity: max_similarity = similarity closest_file = file - + if max_similarity > threshold: # Update the corresponding summary and record the name of the modified file closest_file_path = os.path.join(target_dir, closest_file) closest_file_data = load_yaml(closest_file_path) closest_file_data["files"].append(new_element) - - combined_content = closest_file_data["content"] + " --- " + new_element_data["content"] + + combined_content = closest_file_data["content"] + \ + " --- " + new_element_data["content"] updated_summary = quick_summarize(combined_content) - updated_summary_embedding = embedding_model([updated_summary]).numpy().tolist() - + updated_summary_embedding = embedding_model( + [updated_summary]).numpy().tolist() + closest_file_data["content"] = updated_summary closest_file_data["vector"] = updated_summary_embedding closest_file_data["timestamp"] = time() - + save_yaml(closest_file_path, closest_file_data) else: # Create a new summary for the new_element combined_content = new_element_data["content"] new_summary = quick_summarize(combined_content) - new_summary_embedding = embedding_model([new_summary]).numpy().tolist() - + new_summary_embedding = embedding_model( + [new_summary]).numpy().tolist() + new_summary_data = { "content": new_summary, "vector": new_summary_embedding, "files": [new_element], "timestamp": time() } - + new_summary_filename = f"summary_{len(os.listdir(target_dir))}.yaml" - new_summary_filepath = os.path.join(target_dir, new_summary_filename) + new_summary_filepath = os.path.join( + target_dir, new_summary_filename) save_yaml(new_summary_filepath, new_summary_data) @@ -305,19 +330,20 @@ def quick_summarize(text): if len(text) <= max_chunk_size: prompt = 'Write a detailed summary of the following:\n\n%s\n\nDETAILED SUMMARY:' % text - response = gpt3_completion(prompt) + response = gpt3_5_turbo_chat(prompt) return response else: # Split the text into evenly sized chunks num_chunks = int(np.ceil(len(text) / max_chunk_size)) chunk_size = int(np.ceil(len(text) / num_chunks)) - text_chunks = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)] + text_chunks = [text[i:i + chunk_size] + for i in range(0, len(text), chunk_size)] # Summarize each chunk summaries = [] for chunk in text_chunks: prompt = 'Write a detailed summary of the following:\n\n%s\n\nDETAILED SUMMARY:' % chunk - response = gpt3_completion(prompt) + response = gpt3_5_turbo_chat(prompt) summaries.append(response) # Concatenate the summaries and return the result @@ -325,7 +351,7 @@ def quick_summarize(text): return final_summary -def gpt3_completion(prompt, engine='text-davinci-003', temp=0.0, top_p=1.0, tokens=1000, freq_pen=0.0, pres_pen=0.0, stop=['asdfasdfasdf']): +""" def gpt3_completion(prompt, engine='text-davinci-003', temp=0.0, top_p=1.0, tokens=1000, freq_pen=0.0, pres_pen=0.0, stop=['asdfasdfasdf']): openai.api_key = open_file('key_openai.txt') max_retry = 5 retry = 0 @@ -354,4 +380,39 @@ def gpt3_completion(prompt, engine='text-davinci-003', temp=0.0, top_p=1.0, toke if retry >= max_retry: return "GPT3 error: %s" % oops print('Error communicating with OpenAI:', oops) + sleep(1) """ + + +def gpt3_5_turbo_chat(prompt, model='gpt-3.5-turbo', temp=0.0, top_p=1.0, tokens=1000, freq_pen=0.0, pres_pen=0.0, stop=['asdfasdfasdf']): + openai.api_key = open_file('key_openai.txt') + max_retry = 5 + retry = 0 + prompt = prompt.encode(encoding='ASCII', errors='ignore').decode() + + messages = [{'role': 'user', 'content': prompt}] + + while True: + try: + response = openai.ChatCompletion.create( + model=model, + messages=messages, + temperature=temp, + max_tokens=tokens, + top_p=top_p, + frequency_penalty=freq_pen, + presence_penalty=pres_pen, + stop=stop + ) + text = response['choices'][0]['message']['content'].strip() + filename = '%s_gpt3_5_turbo.txt' % time() + if not os.path.exists('gpt3_logs'): + os.makedirs('gpt3_logs') + save_file('gpt3_logs/%s' % filename, prompt + + '\n\n==========\n\n' + text) + return text + except Exception as oops: + retry += 1 + if retry >= max_retry: + return "GPT-3.5 Turbo error: %s" % oops + print('Error communicating with OpenAI:', oops) sleep(1)