Add interactive search script

interactive_search/README.md

@@ -0,0 +1,26 @@
+# Script for Working with the Marketplace Search API
+
+This directory provides convenience scripts for interacting with the search API of the
+Marketplace, and for running basic evaluations. 
+
+### Interactive Search
+
+To run the interactive search script, use the following command:
+
+```bash
+python interactive_search.py --data-dir ../data/mexican_3_9
+```
+
+### Evaluation
+
+Evaluation is done by randomly sampling a menu item from EACH restaurant in the dataset,
+and for each item, determining the rank of the FIRST restaurant that contains that item.
+
+From these ranks we compute [mean reciprocal rank](https://en.wikipedia.org/wiki/Mean_reciprocal_rank) (MRR) as the evaluation metric.
+
+Values closer to 1.0 are better, with 1.0 being perfect.
+
+To run the evaluation script, use the following command:
+```
+python menu_mrr.py --data-dir ../data/mexican_3_9       
+```

interactive_search/interactive_search.py

@@ -0,0 +1,85 @@
+"""A simple interactive search client for the agentic-economics marketplace."""
+
+import argparse
+import asyncio
+
+from search_launcher import SearchMarketLauncher
+
+
+async def main(
+    data_dir: str,
+    postgres_host: str,
+    postgres_port: int,
+    postgres_password: str,
+    search_algorithm: str = "lexical",
+    show_all_searchable_text: bool = False,
+) -> None:
+    """Run a simple interactive search client for the agentic-economics marketplace."""
+    search_launcher = SearchMarketLauncher(
+        data_dir=data_dir,
+        postgres_host=postgres_host,
+        postgres_port=postgres_port,
+        postgres_password=postgres_password,
+        search_algorithm=search_algorithm,
+    )
+
+    async with search_launcher.start() as _:
+        # Start interactive search loop
+        await search_launcher.interactive_search(
+            show_all_searchable_text=show_all_searchable_text
+        )
+
+        # Shutdown
+        await search_launcher.stop()
+
+
+if __name__ == "__main__":
+    # Add argument parsing for data dir using argparse
+    parser = argparse.ArgumentParser(
+        description="Interactive search client for the multi-agent-marketplace"
+    )
+    parser.add_argument(
+        "--data-dir", help="Path to the dataset directory", required=True
+    )
+    parser.add_argument(
+        "--search-algorithm",
+        default="lexical",
+        help="Search algorithm to use (default: lexical)",
+    )
+    parser.add_argument(
+        "--postgres-host",
+        default="localhost",
+        help="PostgreSQL host (default: localhost)",
+    )
+
+    parser.add_argument(
+        "--postgres-port",
+        type=int,
+        default=5432,
+        help="PostgreSQL port (default: 5432)",
+    )
+
+    parser.add_argument(
+        "--postgres-password",
+        default="postgres",
+        help="PostgreSQL password (default: postgres)",
+    )
+
+    parser.add_argument(
+        "--show-all-searchable-text",
+        action="store_true",
+        help="Show all searchable text for searched businesses for debugging (default: False)",
+    )
+
+    args = parser.parse_args()
+
+    asyncio.run(
+        main(
+            args.data_dir,
+            args.postgres_host,
+            args.postgres_port,
+            args.postgres_password,
+            args.search_algorithm,
+            args.show_all_searchable_text,
+        )
+    )

interactive_search/menu_mrr.py

@@ -0,0 +1,126 @@
+"""Run search queries based on menu items and evaluate using Mean Reciprocal Rank (MRR)."""
+
+import argparse
+import asyncio
+import random
+from pathlib import Path
+
+from magentic_marketplace.experiments.utils.yaml_loader import load_businesses_from_yaml
+from magentic_marketplace.marketplace.shared.models import BusinessAgentProfile
+from search_launcher import SearchMarketLauncher
+
+
+def has_items(items: list[str], business: BusinessAgentProfile) -> bool:
+    """Check if the business has all the specified menu items."""
+    menu_items = set(business.business.menu_features.keys())
+    return set(items).issubset(menu_items)
+
+
+async def main(
+    data_dir: str,
+    postgres_host: str,
+    postgres_port: int,
+    postgres_password: str,
+    order_size: int = 1,
+    search_algorithm: str = "lexical",
+):
+    """Evaluate search functionality using Mean Reciprocal Rank (MRR) based on menu items."""
+    # Start the search market server before running this script
+    search_launcher = SearchMarketLauncher(
+        data_dir=data_dir,
+        postgres_host=postgres_host,
+        postgres_port=postgres_port,
+        postgres_password=postgres_password,
+        search_algorithm=search_algorithm,
+    )
+    async with search_launcher.start() as _:
+        reciprocal_ranks: list[float] = []
+        businesses_dir = Path(args.data_dir) / "businesses"
+
+        businesses = load_businesses_from_yaml(businesses_dir)
+
+        for business in businesses:
+            menu_items = list(business.menu_features.keys())
+            sampled_items = random.sample(menu_items, min(order_size, len(menu_items)))
+
+            query = " ".join(sampled_items)
+            print(f"Searching for: {query}")
+            results = await search_launcher.search(query=query)
+
+            found = False
+            rank = 0
+            for rank in range(len(results)):
+                result = results[rank]
+                if has_items(sampled_items, result):
+                    print(
+                        f"Found matching business at rank {rank + 1}: {result.business.name}"
+                    )
+                    found = True
+                    break
+            if found:
+                reciprocal_ranks.append(1 / (rank + 1))
+            else:
+                print(f"No matching business found in top {len(results)} results.")
+                reciprocal_ranks.append(1 / (len(results) + 1))
+            print()
+
+        print("--- Evaluation Complete ---")
+        print(
+            f"Mean Reciprocal Rank (MRR): {sum(reciprocal_ranks) / len(reciprocal_ranks) if reciprocal_ranks else 0:.4f}"
+        )
+
+        # Shutdown
+        print("Shutting down...")
+        await search_launcher.stop()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Compute Mean Reciprocal Rank (MRR) for search queries based on menu items"
+    )
+    parser.add_argument(
+        "--data-dir", help="Path to the dataset directory", required=True
+    )
+    parser.add_argument(
+        "--order-size",
+        type=int,
+        default=1,
+        help="Number of menu items to include in each search query (default: 1)",
+    )
+    parser.add_argument(
+        "--search-algorithm",
+        type=str,
+        default="lexical",
+        help="Search algorithm to use (default: lexical)",
+    )
+    parser.add_argument(
+        "--postgres-host",
+        default="localhost",
+        help="PostgreSQL host (default: localhost)",
+    )
+
+    parser.add_argument(
+        "--postgres-port",
+        type=int,
+        default=5432,
+        help="PostgreSQL port (default: 5432)",
+    )
+
+    parser.add_argument(
+        "--postgres-password",
+        default="postgres",
+        help="PostgreSQL password (default: postgres)",
+    )
+
+    args = parser.parse_args()
+
+    asyncio.run(
+        main(
+            args.data_dir,
+            args.postgres_host,
+            args.postgres_port,
+            args.postgres_password,
+            args.order_size,
+            args.search_algorithm,
+        )
+    )