Public Dataset

.gitignore

@@ -17,6 +17,7 @@ venv/
 .idea/
 
 state.json
+tmp/
 
 app.config.js
 

desearch/__init__.py

@@ -33,7 +33,6 @@
 print("__version__", __version__)
 
 import os
-from enum import Enum
 
 from openai import AsyncOpenAI
 
@@ -43,44 +42,13 @@
 
 client = AsyncOpenAI(timeout=90.0)
 
+
+MIN_TOTAL_STAKE = 10000
+MIN_ALPHA_STAKE = 20
+
 # Blacklist variables
-ALLOW_NON_REGISTERED = False
-PROMPT_BLACKLIST_STAKE = 20000
-TWITTER_SCRAPPER_BLACKLIST_STAKE = 20000
-ISALIVE_BLACKLIST_STAKE = min(PROMPT_BLACKLIST_STAKE, TWITTER_SCRAPPER_BLACKLIST_STAKE)
 MIN_REQUEST_PERIOD = 2
 MAX_REQUESTS = 30
-# must have the test_key whitelisted to avoid a global blacklist
-testnet_key = ["5EhEZN6soubtKJm8RN7ANx9FGZ2JezxBUFxr45cdsHtDp3Uk"]
-test_key = ["5DcRHcCwD33YsHfj4PX5j2evWLniR1wSWeNmpf5RXaspQT6t"]
-
-valid_validators = [
-    "5FFApaS75bv5pJHfAp2FVLBj9ZaXuFDjEypsaBNc1wCfe52v",
-    "5EhvL1FVkQPpMjZX4MAADcW42i3xPSF1KiCpuaxTYVr28sux",
-    "5CXRfP2ekFhe62r7q3vppRajJmGhTi7vwvb2yr79jveZ282w",
-    "5CaNj3BarTHotEK1n513aoTtFeXcjf6uvKzAyzNuv9cirUoW",
-    "5HK5tp6t2S59DywmHRWPBVJeJ86T61KjurYqeooqj8sREpeN",
-    "5DvTpiniW9s3APmHRYn8FroUWyfnLtrsid5Mtn5EwMXHN2ed",
-    "5G3f8VDTT1ydirT3QffnV2TMrNMR2MkQfGUubQNqZcGSj82T",
-    "5Dz8ShM6rtPw1GBAaqxjycT9LF1TC3iDpzpUH9gKr85Nizo6",
-    "5Hddm3iBFD2GLT5ik7LZnT3XJUnRnN8PoeCFgGQgawUVKNm8",
-    "5HNQURvmjjYhTSksi8Wfsw676b4owGwfLR2BFAQzG7H3HhYf",
-    "5HEo565WAy4Dbq3Sv271SAi7syBSofyfhhwRNjFNSM2gP9M2",
-    "5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3",
-    "5H66kJAzBCv2DC9poHATLQqyt3ag8FLSbHf6rMqTiRcS52rc",
-    "5HbLYXUBy1snPR8nfioQ7GoA9x76EELzEq9j7F32vWUQHm1x",
-    "5FKstHjZkh4v3qAMSBa1oJcHCLjxYZ8SNTSz1opTv4hR7gVB",
-    "5DXTJSPVvf1sow1MU4npJPewEAwhPRb6CWsk4RX9RFt2PRbj",
-    "5EsrMfo7UcPs6AqAotU47VmYGfLHntS9JzhEwbY2EJMcWQxQ",  # server
-    "5Dd8gaRNdhm1YP7G1hcB1N842ecAUQmbLjCRLqH5ycaTGrWv",
-    "5DnXm2tBGAD57ySJv5SfpTfLcsQbSKKp6xZKFWABw3cYUgqg",
-    "5GVpVH7DjYmQY7ckznVnrHncU9knzYJvhY3TfbFY7sPboJB2",
-    "5Fq5v71D4LX8Db1xsmRSy6udQThcZ8sFDqxQFwnUZ1BuqY5A",
-    "5ChuGqW2cxc5AZJ29z6vyTkTncg75L9ovfp8QN8eB8niSD75",
-    "5F27Eqz2PhyMtGMEce898x31DokNqRVxkm5AhDDe6rDGNvoY",
-]
-
-WHITELISTED_KEYS = testnet_key + test_key + valid_validators
 
 BLACKLISTED_KEYS = [
     "5G1NjW9YhXLadMWajvTkfcJy6up3yH2q1YzMXDTi6ijanChe",

neurons/miners/miner.py

@@ -159,45 +159,42 @@ async def _twitter_urls_search(
     async def _web_search(self, synapse: WebSearchSynapse) -> WebSearchSynapse:
         return await self.web_search(synapse)
 
-    def base_blacklist(self, synapse, blacklist_amt=20000) -> Tuple[bool, str]:
+    def base_blacklist(self, synapse) -> Tuple[bool, str]:
         try:
             hotkey = synapse.dendrite.hotkey
             synapse_type = type(synapse).__name__
 
             if hotkey in desearch.BLACKLISTED_KEYS:
                 return True, f"Blacklisted a {synapse_type} request from {hotkey}"
 
-            # if hotkey in desearch.WHITELISTED_KEYS:
-            #     return False, f"accepting {synapse_type} request from {hotkey}"
-
-            # if hotkey not in desearch.valid_validators:
-            #     return (
-            #         True,
-            #         f"Blacklisted a {synapse_type} request from a non-valid hotkey: {hotkey}",
-            #     )
-
             uid = None
-            axon = None
             for _uid, _axon in enumerate(self.metagraph.axons):
                 if _axon.hotkey == hotkey:
                     uid = _uid
-                    axon = _axon
                     break
 
-            if uid is None and desearch.ALLOW_NON_REGISTERED == False:
+            if uid is None:
                 return (
                     True,
                     f"Blacklisted a non registered hotkey's {synapse_type} request from {hotkey}",
                 )
 
             if self.config.subtensor.network == "finney":
-                # check the stake
-                tao = self.metagraph.neurons[uid].stake.tao
-                # metagraph.neurons[uid].S
-                if tao < blacklist_amt:
+                alpha_stake = float(self.metagraph.alpha_stake[uid].item())
+                total_stake = float(self.metagraph.total_stake[uid].item())
+
+                if (
+                    alpha_stake < desearch.MIN_ALPHA_STAKE
+                    or total_stake < desearch.MIN_TOTAL_STAKE
+                ):
                     return (
                         True,
-                        f"Blacklisted a low stake {synapse_type} request: {tao} < {blacklist_amt} from {hotkey}",
+                        (
+                            f"Blacklisted a low stake {synapse_type} request: "
+                            f"alpha_stake={alpha_stake} < {desearch.MIN_ALPHA_STAKE} "
+                            f"or total_stake={total_stake} < {desearch.MIN_TOTAL_STAKE} "
+                            f"from {hotkey}"
+                        ),
                     )
 
             time_window = desearch.MIN_REQUEST_PERIOD * 60
@@ -228,50 +225,40 @@ def base_blacklist(self, synapse, blacklist_amt=20000) -> Tuple[bool, str]:
             bt.logging.error(f"errror in blacklist {traceback.format_exc()}")
 
     def blacklist_is_alive(self, synapse: IsAlive) -> Tuple[bool, str]:
-        blacklist = self.base_blacklist(synapse, desearch.ISALIVE_BLACKLIST_STAKE)
+        blacklist = self.base_blacklist(synapse)
         bt.logging.debug(blacklist[1])
         return blacklist
 
     def blacklist_smart_scraper(
         self, synapse: ScraperStreamingSynapse
     ) -> Tuple[bool, str]:
-        blacklist = self.base_blacklist(
-            synapse, desearch.TWITTER_SCRAPPER_BLACKLIST_STAKE
-        )
+        blacklist = self.base_blacklist(synapse)
         bt.logging.info(blacklist[1])
         return blacklist
 
     def blacklist_twitter_search(
         self, synapse: TwitterSearchSynapse
     ) -> Tuple[bool, str]:
-        blacklist = self.base_blacklist(
-            synapse, desearch.TWITTER_SCRAPPER_BLACKLIST_STAKE
-        )
+        blacklist = self.base_blacklist(synapse)
         bt.logging.info(blacklist[1])
         return blacklist
 
     def blacklist_twitter_id_search(
         self, synapse: TwitterIDSearchSynapse
     ) -> Tuple[bool, str]:
-        blacklist = self.base_blacklist(
-            synapse, desearch.TWITTER_SCRAPPER_BLACKLIST_STAKE
-        )
+        blacklist = self.base_blacklist(synapse)
         bt.logging.info(blacklist[1])
         return blacklist
 
     def blacklist_twitter_urls_search(
         self, synapse: TwitterURLsSearchSynapse
     ) -> Tuple[bool, str]:
-        blacklist = self.base_blacklist(
-            synapse, desearch.TWITTER_SCRAPPER_BLACKLIST_STAKE
-        )
+        blacklist = self.base_blacklist(synapse)
         bt.logging.info(blacklist[1])
         return blacklist
 
     def blacklist_web_search(self, synapse: WebSearchSynapse) -> Tuple[bool, str]:
-        blacklist = self.base_blacklist(
-            synapse, desearch.TWITTER_SCRAPPER_BLACKLIST_STAKE
-        )
+        blacklist = self.base_blacklist(synapse)
         bt.logging.info(blacklist[1])
         return blacklist
 

neurons/validators/api/__init__.py

@@ -0,0 +1,3 @@
+from neurons.validators.api.app import app
+
+__all__ = ["app"]

neurons/validators/api.py → neurons/validators/api/app.py

@@ -26,9 +26,9 @@
     TwitterScraperTweet,
     WebSearchResultList,
 )
+from neurons.validators.api.validator_api import ValidatorAPI
+from neurons.validators.api.validator_service_client import ValidatorServiceClient
 from neurons.validators.env import EXPECTED_ACCESS_KEY, PORT
-from neurons.validators.validator_api import ValidatorAPI
-from neurons.validators.validator_service_client import ValidatorServiceClient
 
 
 async def get_validator_config():
@@ -526,9 +526,7 @@ async def get_tweets_by_urls(
 
         bt.logging.info(f"Fetching tweets for URLs: {urls}")
 
-        results = await api.x_scraper_validator.x_posts_by_urls(
-            urls, uid=request.uid
-        )
+        results = await api.x_scraper_validator.x_posts_by_urls(urls, uid=request.uid)
     except Exception as e:
         bt.logging.error(f"Error fetching tweets by URLs: {e}")
         raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")

neurons/validators/validator_api.py → neurons/validators/api/validator_api.py

@@ -4,11 +4,13 @@
 import bittensor as bt
 
 from desearch.redis.redis_client import close_redis, initialize_redis
-from neurons.validators.advanced_scraper_validator import AdvancedScraperValidator
+from neurons.validators.api.validator_service_client import ValidatorServiceClient
+from neurons.validators.scrapers.advanced_scraper_validator import (
+    AdvancedScraperValidator,
+)
+from neurons.validators.scrapers.web_scraper_validator import WebScraperValidator
+from neurons.validators.scrapers.x_scraper_validator import XScraperValidator
 from neurons.validators.utility_api_client import UtilityAPIClient
-from neurons.validators.validator_service_client import ValidatorServiceClient
-from neurons.validators.web_scraper_validator import WebScraperValidator
-from neurons.validators.x_scraper_validator import XScraperValidator
 
 
 class ValidatorAPI:

neurons/validators/proxy/uid_manager.py

@@ -3,7 +3,7 @@
 
 from bittensor.core.metagraph import AsyncMetagraph
 
-from neurons.validators.weights import EMISSION_CONTROL_HOTKEY
+from neurons.validators.service.weights import EMISSION_CONTROL_HOTKEY
 
 
 class UIDManager: