Start scan from current point after restart, soft-ban detection, and a shutdown function on banned

requirements.txt

@@ -7,5 +7,5 @@ gpsoauth==0.3.0
 coveralls==1.1
 werkzeug==0.11.10
 sqlalchemy==1.0.14
--e git+https://github.com/keyphact/pgoapi.git@39ea20d31b770dd7bc83180d60283e171090e16d#egg=pgoapi
+-e git+https://github.com/keyphact/pgoapi.git@8c1c17637be0aa679d92e582e6c4dd1370a3ac00#egg=pgoapi
 enum34==1.1.6

worker.py

@@ -36,28 +36,23 @@
     if not hasattr(config, setting_name):
         raise RuntimeError('Please set "{}" in config'.format(setting_name))
 
-
 workers = {}
 local_data = threading.local()
 
 
 class MalformedResponse(Exception):
     """Raised when server response is malformed"""
 
-class BannedAccount(Exception):
-    """Raised when account is banned"""
-
 def configure_logger(filename='worker.log'):
     logging.basicConfig(
         filename=filename,
         format=(
-            '[%(asctime)s][%(threadName)10s][%(levelname)8s][L%(lineno)4d] '
+            '[%(asctime)s][%(threadName)10s][%(levelname)5s][%(module)8s] '
             '%(message)s'
         ),
         style='%',
         level=logging.INFO,
     )
-
 logger = logging.getLogger()
 
 
@@ -70,41 +65,47 @@ def __init__(
         name=None,
         worker_no=None,
         points=None,
+        step=None,
+        cycle = None,
+        seen_per_cycle=None,
     ):
         super(Slave, self).__init__(group, target, name)
         self.worker_no = worker_no
         local_data.worker_no = worker_no
         self.points = points
         self.count_points = len(self.points)
-        self.step = 0
-        self.cycle = 0
-        self.seen_per_cycle = 0
+        self.step = step
+        self.cycle = cycle
+        self.seen_per_cycle = seen_per_cycle
         self.total_seen = 0
+        self.banned_count = 0
         self.error_code = None
         self.running = True
+        self.active = True
+        self.permaban = False
         center = self.points[0]
         self.api = PGoApi()
         self.api.activate_signature(config.ENCRYPT_PATH)
         self.api.set_position(center[0], center[1], 100)  # lat, lon, alt
         if hasattr(config, 'PROXIES') and config.PROXIES:
             self.api.set_proxy(config.PROXIES)
+            logger.info('proxy used : %s', config.PROXIES)  
+
 
     def run(self):
         """Wrapper for self.main - runs it a few times before restarting
-
         Also is capable of restarting in case an error occurs.
         """
-        self.cycle = 1
-        self.error_code = None
 
+        self.error_code = None
         username, password, service = utils.get_worker_account(self.worker_no)
-        service = config.ACCOUNTS[self.worker_no][2]
         while True:
             try:
                 loginsuccess = self.api.login(
                     username=username,
                     password=password,
                     provider=service,
+                    app_simulation=True
                 )
                 if not loginsuccess:
                     self.error_code = 'LOGIN FAIL'
@@ -135,19 +136,24 @@ def run(self):
                 self.restart()
                 return
             break
-        while self.cycle <= config.CYCLES_PER_WORKER:
+        while self.active and self.cycle <= config.CYCLES_PER_WORKER:
             if not self.running:
                 self.restart()
                 return
             try:
                 self.main()
             except MalformedResponse:
                 logger.warning('Malformed response received!')
-                self.error_code = 'RESTART'
                 self.restart()
-            except BannedAccount:
-                self.error_code = 'BANNED?'
-                self.restart(30, 90)
+                return
+            except pgoapi_exceptions.NotLoggedInException:
+                logger.warning('Worker not logged in')
+                self.restart()
+                return
+            except pgoapi_exceptions.ServerSideRequestThrottlingException:
+                logger.info('Server throttling - sleeping for a bit (worker)')
+                time.sleep(random.uniform(1, 5))
+                continue
             except Exception:
                 logger.exception('A wild exception appeared!')
                 self.error_code = 'EXCEPTION'
@@ -156,7 +162,12 @@ def run(self):
             if not self.running:
                 self.restart()
                 return
+            if not self.active:
+                return
+
             self.cycle += 1
+            self.step = 0
+            self.seen_per_cycle = 0
             if self.cycle <= config.CYCLES_PER_WORKER:
                 logger.info('Going to sleep for a bit')
                 self.error_code = 'SLEEP'
@@ -165,37 +176,45 @@ def run(self):
                 logger.info('AWAKEN MY MASTERS')
                 self.running = True
                 self.error_code = None
-        self.error_code = 'RESTART'
-        self.restart()
+            else:
+                self.error_code = 'RESTART'
+                self.cycle = 1
+                self.restart()
+                return
+        logger.info('Outside cycle while loop, thread terminate')
+        return
+
 
     def main(self):
         """Heart of the worker - goes over each point and reports sightings"""
         session = db.Session()
-        self.seen_per_cycle = 0
-        self.step = 0
-        for i, point in enumerate(self.points):
+
+        while self.step <  self.count_points:
+            point = self.points[self.step]			
             if not self.running:
                 return
-            logger.info('Visiting point %d (%s %s)', i, point[0], point[1])
+            if not self.active:
+                return
+            logger.info('Visiting point %d (%s %s)', self.step, point[0], point[1])
             self.api.set_position(point[0], point[1], 0)
-            cell_ids = pgoapi_utils.get_cell_ids(point[0], point[1])
+            cell_ids = pgoapi_utils.get_cell_ids(point[0], point[1],500)
             self.api.set_position(point[0], point[1], 100)
             response_dict = self.api.get_map_objects(
                 latitude=pgoapi_utils.f2i(point[0]),
                 longitude=pgoapi_utils.f2i(point[1]),
                 cell_id=cell_ids
             )
-            if not isinstance(response_dict, dict):
-                logger.warning('Response: %s', response_dict)
-                raise MalformedResponse
+            try:
+                map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {})
+            except TypeError as e:
+                logger.exception(e)
+                continue
             if response_dict['status_code'] == 3:
-                logger.warning('Account banned')
-                raise BannedAccount
-            responses = response_dict.get('responses')
-            if not responses:
-                logger.warning('Response: %s', response_dict)
-                raise MalformedResponse
-            map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {})
+                logger.warning('Account is possibly banned')
+                if not self.permaban: self.permaban = True
+                self.banned_count +=1
+                continue
+
             pokemons = []
             forts = []
             if map_objects.get('status') == 1:
@@ -236,6 +255,18 @@ def main(self):
                 len(pokemons),
                 len(forts),
             )
+
+            if self.permaban: self.permaban = False
+
+			#banned posibility count
+            object_seen = len(forts) + len(pokemons)  
+            if self.banned_count >= 0:
+                if object_seen == 0:
+                    self.banned_count += 1
+                    logger.info('banned_count : %d', self.banned_count)  
+                else: self.banned_count = 0
+
+
             # Clear error code and let know that there are Pokemon
             if self.error_code and self.seen_per_cycle:
                 self.error_code = None
@@ -289,22 +320,31 @@ def status(self):
 
     def restart(self, sleep_min=5, sleep_max=20):
         """Sleeps for a bit, then restarts"""
+
         time.sleep(random.randint(sleep_min, sleep_max))
-        start_worker(self.worker_no, self.points)
+        start_worker(self.worker_no, self.points, self.step, self.cycle, self.seen_per_cycle)
 
     def kill(self):
         """Marks worker as not running
-
         It should stop any operation as soon as possible and restart itself.
         """
         self.error_code = 'KILLED'
         self.running = False
 
     def disable(self):
         """Marks worker as disabled"""
+
         self.error_code = 'DISABLED'
-        self.running = False
-
+        self.active = False
+
+    def shutdown(self):
+        """Marks worker as shutdown"""
+
+        if self.permaban:
+            self.error_code = 'PERMANENT BAN'
+        else : 
+            self.error_code = 'IP BAN'
+        self.active = False
 
 def get_status_message(workers, count, start_time, points_stats):
     messages = [workers[i].status.ljust(20) for i in range(count)]
@@ -329,12 +369,15 @@ def get_status_message(workers, count, start_time, points_stats):
     return '\n'.join(output)
 
 
-def start_worker(worker_no, points):
+def start_worker(worker_no, points,step=0,cycle=1,seen_per_cycle=0):
     logger.info('Worker (re)starting up!')
     worker = Slave(
         name='worker-%d' % worker_no,
         worker_no=worker_no,
-        points=points
+        points=points,
+        step = step,
+        seen_per_cycle = seen_per_cycle,
+        cycle = cycle
     )
     if (worker_no not in config.DISABLE_WORKERS):
         worker.daemon = True
@@ -358,6 +401,7 @@ def spawn_workers(workers, status_bar=True):
     }
     last_cleaned_cache = time.time()
     last_workers_checked = time.time()
+    last_proxy_checked = time.time()
     workers_check = [
         (worker, worker.total_seen) for worker in workers.values()
         if worker.running
@@ -367,15 +411,19 @@ def spawn_workers(workers, status_bar=True):
         # Clean cache
         if now - last_cleaned_cache > (15 * 60):  # clean cache
             db.SIGHTING_CACHE.clean_expired()
-            last_cleaned_cache = now
+            last_cleaned_cache = now        
         # Check up on workers
-        if now - last_workers_checked > (5 * 60):
-            # Kill those not doing anything
+        if now - last_workers_checked > (3 * 60):
+            # Kill those not doing anything or shutdown if get banned
             for worker, total_seen in workers_check:
                 if not worker.running:
                     continue
-                if worker.total_seen <= total_seen:
+                if not worker.active:
+                    continue
+                if worker.total_seen <= total_seen and worker.banned_count == 0:
                     worker.kill()
+                if worker.banned_count >= 5:
+                    worker.shutdown()
             # Prepare new list
             workers_check = [
                 (worker, worker.total_seen) for worker in workers.values()
@@ -415,4 +463,4 @@ def parse_args():
     else:
         configure_logger(filename=None)
     logger.setLevel(args.log_level)
-    spawn_workers(workers, status_bar=args.status_bar)
+    spawn_workers(workers, status_bar=args.status_bar)