diff --git a/requirements.txt b/requirements.txt index 46916409c4..621a85d548 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ gpsoauth==0.3.0 coveralls==1.1 werkzeug==0.11.10 sqlalchemy==1.0.14 --e git+https://github.com/keyphact/pgoapi.git@39ea20d31b770dd7bc83180d60283e171090e16d#egg=pgoapi +-e git+https://github.com/keyphact/pgoapi.git@8c1c17637be0aa679d92e582e6c4dd1370a3ac00#egg=pgoapi enum34==1.1.6 diff --git a/worker.py b/worker.py index 9ead7d91eb..fd5e3ce8b4 100644 --- a/worker.py +++ b/worker.py @@ -36,7 +36,6 @@ if not hasattr(config, setting_name): raise RuntimeError('Please set "{}" in config'.format(setting_name)) - workers = {} local_data = threading.local() @@ -44,20 +43,16 @@ class MalformedResponse(Exception): """Raised when server response is malformed""" -class BannedAccount(Exception): - """Raised when account is banned""" - def configure_logger(filename='worker.log'): logging.basicConfig( filename=filename, format=( - '[%(asctime)s][%(threadName)10s][%(levelname)8s][L%(lineno)4d] ' + '[%(asctime)s][%(threadName)10s][%(levelname)5s][%(module)8s] ' '%(message)s' ), style='%', level=logging.INFO, ) - logger = logging.getLogger() @@ -70,41 +65,47 @@ def __init__( name=None, worker_no=None, points=None, + step=None, + cycle = None, + seen_per_cycle=None, ): super(Slave, self).__init__(group, target, name) self.worker_no = worker_no local_data.worker_no = worker_no self.points = points self.count_points = len(self.points) - self.step = 0 - self.cycle = 0 - self.seen_per_cycle = 0 + self.step = step + self.cycle = cycle + self.seen_per_cycle = seen_per_cycle self.total_seen = 0 + self.banned_count = 0 self.error_code = None self.running = True + self.active = True + self.permaban = False center = self.points[0] self.api = PGoApi() self.api.activate_signature(config.ENCRYPT_PATH) self.api.set_position(center[0], center[1], 100) # lat, lon, alt if hasattr(config, 'PROXIES') and config.PROXIES: self.api.set_proxy(config.PROXIES) + logger.info('proxy used : %s', config.PROXIES) + def run(self): """Wrapper for self.main - runs it a few times before restarting - Also is capable of restarting in case an error occurs. """ - self.cycle = 1 - self.error_code = None + self.error_code = None username, password, service = utils.get_worker_account(self.worker_no) - service = config.ACCOUNTS[self.worker_no][2] while True: try: loginsuccess = self.api.login( username=username, password=password, provider=service, + app_simulation=True ) if not loginsuccess: self.error_code = 'LOGIN FAIL' @@ -135,7 +136,7 @@ def run(self): self.restart() return break - while self.cycle <= config.CYCLES_PER_WORKER: + while self.active and self.cycle <= config.CYCLES_PER_WORKER: if not self.running: self.restart() return @@ -143,11 +144,16 @@ def run(self): self.main() except MalformedResponse: logger.warning('Malformed response received!') - self.error_code = 'RESTART' self.restart() - except BannedAccount: - self.error_code = 'BANNED?' - self.restart(30, 90) + return + except pgoapi_exceptions.NotLoggedInException: + logger.warning('Worker not logged in') + self.restart() + return + except pgoapi_exceptions.ServerSideRequestThrottlingException: + logger.info('Server throttling - sleeping for a bit (worker)') + time.sleep(random.uniform(1, 5)) + continue except Exception: logger.exception('A wild exception appeared!') self.error_code = 'EXCEPTION' @@ -156,7 +162,12 @@ def run(self): if not self.running: self.restart() return + if not self.active: + return + self.cycle += 1 + self.step = 0 + self.seen_per_cycle = 0 if self.cycle <= config.CYCLES_PER_WORKER: logger.info('Going to sleep for a bit') self.error_code = 'SLEEP' @@ -165,37 +176,45 @@ def run(self): logger.info('AWAKEN MY MASTERS') self.running = True self.error_code = None - self.error_code = 'RESTART' - self.restart() + else: + self.error_code = 'RESTART' + self.cycle = 1 + self.restart() + return + logger.info('Outside cycle while loop, thread terminate') + return + def main(self): """Heart of the worker - goes over each point and reports sightings""" session = db.Session() - self.seen_per_cycle = 0 - self.step = 0 - for i, point in enumerate(self.points): + + while self.step < self.count_points: + point = self.points[self.step] if not self.running: return - logger.info('Visiting point %d (%s %s)', i, point[0], point[1]) + if not self.active: + return + logger.info('Visiting point %d (%s %s)', self.step, point[0], point[1]) self.api.set_position(point[0], point[1], 0) - cell_ids = pgoapi_utils.get_cell_ids(point[0], point[1]) + cell_ids = pgoapi_utils.get_cell_ids(point[0], point[1],500) self.api.set_position(point[0], point[1], 100) response_dict = self.api.get_map_objects( latitude=pgoapi_utils.f2i(point[0]), longitude=pgoapi_utils.f2i(point[1]), cell_id=cell_ids ) - if not isinstance(response_dict, dict): - logger.warning('Response: %s', response_dict) - raise MalformedResponse + try: + map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {}) + except TypeError as e: + logger.exception(e) + continue if response_dict['status_code'] == 3: - logger.warning('Account banned') - raise BannedAccount - responses = response_dict.get('responses') - if not responses: - logger.warning('Response: %s', response_dict) - raise MalformedResponse - map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {}) + logger.warning('Account is possibly banned') + if not self.permaban: self.permaban = True + self.banned_count +=1 + continue + pokemons = [] forts = [] if map_objects.get('status') == 1: @@ -236,6 +255,18 @@ def main(self): len(pokemons), len(forts), ) + + if self.permaban: self.permaban = False + + #banned posibility count + object_seen = len(forts) + len(pokemons) + if self.banned_count >= 0: + if object_seen == 0: + self.banned_count += 1 + logger.info('banned_count : %d', self.banned_count) + else: self.banned_count = 0 + + # Clear error code and let know that there are Pokemon if self.error_code and self.seen_per_cycle: self.error_code = None @@ -289,12 +320,12 @@ def status(self): def restart(self, sleep_min=5, sleep_max=20): """Sleeps for a bit, then restarts""" + time.sleep(random.randint(sleep_min, sleep_max)) - start_worker(self.worker_no, self.points) + start_worker(self.worker_no, self.points, self.step, self.cycle, self.seen_per_cycle) def kill(self): """Marks worker as not running - It should stop any operation as soon as possible and restart itself. """ self.error_code = 'KILLED' @@ -302,9 +333,18 @@ def kill(self): def disable(self): """Marks worker as disabled""" + self.error_code = 'DISABLED' - self.running = False - + self.active = False + + def shutdown(self): + """Marks worker as shutdown""" + + if self.permaban: + self.error_code = 'PERMANENT BAN' + else : + self.error_code = 'IP BAN' + self.active = False def get_status_message(workers, count, start_time, points_stats): messages = [workers[i].status.ljust(20) for i in range(count)] @@ -329,12 +369,15 @@ def get_status_message(workers, count, start_time, points_stats): return '\n'.join(output) -def start_worker(worker_no, points): +def start_worker(worker_no, points,step=0,cycle=1,seen_per_cycle=0): logger.info('Worker (re)starting up!') worker = Slave( name='worker-%d' % worker_no, worker_no=worker_no, - points=points + points=points, + step = step, + seen_per_cycle = seen_per_cycle, + cycle = cycle ) if (worker_no not in config.DISABLE_WORKERS): worker.daemon = True @@ -358,6 +401,7 @@ def spawn_workers(workers, status_bar=True): } last_cleaned_cache = time.time() last_workers_checked = time.time() + last_proxy_checked = time.time() workers_check = [ (worker, worker.total_seen) for worker in workers.values() if worker.running @@ -367,15 +411,19 @@ def spawn_workers(workers, status_bar=True): # Clean cache if now - last_cleaned_cache > (15 * 60): # clean cache db.SIGHTING_CACHE.clean_expired() - last_cleaned_cache = now + last_cleaned_cache = now # Check up on workers - if now - last_workers_checked > (5 * 60): - # Kill those not doing anything + if now - last_workers_checked > (3 * 60): + # Kill those not doing anything or shutdown if get banned for worker, total_seen in workers_check: if not worker.running: continue - if worker.total_seen <= total_seen: + if not worker.active: + continue + if worker.total_seen <= total_seen and worker.banned_count == 0: worker.kill() + if worker.banned_count >= 5: + worker.shutdown() # Prepare new list workers_check = [ (worker, worker.total_seen) for worker in workers.values() @@ -415,4 +463,4 @@ def parse_args(): else: configure_logger(filename=None) logger.setLevel(args.log_level) - spawn_workers(workers, status_bar=args.status_bar) + spawn_workers(workers, status_bar=args.status_bar) \ No newline at end of file