From 680888407a56420061d930b257b7318c380c3d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Thu, 19 Mar 2015 15:23:49 +0100 Subject: [PATCH 01/11] handle old ideviceinstaller versions with uuid option flag --- .gitignore | 3 ++ device.py | 108 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9765c3f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*pyc +*# +.#* diff --git a/device.py b/device.py index d47da0d..cd7fc98 100644 --- a/device.py +++ b/device.py @@ -67,7 +67,17 @@ def device_info_dict(self): ''' raw device information as dict ''' if (len(self.deviceDict) == 0): - output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid]) + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) + except: + raise + else: + raise + self.deviceDict = plistlib.readPlistFromString(output) return self.deviceDict @@ -78,7 +88,17 @@ def locale(self): ''' the devices locale setting ''' if (self.locale_val == ""): - self.locale_val = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + try: + self.locale_val = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + self.locale_val = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + except: + raise + else: + raise + return self.locale_val @@ -97,7 +117,18 @@ def base_url(self): def free_bytes(self): ''' get the free space left on the device in bytes ''' - output = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + try: + output = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + except: + raise + else: + raise + free_bytes = 0 try: free_bytes = long(output) @@ -111,7 +142,17 @@ def account_info_dict(self): ''' get raw account info from device as dict. ''' if (len(self.accountDict) == 0): - output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + except: + raise + else: + raise + if len(output) > 0: self.accountDict = plistlib.readPlistFromString(output) else: @@ -163,7 +204,17 @@ def accounts(self): def installed_apps(self): ''' list all installed apps as dict. ''' - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + except: + raise + else: + raise + if (len(output)==0): return {} @@ -173,7 +224,17 @@ def installed_apps(self): plist = plistlib.readPlistFromString(output) except Exception: logger.warning("Failed to parse installed apps via xml output. Try to extract data via regex.") - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user"]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o"]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user"]) + except: + raise + else: + raise + regex = re.compile("^(?P.*) - (?P.*) (?P(\d+\.*)+)$",re.MULTILINE) # r = regex.search(output) for i in regex.finditer(output): @@ -216,7 +277,16 @@ def install(self, app_archive_path): ''' result=True try: - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--install", app_archive_path]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--install", app_archive_path]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--install", app_archive_path]) + except: + raise + else: + raise logger.debug('output: %s' % output) if (len(output)==0): result=False @@ -231,7 +301,17 @@ def uninstall(self, bundleId): ''' result=True try: - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--uninstall", bundleId]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--uninstall", bundleId]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--uninstall", bundleId]) + except: + raise + else: + raise + logger.debug('output: %s' % output) if (len(output)==0): result=False @@ -244,7 +324,17 @@ def archive(self, bundleId, app_archive_folder, app_only=True, uninstall=True): ''' archives an app to `app_archive_folder` returns True or False ''' - options = ["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"] + try: + options = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) + except subprocess.CalledProcessError as e: + if e.returncode == 2: + try: + options = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) + except: + raise + else: + raise + if app_only: options.extend(["-o", "app_only"]) if uninstall: From 054162e6f0ea70852b3fa55dd9defb208cd15f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Thu, 19 Mar 2015 15:26:45 +0100 Subject: [PATCH 02/11] dont handle the returncode of check_output --- device.py | 81 +++++++++++++++++++------------------------------------ 1 file changed, 27 insertions(+), 54 deletions(-) diff --git a/device.py b/device.py index cd7fc98..61c02b0 100644 --- a/device.py +++ b/device.py @@ -70,12 +70,9 @@ def device_info_dict(self): try: output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) + except: raise self.deviceDict = plistlib.readPlistFromString(output) @@ -91,12 +88,9 @@ def locale(self): try: self.locale_val = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - self.locale_val = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() - except: - raise - else: + try: + self.locale_val = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + except: raise return self.locale_val @@ -121,12 +115,9 @@ def free_bytes(self): output = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + except: raise free_bytes = 0 @@ -145,12 +136,9 @@ def account_info_dict(self): try: output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + except: raise if len(output) > 0: @@ -207,12 +195,9 @@ def installed_apps(self): try: output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + except: raise if (len(output)==0): @@ -227,12 +212,9 @@ def installed_apps(self): try: output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o"]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user"]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user"]) + except: raise regex = re.compile("^(?P.*) - (?P.*) (?P(\d+\.*)+)$",re.MULTILINE) @@ -280,12 +262,9 @@ def install(self, app_archive_path): try: output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--install", app_archive_path]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--install", app_archive_path]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--install", app_archive_path]) + except: raise logger.debug('output: %s' % output) if (len(output)==0): @@ -304,12 +283,9 @@ def uninstall(self, bundleId): try: output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--uninstall", bundleId]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--uninstall", bundleId]) - except: - raise - else: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--uninstall", bundleId]) + except: raise logger.debug('output: %s' % output) @@ -327,12 +303,9 @@ def archive(self, bundleId, app_archive_folder, app_only=True, uninstall=True): try: options = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) except subprocess.CalledProcessError as e: - if e.returncode == 2: - try: - options = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) - except: - raise - else: + try: + options = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) + except: raise if app_only: From 429e00c462f4286dd58e1ac3b51618015e0cec7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 12:37:46 +0100 Subject: [PATCH 03/11] scheduler.py: print job id on success and choose job type --- scheduler.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/scheduler.py b/scheduler.py index d256d39..fcf7083 100755 --- a/scheduler.py +++ b/scheduler.py @@ -36,7 +36,6 @@ class Scheduler(object): @classmethod def _default_runjob(cls): return { - 'type':'run_app', 'state':'pending', 'jobInfo': { 'appType':'AppStoreApp' @@ -54,7 +53,7 @@ def schedule_job(self, jobDict): return jobId - def schedule_bundleId(self, bundleId, worker=None, device=None, account=None, country=None, executionStrategy=None): + def schedule_bundleId(self, bundleId, job_type, worker=None, device=None, account=None, country=None, executionStrategy=None): jobDict = { 'jobInfo': { 'bundleId':bundleId @@ -70,10 +69,13 @@ def schedule_bundleId(self, bundleId, worker=None, device=None, account=None, co jobDict['jobInfo']['storeCountry'] = country if executionStrategy: jobDict['jobInfo']['executionStrategy'] = executionStrategy + + jobDict['type'] = job_type + return self.schedule_job(jobDict) - def schedule_appId(self, appId, account=None, country=None, executionStrategy=None): + def schedule_appId(self, appId, job_type, account=None, country=None, executionStrategy=None): url = 'http://itunes.apple.com/lookup?id=%s' % appId r = requests.get(url) if r.status_code != 200: @@ -85,11 +87,11 @@ def schedule_appId(self, appId, account=None, country=None, executionStrategy=No if len(results) != 1: logger.error("No app with id %s found", (appId)) return False - return self.schedule_bundleId(results[0]['bundleId'], account=account, country=country, executionStrategy=None) + return self.schedule_bundleId(results[0]['bundleId'], job_type, account=account, country=country, executionStrategy=None) - def schedule_itunes(self, url, account=None, country=None, executionStrategy=None): + def schedule_itunes(self, url, job_type, account=None, country=None, executionStrategy=None): logger.info('Adding apps from iTunes (%s)' % url) r = requests.get(url) if r.status_code != 200: @@ -100,7 +102,7 @@ def schedule_itunes(self, url, account=None, country=None, executionStrategy=Non entries = resDict['feed']['entry'] result = True for entry in entries: - if not self.schedule_bundleId(entry['id']['attributes']['im:bundleId'], account=account, country=country, executionStrategy=None): + if not self.schedule_bundleId(entry['id']['attributes']['im:bundleId'], job_type, account=account, country=country, executionStrategy=None): result = False return result @@ -112,7 +114,7 @@ def main(): parser = argparse.ArgumentParser(description='schedule backend jobs from different sources.') parser.add_argument('-b','--backend', required=True, help='the backend url.') parser.add_argument('-a','--account', help='the accountId to use.') - + parser.add_argument('-t','--job-type', default='install_app', help='could be install_app, run_app or exec_cmd. default is install_app') parser.add_argument('-s','--strategy', help='the execution strategy and duration to use.') # add commands @@ -134,16 +136,22 @@ def main(): def printRes(res): if res: logger.info('done!') + print(res) else: logger.error('error occured (could be partially done)') + + if not args.job_type in ['install_app', 'run_app', 'exec_cmd']: + print('Not a valid job-type!') + return + if 'bundleId' in args and args.bundleId: - res = scheduler.schedule_bundleId(args.bundleId, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_bundleId(args.bundleId, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return if 'appId' in args and args.appId: - res = scheduler.schedule_appId(args.appId, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_appId(args.appId, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return @@ -153,13 +161,13 @@ def printRes(res): if 'itunes_top' in args and args.itunes_top: url = 'https://itunes.apple.com/%s/rss/topfreeapplications/limit=%i/genre=%s/json' % (args.itunes_country, args.itunes_top, genre) - res = scheduler.schedule_itunes(url, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_itunes(url, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return if 'itunes_new' in args and args.itunes_new: url = 'https://itunes.apple.com/%s/rss/newfreeapplications/limit=%i/genre=%s/json' % (args.itunes_country, args.itunes_new, genre) - res = scheduler.schedule_itunes(url, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_itunes(url, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return From 712fdfbea3587722f2fddd1d2cbe134fb7c31e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 15:00:43 +0100 Subject: [PATCH 04/11] device.py: fixed bug in archive() due to ideviceinstaller option flag changes --- device.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/device.py b/device.py index 61c02b0..91b07c0 100644 --- a/device.py +++ b/device.py @@ -300,18 +300,15 @@ def archive(self, bundleId, app_archive_folder, app_only=True, uninstall=True): ''' archives an app to `app_archive_folder` returns True or False ''' - try: - options = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) - except subprocess.CalledProcessError as e: - try: - options = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"]) - except: - raise + options = ["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"] + options_alt = ["ideviceinstaller", "--uuid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"] if app_only: options.extend(["-o", "app_only"]) + options_alt.extend(["-o", "app_only"]) if uninstall: options.extend(["-o", "uninstall"]) + options_alt.extend(["-o", "uninstall"]) if not os.path.exists(app_archive_folder): os.makedirs(app_archive_folder) @@ -323,7 +320,12 @@ def archive(self, bundleId, app_archive_folder, app_only=True, uninstall=True): if (len(output)==0): result=False except subprocess.CalledProcessError as e: - logger.error('archiving app %s failed with: %s ', bundleId, e, output) - result=False + try: + output = subprocess.check_output(options_alt) + logger.debug('output: %s' % output) + if (len(output)==0): + result=False + except subprocess.CalledProcessError as e: + logger.error('archiving app %s failed with: %s ', bundleId, e, output) + result=False return result - From 5f649fd7393340c4647190fba06f26be70114829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 15:29:38 +0100 Subject: [PATCH 05/11] install not on devices which do not match the minimal needed iOS version --- backend.py | 4 ++-- job.py | 33 ++++++++++++++++++++++++++++++++- worker.py | 5 +++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/backend.py b/backend.py index 394c976..748f39f 100644 --- a/backend.py +++ b/backend.py @@ -170,11 +170,11 @@ def get_app_bundleId(self, bundleId, version=None): if r.status_code == 200: appsDict = json.loads(r.text) if len(appsDict) == 1: - return appsDict.values()[0] + return (appsDict.values()[0], appsDict['minimumOsVersion']) logger.debug('%s returned %s results' % (url, len(appsDict))) else: logger.error('%s request failed: %s %s' % (url, r.status_code, r.text)) - return None + return (None, None) def get_app_archive(self, appId, archivePath): diff --git a/job.py b/job.py index e412da6..02fcf06 100644 --- a/job.py +++ b/job.py @@ -3,6 +3,7 @@ import logging import base64 import time +from distutils.version import StrictVersion from enum import Enum from store import AppStore, AppStoreException @@ -13,6 +14,9 @@ class JobExecutionError(Exception): pass +class ProductVersionError(Exception): + pass + class Job(object): @@ -80,6 +84,8 @@ def _install_app(self, pilot): if 'version' in jobInfo: version = jobInfo['version'] + productVersion = self.device.device_info_dict()['ProductVersion'] + #check app type if 'AppStoreApp' == jobInfo['appType']: logger.debug('installing appstore app %s' % bundleId) @@ -100,7 +106,8 @@ def _install_app(self, pilot): alreadyInstalled = True # check the backend for already existing app - app = self.backend.get_app_bundleId(bundleId, version) + (app, minimumOsVersion) = self.backend.get_app_bundleId(bundleId, version) + logger.debug('backend result for bundleId %s: %s' % (bundleId, app)) if app and '_id' in app: self.appId = app['_id'] @@ -115,6 +122,9 @@ def _install_app(self, pilot): elif self.appId: # install from backend + if StrictVersion(minimumOsVersion) > StrictVersion(productVersion): + raise ProductVersionError(minimumOsVersion) + # dirty check for ipa-size < ~50MB if app and 'fileSizeBytes' in app: size = 0 @@ -172,6 +182,8 @@ def _install_app(self, pilot): self.jobDict['appInfo'] = appInfo logger.debug('using appInfo: %s' % str(appInfo)) + if StrictVersion(appInfo['minimum-os-version']) > StrictVersion(productVersion): + raise ProductVersionError(appInfo['minimum-os-version']) ## get account accountId = '' @@ -255,6 +267,19 @@ def execute(self): logger.error("Job execution failed: %s" % str(e)) backendJobData['state'] = Job.STATE.FAILED result = False + except ProductVersionError, e: + logger.warn("Job execution aborted: iOS version to low") + backendJobData['state'] = Job.STATE.PENDING + backendJobData['worker'] = None + backendJobData['device'] = None + + # has to be > 99 to compare with e.g. ProductVersion 7.1.2 + minimumOSVersion = int(''.join(str(e).split('.'))) + if minimumOSVersion < 100: + minimumOSVersion *= 10 + backendJobData['jobInfo']['minimumOSVersion'] = str(minimumOSVersion) + self.backend.post_job(backendJobData) + raise ## set job finished if self.jobId: @@ -387,6 +412,12 @@ def execute(self): backendJobData['state'] = Job.STATE.FAILED self.backend.post_job(backendJobData) return False + except ProductVersionError, e: + logger.warn("Job execution aborted: iOS version to low") + backendJobData['state'] = Job.STATE.PENDING + backendJobData['jobInfo']['minimumOSVersion'] = ''.join(str(e).split('.')) + self.backend.post_job(backendJobData) + raise ## set job finished backendJobData['state'] = Job.STATE.FINISHED diff --git a/worker.py b/worker.py index 251bbae..2067ff6 100755 --- a/worker.py +++ b/worker.py @@ -13,7 +13,7 @@ logger.setLevel(level=logging.INFO) -from job import JobFactory +from job import JobFactory, ProductVersionError from device import iDevice from backend import Backend #from pilot import Pilot @@ -65,7 +65,8 @@ def run(self): logger.error("traceback: %s" % tb) logger.error("Device loop will be stopped now.") self.stop() - + except ProductVersionError as e: + logger.warn('Executing job aborted: Higher OS version (%s) needed, job is set to pending' % e) except Exception as e: logger.error("Executing job failed: %s" % e) tb = traceback.format_exc() From f5b69b43f422752c7100ab8c46e44cf3c80e2c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 15:39:17 +0100 Subject: [PATCH 06/11] bug in device.py --- device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device.py b/device.py index 91b07c0..7e752e0 100644 --- a/device.py +++ b/device.py @@ -68,7 +68,7 @@ def device_info_dict(self): ''' if (len(self.deviceDict) == 0): try: - output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) + output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid]) except subprocess.CalledProcessError as e: try: output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) From a41285c0d21b7c4c91910923e83da9cccab234c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 16:21:44 +0100 Subject: [PATCH 07/11] job.py: log error messages in the database (e.g. for statistics) --- job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/job.py b/job.py index 02fcf06..ae85b62 100644 --- a/job.py +++ b/job.py @@ -266,6 +266,7 @@ def execute(self): except JobExecutionError, e: logger.error("Job execution failed: %s" % str(e)) backendJobData['state'] = Job.STATE.FAILED + backendJobData['error_message'] = str(e) result = False except ProductVersionError, e: logger.warn("Job execution aborted: iOS version to low") @@ -410,6 +411,7 @@ def execute(self): except JobExecutionError, e: logger.error("Job execution failed: %s" % str(e)) backendJobData['state'] = Job.STATE.FAILED + backendJobData['error_message'] = str(e) self.backend.post_job(backendJobData) return False except ProductVersionError, e: From da46b734bdcc2c02829e84494d2147259363552e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 16:49:10 +0100 Subject: [PATCH 08/11] fixed bug in backend.py --- backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend.py b/backend.py index 748f39f..35f2951 100644 --- a/backend.py +++ b/backend.py @@ -170,7 +170,11 @@ def get_app_bundleId(self, bundleId, version=None): if r.status_code == 200: appsDict = json.loads(r.text) if len(appsDict) == 1: - return (appsDict.values()[0], appsDict['minimumOsVersion']) + if not 'minimumOsVersion' in appsDict: + minimumOsVersion = 0 + else: + minimumOsVersion = appsDict['minimumOsVersion'] + return (appsDict.values()[0], minimumOsVersion) logger.debug('%s returned %s results' % (url, len(appsDict))) else: logger.error('%s request failed: %s %s' % (url, r.status_code, r.text)) From 3911609be664f525d75134c23e33667a7223c339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Tue, 24 Mar 2015 17:04:26 +0100 Subject: [PATCH 09/11] fix wasnt a real fix. this works now --- backend.py | 2 +- job.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend.py b/backend.py index 35f2951..ca75972 100644 --- a/backend.py +++ b/backend.py @@ -171,7 +171,7 @@ def get_app_bundleId(self, bundleId, version=None): appsDict = json.loads(r.text) if len(appsDict) == 1: if not 'minimumOsVersion' in appsDict: - minimumOsVersion = 0 + minimumOsVersion = '0' else: minimumOsVersion = appsDict['minimumOsVersion'] return (appsDict.values()[0], minimumOsVersion) diff --git a/job.py b/job.py index ae85b62..3210005 100644 --- a/job.py +++ b/job.py @@ -122,7 +122,11 @@ def _install_app(self, pilot): elif self.appId: # install from backend - if StrictVersion(minimumOsVersion) > StrictVersion(productVersion): + productVersion_alt = int(''.join(productVersion.split('.'))) + # has to be > 99 to compare with e.g. ProductVersion 7.1.2 + if productVersion_alt < 100: + productVersion_alt *= 10 + if int(minimumOsVersion) > productVersion_alt: raise ProductVersionError(minimumOsVersion) # dirty check for ipa-size < ~50MB From 6594dd3ab0921bbdb3dd18195708923698eb98de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Thu, 26 Mar 2015 10:32:20 +0100 Subject: [PATCH 10/11] store.py: choose User-Agent by device --- job.py | 8 +++++++- store.py | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/job.py b/job.py index 3210005..c6099bc 100644 --- a/job.py +++ b/job.py @@ -170,10 +170,16 @@ def _install_app(self, pilot): storeCountry = 'de' if 'storeCountry' in jobInfo: storeCountry = jobInfo['storeCountry'] + if 'DeviceClass' in self.device.device_info_dict(): + deviceClass = self.device.device_info_dict()['DeviceClass'] + else: + deviceClass = 'iPhone' + + logger.debug('User-Agent: %s' % deviceClass) ## get appInfo logger.debug('fetch appInfo from iTunesStore') - store = AppStore(storeCountry) + store = AppStore(storeCountry, deviceClass) trackId = 0 appInfo = {} try: diff --git a/store.py b/store.py index 996eab6..d186d5a 100644 --- a/store.py +++ b/store.py @@ -1,6 +1,7 @@ from bs4 import BeautifulSoup import urllib2 import json +import plistlib class AppStoreException(Exception): pass @@ -20,7 +21,15 @@ class AppStore(object): def __do_request(self, url): request = urllib2.Request(url) - request.add_header('User-Agent', 'iTunes-iPhone/5.1.1 (3)') + if self.deviceClass == "iPhone": + request.add_header('User-Agent', 'iTunes-iPhone/5.1.1 (5; 16GB)') + elif self.deviceClass == "iPad": + request.add_header('User-Agent', 'iTunes-iPad/5.1.1 (16GB)') + elif self.deviceClass == "iPod": + request.add_header('User-Agent', 'iTunes-iPod/5.1.1 (5; 16GB)') + else: + raise AppStoreException("Invalid User-Agent: %s" % self.deviceClass) + response = None try: response = urllib2.urlopen(request, timeout=15) @@ -32,8 +41,9 @@ def __do_request(self, url): return response - def __init__(self, country="de"): + def __init__(self, country="de", deviceClass="iPhone"): self.country = country + self.deviceClass = deviceClass def get_app_info(self, appId): @@ -112,4 +122,4 @@ def countryForStoreFrontId(storeFrontId): if storeFrontId in AppStore.storeFrontIdToCountryDict: return AppStore.storeFrontIdToCountryDict[storeFrontId] else: - return None \ No newline at end of file + return None From b9d045f9eb8c935f2341cc738aca5435c8f63e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karsten=20K=C3=B6nig?= Date: Thu, 26 Mar 2015 11:13:00 +0100 Subject: [PATCH 11/11] job.py: try different device classes for a specific job --- job.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/job.py b/job.py index c6099bc..4b81bca 100644 --- a/job.py +++ b/job.py @@ -186,8 +186,19 @@ def _install_app(self, pilot): trackId = store.get_trackId_for_bundleId(bundleId) appInfo = store.get_app_info(trackId) except AppStoreException as e: + if deviceClass == 'iPad': + device_type = 0b100 + elif deviceClass == 'iPhone': + device_type = 0b10 + else: + device_type == 0b1 + + self.jobDict['compatible_devices'] ^= device_type logger.error('unable to get appInfo: %s ', e) - raise JobExecutionError('unable to get appInfo: AppStoreException') + if self.jobDict['compatible_devices'] != 0: + raise + else: + raise JobExecutionError('unable to get appInfo: AppStoreException') self.jobDict['appInfo'] = appInfo logger.debug('using appInfo: %s' % str(appInfo)) @@ -290,7 +301,15 @@ def execute(self): minimumOSVersion *= 10 backendJobData['jobInfo']['minimumOSVersion'] = str(minimumOSVersion) self.backend.post_job(backendJobData) - raise + return False + except AppStoreException: + logger.warn("Job execution aborted: Job seems to be not compatible with device") + backendJobData['state'] = Job.STATE.PENDING + backendJobData['worker'] = None + backendJobData['device'] = None + backendJobData['compatible_devices'] = self.jobDict['compatible_devices'] + self.backend.post_job(backendJobData) + return False ## set job finished if self.jobId: @@ -424,12 +443,6 @@ def execute(self): backendJobData['error_message'] = str(e) self.backend.post_job(backendJobData) return False - except ProductVersionError, e: - logger.warn("Job execution aborted: iOS version to low") - backendJobData['state'] = Job.STATE.PENDING - backendJobData['jobInfo']['minimumOSVersion'] = ''.join(str(e).split('.')) - self.backend.post_job(backendJobData) - raise ## set job finished backendJobData['state'] = Job.STATE.FINISHED