diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9765c3f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*pyc +*# +.#* diff --git a/backend.py b/backend.py index 394c976..ca75972 100644 --- a/backend.py +++ b/backend.py @@ -170,11 +170,15 @@ def get_app_bundleId(self, bundleId, version=None): if r.status_code == 200: appsDict = json.loads(r.text) if len(appsDict) == 1: - return appsDict.values()[0] + if not 'minimumOsVersion' in appsDict: + minimumOsVersion = '0' + else: + minimumOsVersion = appsDict['minimumOsVersion'] + return (appsDict.values()[0], minimumOsVersion) logger.debug('%s returned %s results' % (url, len(appsDict))) else: logger.error('%s request failed: %s %s' % (url, r.status_code, r.text)) - return None + return (None, None) def get_app_archive(self, appId, archivePath): diff --git a/device.py b/device.py index d47da0d..7e752e0 100644 --- a/device.py +++ b/device.py @@ -67,7 +67,14 @@ def device_info_dict(self): ''' raw device information as dict ''' if (len(self.deviceDict) == 0): - output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid]) + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid]) + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid]) + except: + raise + self.deviceDict = plistlib.readPlistFromString(output) return self.deviceDict @@ -78,7 +85,14 @@ def locale(self): ''' the devices locale setting ''' if (self.locale_val == ""): - self.locale_val = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + try: + self.locale_val = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + except subprocess.CalledProcessError as e: + try: + self.locale_val = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.international", "--key", "Locale"]).strip() + except: + raise + return self.locale_val @@ -97,7 +111,15 @@ def base_url(self): def free_bytes(self): ''' get the free space left on the device in bytes ''' - output = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + try: + output = subprocess.check_output(["ideviceinfo", "--udid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinfo", "--uuid", self.udid, "--domain", "com.apple.disk_usage", "--key", "TotalDataAvailable"]) + except: + raise + free_bytes = 0 try: free_bytes = long(output) @@ -111,7 +133,14 @@ def account_info_dict(self): ''' get raw account info from device as dict. ''' if (len(self.accountDict) == 0): - output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--udid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinfo", "--xml", "--uuid", self.udid, "--domain", "com.apple.mobile.iTunes.store", "--key", "KnownAccounts"]) + except: + raise + if len(output) > 0: self.accountDict = plistlib.readPlistFromString(output) else: @@ -163,7 +192,14 @@ def accounts(self): def installed_apps(self): ''' list all installed apps as dict. ''' - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user", "-o", "xml"]) + except: + raise + if (len(output)==0): return {} @@ -173,7 +209,14 @@ def installed_apps(self): plist = plistlib.readPlistFromString(output) except Exception: logger.warning("Failed to parse installed apps via xml output. Try to extract data via regex.") - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user"]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--list-apps", "-o", "list_user", "-o"]) + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--list-apps", "-o", "list_user"]) + except: + raise + regex = re.compile("^(?P.*) - (?P.*) (?P(\d+\.*)+)$",re.MULTILINE) # r = regex.search(output) for i in regex.finditer(output): @@ -216,7 +259,13 @@ def install(self, app_archive_path): ''' result=True try: - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--install", app_archive_path]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--install", app_archive_path]) + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--install", app_archive_path]) + except: + raise logger.debug('output: %s' % output) if (len(output)==0): result=False @@ -231,7 +280,14 @@ def uninstall(self, bundleId): ''' result=True try: - output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--uninstall", bundleId]) + try: + output = subprocess.check_output(["ideviceinstaller", "--udid", self.udid, "--uninstall", bundleId]) + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output(["ideviceinstaller", "--uuid", self.udid, "--uninstall", bundleId]) + except: + raise + logger.debug('output: %s' % output) if (len(output)==0): result=False @@ -244,11 +300,15 @@ def archive(self, bundleId, app_archive_folder, app_only=True, uninstall=True): ''' archives an app to `app_archive_folder` returns True or False ''' - options = ["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"] + options = ["ideviceinstaller", "--udid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"] + options_alt = ["ideviceinstaller", "--uuid", self.udid, "--archive", bundleId, "-o", "copy="+app_archive_folder, "-o", "remove"] + if app_only: options.extend(["-o", "app_only"]) + options_alt.extend(["-o", "app_only"]) if uninstall: options.extend(["-o", "uninstall"]) + options_alt.extend(["-o", "uninstall"]) if not os.path.exists(app_archive_folder): os.makedirs(app_archive_folder) @@ -260,7 +320,12 @@ def archive(self, bundleId, app_archive_folder, app_only=True, uninstall=True): if (len(output)==0): result=False except subprocess.CalledProcessError as e: - logger.error('archiving app %s failed with: %s ', bundleId, e, output) - result=False + try: + output = subprocess.check_output(options_alt) + logger.debug('output: %s' % output) + if (len(output)==0): + result=False + except subprocess.CalledProcessError as e: + logger.error('archiving app %s failed with: %s ', bundleId, e, output) + result=False return result - diff --git a/job.py b/job.py index e412da6..4b81bca 100644 --- a/job.py +++ b/job.py @@ -3,6 +3,7 @@ import logging import base64 import time +from distutils.version import StrictVersion from enum import Enum from store import AppStore, AppStoreException @@ -13,6 +14,9 @@ class JobExecutionError(Exception): pass +class ProductVersionError(Exception): + pass + class Job(object): @@ -80,6 +84,8 @@ def _install_app(self, pilot): if 'version' in jobInfo: version = jobInfo['version'] + productVersion = self.device.device_info_dict()['ProductVersion'] + #check app type if 'AppStoreApp' == jobInfo['appType']: logger.debug('installing appstore app %s' % bundleId) @@ -100,7 +106,8 @@ def _install_app(self, pilot): alreadyInstalled = True # check the backend for already existing app - app = self.backend.get_app_bundleId(bundleId, version) + (app, minimumOsVersion) = self.backend.get_app_bundleId(bundleId, version) + logger.debug('backend result for bundleId %s: %s' % (bundleId, app)) if app and '_id' in app: self.appId = app['_id'] @@ -115,6 +122,13 @@ def _install_app(self, pilot): elif self.appId: # install from backend + productVersion_alt = int(''.join(productVersion.split('.'))) + # has to be > 99 to compare with e.g. ProductVersion 7.1.2 + if productVersion_alt < 100: + productVersion_alt *= 10 + if int(minimumOsVersion) > productVersion_alt: + raise ProductVersionError(minimumOsVersion) + # dirty check for ipa-size < ~50MB if app and 'fileSizeBytes' in app: size = 0 @@ -156,22 +170,41 @@ def _install_app(self, pilot): storeCountry = 'de' if 'storeCountry' in jobInfo: storeCountry = jobInfo['storeCountry'] + if 'DeviceClass' in self.device.device_info_dict(): + deviceClass = self.device.device_info_dict()['DeviceClass'] + else: + deviceClass = 'iPhone' + + logger.debug('User-Agent: %s' % deviceClass) ## get appInfo logger.debug('fetch appInfo from iTunesStore') - store = AppStore(storeCountry) + store = AppStore(storeCountry, deviceClass) trackId = 0 appInfo = {} try: trackId = store.get_trackId_for_bundleId(bundleId) appInfo = store.get_app_info(trackId) except AppStoreException as e: + if deviceClass == 'iPad': + device_type = 0b100 + elif deviceClass == 'iPhone': + device_type = 0b10 + else: + device_type == 0b1 + + self.jobDict['compatible_devices'] ^= device_type logger.error('unable to get appInfo: %s ', e) - raise JobExecutionError('unable to get appInfo: AppStoreException') + if self.jobDict['compatible_devices'] != 0: + raise + else: + raise JobExecutionError('unable to get appInfo: AppStoreException') self.jobDict['appInfo'] = appInfo logger.debug('using appInfo: %s' % str(appInfo)) + if StrictVersion(appInfo['minimum-os-version']) > StrictVersion(productVersion): + raise ProductVersionError(appInfo['minimum-os-version']) ## get account accountId = '' @@ -254,7 +287,29 @@ def execute(self): except JobExecutionError, e: logger.error("Job execution failed: %s" % str(e)) backendJobData['state'] = Job.STATE.FAILED + backendJobData['error_message'] = str(e) result = False + except ProductVersionError, e: + logger.warn("Job execution aborted: iOS version to low") + backendJobData['state'] = Job.STATE.PENDING + backendJobData['worker'] = None + backendJobData['device'] = None + + # has to be > 99 to compare with e.g. ProductVersion 7.1.2 + minimumOSVersion = int(''.join(str(e).split('.'))) + if minimumOSVersion < 100: + minimumOSVersion *= 10 + backendJobData['jobInfo']['minimumOSVersion'] = str(minimumOSVersion) + self.backend.post_job(backendJobData) + return False + except AppStoreException: + logger.warn("Job execution aborted: Job seems to be not compatible with device") + backendJobData['state'] = Job.STATE.PENDING + backendJobData['worker'] = None + backendJobData['device'] = None + backendJobData['compatible_devices'] = self.jobDict['compatible_devices'] + self.backend.post_job(backendJobData) + return False ## set job finished if self.jobId: @@ -385,6 +440,7 @@ def execute(self): except JobExecutionError, e: logger.error("Job execution failed: %s" % str(e)) backendJobData['state'] = Job.STATE.FAILED + backendJobData['error_message'] = str(e) self.backend.post_job(backendJobData) return False diff --git a/scheduler.py b/scheduler.py index d256d39..fcf7083 100755 --- a/scheduler.py +++ b/scheduler.py @@ -36,7 +36,6 @@ class Scheduler(object): @classmethod def _default_runjob(cls): return { - 'type':'run_app', 'state':'pending', 'jobInfo': { 'appType':'AppStoreApp' @@ -54,7 +53,7 @@ def schedule_job(self, jobDict): return jobId - def schedule_bundleId(self, bundleId, worker=None, device=None, account=None, country=None, executionStrategy=None): + def schedule_bundleId(self, bundleId, job_type, worker=None, device=None, account=None, country=None, executionStrategy=None): jobDict = { 'jobInfo': { 'bundleId':bundleId @@ -70,10 +69,13 @@ def schedule_bundleId(self, bundleId, worker=None, device=None, account=None, co jobDict['jobInfo']['storeCountry'] = country if executionStrategy: jobDict['jobInfo']['executionStrategy'] = executionStrategy + + jobDict['type'] = job_type + return self.schedule_job(jobDict) - def schedule_appId(self, appId, account=None, country=None, executionStrategy=None): + def schedule_appId(self, appId, job_type, account=None, country=None, executionStrategy=None): url = 'http://itunes.apple.com/lookup?id=%s' % appId r = requests.get(url) if r.status_code != 200: @@ -85,11 +87,11 @@ def schedule_appId(self, appId, account=None, country=None, executionStrategy=No if len(results) != 1: logger.error("No app with id %s found", (appId)) return False - return self.schedule_bundleId(results[0]['bundleId'], account=account, country=country, executionStrategy=None) + return self.schedule_bundleId(results[0]['bundleId'], job_type, account=account, country=country, executionStrategy=None) - def schedule_itunes(self, url, account=None, country=None, executionStrategy=None): + def schedule_itunes(self, url, job_type, account=None, country=None, executionStrategy=None): logger.info('Adding apps from iTunes (%s)' % url) r = requests.get(url) if r.status_code != 200: @@ -100,7 +102,7 @@ def schedule_itunes(self, url, account=None, country=None, executionStrategy=Non entries = resDict['feed']['entry'] result = True for entry in entries: - if not self.schedule_bundleId(entry['id']['attributes']['im:bundleId'], account=account, country=country, executionStrategy=None): + if not self.schedule_bundleId(entry['id']['attributes']['im:bundleId'], job_type, account=account, country=country, executionStrategy=None): result = False return result @@ -112,7 +114,7 @@ def main(): parser = argparse.ArgumentParser(description='schedule backend jobs from different sources.') parser.add_argument('-b','--backend', required=True, help='the backend url.') parser.add_argument('-a','--account', help='the accountId to use.') - + parser.add_argument('-t','--job-type', default='install_app', help='could be install_app, run_app or exec_cmd. default is install_app') parser.add_argument('-s','--strategy', help='the execution strategy and duration to use.') # add commands @@ -134,16 +136,22 @@ def main(): def printRes(res): if res: logger.info('done!') + print(res) else: logger.error('error occured (could be partially done)') + + if not args.job_type in ['install_app', 'run_app', 'exec_cmd']: + print('Not a valid job-type!') + return + if 'bundleId' in args and args.bundleId: - res = scheduler.schedule_bundleId(args.bundleId, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_bundleId(args.bundleId, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return if 'appId' in args and args.appId: - res = scheduler.schedule_appId(args.appId, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_appId(args.appId, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return @@ -153,13 +161,13 @@ def printRes(res): if 'itunes_top' in args and args.itunes_top: url = 'https://itunes.apple.com/%s/rss/topfreeapplications/limit=%i/genre=%s/json' % (args.itunes_country, args.itunes_top, genre) - res = scheduler.schedule_itunes(url, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_itunes(url, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return if 'itunes_new' in args and args.itunes_new: url = 'https://itunes.apple.com/%s/rss/newfreeapplications/limit=%i/genre=%s/json' % (args.itunes_country, args.itunes_new, genre) - res = scheduler.schedule_itunes(url, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) + res = scheduler.schedule_itunes(url, args.job_type, account=args.account, country=args.itunes_country, executionStrategy=args.strategy) printRes(res) return diff --git a/store.py b/store.py index 996eab6..d186d5a 100644 --- a/store.py +++ b/store.py @@ -1,6 +1,7 @@ from bs4 import BeautifulSoup import urllib2 import json +import plistlib class AppStoreException(Exception): pass @@ -20,7 +21,15 @@ class AppStore(object): def __do_request(self, url): request = urllib2.Request(url) - request.add_header('User-Agent', 'iTunes-iPhone/5.1.1 (3)') + if self.deviceClass == "iPhone": + request.add_header('User-Agent', 'iTunes-iPhone/5.1.1 (5; 16GB)') + elif self.deviceClass == "iPad": + request.add_header('User-Agent', 'iTunes-iPad/5.1.1 (16GB)') + elif self.deviceClass == "iPod": + request.add_header('User-Agent', 'iTunes-iPod/5.1.1 (5; 16GB)') + else: + raise AppStoreException("Invalid User-Agent: %s" % self.deviceClass) + response = None try: response = urllib2.urlopen(request, timeout=15) @@ -32,8 +41,9 @@ def __do_request(self, url): return response - def __init__(self, country="de"): + def __init__(self, country="de", deviceClass="iPhone"): self.country = country + self.deviceClass = deviceClass def get_app_info(self, appId): @@ -112,4 +122,4 @@ def countryForStoreFrontId(storeFrontId): if storeFrontId in AppStore.storeFrontIdToCountryDict: return AppStore.storeFrontIdToCountryDict[storeFrontId] else: - return None \ No newline at end of file + return None diff --git a/worker.py b/worker.py index 251bbae..2067ff6 100755 --- a/worker.py +++ b/worker.py @@ -13,7 +13,7 @@ logger.setLevel(level=logging.INFO) -from job import JobFactory +from job import JobFactory, ProductVersionError from device import iDevice from backend import Backend #from pilot import Pilot @@ -65,7 +65,8 @@ def run(self): logger.error("traceback: %s" % tb) logger.error("Device loop will be stopped now.") self.stop() - + except ProductVersionError as e: + logger.warn('Executing job aborted: Higher OS version (%s) needed, job is set to pending' % e) except Exception as e: logger.error("Executing job failed: %s" % e) tb = traceback.format_exc()