diff --git a/anchore_engine/clients/docker_registry.py b/anchore_engine/clients/docker_registry.py index 2108d7d39..a43dbf0ab 100644 --- a/anchore_engine/clients/docker_registry.py +++ b/anchore_engine/clients/docker_registry.py @@ -8,7 +8,8 @@ import anchore_engine.configuration.localconfig import anchore_engine.auth.common from anchore_engine.subsys import logger -from anchore_engine.clients.skopeo_wrapper import get_image_manifest_skopeo, get_repo_tags_skopeo +from anchore_engine.clients.skopeo_wrapper import get_image_manifest_skopeo, get_repo_tags_skopeo, get_image_manifest_v2 as skopeo_get_image_manifest +from anchore_engine.utils import ImageInfo def get_image_manifest_docker_registry(url, registry, repo, tag, user=None, pw=None, verify=True): @@ -279,3 +280,48 @@ def get_image_manifest(userId, image_info, registry_creds): raise err else: raise Exception("could not get manifest/digest for image ({}) from registry ({}) - error: {}".format(fulltag, url, err)) + + +def get_image_manifest_v2(userId, image_info, registry_creds, default=None) -> ImageInfo: + logger.debug("get_image_manifest_list input: " + str(userId) + " : " + str(image_info) + " : " + str(time.time())) + logger.info("nightfury get_image_manifest_list input: " + str(userId) + " : " + str(image_info) + " : " + str(time.time())) + + registry = image_info['registry'] + try: + user, pw, registry_verify = anchore_engine.auth.common.get_creds_by_registry(registry, image_info['repo'], registry_creds=registry_creds) + except Exception as err: + raise err + + if registry == 'docker.io': + url = "https://index.docker.io" + if not re.match(".*/.*", image_info['repo']): + repo = "library/"+image_info['repo'] + else: + repo = image_info['repo'] + else: + url = "https://"+registry + repo = image_info['repo'] + + if image_info['digest']: + tag = None + input_digest = image_info['digest'] + fulltag = "{}/{}@{}".format(registry, repo, input_digest) + else: + input_digest = None + tag = image_info['tag'] + fulltag = "{}/{}:{}".format(registry, repo, tag) + + logger.debug("trying to get v2 manifest/digest for image ("+str(fulltag)+")") + logger.info("nightfury trying to get v2 manifest/digest for image ("+str(fulltag)+")") + try: + if tag: + result = skopeo_get_image_manifest(url, registry, repo, intag=tag, user=user, pw=pw, verify=registry_verify) + elif input_digest: + result = skopeo_get_image_manifest(url, registry, repo, indigest=input_digest, user=user, pw=pw, verify=registry_verify) + else: + raise Exception("neither tag nor digest was given as input") + except Exception: + logger.exception("could not get manifest/digest for image ({}) from registry ({})".format(fulltag, url)) + raise + + return result diff --git a/anchore_engine/clients/services/catalog.py b/anchore_engine/clients/services/catalog.py index 353b029f6..494e90265 100644 --- a/anchore_engine/clients/services/catalog.py +++ b/anchore_engine/clients/services/catalog.py @@ -53,10 +53,10 @@ def get_image(self, imageDigest): def get_image_by_id(self, imageId): return self.call_api(http.anchy_get, 'images', query_params={'imageId': imageId}) - def list_images(self, tag=None, digest=None, imageId=None, registry_lookup=False, history=False, image_status='active', analysis_status=None): + def list_images(self, tag=None, digest=None, imageId=None, registry_lookup=False, history=False, image_status='active', analysis_status=None, architecture=None): return self.call_api(http.anchy_get, 'images', query_params={'tag': tag, 'history': history, 'registry_lookup': registry_lookup, - 'digest': digest, 'imageId': imageId, 'image_status': image_status, 'analysis_status': analysis_status}) + 'digest': digest, 'imageId': imageId, 'image_status': image_status, 'analysis_status': analysis_status, 'architecture': architecture}) def update_image(self, imageDigest, image_record=None): payload = {} diff --git a/anchore_engine/clients/skopeo_wrapper.py b/anchore_engine/clients/skopeo_wrapper.py index d9488a228..cdc447d3f 100644 --- a/anchore_engine/clients/skopeo_wrapper.py +++ b/anchore_engine/clients/skopeo_wrapper.py @@ -4,7 +4,7 @@ import tempfile import anchore_engine.configuration.localconfig -from anchore_engine.utils import run_command, run_command_list, manifest_to_digest, AnchoreException +from anchore_engine.utils import run_command, run_command_list, manifest_to_digest, AnchoreException, ManifestDigestArch, ImageInfo from anchore_engine.subsys import logger from anchore_engine.common.errors import AnchoreError @@ -331,6 +331,135 @@ def get_image_manifest_skopeo(url, registry, repo, intag=None, indigest=None, to return manifest, digest, topdigest, topmanifest + +def get_digest_arch(pullstring, user=None, pw=None, verify=True): + ret = None + try: + proc_env = os.environ.copy() + if user and pw: + proc_env['SKOPUSER'] = user + proc_env['SKOPPASS'] = pw + credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"' + else: + credstr = "" + + if verify: + tlsverifystr = "--tls-verify=true" + else: + tlsverifystr = "--tls-verify=false" + + localconfig = anchore_engine.configuration.localconfig.get_config() + global_timeout = localconfig.get('skopeo_global_timeout', 0) + try: + global_timeout = int(global_timeout) + if global_timeout < 0: + global_timeout = 0 + except: + global_timeout = 0 + + if global_timeout: + global_timeout_str = "--command-timeout {}s".format(global_timeout) + else: + global_timeout_str = "" + + os_override_strs = ["", "--override-os windows"] + try: + success = False + for os_override_str in os_override_strs: + cmd = ["/bin/sh", "-c", + "skopeo {} {} inspect {} {} docker://{}".format(global_timeout_str, os_override_str, + tlsverifystr, credstr, pullstring)] + cmdstr = ' '.join(cmd) + try: + rc, sout, serr = run_command_list(cmd, env=proc_env) + if rc != 0: + skopeo_error = SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) + if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name: + raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr) + else: + logger.debug( + "command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str( + serr).strip()) + success = True + except Exception as err: + logger.error("command failed with exception - " + str(err)) + raise err + + if success: + sout = str(sout, 'utf-8') if sout else None + ret = sout + break + + if not success: + logger.error("could not retrieve manifest") + raise Exception("could not retrieve manifest") + + except Exception as err: + raise err + except Exception as err: + raise err + + return ret + +def _get_image_manifest_digest(registry, repo, tag=None, digest=None, user=None, pw=None, verify=True): + if digest: + pull_string = registry + "/" + repo + "@" + digest + elif tag: + pull_string = registry + "/" + repo + ":" + tag + else: + raise Exception("invalid input - must supply either an intag or indigest") + + try: + raw_manifest = get_image_manifest_skopeo_raw(pull_string, user=user, pw=pw, verify=verify) + s_manifest = json.loads(raw_manifest) + s_digest = manifest_to_digest(raw_manifest) + return s_manifest, s_digest + except Exception as err: + logger.warn("CMD failed - exception: " + str(err)) + raise err + + +def get_image_manifest_v2(url, registry, repo, intag=None, indigest=None, user=None, pw=None, verify=True, default=None): + try: + s_manifest, s_digest = _get_image_manifest_digest(registry, repo, intag, indigest, user, pw, verify) + result = ImageInfo(parent=ManifestDigestArch(s_manifest, s_digest, None), children=[]) + + if s_manifest.get('schemaVersion') == 2 and s_manifest.get('mediaType') == 'application/vnd.docker.distribution.manifest.list.v2+json': + for entry in s_manifest.get('manifests'): + child_platform = entry.get('platform') + child_digest = entry.get('digest') + if child_digest and child_platform and child_platform.get('os') in ['linux', 'windows']: + child_arch = child_platform.get('architecture') + child_variant = child_platform.get('variant') + arch_var = '{}_{}'.format(child_arch, child_variant) if child_variant else child_arch + if not default: + s_manifest, s_digest = _get_image_manifest_digest(registry, repo, None, child_digest, user, pw, verify) + result.children.append(ManifestDigestArch(s_manifest, s_digest, arch_var)) + elif child_arch in default: + s_manifest, s_digest = _get_image_manifest_digest(registry, repo, None, child_digest, user, pw, verify) + result.children.append(ManifestDigestArch(s_manifest, s_digest, arch_var)) + break + # else: + # continue + else: + try: + pull_string = registry + "/" + repo + "@" + s_digest + raw_inspect_out = get_digest_arch(pull_string, user=user, pw=pw, verify=verify) + inspect_out = json.loads(raw_inspect_out) + result.parent.arch = inspect_out.get('Architecture') + except Exception as err: + logger.warn("CMD failed - exception: " + str(err)) + raise err + except Exception as err: + logger.exception("Failed to fetch skopeo image manifest", err) + raise err + + if not result: + raise SkopeoError(msg="No digest/manifest from skopeo") + + return result + + class SkopeoError(AnchoreException): def __init__(self, cmd=None, rc=None, err=None, out=None, msg='Error encountered in skopeo operation'): diff --git a/anchore_engine/common/images.py b/anchore_engine/common/images.py index 4a037f747..529543448 100644 --- a/anchore_engine/common/images.py +++ b/anchore_engine/common/images.py @@ -6,7 +6,8 @@ from anchore_engine import db from anchore_engine.clients import docker_registry from anchore_engine.subsys import logger - +from copy import deepcopy +from anchore_engine.utils import ManifestDigestArch def lookup_registry_image(userId, image_info, registry_creds): digest = None @@ -69,6 +70,68 @@ def get_image_info(userId, image_type, input_string, registry_lookup=False, regi return ret +def _make_image_info_dict(common_info, parent: ManifestDigestArch, child: ManifestDigestArch): + image_info = deepcopy(common_info) + image_info['digest'] = child.digest + image_info['fulldigest'] = image_info['registry'] + "/" + image_info['repo'] + "@" + child.digest + image_info['manifest'] = child.manifest + image_info['architecture'] = child.arch + image_info['parentmanifest'] = parent.manifest + image_info['parentdigest'] = parent.digest + + # if we got a manifest, and the image_info does not yet contain an imageId, try to get it from the manifest + if image_info['manifest'] and not image_info['imageId']: + try: + imageId = re.sub("^sha256:", "", child.manifest['config']['digest']) + image_info['imageId'] = imageId + except Exception as err: + logger.debug("could not extract imageId from fetched manifest - exception: " + str(err)) + logger.debug("using digest hash as imageId due to incomplete manifest (" + str( + image_info['fulldigest']) + ")") + htype, image_info['imageId'] = image_info['digest'].split(":", 1) + + return image_info + + +def get_image_infos(userId, image_type, input_string, registry_lookup=False, registry_creds=[]): + ret = [] + + if image_type == 'docker': + try: + common_info = anchore_engine.utils.parse_dockerimage_string(input_string) + except Exception as err: + raise anchore_engine.common.helpers.make_anchore_exception(err, + input_message="cannot handle image input string", + input_httpcode=400) + + # ret.update(common_info) + + if registry_lookup and common_info['registry'] != 'localbuild': + # digest, manifest = lookup_registry_image(userId, image_info, registry_creds) + try: + result = docker_registry.get_image_manifest_v2(userId, common_info, registry_creds) + except Exception as err: + raise anchore_engine.common.helpers.make_anchore_exception(err, + input_message="cannot fetch image digest/manifest from registry", + input_httpcode=400) + + parent = result.parent + if result.children: + ret = [_make_image_info_dict(common_info, parent, child) for child in result.children] + else: + ret.append(_make_image_info_dict(common_info, parent, parent)) + else: + image_info = deepcopy(common_info) + image_info['manifest'] = {} + image_info['parentmanifest'] = {} + image_info['architecture'] = 'unknown' + ret.append(image_info) + else: + raise Exception("image type (" + str(image_type) + ") not supported") + + return ret + + def clean_docker_image_details_for_update(image_details): ret = [] @@ -81,7 +144,7 @@ def clean_docker_image_details_for_update(image_details): return ret -def make_image_record(userId, image_type, input_string, image_metadata={}, registry_lookup=True, registry_creds=[]): +def make_image_record(userId, image_type, input_string, image_metadata={}, registry_lookup=True, registry_creds=[], arch=None): if image_type == 'docker': try: dockerfile = image_metadata.get('dockerfile', None) @@ -116,7 +179,7 @@ def make_image_record(userId, image_type, input_string, image_metadata={}, regis parentdigest = image_metadata.get('parentdigest', None) created_at = image_metadata.get('created_at', None) - return make_docker_image(userId, input_string=input_string, tag=tag, digest=digest, imageId=imageId, parentdigest=parentdigest, created_at=created_at, dockerfile=dockerfile, dockerfile_mode=dockerfile_mode, registry_lookup=registry_lookup, registry_creds=registry_creds, annotations=annotations) + return make_docker_image(userId, input_string=input_string, tag=tag, digest=digest, imageId=imageId, parentdigest=parentdigest, created_at=created_at, dockerfile=dockerfile, dockerfile_mode=dockerfile_mode, registry_lookup=registry_lookup, registry_creds=registry_creds, annotations=annotations, arch=arch) else: raise Exception("image type ("+str(image_type)+") not supported") @@ -124,7 +187,7 @@ def make_image_record(userId, image_type, input_string, image_metadata={}, regis return None -def make_docker_image(userId, input_string=None, tag=None, digest=None, imageId=None, parentdigest=None, created_at=None, dockerfile=None, dockerfile_mode=None, registry_lookup=True, registry_creds=[], annotations={}): +def make_docker_image(userId, input_string=None, tag=None, digest=None, imageId=None, parentdigest=None, created_at=None, dockerfile=None, dockerfile_mode=None, registry_lookup=True, registry_creds=[], annotations={}, arch=None): ret = {} if input_string: @@ -152,6 +215,7 @@ def make_docker_image(userId, input_string=None, tag=None, digest=None, imageId= new_input['userId'] = userId new_input['image_type'] = 'docker' new_input['dockerfile_mode'] = dockerfile_mode + new_input['arch'] = arch if not parentdigest: parentdigest = imageDigest diff --git a/anchore_engine/db/db_catalog_image.py b/anchore_engine/db/db_catalog_image.py index e02874213..f1f9b4d5b 100644 --- a/anchore_engine/db/db_catalog_image.py +++ b/anchore_engine/db/db_catalog_image.py @@ -159,23 +159,24 @@ def get_created_at(record): return record['created_at'] return 0 -def get_byimagefilter(userId, image_type, dbfilter={}, onlylatest=False, image_status='active', analysis_status=None, session=None): +def get_byimagefilter(userId, image_type, dbfilter={}, onlylatest=False, image_status='active', analysis_status=None, arch=None, session=None): if not session: session = db.Session ret = [] ret_results = [] + latest_tuple_dict = dict() if image_type == 'docker': results = db.db_catalog_image_docker.get_byfilter(userId, session=session, **dbfilter) - latest = None for result in results: imageDigest = result['imageDigest'] dbobj = get(imageDigest, userId, session=session) - if (image_status is None or dbobj['image_status'] == image_status) and (analysis_status is None or dbobj['analysis_status'] == analysis_status): - if not latest: - latest = dbobj + if (image_status is None or dbobj['image_status'] == image_status) and (analysis_status is None or dbobj['analysis_status'] == analysis_status) and (arch and arch == dbobj['arch']): + tag_arch_tuple = (result['registry'], result('repo'), result['tag'], dbobj['arch']) + if tag_arch_tuple not in latest_tuple_dict: + latest_tuple_dict[tag_arch_tuple] = dbobj ret_results.append(dbobj) @@ -183,34 +184,36 @@ def get_byimagefilter(userId, image_type, dbfilter={}, onlylatest=False, image_s if not onlylatest: ret = ret_results else: - if latest: - ret = [latest] + if latest_tuple_dict: + ret = list(latest_tuple_dict.values()) return ret def get_all_tagsummary(userId, session=None, image_status=None): query = session.query(CatalogImage.imageDigest, - CatalogImage.parentDigest, - CatalogImageDocker.registry, - CatalogImageDocker.repo, - CatalogImageDocker.tag, - CatalogImage.analysis_status, - CatalogImageDocker.created_at, - CatalogImageDocker.imageId, - CatalogImage.analyzed_at, - CatalogImageDocker.tag_detected_at, - CatalogImage.image_status).filter(and_(CatalogImage.userId == userId, - CatalogImage.imageDigest == CatalogImageDocker.imageDigest, - CatalogImageDocker.userId == userId)) + CatalogImage.arch, + CatalogImage.parentDigest, + CatalogImageDocker.registry, + CatalogImageDocker.repo, + CatalogImageDocker.tag, + CatalogImage.analysis_status, + CatalogImageDocker.created_at, + CatalogImageDocker.imageId, + CatalogImage.analyzed_at, + CatalogImageDocker.tag_detected_at, + CatalogImage.image_status).filter(and_(CatalogImage.userId == userId, + CatalogImage.imageDigest == CatalogImageDocker.imageDigest, + CatalogImageDocker.userId == userId)) if image_status and isinstance(image_status, list) and 'all' not in image_status: # filter only if specific states are input and != all query = query.filter(CatalogImage.image_status.in_(image_status)) ret = [] - for idig, pdig, reg, repo, tag, astat, cat, iid, anat, dat, istat in query: + for idig, arch, pdig, reg, repo, tag, astat, cat, iid, anat, dat, istat in query: ret.append({ 'imageDigest': idig, + 'architecture': arch, 'parentDigest': pdig, 'fulltag': reg + "/" + repo + ":" + tag, 'analysis_status': astat, diff --git a/anchore_engine/services/apiext/api/controllers/images.py b/anchore_engine/services/apiext/api/controllers/images.py index a38b8f89c..53dd67e3e 100644 --- a/anchore_engine/services/apiext/api/controllers/images.py +++ b/anchore_engine/services/apiext/api/controllers/images.py @@ -1073,12 +1073,12 @@ def get_image_vulnerabilities_by_type_imageId(imageId, vtype): # return(return_object, httpcode) -def do_list_images(account, filter_tag=None, filter_digest=None, history=False, image_status=None, analysis_status=None): +def do_list_images(account, filter_tag=None, filter_digest=None, history=False, image_status=None, analysis_status=None, architecture=None): client = internal_client_for(CatalogClient, account) try: # Query param fulltag has precedence for search - image_records = client.list_images(tag=filter_tag, digest=filter_digest, history=history, image_status=image_status, analysis_status=analysis_status) + image_records = client.list_images(tag=filter_tag, digest=filter_digest, history=history, image_status=image_status, analysis_status=analysis_status, architecture=architecture) return [make_response_image(image_record, include_detail=True) for image_record in image_records] @@ -1176,71 +1176,77 @@ def analyze_image(account, source, force=False, enable_subscriptions=None, annot raise ValueError("The source property must have at least one of tag, digest, or archive set to non-null") # add the image to the catalog - image_record = client.add_image(tag=tag, digest=digest, dockerfile=dockerfile, annotations=annotations, + # image_record = client.add_image(tag=tag, digest=digest, dockerfile=dockerfile, annotations=annotations, + # created_at=ts, from_archive=is_from_archive, allow_dockerfile_update=force) + image_records = client.add_image(tag=tag, digest=digest, dockerfile=dockerfile, annotations=annotations, created_at=ts, from_archive=is_from_archive, allow_dockerfile_update=force) - imageDigest = image_record['imageDigest'] + results = [] + for image_record in image_records: + imageDigest = image_record['imageDigest'] - # finally, do any state updates and return - if image_record: - logger.debug("added image: " + str(imageDigest)) + # finally, do any state updates and return + if image_record: + logger.debug("added image: " + str(imageDigest)) - # auto-subscribe for NOW - for image_detail in image_record['image_detail']: - fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] + # auto-subscribe for NOW + for image_detail in image_record['image_detail']: + fulltag = image_detail['registry'] + "/" + image_detail['repo'] + ":" + image_detail['tag'] - foundtypes = [] - try: - subscription_records = client.get_subscription(subscription_key=fulltag) - except Exception as err: - subscription_records = [] - - for subscription_record in subscription_records: - if subscription_record['subscription_key'] == fulltag: - foundtypes.append(subscription_record['subscription_type']) - - sub_types = anchore_engine.common.subscription_types - for sub_type in sub_types: - if sub_type in ['repo_update']: - continue - if sub_type not in foundtypes: - try: - default_active = False - if enable_subscriptions and sub_type in enable_subscriptions: - logger.debug("auto-subscribing image: " + str(sub_type)) - default_active = True - client.add_subscription({'active': default_active, 'subscription_type': sub_type, - 'subscription_key': fulltag}) - except: + foundtypes = [] + try: + subscription_records = client.get_subscription(subscription_key=fulltag) + except Exception as err: + subscription_records = [] + + for subscription_record in subscription_records: + if subscription_record['subscription_key'] == fulltag: + foundtypes.append(subscription_record['subscription_type']) + + sub_types = anchore_engine.common.subscription_types + for sub_type in sub_types: + if sub_type in ['repo_update']: + continue + if sub_type not in foundtypes: try: - client.update_subscription({'subscription_type': sub_type, 'subscription_key': fulltag}) + default_active = False + if enable_subscriptions and sub_type in enable_subscriptions: + logger.debug("auto-subscribing image: " + str(sub_type)) + default_active = True + client.add_subscription({'active': default_active, 'subscription_type': sub_type, + 'subscription_key': fulltag}) except: - pass - else: - if enable_subscriptions and sub_type in enable_subscriptions: - client.update_subscription({'active': True, 'subscription_type': sub_type, 'subscription_key': fulltag}) - - # set the state of the image appropriately - currstate = image_record['analysis_status'] - if not currstate: - newstate = taskstate.init_state('analyze', None) - elif force or currstate == taskstate.fault_state('analyze'): - newstate = taskstate.reset_state('analyze') - elif image_record['image_status'] != taskstate.base_state('image_status'): - newstate = taskstate.reset_state('analyze') - else: - newstate = currstate - - if (currstate != newstate) or (force): - logger.debug("state change detected: " + str(currstate) + " : " + str(newstate)) - image_record.update({'image_status': taskstate.reset_state('image_status'), 'analysis_status': newstate}) - updated_image_record = client.update_image(imageDigest, image_record) - if updated_image_record: - image_record = updated_image_record[0] - else: - logger.debug("no state change detected: " + str(currstate) + " : " + str(newstate)) + try: + client.update_subscription({'subscription_type': sub_type, 'subscription_key': fulltag}) + except: + pass + else: + if enable_subscriptions and sub_type in enable_subscriptions: + client.update_subscription({'active': True, 'subscription_type': sub_type, 'subscription_key': fulltag}) + + # set the state of the image appropriately + currstate = image_record['analysis_status'] + if not currstate: + newstate = taskstate.init_state('analyze', None) + elif force or currstate == taskstate.fault_state('analyze'): + newstate = taskstate.reset_state('analyze') + elif image_record['image_status'] != taskstate.base_state('image_status'): + newstate = taskstate.reset_state('analyze') + else: + newstate = currstate + + if (currstate != newstate) or (force): + logger.debug("state change detected: " + str(currstate) + " : " + str(newstate)) + image_record.update({'image_status': taskstate.reset_state('image_status'), 'analysis_status': newstate}) + updated_image_record = client.update_image(imageDigest, image_record) + if updated_image_record: + image_record = updated_image_record[0] + else: + logger.debug("no state change detected: " + str(currstate) + " : " + str(newstate)) + + results.append(make_response_image(image_record, include_detail=True)) - return [make_response_image(image_record, include_detail=True)] + return results except Exception as err: logger.debug("operation exception: " + str(err)) raise err diff --git a/anchore_engine/services/apiext/swagger/swagger.yaml b/anchore_engine/services/apiext/swagger/swagger.yaml index f1c8f64a6..0c9d1d97f 100644 --- a/anchore_engine/services/apiext/swagger/swagger.yaml +++ b/anchore_engine/services/apiext/swagger/swagger.yaml @@ -427,6 +427,11 @@ paths: required: false type: string description: "Full docker-pull string to filter results by (e.g. docker.io/library/nginx:latest, or myhost.com:5000/testimages:v1.1.1)" + - name: architecture + in: query + required: false + type: string + description: Filter by just architecture such as amd64 or or architecture and variant separated by _ such as arm_v6 - name: image_status in: query required: false @@ -3160,6 +3165,8 @@ definitions: properties: imageDigest: type: string + architecture: + type: string parentDigest: type: string imageId: diff --git a/anchore_engine/services/catalog/api/controllers/default_controller.py b/anchore_engine/services/catalog/api/controllers/default_controller.py index 20eff5c66..2be2f50e4 100644 --- a/anchore_engine/services/catalog/api/controllers/default_controller.py +++ b/anchore_engine/services/catalog/api/controllers/default_controller.py @@ -59,10 +59,10 @@ def image_tags_get(image_status=None): @authorizer.requires_account(with_types=INTERNAL_SERVICE_ALLOWED) -def list_images(tag=None, digest=None, imageId=None, registry_lookup=False, history=False, image_status='active', analysis_status=None): +def list_images(tag=None, digest=None, imageId=None, registry_lookup=False, history=False, image_status='active', analysis_status=None, architecture=None): try: request_inputs = anchore_engine.apis.do_request_prep(connexion.request, - default_params={'tag': tag, 'digest': digest, 'imageId': imageId, 'registry_lookup': registry_lookup, 'history': history, 'image_status': image_status, 'analysis_status': analysis_status}) + default_params={'tag': tag, 'digest': digest, 'imageId': imageId, 'registry_lookup': registry_lookup, 'history': history, 'image_status': image_status, 'analysis_status': analysis_status, 'architecture': architecture}) with db.session_scope() as session: return_object, httpcode = anchore_engine.services.catalog.catalog_impl.image(session, request_inputs) diff --git a/anchore_engine/services/catalog/catalog_impl.py b/anchore_engine/services/catalog/catalog_impl.py index b44cd71e5..f037334c7 100644 --- a/anchore_engine/services/catalog/catalog_impl.py +++ b/anchore_engine/services/catalog/catalog_impl.py @@ -242,6 +242,11 @@ def image(dbsession, request_inputs, bodycontent=None): if params and 'history' in params: history = params['history'] + architecture = None + if params: + architecture = params.get('architecture') + + image_status = params.get('image_status') if params else None analysis_status = params.get('analysis_status') if params else None @@ -301,10 +306,10 @@ def image(dbsession, request_inputs, bodycontent=None): logger.debug("image DB lookup filter: " + json.dumps(dbfilter, indent=4)) if history: - image_records = db_catalog_image.get_byimagefilter(userId, 'docker', dbfilter=dbfilter, image_status=image_status_filter, analysis_status=analysis_status_filter, session=dbsession) + image_records = db_catalog_image.get_byimagefilter(userId, 'docker', dbfilter=dbfilter, image_status=image_status_filter, analysis_status=analysis_status_filter, arch=architecture, session=dbsession) else: image_records = db_catalog_image.get_byimagefilter(userId, 'docker', dbfilter=dbfilter, image_status=image_status_filter, analysis_status=analysis_status_filter, - onlylatest=True, session=dbsession) + onlylatest=True, arch=architecture, session=dbsession) if image_records: return_object = image_records @@ -347,7 +352,7 @@ def image(dbsession, request_inputs, bodycontent=None): annotations = jsondata['annotations'] - image_record = {} + added_image_records = [] try: registry_creds = db_registries.get_byuserId(userId, session=dbsession) try: @@ -375,7 +380,8 @@ def image(dbsession, request_inputs, bodycontent=None): logger.debug("INPUT IMAGE INFO: {}".format(image_info)) logger.debug("INPUT IMAGE INFO OVERRIDES: {}".format(image_info_overrides)) try: - image_info = anchore_engine.common.images.get_image_info(userId, 'docker', input_string, registry_lookup=True, registry_creds=registry_creds) + # image_info = anchore_engine.common.images.get_image_info(userId, 'docker', input_string, registry_lookup=True, registry_creds=registry_creds) + image_info_list = anchore_engine.common.images.get_image_infos(userId, 'docker', input_string, registry_lookup=True, registry_creds=registry_creds) except Exception as err: fail_event = anchore_engine.subsys.events.ImageRegistryLookupFailed(user_id=userId, image_pull_string=input_string, data=err.__dict__) try: @@ -385,32 +391,39 @@ def image(dbsession, request_inputs, bodycontent=None): raise err if image_info_overrides: - image_info.update(image_info_overrides) + # image_info.update(image_info_overrides) + for item in image_info_list: + item.update(image_info_overrides) - logger.debug("INPUT FINAL IMAGE INFO: {}".format(image_info)) + logger.debug("INPUT FINAL IMAGE INFOs: {}".format(json.dumps(image_info_list, indent=2))) - manifest = None - try: - if 'manifest' in image_info: - manifest = json.dumps(image_info['manifest']) - else: - raise Exception("no manifest from get_image_info") - except Exception as err: - raise Exception("could not fetch/parse manifest - exception: " + str(err)) + errs = [] + for item in image_info_list: + + manifest = None + try: + if 'manifest' in item: + manifest = json.dumps(item['manifest']) + else: + raise Exception("no manifest from get_image_info") + except Exception as err: + # raise Exception("could not fetch/parse manifest - exception: " + str(err)) + continue - parent_manifest = json.dumps(image_info.get('parentmanifest', {})) + parent_manifest = json.dumps(item.get('parentmanifest', {})) - logger.debug("ADDING/UPDATING IMAGE IN IMAGE POST: " + str(image_info)) + logger.debug("ADDING/UPDATING IMAGE IN IMAGE POST: " + str(item)) - # Check for dockerfile updates to an existing image - if not allow_dockerfile_update and dockerfile and dockerfile_mode.lower() == 'actual': - found_img = db_catalog_image.get(imageDigest=image_info['digest'], userId=userId, session=dbsession) - if found_img: - raise BadRequest('Cannot specify dockerfile for an image that already exists unless using force=True for re-analysis', detail={'digest': image_info['digest'], 'tag': image_info['fulltag']}) + # Check for dockerfile updates to an existing image + if not allow_dockerfile_update and dockerfile and dockerfile_mode.lower() == 'actual': + found_img = db_catalog_image.get(imageDigest=item['digest'], userId=userId, session=dbsession) + if found_img: + raise BadRequest('Cannot specify dockerfile for an image that already exists unless using force=True for re-analysis', detail={'digest': item['digest'], 'tag': item['fulltag']}) - image_records = add_or_update_image(dbsession, userId, image_info['imageId'], tags=[image_info['fulltag']], digests=[image_info['fulldigest']], parentdigest=image_info.get('parentdigest', None), created_at=image_info.get('created_at_override', None), dockerfile=dockerfile, dockerfile_mode=dockerfile_mode, manifest=manifest, parent_manifest=parent_manifest, annotations=annotations) - if image_records: - image_record = image_records[0] + image_records = add_or_update_image(dbsession, userId, item['imageId'], tags=[item['fulltag']], digests=[item['fulldigest']], parentdigest=item.get('parentdigest', None), created_at=item.get('created_at_override', None), dockerfile=dockerfile, dockerfile_mode=dockerfile_mode, manifest=manifest, parent_manifest=parent_manifest, annotations=annotations, arch=item['architecture']) + if image_records: + # image_record = image_records[0] + added_image_records.append(image_records[0]) except AnchoreApiError: raise @@ -419,9 +432,9 @@ def image(dbsession, request_inputs, bodycontent=None): httpcode = 404 raise err - if image_record: + if added_image_records: httpcode = 200 - return_object = image_record + return_object = added_image_records else: httpcode = 404 raise Exception("could not add input image") @@ -1356,7 +1369,7 @@ def perform_policy_evaluation(userId, imageDigest, dbsession, evaltag=None, poli return curr_evaluation_record, curr_evaluation_result -def add_or_update_image(dbsession, userId, imageId, tags=[], digests=[], parentdigest=None, created_at=None, anchore_data=None, dockerfile=None, dockerfile_mode=None, manifest=None, annotations={}, parent_manifest=None): +def add_or_update_image(dbsession, userId, imageId, tags=[], digests=[], parentdigest=None, created_at=None, anchore_data=None, dockerfile=None, dockerfile_mode=None, manifest=None, annotations={}, parent_manifest=None, arch=None): ret = [] logger.debug("adding based on input tags/digests for imageId ("+str(imageId)+") tags="+str(tags)+" digests="+str(digests)) obj_store = anchore_engine.subsys.object_store.manager.get_manager() @@ -1413,7 +1426,7 @@ def add_or_update_image(dbsession, userId, imageId, tags=[], digests=[], parentd fulldigest = registry + "/" + repo + "@" + d for t in tags: fulltag = registry + "/" + repo + ":" + t - new_image_record = anchore_engine.common.images.make_image_record(userId, 'docker', None, image_metadata={'tag':fulltag, 'digest':fulldigest, 'imageId':imageId, 'parentdigest': parentdigest, 'created_at': created_at, 'dockerfile':dockerfile, 'dockerfile_mode': dockerfile_mode, 'annotations': annotations}, registry_lookup=False, registry_creds=(None, None)) + new_image_record = anchore_engine.common.images.make_image_record(userId, 'docker', None, image_metadata={'tag':fulltag, 'digest':fulldigest, 'imageId':imageId, 'parentdigest': parentdigest, 'created_at': created_at, 'dockerfile':dockerfile, 'dockerfile_mode': dockerfile_mode, 'annotations': annotations}, registry_lookup=False, registry_creds=(None, None), arch=arch) imageDigest = new_image_record['imageDigest'] image_record = db_catalog_image.get(imageDigest, userId, session=dbsession) if not image_record: diff --git a/anchore_engine/services/catalog/swagger/swagger.yaml b/anchore_engine/services/catalog/swagger/swagger.yaml index c1909d3e7..818953da9 100644 --- a/anchore_engine/services/catalog/swagger/swagger.yaml +++ b/anchore_engine/services/catalog/swagger/swagger.yaml @@ -134,6 +134,10 @@ paths: type: string description: "tag of image to get" required: false + - name: 'architecture' + in: query + required: false + type: string - name: 'digest' in: query type: string diff --git a/anchore_engine/utils.py b/anchore_engine/utils.py index 42cd7f582..d4c909cb1 100644 --- a/anchore_engine/utils.py +++ b/anchore_engine/utils.py @@ -21,7 +21,7 @@ from anchore_engine.subsys import logger from anchore_engine.util.docker import parse_dockerimage_string - +from typing import List K_BYTES = 1024 M_BYTES = 1024 * K_BYTES @@ -722,3 +722,18 @@ def mapped_parser_item_iterator(input_stream, item_path): """ events = map(ijson_decimal_to_float, ijpython.parse(input_stream)) return ijcommon.items(events, item_path) + + +class ManifestDigestArch(object): + + def __init__(self, manifest, digest, arch=None): + self.manifest = manifest + self.digest = digest + self.arch = arch + + +class ImageInfo(object): + + def __init__(self, parent: ManifestDigestArch, children: List[ManifestDigestArch] = None): + self.parent = parent + self.children = children