Skip to content
This repository was archived by the owner on Jan 27, 2023. It is now read-only.

Draft: PoC multi-arch support #626

Draft
nightfurys wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

Draft: PoC multi-arch support #626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f0077f2
support multi-arch in analyze tag path
Sep 18, 2020
0d67e87
include variant into architecture representation when available for u…
Sep 22, 2020
965610a
Update summary tags to return arch
Sep 22, 2020
befa29e
Update get images
Sep 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 47 additions & 1 deletion anchore_engine/clients/docker_registry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
import anchore_engine.configuration.localconfig
import anchore_engine.auth.common
from anchore_engine.subsys import logger
from anchore_engine.clients.skopeo_wrapper import get_image_manifest_skopeo, get_repo_tags_skopeo
from anchore_engine.clients.skopeo_wrapper import get_image_manifest_skopeo, get_repo_tags_skopeo, get_image_manifest_v2 as skopeo_get_image_manifest
from anchore_engine.utils import ImageInfo


def get_image_manifest_docker_registry(url, registry, repo, tag, user=None, pw=None, verify=True):
Expand Down Expand Up @@ -279,3 +280,48 @@ def get_image_manifest(userId, image_info, registry_creds):
raise err
else:
raise Exception("could not get manifest/digest for image ({}) from registry ({}) - error: {}".format(fulltag, url, err))


def get_image_manifest_v2(userId, image_info, registry_creds, default=None) -> ImageInfo:
logger.debug("get_image_manifest_list input: " + str(userId) + " : " + str(image_info) + " : " + str(time.time()))
logger.info("nightfury get_image_manifest_list input: " + str(userId) + " : " + str(image_info) + " : " + str(time.time()))

registry = image_info['registry']
try:
user, pw, registry_verify = anchore_engine.auth.common.get_creds_by_registry(registry, image_info['repo'], registry_creds=registry_creds)
except Exception as err:
raise err

if registry == 'docker.io':
url = "https://index.docker.io"
if not re.match(".*/.*", image_info['repo']):
repo = "library/"+image_info['repo']
else:
repo = image_info['repo']
else:
url = "https://"+registry
repo = image_info['repo']

if image_info['digest']:
tag = None
input_digest = image_info['digest']
fulltag = "{}/{}@{}".format(registry, repo, input_digest)
else:
input_digest = None
tag = image_info['tag']
fulltag = "{}/{}:{}".format(registry, repo, tag)

logger.debug("trying to get v2 manifest/digest for image ("+str(fulltag)+")")
logger.info("nightfury trying to get v2 manifest/digest for image ("+str(fulltag)+")")
try:
if tag:
result = skopeo_get_image_manifest(url, registry, repo, intag=tag, user=user, pw=pw, verify=registry_verify)
elif input_digest:
result = skopeo_get_image_manifest(url, registry, repo, indigest=input_digest, user=user, pw=pw, verify=registry_verify)
else:
raise Exception("neither tag nor digest was given as input")
except Exception:
logger.exception("could not get manifest/digest for image ({}) from registry ({})".format(fulltag, url))
raise

return result
4 changes: 2 additions & 2 deletions anchore_engine/clients/services/catalog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ def get_image(self, imageDigest):
def get_image_by_id(self, imageId):
return self.call_api(http.anchy_get, 'images', query_params={'imageId': imageId})

def list_images(self, tag=None, digest=None, imageId=None, registry_lookup=False, history=False, image_status='active', analysis_status=None):
def list_images(self, tag=None, digest=None, imageId=None, registry_lookup=False, history=False, image_status='active', analysis_status=None, architecture=None):
return self.call_api(http.anchy_get, 'images',
query_params={'tag': tag, 'history': history, 'registry_lookup': registry_lookup,
'digest': digest, 'imageId': imageId, 'image_status': image_status, 'analysis_status': analysis_status})
'digest': digest, 'imageId': imageId, 'image_status': image_status, 'analysis_status': analysis_status, 'architecture': architecture})

def update_image(self, imageDigest, image_record=None):
payload = {}
Expand Down
131 changes: 130 additions & 1 deletion anchore_engine/clients/skopeo_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import tempfile

import anchore_engine.configuration.localconfig
from anchore_engine.utils import run_command, run_command_list, manifest_to_digest, AnchoreException
from anchore_engine.utils import run_command, run_command_list, manifest_to_digest, AnchoreException, ManifestDigestArch, ImageInfo
from anchore_engine.subsys import logger
from anchore_engine.common.errors import AnchoreError

Expand Down Expand Up @@ -331,6 +331,135 @@ def get_image_manifest_skopeo(url, registry, repo, intag=None, indigest=None, to

return manifest, digest, topdigest, topmanifest


def get_digest_arch(pullstring, user=None, pw=None, verify=True):
ret = None
try:
proc_env = os.environ.copy()
if user and pw:
Copy link
Contributor

@Toure Toure Sep 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I noticed that this is used through the entire module which means it should be broken out into a separate function like

def set_auth_info(user=None, password=None, verify=False):
        proc_env = os.environ.copy()
        if user and pw:
            proc_env['SKOPUSER'] = user
            proc_env['SKOPPASS'] = pw
            credstr = "--creds \"${SKOPUSER}\":\"${SKOPPASS}\""
        else:
            credstr = ""
        if verify:
            tlsverifystr = "--tls-verify=true"
        else:
            tlsverifystr = "--tls-verify=false"

        return (proc_env, credstr, tlsverifystr)

This would reduce the amount of code and give us reusable functionality for future expansion.

proc_env['SKOPUSER'] = user
proc_env['SKOPPASS'] = pw
credstr = '--creds \"${SKOPUSER}\":\"${SKOPPASS}\"'
else:
credstr = ""

if verify:
tlsverifystr = "--tls-verify=true"
else:
tlsverifystr = "--tls-verify=false"

localconfig = anchore_engine.configuration.localconfig.get_config()
global_timeout = localconfig.get('skopeo_global_timeout', 0)
try:
global_timeout = int(global_timeout)
if global_timeout < 0:
global_timeout = 0
except:
global_timeout = 0

if global_timeout:
global_timeout_str = "--command-timeout {}s".format(global_timeout)
else:
global_timeout_str = ""

os_override_strs = ["", "--override-os windows"]
try:
success = False
for os_override_str in os_override_strs:
cmd = ["/bin/sh", "-c",
"skopeo {} {} inspect {} {} docker://{}".format(global_timeout_str, os_override_str,
tlsverifystr, credstr, pullstring)]
cmdstr = ' '.join(cmd)
try:
rc, sout, serr = run_command_list(cmd, env=proc_env)
if rc != 0:
skopeo_error = SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
if skopeo_error.error_code != AnchoreError.OSARCH_MISMATCH.name:
raise SkopeoError(cmd=cmd, rc=rc, out=sout, err=serr)
else:
logger.debug(
"command succeeded: cmd=" + str(cmdstr) + " stdout=" + str(sout).strip() + " stderr=" + str(
serr).strip())
success = True
except Exception as err:
logger.error("command failed with exception - " + str(err))
raise err

if success:
sout = str(sout, 'utf-8') if sout else None
ret = sout
break

if not success:
logger.error("could not retrieve manifest")
raise Exception("could not retrieve manifest")

except Exception as err:
raise err
Copy link
Contributor

@alfredodeza alfredodeza Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't have any effect and it is equal to not having it at all

except Exception as err:
raise err

return ret

def _get_image_manifest_digest(registry, repo, tag=None, digest=None, user=None, pw=None, verify=True):
if digest:
pull_string = registry + "/" + repo + "@" + digest
elif tag:
pull_string = registry + "/" + repo + ":" + tag
else:
raise Exception("invalid input - must supply either an intag or indigest")

try:
raw_manifest = get_image_manifest_skopeo_raw(pull_string, user=user, pw=pw, verify=verify)
s_manifest = json.loads(raw_manifest)
s_digest = manifest_to_digest(raw_manifest)
return s_manifest, s_digest
except Exception as err:
logger.warn("CMD failed - exception: " + str(err))
Copy link
Contributor

@alfredodeza alfredodeza Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this doesn't add anything that the Exception wouldn't already show. Does this imply there is something beyond str(err) that is useful to log? It seems this could do well without a try/except block

raise err


def get_image_manifest_v2(url, registry, repo, intag=None, indigest=None, user=None, pw=None, verify=True, default=None):
Copy link
Contributor

@Toure Toure Sep 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

default should be set to False as it is a bool.

try:
s_manifest, s_digest = _get_image_manifest_digest(registry, repo, intag, indigest, user, pw, verify)
result = ImageInfo(parent=ManifestDigestArch(s_manifest, s_digest, None), children=[])

if s_manifest.get('schemaVersion') == 2 and s_manifest.get('mediaType') == 'application/vnd.docker.distribution.manifest.list.v2+json':
for entry in s_manifest.get('manifests'):
child_platform = entry.get('platform')
child_digest = entry.get('digest')
if child_digest and child_platform and child_platform.get('os') in ['linux', 'windows']:
child_arch = child_platform.get('architecture')
child_variant = child_platform.get('variant')
arch_var = '{}_{}'.format(child_arch, child_variant) if child_variant else child_arch
if not default:
s_manifest, s_digest = _get_image_manifest_digest(registry, repo, None, child_digest, user, pw, verify)
result.children.append(ManifestDigestArch(s_manifest, s_digest, arch_var))
elif child_arch in default:
s_manifest, s_digest = _get_image_manifest_digest(registry, repo, None, child_digest, user, pw, verify)
result.children.append(ManifestDigestArch(s_manifest, s_digest, arch_var))
break
# else:
# continue
else:
try:
pull_string = registry + "/" + repo + "@" + s_digest
raw_inspect_out = get_digest_arch(pull_string, user=user, pw=pw, verify=verify)
inspect_out = json.loads(raw_inspect_out)
result.parent.arch = inspect_out.get('Architecture')
except Exception as err:
logger.warn("CMD failed - exception: " + str(err))
raise err
except Exception as err:
logger.exception("Failed to fetch skopeo image manifest", err)
raise err

if not result:
raise SkopeoError(msg="No digest/manifest from skopeo")

return result


class SkopeoError(AnchoreException):

def __init__(self, cmd=None, rc=None, err=None, out=None, msg='Error encountered in skopeo operation'):
Expand Down
72 changes: 68 additions & 4 deletions anchore_engine/common/images.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
from anchore_engine import db
from anchore_engine.clients import docker_registry
from anchore_engine.subsys import logger

from copy import deepcopy
from anchore_engine.utils import ManifestDigestArch

def lookup_registry_image(userId, image_info, registry_creds):
digest = None
Expand Down Expand Up @@ -69,6 +70,68 @@ def get_image_info(userId, image_type, input_string, registry_lookup=False, regi
return ret


def _make_image_info_dict(common_info, parent: ManifestDigestArch, child: ManifestDigestArch):
image_info = deepcopy(common_info)
image_info['digest'] = child.digest
image_info['fulldigest'] = image_info['registry'] + "/" + image_info['repo'] + "@" + child.digest
image_info['manifest'] = child.manifest
image_info['architecture'] = child.arch
image_info['parentmanifest'] = parent.manifest
image_info['parentdigest'] = parent.digest

# if we got a manifest, and the image_info does not yet contain an imageId, try to get it from the manifest
if image_info['manifest'] and not image_info['imageId']:
try:
imageId = re.sub("^sha256:", "", child.manifest['config']['digest'])
image_info['imageId'] = imageId
except Exception as err:
logger.debug("could not extract imageId from fetched manifest - exception: " + str(err))
logger.debug("using digest hash as imageId due to incomplete manifest (" + str(
image_info['fulldigest']) + ")")
htype, image_info['imageId'] = image_info['digest'].split(":", 1)

return image_info


def get_image_infos(userId, image_type, input_string, registry_lookup=False, registry_creds=[]):
ret = []

if image_type == 'docker':
try:
common_info = anchore_engine.utils.parse_dockerimage_string(input_string)
except Exception as err:
raise anchore_engine.common.helpers.make_anchore_exception(err,
input_message="cannot handle image input string",
input_httpcode=400)

# ret.update(common_info)

if registry_lookup and common_info['registry'] != 'localbuild':
# digest, manifest = lookup_registry_image(userId, image_info, registry_creds)
try:
result = docker_registry.get_image_manifest_v2(userId, common_info, registry_creds)
except Exception as err:
raise anchore_engine.common.helpers.make_anchore_exception(err,
input_message="cannot fetch image digest/manifest from registry",
input_httpcode=400)

parent = result.parent
if result.children:
ret = [_make_image_info_dict(common_info, parent, child) for child in result.children]
else:
ret.append(_make_image_info_dict(common_info, parent, parent))
else:
image_info = deepcopy(common_info)
image_info['manifest'] = {}
image_info['parentmanifest'] = {}
image_info['architecture'] = 'unknown'
ret.append(image_info)
else:
raise Exception("image type (" + str(image_type) + ") not supported")

return ret


def clean_docker_image_details_for_update(image_details):
ret = []

Expand All @@ -81,7 +144,7 @@ def clean_docker_image_details_for_update(image_details):
return ret


def make_image_record(userId, image_type, input_string, image_metadata={}, registry_lookup=True, registry_creds=[]):
def make_image_record(userId, image_type, input_string, image_metadata={}, registry_lookup=True, registry_creds=[], arch=None):
if image_type == 'docker':
try:
dockerfile = image_metadata.get('dockerfile', None)
Expand Down Expand Up @@ -116,15 +179,15 @@ def make_image_record(userId, image_type, input_string, image_metadata={}, regis
parentdigest = image_metadata.get('parentdigest', None)
created_at = image_metadata.get('created_at', None)

return make_docker_image(userId, input_string=input_string, tag=tag, digest=digest, imageId=imageId, parentdigest=parentdigest, created_at=created_at, dockerfile=dockerfile, dockerfile_mode=dockerfile_mode, registry_lookup=registry_lookup, registry_creds=registry_creds, annotations=annotations)
return make_docker_image(userId, input_string=input_string, tag=tag, digest=digest, imageId=imageId, parentdigest=parentdigest, created_at=created_at, dockerfile=dockerfile, dockerfile_mode=dockerfile_mode, registry_lookup=registry_lookup, registry_creds=registry_creds, annotations=annotations, arch=arch)

else:
raise Exception("image type ("+str(image_type)+") not supported")

return None


def make_docker_image(userId, input_string=None, tag=None, digest=None, imageId=None, parentdigest=None, created_at=None, dockerfile=None, dockerfile_mode=None, registry_lookup=True, registry_creds=[], annotations={}):
def make_docker_image(userId, input_string=None, tag=None, digest=None, imageId=None, parentdigest=None, created_at=None, dockerfile=None, dockerfile_mode=None, registry_lookup=True, registry_creds=[], annotations={}, arch=None):
ret = {}

if input_string:
Expand Down Expand Up @@ -152,6 +215,7 @@ def make_docker_image(userId, input_string=None, tag=None, digest=None, imageId=
new_input['userId'] = userId
new_input['image_type'] = 'docker'
new_input['dockerfile_mode'] = dockerfile_mode
new_input['arch'] = arch

if not parentdigest:
parentdigest = imageDigest
Expand Down
Loading