diff --git a/ChangeLog.md b/ChangeLog.md index a56b371..311ddfa 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -14,11 +14,14 @@ * CaPyCLI now supports color console output also when running in GitLab CI. * `bom map` fix: In few cases with --nocache, it added mixed matches to output BOM, now we assure that only the best mapping results are added. +* `project createbom` stores release relations (`CONTAINED`, `SIDE_BY_SIDE` etc.) as capycli:projectRelation +* `project update`: optimized handling of release mainline state and release relation. Now states + provided in the SBOM are used and slowdowns/crashes introduced in 2.7.0 (#121) fixed again. ## 2.7.0 * fix for `bom findsources` for some JavaScript SBOMs. -* `bom show` command also lists purl and source code download url in verbose mode. +* `bom show` command also lists purl and source code download url in verbose mode. If one of the values is missing and `--forceerror` has been specified, error code 97 is returned. * `bom show` command also lists license information in verbose mode, but only for CycloneDX 1.6 and later. diff --git a/capycli/common/capycli_bom_support.py b/capycli/common/capycli_bom_support.py index ea3c1e8..952ccd2 100644 --- a/capycli/common/capycli_bom_support.py +++ b/capycli/common/capycli_bom_support.py @@ -66,6 +66,7 @@ class CycloneDxSupport(): CDX_PROP_CLEARING_STATE = "capycli:clearingState" CDX_PROP_CATEGORIES = "capycli:categories" CDX_PROP_PROJ_STATE = "capycli:projectClearingState" + CDX_PROP_PROJ_RELATION = "capycli:projectRelation" CDX_PROP_PROFILE = "siemens:profile" @staticmethod diff --git a/capycli/project/create_bom.py b/capycli/project/create_bom.py index 86d9aa7..54319fe 100644 --- a/capycli/project/create_bom.py +++ b/capycli/project/create_bom.py @@ -8,7 +8,7 @@ import logging import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple from cyclonedx.model import ExternalReferenceType, HashAlgorithm from cyclonedx.model.bom import Bom @@ -36,14 +36,13 @@ def get_external_id(self, name: str, release_details: Dict[str, Any]) -> str: return release_details["externalIds"].get(name, "") - def get_clearing_state(self, proj: Dict[str, Any], href: str) -> str: - """Returns the clearing state of the given component/release""" + def get_linked_state(self, proj: Dict[str, Any], href: str) -> Tuple[str, str]: + """Returns project mainline state and relation of the given release""" rel = proj["linkedReleases"] for key in rel: if key["release"] == href: - return key["mainlineState"] - - return "" + return (key["mainlineState"], key["relation"]) + return ("", "") def create_project_bom(self, project: Dict[str, Any]) -> List[Component]: bom: List[Component] = [] @@ -112,9 +111,10 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]: print_red(" ERROR: unable to access project:" + repr(swex)) sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) - state = self.get_clearing_state(project, href) - if state: - CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state) + mainline_state, relation = self.get_linked_state(project, href) + if mainline_state and relation: + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, mainline_state) + CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_RELATION, relation) sw360_id = self.client.get_id_from_href(href) CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id) diff --git a/capycli/project/create_project.py b/capycli/project/create_project.py index bf1b4da..564e380 100644 --- a/capycli/project/create_project.py +++ b/capycli/project/create_project.py @@ -33,9 +33,9 @@ def __init__(self, onlyUpdateProject: bool = False) -> None: self.onlyUpdateProject = onlyUpdateProject self.project_mainline_state: str = "" - def bom_to_release_list(self, sbom: Bom) -> List[str]: - """Creates a list with linked releases""" - linkedReleases = [] + def bom_to_release_list(self, sbom: Bom) -> Dict[str, Any]: + """Creates a list with linked releases from the SBOM.""" + linkedReleases: Dict[str, Any] = {} for cx_comp in sbom.components: rid = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_SW360ID) @@ -45,31 +45,39 @@ def bom_to_release_list(self, sbom: Bom) -> List[str]: + ", " + str(cx_comp.version)) continue - linkedReleases.append(rid) + linkedReleases[rid] = {} + + mainlineState = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE) + if mainlineState: + linkedReleases[rid]["mainlineState"] = mainlineState + relation = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION) + if relation: + # No typo. In project structure, it's "relation", while release update API uses "releaseRelation". + linkedReleases[rid]["releaseRelation"] = relation return linkedReleases - def get_release_project_mainline_states(self, project: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]: - pms: List[Dict[str, Any]] = [] + def merge_project_mainline_states(self, data: Dict[str, Any], project: Optional[Dict[str, Any]]) -> None: if not project: - return pms + return if "linkedReleases" not in project: - return pms + return for release in project["linkedReleases"]: # NOT ["sw360:releases"] pms_release = release.get("release", "") if not pms_release: continue + pms_release = pms_release.split("/")[-1] + if pms_release not in data: + continue pms_state = release.get("mainlineState", "OPEN") pms_relation = release.get("relation", "UNKNOWN") - pms_entry: Dict[str, Any] = {} - pms_entry["release"] = pms_release - pms_entry["mainlineState"] = pms_state - pms_entry["new_relation"] = pms_relation - pms.append(pms_entry) - return pms + if "mainlineState" not in data[pms_release]: + data[pms_release]["mainlineState"] = pms_state + if "releaseRelation" not in data[pms_release]: + data[pms_release]["releaseRelation"] = pms_relation def update_project(self, project_id: str, project: Optional[Dict[str, Any]], sbom: Bom, project_info: Dict[str, Any]) -> None: @@ -79,7 +87,7 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]], sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) data = self.bom_to_release_list(sbom) - pms = self.get_release_project_mainline_states(project) + self.merge_project_mainline_states(data, project) ignore_update_elements = ["name", "version"] # remove elements from list because they are handled separately @@ -90,13 +98,17 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]], try: print_text(" " + str(len(data)) + " releases in SBOM") + update_mode = self.onlyUpdateProject if project and "_embedded" in project and "sw360:releases" in project["_embedded"]: print_text( " " + str(len(project["_embedded"]["sw360:releases"])) + " releases in project before update") + else: + # Workaround for SW360 API bug: add releases will hang forever for empty projects + update_mode = False # note: type in sw360python, 1.4.0 is wrong - we are using the correct one! - result = self.client.update_project_releases(data, project_id, add=self.onlyUpdateProject) # type: ignore + result = self.client.update_project_releases(data, project_id, add=update_mode) # type: ignore if not result: print_red(" Error updating project releases!") project = self.client.get_project(project_id) @@ -114,20 +126,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]], if not result2: print_red(" Error updating project!") - if pms and project: - print_text(" Restoring original project mainline states...") - for pms_entry in pms: - update_release = False - for r in project.get("linkedReleases", []): - if r["release"] == pms_entry["release"]: - update_release = True - break - - if update_release: - rid = self.client.get_id_from_href(pms_entry["release"]) - self.client.update_project_release_relationship( - project_id, rid, pms_entry["mainlineState"], pms_entry["new_relation"], "") - except SW360Error as swex: if swex.response is None: print_red(" Unknown error: " + swex.message) @@ -353,6 +351,7 @@ def run(self, args: Any) -> None: sys.exit(ResultCode.RESULT_FILE_NOT_FOUND) is_update_version = False + project = None if args.old_version and args.old_version != "": print_text("Project version will be updated with version: " + args.old_version) @@ -413,7 +412,8 @@ def run(self, args: Any) -> None: if self.project_id: print("Updating project...") try: - project = self.client.get_project(self.project_id) + if project is None: + project = self.client.get_project(self.project_id) except SW360Error as swex: print_red(" ERROR: unable to access project:" + repr(swex)) sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360) diff --git a/tests/fixtures/sbom_for_create_project.json b/tests/fixtures/sbom_for_create_project.json index 38040b4..e97c4a6 100644 --- a/tests/fixtures/sbom_for_create_project.json +++ b/tests/fixtures/sbom_for_create_project.json @@ -66,6 +66,10 @@ "name": "siemens:primaryLanguage", "value": "Python" }, + { + "name": "capycli:projectRelation", + "value": "DYNAMICALLY_LINKED" + }, { "name": "siemens:sw360Id", "value": "a5cae39f39db4e2587a7d760f59ce3d0" @@ -79,4 +83,4 @@ "dependsOn": [] } ] -} \ No newline at end of file +} diff --git a/tests/test_base.py b/tests/test_base.py index 9c3e811..0afa462 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -214,10 +214,10 @@ def get_project_for_test() -> Dict[str, Any]: { "createdBy": "thomas.graf@siemens.com", "release": "https://my.server.com/resource/api/releases/r002", - "mainlineState": "SPECIFIC", + "mainlineState": "MAINLINE", "comment": "Automatically updated by SCC", "createdOn": "2023-03-14", - "relation": "UNKNOWN" + "relation": "DYNAMICALLY_LINKED" } ], "_links": { diff --git a/tests/test_create_bom.py b/tests/test_create_bom.py index 22e50c0..640f6e4 100644 --- a/tests/test_create_bom.py +++ b/tests/test_create_bom.py @@ -230,6 +230,16 @@ def test_project_by_id(self) -> None: assert len(ext_refs_vcs) == 1 assert str(ext_refs_vcs[0].url) == release["repository"]["url"] + prj_ml_state = CycloneDxSupport.get_property(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE) + assert prj_ml_state.value == "MAINLINE" + releaseRelation = CycloneDxSupport.get_property(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION) + assert releaseRelation.value == "DYNAMICALLY_LINKED" + + prj_ml_state = CycloneDxSupport.get_property(cdx_bom.components[1], CycloneDxSupport.CDX_PROP_PROJ_STATE) + assert prj_ml_state.value == "SPECIFIC" + releaseRelation = CycloneDxSupport.get_property(cdx_bom.components[1], CycloneDxSupport.CDX_PROP_PROJ_RELATION) + assert releaseRelation.value == "UNKNOWN" + assert cdx_bom.metadata.component is not None if cdx_bom.metadata.component: assert cdx_bom.metadata.component.name == project["name"] diff --git a/tests/test_create_project.py b/tests/test_create_project.py index be6943c..02a8b1f 100644 --- a/tests/test_create_project.py +++ b/tests/test_create_project.py @@ -8,7 +8,7 @@ import json import os -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, Tuple import responses import responses.matchers @@ -44,7 +44,7 @@ def match(request: Any) -> Tuple[bool, str]: return match -def update_release_matcher(releases: List[str]) -> Any: +def update_release_matcher(releases: Dict[str, Any]) -> Any: """ Matches the updated releases. @@ -66,10 +66,15 @@ def match(request: Any) -> Tuple[bool, str]: reason = ("Number of releases does not match, got " + str(len(json_body)) + " expected: " + str(len(releases))) else: - for rel in releases: + for rel, rel_data in releases.items(): if rel not in request_body: result = False reason = ("Release " + rel + " not found in: " + request_body) + if rel_data != json_body[rel]: + result = False + reason = ("Release[" + rel + "] = '" + str(json_body[rel]) + "' does not match expected " + + str(rel_data)) + break return result, reason return match @@ -426,9 +431,14 @@ def test_project_update(self) -> None: "visibility": "EVERYONE", "_links": { "self": { - "href": TestBase.MYURL + "resource/api/projects/376576" + "href": TestBase.MYURL + "resource/api/projects/007" } }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "UNKNOWN", + }], "_embedded": { "sw360:releases": [{ "name": "Angular 2.3.0", @@ -456,14 +466,11 @@ def test_project_update(self) -> None: "veraion": "1.9.0", "businessUnit": "SI", "description": "CaPyCLI", - "linkedReleases": { - "a5cae39f39db4e2587a7d760f59ce3d0": { - "mainlineState": "SPECIFIC", - "releaseRelation": "DYNAMICALLY_LINKED", - "setMainlineState": True, - "setReleaseRelation": True - } - }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "DYNAMICALLY_LINKED", + }], "_links": { "self": { "href": self.MYURL + "resource/api/projects/007" @@ -482,7 +489,8 @@ def test_project_update(self) -> None: } }, match=[ - update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"]) + update_release_matcher({"a5cae39f39db4e2587a7d760f59ce3d0": { + "releaseRelation": "DYNAMICALLY_LINKED"}}) ], status=201, content_type="application/json", @@ -499,14 +507,175 @@ def test_project_update(self) -> None: "veraion": "1.9.0", "businessUnit": "SI", "description": "CaPyCLI", - "linkedReleases": { - "a5cae39f39db4e2587a7d760f59ce3d0": { - "mainlineState": "SPECIFIC", - "releaseRelation": "DYNAMICALLY_LINKED", - "setMainlineState": True, - "setReleaseRelation": True + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "DYNAMICALLY_LINKED", + }], + "_links": { + "self": { + "href": self.MYURL + "resource/api/projects/007" + } + }, + "_embedded": { + "sw360:releases": [{ + "name": "Angular 2.3.0", + "version": "2.3.0", + "_links": { + "self": { + "href": "https://sw360.org/api/releases/3765276512" + } + } + }] + } + }, + match=[ + min_json_matcher( + { + "businessUnit": "SI", + "description": "CaPyCLI", + "ownerGroup": "SI", + "projectOwner": "thomas.graf@siemens.com", + "projectResponsible": "thomas.graf@siemens.com", + "projectType": "INNER_SOURCE", + "tag": "SI BP DB Demo", + "visibility": "EVERYONE" + }) + ], + status=201, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + out = self.capture_stdout(sut.run, args) + self.assertTrue(self.INPUTFILE in out) + + @responses.activate + def test_project_copy_from(self) -> None: + """copy project 007 to 017""" + sut = CreateProject() + + # create argparse command line argument object + args = AppArguments() + args.command = [] + args.command.append("project") + args.command.append("create") + args.sw360_token = TestBase.MYTOKEN + args.sw360_url = TestBase.MYURL + args.version = "2.0.0" + args.copy_from = "007" + args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", self.INPUTFILE) + args.verbose = True + args.debug = True + + self.add_login_response() + + new_project_json = { + "name": "CaPyCLI", + "version": "2.0.0", + "securityResponsibles": [], + "considerReleasesFromExternalList": False, + "projectType": "PRODUCT", + "visibility": "EVERYONE", + "_links": { + "self": { + "href": TestBase.MYURL + "resource/api/projects/017" + } + }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/a5cae39f39db4e2587a7d760f59ce3d0", + "mainlineState": "SPECIFIC", + "relation": "UNKNOWN", + }], + "_embedded": { + "sw360:releases": [{ + "name": "charset-normalizer", + "version": "3.1.0", + "_links": { + "self": { + "href": "https://sw360.org/api/releases/a5cae39f39db4e2587a7d760f59ce3d0", + } + } + }] + } + } + + responses.add( + responses.POST, + url=self.MYURL + "resource/api/projects/duplicate/007", + json=new_project_json, + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + responses.add( + responses.GET, + url=self.MYURL + "resource/api/projects/017", + json=new_project_json, + status=200, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # update project releases + responses.add( + responses.POST, + url=self.MYURL + "resource/api/projects/017/releases", + json={ + # server returns complete project, here we only mock a part of it + "name": "CaPyCLI", + "veraion": "1.9.0", + "businessUnit": "SI", + "description": "CaPyCLI", + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "DYNAMICALLY_LINKED", + }], + "_links": { + "self": { + "href": self.MYURL + "resource/api/projects/007" } }, + "_embedded": { + "sw360:releases": [{ + "name": "Angular 2.3.0", + "version": "2.3.0", + "_links": { + "self": { + "href": "https://sw360.org/api/releases/3765276512" + } + } + }] + } + }, + match=[ + update_release_matcher({"a5cae39f39db4e2587a7d760f59ce3d0": { + "mainlineState": "SPECIFIC", # from project 007 + "releaseRelation": "DYNAMICALLY_LINKED" # from SBOM + }}) + ], + status=201, + content_type="application/json", + adding_headers={"Authorization": "Token " + self.MYTOKEN}, + ) + + # update project + responses.add( + responses.PATCH, + url=self.MYURL + "resource/api/projects/017", + json={ + # server returns complete project, here we only mock a part of it + "name": "CaPyCLI", + "veraion": "1.9.0", + "businessUnit": "SI", + "description": "CaPyCLI", + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "DYNAMICALLY_LINKED", + }], "_links": { "self": { "href": self.MYURL + "resource/api/projects/007" @@ -608,14 +777,11 @@ def xtest_project_update_old_version(self) -> None: "veraion": "1.9.9", "businessUnit": "SI", "description": "CaPyCLI", - "linkedReleases": { - "a5cae39f39db4e2587a7d760f59ce3d0": { - "mainlineState": "SPECIFIC", - "releaseRelation": "DYNAMICALLY_LINKED", - "setMainlineState": True, - "setReleaseRelation": True - } - }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "DYNAMICALLY_LINKED", + }], "_links": { "self": { "href": self.MYURL + "resource/api/projects/008" @@ -634,7 +800,8 @@ def xtest_project_update_old_version(self) -> None: } }, match=[ - update_release_matcher(["a5cae39f39db4e2587a7d760f59ce3d0"]) + update_release_matcher({"a5cae39f39db4e2587a7d760f59ce3d0": { + "releaseRelation": "DYNAMICALLY_LINKED"}}) ], status=201, content_type="application/json", @@ -651,14 +818,11 @@ def xtest_project_update_old_version(self) -> None: "veraion": "1.9.0", "businessUnit": "SI", "description": "CaPyCLI", - "linkedReleases": { - "a5cae39f39db4e2587a7d760f59ce3d0": { - "mainlineState": "SPECIFIC", - "releaseRelation": "DYNAMICALLY_LINKED", - "setMainlineState": True, - "setReleaseRelation": True - } - }, + "linkedReleases": [{ + "release": "https://sw360.org/api/releases/3765276512", + "mainlineState": "SPECIFIC", + "relation": "DYNAMICALLY_LINKED", + }], "_links": { "self": { "href": self.MYURL + "resource/api/projects/007" diff --git a/tests/test_show_project.py b/tests/test_show_project.py index 0349602..b6c5146 100644 --- a/tests/test_show_project.py +++ b/tests/test_show_project.py @@ -168,7 +168,7 @@ def test_project_show_by_id(self) -> None: self.assertTrue("Project owner: thomas.graf@siemens.com" in out) self.assertTrue("Clearing state: IN_PROGRESS" in out) self.assertTrue("No linked projects" in out) - self.assertTrue("cli-support, 1.3 = SPECIFIC, APPROVED" in out) + self.assertTrue("cli-support, 1.3 = MAINLINE, APPROVED" in out) self.assertTrue("wheel, 0.38.4 = SPECIFIC, APPROVED" in out) @responses.activate @@ -337,7 +337,7 @@ def test_project_show_with_subproject(self) -> None: self.assertTrue("Clearing state: IN_PROGRESS" in out) self.assertTrue("Linked projects:" in out) self.assertTrue("sub-project-dummy, 2.0.1" in out) - self.assertTrue("cli-support, 1.3 = SPECIFIC, APPROVED" in out) + self.assertTrue("cli-support, 1.3 = MAINLINE, APPROVED" in out) self.assertTrue("wheel, 0.38.4 = SPECIFIC, APPROVED" in out) self.assertTrue(os.path.isfile(self.OUTPUTFILE), "no output file generated")