avoid "Restoring original project mainline states" during "project update"

ChangeLog.md

@@ -14,11 +14,14 @@
 * CaPyCLI now supports color console output also when running in GitLab CI.
 * `bom map` fix: In few cases with --nocache, it added mixed matches to output
   BOM, now we assure that only the best mapping results are added.
+* `project createbom` stores release relations (`CONTAINED`, `SIDE_BY_SIDE` etc.) as capycli:projectRelation
+* `project update`: optimized handling of release mainline state and release relation. Now states
+  provided in the SBOM are used and slowdowns/crashes introduced in 2.7.0 (#121) fixed again.
 
 ## 2.7.0
 
 * fix for `bom findsources` for some JavaScript SBOMs.
-* `bom show` command also lists purl and source code download url in verbose mode.  
+* `bom show` command also lists purl and source code download url in verbose mode.
   If one of the values is missing and `--forceerror` has been specified, error code 97 is returned.
 * `bom show` command also lists license information in verbose mode, but
   only for CycloneDX 1.6 and later.

capycli/common/capycli_bom_support.py

@@ -66,6 +66,7 @@ class CycloneDxSupport():
     CDX_PROP_CLEARING_STATE = "capycli:clearingState"
     CDX_PROP_CATEGORIES = "capycli:categories"
     CDX_PROP_PROJ_STATE = "capycli:projectClearingState"
+    CDX_PROP_PROJ_RELATION = "capycli:projectRelation"
     CDX_PROP_PROFILE = "siemens:profile"
 
     @staticmethod

capycli/project/create_bom.py

@@ -8,7 +8,7 @@
 
 import logging
 import sys
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Tuple
 
 from cyclonedx.model import ExternalReferenceType, HashAlgorithm
 from cyclonedx.model.bom import Bom
@@ -36,14 +36,13 @@ def get_external_id(self, name: str, release_details: Dict[str, Any]) -> str:
 
         return release_details["externalIds"].get(name, "")
 
-    def get_clearing_state(self, proj: Dict[str, Any], href: str) -> str:
-        """Returns the clearing state of the given component/release"""
+    def get_linked_state(self, proj: Dict[str, Any], href: str) -> Tuple[str, str]:
+        """Returns project mainline state and relation of the given release"""
         rel = proj["linkedReleases"]
         for key in rel:
             if key["release"] == href:
-                return key["mainlineState"]
-
-        return ""
+                return (key["mainlineState"], key["relation"])
+        return ("", "")
 
     def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
         bom: List[Component] = []
@@ -112,9 +111,10 @@ def create_project_bom(self, project: Dict[str, Any]) -> List[Component]:
                 print_red("    ERROR: unable to access project:" + repr(swex))
                 sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
 
-            state = self.get_clearing_state(project, href)
-            if state:
-                CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, state)
+            mainline_state, relation = self.get_linked_state(project, href)
+            if mainline_state and relation:
+                CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_STATE, mainline_state)
+                CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_PROJ_RELATION, relation)
 
             sw360_id = self.client.get_id_from_href(href)
             CycloneDxSupport.set_property(rel_item, CycloneDxSupport.CDX_PROP_SW360ID, sw360_id)

capycli/project/create_project.py

@@ -33,9 +33,9 @@ def __init__(self, onlyUpdateProject: bool = False) -> None:
         self.onlyUpdateProject = onlyUpdateProject
         self.project_mainline_state: str = ""
 
-    def bom_to_release_list(self, sbom: Bom) -> List[str]:
-        """Creates a list with linked releases"""
-        linkedReleases = []
+    def bom_to_release_list(self, sbom: Bom) -> Dict[str, Any]:
+        """Creates a list with linked releases from the SBOM."""
+        linkedReleases: Dict[str, Any] = {}
 
         for cx_comp in sbom.components:
             rid = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_SW360ID)
@@ -45,31 +45,39 @@ def bom_to_release_list(self, sbom: Bom) -> List[str]:
                     + ", " + str(cx_comp.version))
                 continue
 
-            linkedReleases.append(rid)
+            linkedReleases[rid] = {}
+
+            mainlineState = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE)
+            if mainlineState:
+                linkedReleases[rid]["mainlineState"] = mainlineState
+            relation = CycloneDxSupport.get_property_value(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION)
+            if relation:
+                # No typo. In project structure, it's "relation", while release update API uses "releaseRelation".
+                linkedReleases[rid]["releaseRelation"] = relation
 
         return linkedReleases
 
-    def get_release_project_mainline_states(self, project: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        pms: List[Dict[str, Any]] = []
+    def merge_project_mainline_states(self, data: Dict[str, Any], project: Optional[Dict[str, Any]]) -> None:
         if not project:
-            return pms
+            return
 
         if "linkedReleases" not in project:
-            return pms
+            return
 
         for release in project["linkedReleases"]:  # NOT ["sw360:releases"]
             pms_release = release.get("release", "")
             if not pms_release:
                 continue
+            pms_release = pms_release.split("/")[-1]
+            if pms_release not in data:
+                continue
             pms_state = release.get("mainlineState", "OPEN")
             pms_relation = release.get("relation", "UNKNOWN")
-            pms_entry: Dict[str, Any] = {}
-            pms_entry["release"] = pms_release
-            pms_entry["mainlineState"] = pms_state
-            pms_entry["new_relation"] = pms_relation
-            pms.append(pms_entry)
 
-        return pms
+            if "mainlineState" not in data[pms_release]:
+                data[pms_release]["mainlineState"] = pms_state
+            if "releaseRelation" not in data[pms_release]:
+                data[pms_release]["releaseRelation"] = pms_relation
 
     def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
                        sbom: Bom, project_info: Dict[str, Any]) -> None:
@@ -79,7 +87,7 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
             sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)
 
         data = self.bom_to_release_list(sbom)
-        pms = self.get_release_project_mainline_states(project)
+        self.merge_project_mainline_states(data, project)
 
         ignore_update_elements = ["name", "version"]
         # remove elements from list because they are handled separately
@@ -90,13 +98,17 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
         try:
             print_text("  " + str(len(data)) + " releases in SBOM")
 
+            update_mode = self.onlyUpdateProject
             if project and "_embedded" in project and "sw360:releases" in project["_embedded"]:
                 print_text(
                     "  " + str(len(project["_embedded"]["sw360:releases"])) +
                     " releases in project before update")
+            else:
+                # Workaround for SW360 API bug: add releases will hang forever for empty projects
+                update_mode = False
 
             # note: type in sw360python, 1.4.0 is wrong - we are using the correct one!
-            result = self.client.update_project_releases(data, project_id, add=self.onlyUpdateProject)  # type: ignore
+            result = self.client.update_project_releases(data, project_id, add=update_mode)  # type: ignore
             if not result:
                 print_red("  Error updating project releases!")
             project = self.client.get_project(project_id)
@@ -114,20 +126,6 @@ def update_project(self, project_id: str, project: Optional[Dict[str, Any]],
                 if not result2:
                     print_red("  Error updating project!")
 
-            if pms and project:
-                print_text("  Restoring original project mainline states...")
-                for pms_entry in pms:
-                    update_release = False
-                    for r in project.get("linkedReleases", []):
-                        if r["release"] == pms_entry["release"]:
-                            update_release = True
-                            break
-
-                    if update_release:
-                        rid = self.client.get_id_from_href(pms_entry["release"])
-                        self.client.update_project_release_relationship(
-                            project_id, rid, pms_entry["mainlineState"], pms_entry["new_relation"], "")
-
         except SW360Error as swex:
             if swex.response is None:
                 print_red("  Unknown error: " + swex.message)
@@ -353,6 +351,7 @@ def run(self, args: Any) -> None:
             sys.exit(ResultCode.RESULT_FILE_NOT_FOUND)
 
         is_update_version = False
+        project = None
 
         if args.old_version and args.old_version != "":
             print_text("Project version will be updated with version: " + args.old_version)
@@ -413,7 +412,8 @@ def run(self, args: Any) -> None:
         if self.project_id:
             print("Updating project...")
             try:
-                project = self.client.get_project(self.project_id)
+                if project is None:
+                    project = self.client.get_project(self.project_id)
             except SW360Error as swex:
                 print_red("  ERROR: unable to access project:" + repr(swex))
                 sys.exit(ResultCode.RESULT_ERROR_ACCESSING_SW360)

tests/fixtures/sbom_for_create_project.json

@@ -66,6 +66,10 @@
           "name": "siemens:primaryLanguage",
           "value": "Python"
         },
+        {
+          "name": "capycli:projectRelation",
+          "value": "DYNAMICALLY_LINKED"
+        },
         {
           "name": "siemens:sw360Id",
           "value": "a5cae39f39db4e2587a7d760f59ce3d0"
@@ -79,4 +83,4 @@
       "dependsOn": []
     }
   ]
-}
+}

tests/test_base.py

@@ -214,10 +214,10 @@ def get_project_for_test() -> Dict[str, Any]:
                 {
                     "createdBy": "thomas.graf@siemens.com",
                     "release": "https://my.server.com/resource/api/releases/r002",
-                    "mainlineState": "SPECIFIC",
+                    "mainlineState": "MAINLINE",
                     "comment": "Automatically updated by SCC",
                     "createdOn": "2023-03-14",
-                    "relation": "UNKNOWN"
+                    "relation": "DYNAMICALLY_LINKED"
                 }
             ],
             "_links": {

tests/test_create_bom.py

@@ -230,6 +230,16 @@ def test_project_by_id(self) -> None:
         assert len(ext_refs_vcs) == 1
         assert str(ext_refs_vcs[0].url) == release["repository"]["url"]
 
+        prj_ml_state = CycloneDxSupport.get_property(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_STATE)
+        assert prj_ml_state.value == "MAINLINE"
+        releaseRelation = CycloneDxSupport.get_property(cx_comp, CycloneDxSupport.CDX_PROP_PROJ_RELATION)
+        assert releaseRelation.value == "DYNAMICALLY_LINKED"
+
+        prj_ml_state = CycloneDxSupport.get_property(cdx_bom.components[1], CycloneDxSupport.CDX_PROP_PROJ_STATE)
+        assert prj_ml_state.value == "SPECIFIC"
+        releaseRelation = CycloneDxSupport.get_property(cdx_bom.components[1], CycloneDxSupport.CDX_PROP_PROJ_RELATION)
+        assert releaseRelation.value == "UNKNOWN"
+
         assert cdx_bom.metadata.component is not None
         if cdx_bom.metadata.component:
             assert cdx_bom.metadata.component.name == project["name"]