feat: adding MVO to structural tests

README.md

@@ -4,6 +4,7 @@ Securesign project structural and acceptance tests. Based on
 * Securesign operator: https://github.com/securesign/secure-sign-operator
 * Securesign Ansible collection: https://github.com/securesign/artifact-signer-ansible
 * Policy Controller operator: https://github.com/securesign/policy-controller-operator
+* Model Validation operator: https://github.com/securesign/model-validation-operator
 
 ## Automation
 Current automation is done via Github actions here: https://github.com/securesign/releases/actions/workflows/structural.yml
@@ -61,6 +62,11 @@ To run policy controller operator tests use:
 go test -v ./test/acceptance/policy_controller/... --ginkgo.v
 ```
 
+To run model validation operator tests use:
+```
+go test -v ./test/acceptance/model_transparency/... --ginkgo.v
+```
+
 ## Repository List
 The [repositories.json](testdata/repositories.json) file is used to check of all images are published correctly. To pull the list of repositories from Pyxis API:
 
@@ -84,4 +90,4 @@ Downloading one artifact:
     -H "Accept: application/vnd.github+json" \
     -H "Authorization: Bearer ghp_Ae" \
     -H "X-GitHub-Api-Version: 2022-11-28" \
-    https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts/2442056100/zip
+    https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts/2442056100/zip

test/acceptance/model_transparency/fbc_images_test.go

@@ -0,0 +1,114 @@
+package acceptance
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/securesign/structural-tests/test/support"
+	"github.com/securesign/structural-tests/test/support/olm"
+)
+
+const (
+	olmPackage          = "model-validation-operator"
+	operatorBundleImage = "registry.redhat.io/rhtas/model-validation-operator-bundle"
+	fbcCatalogFileName  = "catalog.json"
+	fbcCatalogPath      = "/configs/model-validation-operator/catalog.json"
+	defaultChannel      = "tech-preview"
+)
+
+var _ = Describe("File-based catalog images", Ordered, func() {
+
+	defer GinkgoRecover()
+	var ocps []TableEntry
+	var bundleImage string
+
+	snapshotData, err := support.ParseSnapshotData()
+	Expect(err).NotTo(HaveOccurred())
+	for key, snapshotImage := range snapshotData.Images {
+		if strings.Index(key, "mvo-fbc-") == 0 {
+			ocps = append(ocps, Entry(key, key, snapshotImage))
+		}
+	}
+	Expect(ocps).NotTo(BeEmpty())
+
+	bundleImage = snapshotData.Images[support.ModelValidationOperatorBundleImageKey]
+
+	DescribeTableSubtree("ocp",
+		func(key, fbcImage string) {
+
+			var bundles []olm.Bundle
+			var channels []olm.Channel
+			var packages []olm.Package
+
+			It("extract catalog.json", func() {
+				dir, err := os.MkdirTemp("", key)
+				Expect(err).NotTo(HaveOccurred())
+				defer os.RemoveAll(dir)
+
+				Expect(support.FileFromImage(context.Background(), fbcImage, fbcCatalogPath, dir)).To(Succeed())
+				file, err := os.Open(dir + "/" + fbcCatalogFileName)
+				Expect(err).NotTo(HaveOccurred())
+				defer file.Close()
+
+				catalog, err := olm.ParseCatalogJSON(file)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(catalog).NotTo(BeNil())
+
+				for _, obj := range catalog {
+					switch typedObj := obj.(type) {
+					case olm.Bundle:
+						bundles = append(bundles, typedObj)
+					case olm.Channel:
+						channels = append(channels, typedObj)
+					case olm.Package:
+						packages = append(packages, typedObj)
+					}
+				}
+
+				Expect(bundles).ToNot(BeEmpty())
+				Expect(channels).ToNot(BeEmpty())
+				Expect(packages).ToNot(BeEmpty())
+			})
+
+			It("extract bundle-image from snapshot.json", func() {
+				snapshotData, err := support.ParseSnapshotData()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(snapshotData.Images).NotTo(BeEmpty())
+
+			})
+
+			It("verify package", func() {
+				for _, p := range packages {
+					Expect(p.Name).To(Equal(olmPackage))
+					Expect(p.DefaultChannel).To(Equal(defaultChannel))
+				}
+			})
+
+			It("verify channels", func() {
+				expectedChannels := []string{"tech-preview"}
+				Expect(channels).To(HaveLen(len(expectedChannels)))
+
+				for _, channel := range channels {
+					Expect(channel.Package).To(Equal(olmPackage))
+					Expect(expectedChannels).To(ContainElement(channel.Name))
+				}
+			})
+
+			It("contains operator-bundle", func() {
+				bundleImageHash := support.ExtractHash(bundleImage)
+				exists := false
+
+				for _, bundle := range bundles {
+					Expect(bundle.Package).To(Equal(olmPackage))
+					if bundle.Image == fmt.Sprintf("%s@sha256:%s", operatorBundleImage, bundleImageHash) {
+						exists = true
+					}
+				}
+				Expect(exists).To(BeTrue(), fmt.Sprintf("olm bundle with %s hash not found", bundleImageHash))
+			})
+		}, ocps)
+})

test/acceptance/model_transparency/mvo_acceptance_suite_test.go

@@ -0,0 +1,18 @@
+package acceptance
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+)
+
+func TestAcceptance(t *testing.T) {
+	format.MaxLength = 0
+
+	RegisterFailHandler(Fail)
+	log.SetLogger(GinkgoLogr)
+	RunSpecs(t, "Model Transparency Acceptance Tests Suite")
+}

test/acceptance/model_transparency/operator_images_test.go

@@ -0,0 +1,128 @@
+package acceptance
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/securesign/structural-tests/test/support"
+)
+
+var ErrNotFoundInRegistry = errors.New("not found in registry")
+
+var _ = Describe("Model Validation Operator", Ordered, func() {
+
+	var (
+		snapshotData      support.SnapshotData
+		repositories      *support.RepositoryList
+		operatorMvoImages support.OperatorMap
+		operator          string
+	)
+
+	BeforeAll(func() {
+		var err error
+		snapshotData, err = support.ParseSnapshotData()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(snapshotData.Images).NotTo(BeEmpty(), "No images were detected in snapshot file")
+
+		repositories, err = support.LoadRepositoryList()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(repositories.Data).NotTo(BeEmpty(), "No images were detected in repositories file")
+	})
+
+	It("get operator image", func() {
+		operator = snapshotData.Images[support.ModelValidationOperatorImageKey]
+		Expect(operator).NotTo(BeEmpty(), "Operator image not detected in snapshot file")
+		log.Printf("Using %s\n", operator)
+	})
+
+	It("get all MV images used by this operator", func() {
+		valuesFile, err := support.RunImage(operator, []string{}, []string{"-h"})
+		Expect(err).NotTo(HaveOccurred())
+
+		operatorMvoImages = support.ParseMVOperatorImages(valuesFile)
+
+		Expect(operatorMvoImages).NotTo(BeEmpty())
+		support.LogMap(fmt.Sprintf("Operator MVO images (%d):", len(operatorMvoImages)), operatorMvoImages)
+	})
+
+	It("operator images are listed in registry.redhat.io", func() {
+		var errs []error
+
+		for _, image := range operatorMvoImages {
+			if repositories.FindByImage(image) == nil {
+				errs = append(errs, fmt.Errorf("%w: %s", ErrNotFoundInRegistry, image))
+			}
+		}
+		Expect(errs).To(BeEmpty())
+	})
+
+	It("operator MVO images are all valid", func() {
+
+		Expect(support.GetMapKeys(operatorMvoImages)).To(ContainElements(support.MandatoryMvoOperatorImageKeys()))
+		Expect(len(operatorMvoImages)).To(BeNumerically("==", len(support.MandatoryMvoOperatorImageKeys())))
+		Expect(operatorMvoImages).To(HaveEach(MatchRegexp(support.TasImageDefinitionRegexp)))
+	})
+
+	It("all image hashes are also defined in releases snapshot", func() {
+
+		mapped := make(map[string]string)
+		for _, imageKey := range support.MandatoryMvoOperatorImageKeys() {
+			oSha := support.ExtractHash(operatorMvoImages[imageKey])
+			if _, keyExist := snapshotData.Images[imageKey]; !keyExist {
+				mapped[imageKey] = "MISSING"
+				continue
+			}
+			sSha := support.ExtractHash(snapshotData.Images[imageKey])
+			if oSha == sSha {
+				mapped[imageKey] = "match"
+			} else {
+				mapped[imageKey] = "DIFFERENT HASHES"
+			}
+		}
+		Expect(mapped).To(HaveEach("match"), "Operator images are missing or have different hashes in snapshot file")
+	})
+
+	It("image hashes are all unique", func() {
+
+		operatorHashes := support.ExtractHashes(support.GetMapValues(operatorMvoImages))
+		mapped := make(map[string]int)
+		for _, hash := range operatorHashes {
+			_, exist := mapped[hash]
+			if exist {
+				mapped[hash]++
+			} else {
+				mapped[hash] = 1
+			}
+		}
+		Expect(mapped).To(HaveEach(1))
+		Expect(operatorMvoImages).To(HaveLen(len(mapped)))
+	})
+
+	It("operator-bundle use the right operator", func() {
+		dir, err := os.MkdirTemp("", "bundle")
+		Expect(err).NotTo(HaveOccurred())
+		defer os.RemoveAll(dir)
+
+		Expect(support.FileFromImage(
+			context.Background(),
+			snapshotData.Images[support.ModelValidationOperatorBundleImageKey],
+			support.ModelValidationOperatorBundleClusterServiceVersionPath, dir),
+		).To(Succeed())
+		fileContent, err := os.ReadFile(filepath.Join(dir, support.ModelValidationOperatorBundleClusterServiceVersionFile))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fileContent).NotTo(BeEmpty())
+
+		operatorHash := support.ExtractHash(snapshotData.Images[support.ModelValidationOperatorImageKey])
+		re := regexp.MustCompile(`(\w+:\s*[\w./-]+operator[\w-]*@sha256:` + operatorHash + `)`)
+		matches := re.FindAllString(string(fileContent), -1)
+		Expect(matches).NotTo(BeEmpty())
+		support.LogArray("Operator images found in operator-bundle:", matches)
+	})
+})

test/acceptance/model_transparency/releases_images_test.go

@@ -0,0 +1,43 @@
+package acceptance
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/securesign/structural-tests/test/support"
+)
+
+var _ = Describe("Model Validation Operator Releases", Ordered, func() {
+
+	var (
+		snapshotData support.SnapshotData
+	)
+
+	It("snapshot.json file exist and is parseable", func() {
+		var err error
+		snapshotData, err = support.ParseSnapshotData()
+		Expect(err).NotTo(HaveOccurred())
+		support.LogMap(fmt.Sprintf("Snapshot images (%d):", len(snapshotData.Images)), snapshotData.Images)
+		Expect(snapshotData.Images).NotTo(BeEmpty(), "No images were detected in snapshot file")
+	})
+
+	It("snapshot.json file contains valid images", func() {
+		Expect(snapshotData.Images).To(HaveEach(MatchRegexp(support.SnapshotImageDefinitionRegexp)))
+	})
+
+	It("snapshot.json file image snapshots are all unique", func() {
+		snapshotHashes := support.ExtractHashes(support.GetMapValues(snapshotData.Images))
+		mapped := make(map[string]int)
+		for _, hash := range snapshotHashes {
+			_, exist := mapped[hash]
+			if exist {
+				mapped[hash]++
+			} else {
+				mapped[hash] = 1
+			}
+		}
+		Expect(mapped).To(HaveEach(1))
+		Expect(len(snapshotData.Images)).To(BeNumerically("==", len(mapped)))
+	})
+})

test/support/acceptance.go

@@ -70,6 +70,24 @@ func ParsePCOperatorImages(valuesFile string) (OperatorMap, OperatorMap) {
 	return operatorPcoImages, operatorOtherImages
 }
 
+func ParseMVOperatorImages(valuesFile string) OperatorMap {
+	operatorMvoImages := make(OperatorMap)
+
+	const reFirstCapture = 1
+
+	helpRe := regexp.MustCompile(`(?s)-{1,2}model-transparency-cli-image\b.*?\(default\s+"([^"]+)"\)`)
+	m := helpRe.FindStringSubmatch(valuesFile)
+	if len(m) > reFirstCapture {
+		img := strings.TrimSpace(m[reFirstCapture])
+		if img != "" {
+			operatorMvoImages["model-transparency-image"] = img
+			return operatorMvoImages
+		}
+	}
+
+	return operatorMvoImages
+}
+
 func MapAnsibleImages(ansibleDefinitionFileContent []byte) (AnsibleMap, error) {
 	var ansibleImages AnsibleMap
 	err := yaml.Unmarshal(ansibleDefinitionFileContent, &ansibleImages)

test/support/snapshot_map_type.go

@@ -11,7 +11,7 @@ type SnapshotData struct {
 	Others map[string]string
 }
 
-var imageRegexp = regexp.MustCompile(`^(fbc-[\w-]+|pco-fbc-[\w-]+|[\w-]+-image)$`)
+var imageRegexp = regexp.MustCompile(`^(fbc-[\w-]+|pco-fbc-[\w-]+|mvo-fbc-[\w-]+|[\w-]+-image)$`)
 
 func (data *SnapshotData) UnmarshalJSON(b []byte) error {
 	var raw map[string]interface{}