diff --git a/README.md b/README.md index d099c89..70c855e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Securesign project structural and acceptance tests. Based on * Securesign operator: https://github.com/securesign/secure-sign-operator * Securesign Ansible collection: https://github.com/securesign/artifact-signer-ansible * Policy Controller operator: https://github.com/securesign/policy-controller-operator +* Model Validation operator: https://github.com/securesign/model-validation-operator ## Automation Current automation is done via Github actions here: https://github.com/securesign/releases/actions/workflows/structural.yml @@ -61,6 +62,11 @@ To run policy controller operator tests use: go test -v ./test/acceptance/policy_controller/... --ginkgo.v ``` +To run model validation operator tests use: +``` +go test -v ./test/acceptance/model_transparency/... --ginkgo.v +``` + ## Repository List The [repositories.json](testdata/repositories.json) file is used to check of all images are published correctly. To pull the list of repositories from Pyxis API: @@ -84,4 +90,4 @@ Downloading one artifact: -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer ghp_Ae" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts/2442056100/zip \ No newline at end of file + https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts/2442056100/zip diff --git a/test/acceptance/model_transparency/fbc_images_test.go b/test/acceptance/model_transparency/fbc_images_test.go new file mode 100644 index 0000000..bd8559e --- /dev/null +++ b/test/acceptance/model_transparency/fbc_images_test.go @@ -0,0 +1,114 @@ +package acceptance + +import ( + "context" + "fmt" + "os" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/securesign/structural-tests/test/support" + "github.com/securesign/structural-tests/test/support/olm" +) + +const ( + olmPackage = "model-validation-operator" + operatorBundleImage = "registry.redhat.io/rhtas/model-validation-operator-bundle" + fbcCatalogFileName = "catalog.json" + fbcCatalogPath = "/configs/model-validation-operator/catalog.json" + defaultChannel = "tech-preview" +) + +var _ = Describe("File-based catalog images", Ordered, func() { + + defer GinkgoRecover() + var ocps []TableEntry + var bundleImage string + + snapshotData, err := support.ParseSnapshotData() + Expect(err).NotTo(HaveOccurred()) + for key, snapshotImage := range snapshotData.Images { + if strings.Index(key, "mvo-fbc-") == 0 { + ocps = append(ocps, Entry(key, key, snapshotImage)) + } + } + Expect(ocps).NotTo(BeEmpty()) + + bundleImage = snapshotData.Images[support.ModelValidationOperatorBundleImageKey] + + DescribeTableSubtree("ocp", + func(key, fbcImage string) { + + var bundles []olm.Bundle + var channels []olm.Channel + var packages []olm.Package + + It("extract catalog.json", func() { + dir, err := os.MkdirTemp("", key) + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(dir) + + Expect(support.FileFromImage(context.Background(), fbcImage, fbcCatalogPath, dir)).To(Succeed()) + file, err := os.Open(dir + "/" + fbcCatalogFileName) + Expect(err).NotTo(HaveOccurred()) + defer file.Close() + + catalog, err := olm.ParseCatalogJSON(file) + Expect(err).NotTo(HaveOccurred()) + Expect(catalog).NotTo(BeNil()) + + for _, obj := range catalog { + switch typedObj := obj.(type) { + case olm.Bundle: + bundles = append(bundles, typedObj) + case olm.Channel: + channels = append(channels, typedObj) + case olm.Package: + packages = append(packages, typedObj) + } + } + + Expect(bundles).ToNot(BeEmpty()) + Expect(channels).ToNot(BeEmpty()) + Expect(packages).ToNot(BeEmpty()) + }) + + It("extract bundle-image from snapshot.json", func() { + snapshotData, err := support.ParseSnapshotData() + Expect(err).NotTo(HaveOccurred()) + Expect(snapshotData.Images).NotTo(BeEmpty()) + + }) + + It("verify package", func() { + for _, p := range packages { + Expect(p.Name).To(Equal(olmPackage)) + Expect(p.DefaultChannel).To(Equal(defaultChannel)) + } + }) + + It("verify channels", func() { + expectedChannels := []string{"tech-preview"} + Expect(channels).To(HaveLen(len(expectedChannels))) + + for _, channel := range channels { + Expect(channel.Package).To(Equal(olmPackage)) + Expect(expectedChannels).To(ContainElement(channel.Name)) + } + }) + + It("contains operator-bundle", func() { + bundleImageHash := support.ExtractHash(bundleImage) + exists := false + + for _, bundle := range bundles { + Expect(bundle.Package).To(Equal(olmPackage)) + if bundle.Image == fmt.Sprintf("%s@sha256:%s", operatorBundleImage, bundleImageHash) { + exists = true + } + } + Expect(exists).To(BeTrue(), fmt.Sprintf("olm bundle with %s hash not found", bundleImageHash)) + }) + }, ocps) +}) diff --git a/test/acceptance/model_transparency/mvo_acceptance_suite_test.go b/test/acceptance/model_transparency/mvo_acceptance_suite_test.go new file mode 100644 index 0000000..b17c42c --- /dev/null +++ b/test/acceptance/model_transparency/mvo_acceptance_suite_test.go @@ -0,0 +1,18 @@ +package acceptance + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func TestAcceptance(t *testing.T) { + format.MaxLength = 0 + + RegisterFailHandler(Fail) + log.SetLogger(GinkgoLogr) + RunSpecs(t, "Model Transparency Acceptance Tests Suite") +} diff --git a/test/acceptance/model_transparency/operator_images_test.go b/test/acceptance/model_transparency/operator_images_test.go new file mode 100644 index 0000000..47b09dc --- /dev/null +++ b/test/acceptance/model_transparency/operator_images_test.go @@ -0,0 +1,128 @@ +package acceptance + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/securesign/structural-tests/test/support" +) + +var ErrNotFoundInRegistry = errors.New("not found in registry") + +var _ = Describe("Model Validation Operator", Ordered, func() { + + var ( + snapshotData support.SnapshotData + repositories *support.RepositoryList + operatorMvoImages support.OperatorMap + operator string + ) + + BeforeAll(func() { + var err error + snapshotData, err = support.ParseSnapshotData() + Expect(err).NotTo(HaveOccurred()) + Expect(snapshotData.Images).NotTo(BeEmpty(), "No images were detected in snapshot file") + + repositories, err = support.LoadRepositoryList() + Expect(err).NotTo(HaveOccurred()) + Expect(repositories.Data).NotTo(BeEmpty(), "No images were detected in repositories file") + }) + + It("get operator image", func() { + operator = snapshotData.Images[support.ModelValidationOperatorImageKey] + Expect(operator).NotTo(BeEmpty(), "Operator image not detected in snapshot file") + log.Printf("Using %s\n", operator) + }) + + It("get all MV images used by this operator", func() { + valuesFile, err := support.RunImage(operator, []string{}, []string{"-h"}) + Expect(err).NotTo(HaveOccurred()) + + operatorMvoImages = support.ParseMVOperatorImages(valuesFile) + + Expect(operatorMvoImages).NotTo(BeEmpty()) + support.LogMap(fmt.Sprintf("Operator MVO images (%d):", len(operatorMvoImages)), operatorMvoImages) + }) + + It("operator images are listed in registry.redhat.io", func() { + var errs []error + + for _, image := range operatorMvoImages { + if repositories.FindByImage(image) == nil { + errs = append(errs, fmt.Errorf("%w: %s", ErrNotFoundInRegistry, image)) + } + } + Expect(errs).To(BeEmpty()) + }) + + It("operator MVO images are all valid", func() { + + Expect(support.GetMapKeys(operatorMvoImages)).To(ContainElements(support.MandatoryMvoOperatorImageKeys())) + Expect(len(operatorMvoImages)).To(BeNumerically("==", len(support.MandatoryMvoOperatorImageKeys()))) + Expect(operatorMvoImages).To(HaveEach(MatchRegexp(support.TasImageDefinitionRegexp))) + }) + + It("all image hashes are also defined in releases snapshot", func() { + + mapped := make(map[string]string) + for _, imageKey := range support.MandatoryMvoOperatorImageKeys() { + oSha := support.ExtractHash(operatorMvoImages[imageKey]) + if _, keyExist := snapshotData.Images[imageKey]; !keyExist { + mapped[imageKey] = "MISSING" + continue + } + sSha := support.ExtractHash(snapshotData.Images[imageKey]) + if oSha == sSha { + mapped[imageKey] = "match" + } else { + mapped[imageKey] = "DIFFERENT HASHES" + } + } + Expect(mapped).To(HaveEach("match"), "Operator images are missing or have different hashes in snapshot file") + }) + + It("image hashes are all unique", func() { + + operatorHashes := support.ExtractHashes(support.GetMapValues(operatorMvoImages)) + mapped := make(map[string]int) + for _, hash := range operatorHashes { + _, exist := mapped[hash] + if exist { + mapped[hash]++ + } else { + mapped[hash] = 1 + } + } + Expect(mapped).To(HaveEach(1)) + Expect(operatorMvoImages).To(HaveLen(len(mapped))) + }) + + It("operator-bundle use the right operator", func() { + dir, err := os.MkdirTemp("", "bundle") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(dir) + + Expect(support.FileFromImage( + context.Background(), + snapshotData.Images[support.ModelValidationOperatorBundleImageKey], + support.ModelValidationOperatorBundleClusterServiceVersionPath, dir), + ).To(Succeed()) + fileContent, err := os.ReadFile(filepath.Join(dir, support.ModelValidationOperatorBundleClusterServiceVersionFile)) + Expect(err).NotTo(HaveOccurred()) + Expect(fileContent).NotTo(BeEmpty()) + + operatorHash := support.ExtractHash(snapshotData.Images[support.ModelValidationOperatorImageKey]) + re := regexp.MustCompile(`(\w+:\s*[\w./-]+operator[\w-]*@sha256:` + operatorHash + `)`) + matches := re.FindAllString(string(fileContent), -1) + Expect(matches).NotTo(BeEmpty()) + support.LogArray("Operator images found in operator-bundle:", matches) + }) +}) diff --git a/test/acceptance/model_transparency/releases_images_test.go b/test/acceptance/model_transparency/releases_images_test.go new file mode 100644 index 0000000..fac6810 --- /dev/null +++ b/test/acceptance/model_transparency/releases_images_test.go @@ -0,0 +1,43 @@ +package acceptance + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/securesign/structural-tests/test/support" +) + +var _ = Describe("Model Validation Operator Releases", Ordered, func() { + + var ( + snapshotData support.SnapshotData + ) + + It("snapshot.json file exist and is parseable", func() { + var err error + snapshotData, err = support.ParseSnapshotData() + Expect(err).NotTo(HaveOccurred()) + support.LogMap(fmt.Sprintf("Snapshot images (%d):", len(snapshotData.Images)), snapshotData.Images) + Expect(snapshotData.Images).NotTo(BeEmpty(), "No images were detected in snapshot file") + }) + + It("snapshot.json file contains valid images", func() { + Expect(snapshotData.Images).To(HaveEach(MatchRegexp(support.SnapshotImageDefinitionRegexp))) + }) + + It("snapshot.json file image snapshots are all unique", func() { + snapshotHashes := support.ExtractHashes(support.GetMapValues(snapshotData.Images)) + mapped := make(map[string]int) + for _, hash := range snapshotHashes { + _, exist := mapped[hash] + if exist { + mapped[hash]++ + } else { + mapped[hash] = 1 + } + } + Expect(mapped).To(HaveEach(1)) + Expect(len(snapshotData.Images)).To(BeNumerically("==", len(mapped))) + }) +}) diff --git a/test/support/acceptance.go b/test/support/acceptance.go index 3b2c50b..4979d26 100644 --- a/test/support/acceptance.go +++ b/test/support/acceptance.go @@ -70,6 +70,24 @@ func ParsePCOperatorImages(valuesFile string) (OperatorMap, OperatorMap) { return operatorPcoImages, operatorOtherImages } +func ParseMVOperatorImages(valuesFile string) OperatorMap { + operatorMvoImages := make(OperatorMap) + + const reFirstCapture = 1 + + helpRe := regexp.MustCompile(`(?s)-{1,2}model-transparency-cli-image\b.*?\(default\s+"([^"]+)"\)`) + m := helpRe.FindStringSubmatch(valuesFile) + if len(m) > reFirstCapture { + img := strings.TrimSpace(m[reFirstCapture]) + if img != "" { + operatorMvoImages["model-transparency-image"] = img + return operatorMvoImages + } + } + + return operatorMvoImages +} + func MapAnsibleImages(ansibleDefinitionFileContent []byte) (AnsibleMap, error) { var ansibleImages AnsibleMap err := yaml.Unmarshal(ansibleDefinitionFileContent, &ansibleImages) diff --git a/test/support/snapshot_map_type.go b/test/support/snapshot_map_type.go index 31a3574..056e4cf 100644 --- a/test/support/snapshot_map_type.go +++ b/test/support/snapshot_map_type.go @@ -11,7 +11,7 @@ type SnapshotData struct { Others map[string]string } -var imageRegexp = regexp.MustCompile(`^(fbc-[\w-]+|pco-fbc-[\w-]+|[\w-]+-image)$`) +var imageRegexp = regexp.MustCompile(`^(fbc-[\w-]+|pco-fbc-[\w-]+|mvo-fbc-[\w-]+|[\w-]+-image)$`) func (data *SnapshotData) UnmarshalJSON(b []byte) error { var raw map[string]interface{} diff --git a/test/support/test_constants.go b/test/support/test_constants.go index 4e3b03c..01576f0 100644 --- a/test/support/test_constants.go +++ b/test/support/test_constants.go @@ -9,6 +9,8 @@ const ( PolicyControllerOperatorImageKey = "policy-controller-operator-image" PolicyControllerOperatorBundleImageKey = "policy-controller-operator-bundle-image" + ModelValidationOperatorImageKey = "model-validation-operator-image" + ModelValidationOperatorBundleImageKey = "model-validation-operator-bundle-image" OperatorImageKey = "rhtas-operator-image" OperatorBundleImageKey = "rhtas-operator-bundle-image" AnsibleCollectionKey = "artifact-signer-ansible.collection.url" @@ -17,8 +19,10 @@ const ( AnsibleArtifactsURL = "https://api.github.com/repos/securesign/artifact-signer-ansible/actions/artifacts" OperatorBundleClusterServiceVersionFile = "rhtas-operator.clusterserviceversion.yaml" PolicyControllerOperatorBundleClusterServiceVersionFile = "policy-controller-operator.clusterserviceversion.yaml" + ModelValidationOperatorBundleClusterServiceVersionFile = "model-validation-operator.clusterserviceversion.yaml" OperatorBundleClusterServiceVersionPath = "manifests/" + OperatorBundleClusterServiceVersionFile PolicyControllerOperatorBundleClusterServiceVersionPath = "manifests/" + PolicyControllerOperatorBundleClusterServiceVersionFile + ModelValidationOperatorBundleClusterServiceVersionPath = "manifests/" + ModelValidationOperatorBundleClusterServiceVersionFile TasImageDefinitionRegexp = `^registry.redhat.io/rhtas/[\w/-]+@sha256:\w{64}$` OtherImageDefinitionRegexp = `^(registry.redhat.io|registry.access.redhat.com)` @@ -66,6 +70,12 @@ func OtherPCOOperatorImageKeys() []string { } } +func MandatoryMvoOperatorImageKeys() []string { + return []string{ + "model-transparency-image", + } +} + func OtherOperatorImageKeys() []string { return []string{ "trillian-netcat-image", diff --git a/testdata/repositories.json b/testdata/repositories.json index c805a59..2ef38b3 100644 --- a/testdata/repositories.json +++ b/testdata/repositories.json @@ -50,6 +50,31 @@ "_id": "688a24d736269c658560f934", "published": false }, + { + "repository": "rhtas/rhtas-console-rhel9", + "_id": "68a380bc0b357c7050adfe6a", + "published": false + }, + { + "repository": "rhtas/rhtas-console-ui-rhel9", + "_id": "68a38124312ed425e3215226", + "published": false + }, + { + "repository": "rhtas/model-validation-rhel9-operator", + "_id": "68c0295377ceb0bccde13cb6", + "published": false + }, + { + "repository": "rhtas/model-transparency-rhel9", + "_id": "68c02bb7097737a4a148b6c6", + "published": false + }, + { + "repository": "rhtas/model-validation-operator-bundle", + "_id": "68c02cbc097737a4a148bbaa", + "published": false + }, { "repository": "rhtas/rhtas-rhel9-operator", "_id": "65e79775f4abd6689b4f056c",