From 9fd5004f35cd6d01a9cb2ebb95668044f0f9d34c Mon Sep 17 00:00:00 2001 From: Yan Song Date: Sat, 6 May 2023 11:16:25 +0000 Subject: [PATCH 1/4] solver: fix unnecessary force compression for provenance Signed-off-by: Yan Song --- solver/llbsolver/proc/provenance.go | 5 +++-- solver/llbsolver/proc/sbom.go | 3 ++- solver/llbsolver/provenance.go | 4 +++- solver/llbsolver/solver.go | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go index 1af3af196..2edbdc39e 100644 --- a/solver/llbsolver/proc/provenance.go +++ b/solver/llbsolver/proc/provenance.go @@ -11,11 +11,12 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/compression" "github.com/pkg/errors" ) func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { - return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, comp compression.Config) (*llbsolver.Result, error) { ps, err := exptypes.ParsePlatforms(res.Metadata) if err != nil { return nil, err @@ -41,7 +42,7 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { return nil, errors.Errorf("could not find ref %s", p.ID) } - pc, err := llbsolver.NewProvenanceCreator(ctx, cp, ref, attrs, j) + pc, err := llbsolver.NewProvenanceCreator(ctx, cp, ref, attrs, j, comp) if err != nil { return nil, err } diff --git a/solver/llbsolver/proc/sbom.go b/solver/llbsolver/proc/sbom.go index 2d7e969ba..c0837218a 100644 --- a/solver/llbsolver/proc/sbom.go +++ b/solver/llbsolver/proc/sbom.go @@ -10,11 +10,12 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/compression" "github.com/pkg/errors" ) func SBOMProcessor(scannerRef string, useCache bool) llbsolver.Processor { - return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, comp compression.Config) (*llbsolver.Result, error) { // skip sbom generation if we already have an sbom if sbom.HasSBOM(res.Result) { return res, nil diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index b30581c85..f02e958e7 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -20,6 +20,7 @@ import ( "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" + "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -372,7 +373,7 @@ type ProvenanceCreator struct { addLayers func() error } -func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solver.ResultProxy, attrs map[string]string, j *solver.Job) (*ProvenanceCreator, error) { +func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solver.ResultProxy, attrs map[string]string, j *solver.Job, comp compression.Config) (*ProvenanceCreator, error) { var reproducible bool if v, ok := attrs["reproducible"]; ok { b, err := strconv.ParseBool(v) @@ -449,6 +450,7 @@ func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solve ResolveRemotes: resolveRemotes, Mode: solver.CacheExportModeRemoteOnly, ExportRoots: true, + CompressionOpt: &comp, }); err != nil { return err } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 2f7ba61e5..ffe7ed570 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -93,7 +93,7 @@ type Solver struct { // Processor defines a processing function to be applied after solving, but // before exporting -type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job) (*Result, error) +type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job, comp compression.Config) (*Result, error) func New(opt Opt) (*Solver, error) { s := &Solver{ @@ -200,7 +200,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend } makeProvenance := func(res solver.ResultProxy, cap *provenance.Capture) (*controlapi.Descriptor, func(), error) { - prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j) + prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j, exp.Exporter.Config().Compression()) if err != nil { return nil, nil, err } @@ -501,7 +501,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } for _, post := range post { - res2, err := post(ctx, resProv, s, j) + res2, err := post(ctx, resProv, s, j, exp.Exporter.Config().Compression()) if err != nil { return nil, err } From 76aeaa55ff4e1c575644bffbf0a43bea92215fbe Mon Sep 17 00:00:00 2001 From: Yan Song Date: Sat, 6 May 2023 11:22:27 +0000 Subject: [PATCH 2/4] nydus: set default compression option to avoid recomputing Signed-off-by: Yan Song --- nydus/util/util.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nydus/util/util.go b/nydus/util/util.go index f65cfe520..6524ebf78 100644 --- a/nydus/util/util.go +++ b/nydus/util/util.go @@ -15,12 +15,14 @@ type compressorKey struct{} type chunkDictDigestKey struct{} func WithContext(ctx context.Context, fsVersion string, compressor string, chunkDictDigest digest.Digest) context.Context { - if fsVersion == "" { - fsVersion = "5" + if fsVersion != "" { + ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion) + } + + if compressor != "" { + ctx = context.WithValue(ctx, compressorKey{}, compressor) } - ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion) - ctx = context.WithValue(ctx, compressorKey{}, compressor) if chunkDictDigest != "" { ctx = context.WithValue(ctx, chunkDictDigestKey{}, chunkDictDigest.String()) } @@ -29,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk } func GetContext(ctx context.Context) (string, string, string) { - fsVersion := "" - compressor := "" + fsVersion := "5" + compressor := "lz4_block" chunkDictDigest := "" ctxValue := ctx.Value(fsVersionKey{}) From b2ef881dd4b2201bc5b630a0af91ea33c13d45bc Mon Sep 17 00:00:00 2001 From: Yan Song Date: Thu, 25 May 2023 09:18:46 +0000 Subject: [PATCH 3/4] nydus: use rafs v6 as default fs version - Use `--nydus-fs-version=6` as default; - Use `--nydus-compressor=zstd` as default; - Upgrade suggested nydus version to v2.1.6; Signed-off-by: Yan Song --- Dockerfile | 2 +- docs/nydus.md | 4 ++-- nydus/nydus.go | 4 ++-- nydus/util/util.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8869138b5..9c8a54f19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ARG CNI_VERSION=v1.1.1 ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0 ARG NERDCTL_VERSION=v1.0.0 ARG DNSNAME_VERSION=v1.3.1 -ARG NYDUS_VERSION=v2.1.0 +ARG NYDUS_VERSION=v2.1.6 ARG ALPINE_VERSION=3.17 diff --git a/docs/nydus.md b/docs/nydus.md index 0334c76c5..80c39a8ef 100644 --- a/docs/nydus.md +++ b/docs/nydus.md @@ -16,7 +16,7 @@ go build -tags=nydus -o ./bin/buildkitd ./cmd/buildkitd ### Building Nydus with BuildKit -Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.1 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd: +Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.6 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd: ``` env NYDUS_BUILDER=/path/to/nydus-image buildkitd ... @@ -33,7 +33,7 @@ buildctl build ... \ Available options: -- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `5`; +- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `6`; - `nydus-compressor`: Specify nydus image compressor, possible values: `none`, `lz4_block`, `zstd`, default `lz4_block`; - `nydus-chunk-dict-image`: Specify nydus chunk dict image reference for data de-duplication; diff --git a/nydus/nydus.go b/nydus/nydus.go index 2d808d2f0..5e80b6671 100644 --- a/nydus/nydus.go +++ b/nydus/nydus.go @@ -170,10 +170,10 @@ func loadChunkDict(ctx context.Context, registryHosts docker.RegistryHosts, sm * } if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] == "" { - bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "5" + bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "6" } if nydusFSVersion == "" { - nydusFSVersion = "5" + nydusFSVersion = "6" } if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] != nydusFSVersion { diff --git a/nydus/util/util.go b/nydus/util/util.go index 6524ebf78..22f58f641 100644 --- a/nydus/util/util.go +++ b/nydus/util/util.go @@ -31,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk } func GetContext(ctx context.Context) (string, string, string) { - fsVersion := "5" - compressor := "lz4_block" + fsVersion := "6" + compressor := "zstd" chunkDictDigest := "" ctxValue := ctx.Value(fsVersionKey{}) From 20342e14212439900b3888ae8c7b00d35cc57b33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 18:22:51 +0000 Subject: [PATCH 4/4] build(deps): bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/buildx-image.yml | 2 +- .github/workflows/nydus.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8d97ec4e..1cf62c7fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -381,7 +381,7 @@ jobs: - name: Login to DockerHub if: needs.release-base.outputs.push == 'push' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -488,7 +488,7 @@ jobs: buildkitd-flags: --debug - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 if: needs.frontend-base.outputs.push == 'push' with: username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/.github/workflows/buildx-image.yml b/.github/workflows/buildx-image.yml index d9c655480..b8562ec26 100644 --- a/.github/workflows/buildx-image.yml +++ b/.github/workflows/buildx-image.yml @@ -52,7 +52,7 @@ jobs: - name: Login to DockerHub if: github.event.inputs.dry-run != 'true' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/nydus.yml b/.github/workflows/nydus.yml index 81f673eb8..f39811715 100644 --- a/.github/workflows/nydus.yml +++ b/.github/workflows/nydus.yml @@ -26,7 +26,7 @@ jobs: sudo cp nydus-static/nydusify nydus-static/nydus-image /usr/local/bin/ sudo cp nydus-static/nydusd /usr/local/bin/nydusd - name: Log in to the container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }}