From 9fd5004f35cd6d01a9cb2ebb95668044f0f9d34c Mon Sep 17 00:00:00 2001 From: Yan Song Date: Sat, 6 May 2023 11:16:25 +0000 Subject: [PATCH 1/4] solver: fix unnecessary force compression for provenance Signed-off-by: Yan Song --- solver/llbsolver/proc/provenance.go | 5 +++-- solver/llbsolver/proc/sbom.go | 3 ++- solver/llbsolver/provenance.go | 4 +++- solver/llbsolver/solver.go | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go index 1af3af196..2edbdc39e 100644 --- a/solver/llbsolver/proc/provenance.go +++ b/solver/llbsolver/proc/provenance.go @@ -11,11 +11,12 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/compression" "github.com/pkg/errors" ) func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { - return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, comp compression.Config) (*llbsolver.Result, error) { ps, err := exptypes.ParsePlatforms(res.Metadata) if err != nil { return nil, err @@ -41,7 +42,7 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { return nil, errors.Errorf("could not find ref %s", p.ID) } - pc, err := llbsolver.NewProvenanceCreator(ctx, cp, ref, attrs, j) + pc, err := llbsolver.NewProvenanceCreator(ctx, cp, ref, attrs, j, comp) if err != nil { return nil, err } diff --git a/solver/llbsolver/proc/sbom.go b/solver/llbsolver/proc/sbom.go index 2d7e969ba..c0837218a 100644 --- a/solver/llbsolver/proc/sbom.go +++ b/solver/llbsolver/proc/sbom.go @@ -10,11 +10,12 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/compression" "github.com/pkg/errors" ) func SBOMProcessor(scannerRef string, useCache bool) llbsolver.Processor { - return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, comp compression.Config) (*llbsolver.Result, error) { // skip sbom generation if we already have an sbom if sbom.HasSBOM(res.Result) { return res, nil diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index b30581c85..f02e958e7 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -20,6 +20,7 @@ import ( "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" + "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -372,7 +373,7 @@ type ProvenanceCreator struct { addLayers func() error } -func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solver.ResultProxy, attrs map[string]string, j *solver.Job) (*ProvenanceCreator, error) { +func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solver.ResultProxy, attrs map[string]string, j *solver.Job, comp compression.Config) (*ProvenanceCreator, error) { var reproducible bool if v, ok := attrs["reproducible"]; ok { b, err := strconv.ParseBool(v) @@ -449,6 +450,7 @@ func NewProvenanceCreator(ctx context.Context, cp *provenance.Capture, res solve ResolveRemotes: resolveRemotes, Mode: solver.CacheExportModeRemoteOnly, ExportRoots: true, + CompressionOpt: &comp, }); err != nil { return err } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 2f7ba61e5..ffe7ed570 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -93,7 +93,7 @@ type Solver struct { // Processor defines a processing function to be applied after solving, but // before exporting -type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job) (*Result, error) +type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job, comp compression.Config) (*Result, error) func New(opt Opt) (*Solver, error) { s := &Solver{ @@ -200,7 +200,7 @@ func (s *Solver) recordBuildHistory(ctx context.Context, id string, req frontend } makeProvenance := func(res solver.ResultProxy, cap *provenance.Capture) (*controlapi.Descriptor, func(), error) { - prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j) + prc, err := NewProvenanceCreator(ctx2, cap, res, attrs, j, exp.Exporter.Config().Compression()) if err != nil { return nil, nil, err } @@ -501,7 +501,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } for _, post := range post { - res2, err := post(ctx, resProv, s, j) + res2, err := post(ctx, resProv, s, j, exp.Exporter.Config().Compression()) if err != nil { return nil, err } From 76aeaa55ff4e1c575644bffbf0a43bea92215fbe Mon Sep 17 00:00:00 2001 From: Yan Song Date: Sat, 6 May 2023 11:22:27 +0000 Subject: [PATCH 2/4] nydus: set default compression option to avoid recomputing Signed-off-by: Yan Song --- nydus/util/util.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nydus/util/util.go b/nydus/util/util.go index f65cfe520..6524ebf78 100644 --- a/nydus/util/util.go +++ b/nydus/util/util.go @@ -15,12 +15,14 @@ type compressorKey struct{} type chunkDictDigestKey struct{} func WithContext(ctx context.Context, fsVersion string, compressor string, chunkDictDigest digest.Digest) context.Context { - if fsVersion == "" { - fsVersion = "5" + if fsVersion != "" { + ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion) + } + + if compressor != "" { + ctx = context.WithValue(ctx, compressorKey{}, compressor) } - ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion) - ctx = context.WithValue(ctx, compressorKey{}, compressor) if chunkDictDigest != "" { ctx = context.WithValue(ctx, chunkDictDigestKey{}, chunkDictDigest.String()) } @@ -29,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk } func GetContext(ctx context.Context) (string, string, string) { - fsVersion := "" - compressor := "" + fsVersion := "5" + compressor := "lz4_block" chunkDictDigest := "" ctxValue := ctx.Value(fsVersionKey{}) From b2ef881dd4b2201bc5b630a0af91ea33c13d45bc Mon Sep 17 00:00:00 2001 From: Yan Song Date: Thu, 25 May 2023 09:18:46 +0000 Subject: [PATCH 3/4] nydus: use rafs v6 as default fs version - Use `--nydus-fs-version=6` as default; - Use `--nydus-compressor=zstd` as default; - Upgrade suggested nydus version to v2.1.6; Signed-off-by: Yan Song --- Dockerfile | 2 +- docs/nydus.md | 4 ++-- nydus/nydus.go | 4 ++-- nydus/util/util.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8869138b5..9c8a54f19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ARG CNI_VERSION=v1.1.1 ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0 ARG NERDCTL_VERSION=v1.0.0 ARG DNSNAME_VERSION=v1.3.1 -ARG NYDUS_VERSION=v2.1.0 +ARG NYDUS_VERSION=v2.1.6 ARG ALPINE_VERSION=3.17 diff --git a/docs/nydus.md b/docs/nydus.md index 0334c76c5..80c39a8ef 100644 --- a/docs/nydus.md +++ b/docs/nydus.md @@ -16,7 +16,7 @@ go build -tags=nydus -o ./bin/buildkitd ./cmd/buildkitd ### Building Nydus with BuildKit -Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.1 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd: +Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.6 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd: ``` env NYDUS_BUILDER=/path/to/nydus-image buildkitd ... @@ -33,7 +33,7 @@ buildctl build ... \ Available options: -- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `5`; +- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `6`; - `nydus-compressor`: Specify nydus image compressor, possible values: `none`, `lz4_block`, `zstd`, default `lz4_block`; - `nydus-chunk-dict-image`: Specify nydus chunk dict image reference for data de-duplication; diff --git a/nydus/nydus.go b/nydus/nydus.go index 2d808d2f0..5e80b6671 100644 --- a/nydus/nydus.go +++ b/nydus/nydus.go @@ -170,10 +170,10 @@ func loadChunkDict(ctx context.Context, registryHosts docker.RegistryHosts, sm * } if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] == "" { - bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "5" + bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "6" } if nydusFSVersion == "" { - nydusFSVersion = "5" + nydusFSVersion = "6" } if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] != nydusFSVersion { diff --git a/nydus/util/util.go b/nydus/util/util.go index 6524ebf78..22f58f641 100644 --- a/nydus/util/util.go +++ b/nydus/util/util.go @@ -31,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk } func GetContext(ctx context.Context) (string, string, string) { - fsVersion := "5" - compressor := "lz4_block" + fsVersion := "6" + compressor := "zstd" chunkDictDigest := "" ctxValue := ctx.Value(fsVersionKey{}) From 8afd2e8f16ed123d5ad9a16863128596b510215a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 18:49:33 +0000 Subject: [PATCH 4/4] build(deps): bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8d97ec4e..554941e4e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -295,7 +295,7 @@ jobs: .join(','); - name: Send to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: ${{ steps.files.outputs.result }}