build(deps): bump actions/github-script from 6 to 7

.github/workflows/build.yml

@@ -284,7 +284,7 @@ jobs:
           path: ./coverage
       -
         name: List coverage files
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         id: files
         with:
           result-encoding: string

.github/workflows/dockerd.yml

@@ -30,7 +30,7 @@ jobs:
           echo "DOCKER_VERSION=$version" >> $GITHUB_ENV
       -
         name: Check build
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         id: build
         with:
           result-encoding: string

.github/workflows/nydus.yml

@@ -20,8 +20,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Install Nydus binaries
         run: |
-          NYDUS_VERSION=$(curl --silent "https://api.github.com/repos/dragonflyoss/image-service/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')
-          wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-linux-amd64.tgz
+          NYDUS_VERSION=$(curl --silent "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')
+          wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-linux-amd64.tgz
           tar xzf nydus-static-$NYDUS_VERSION-linux-amd64.tgz
           sudo cp nydus-static/nydusify nydus-static/nydus-image /usr/local/bin/
           sudo cp nydus-static/nydusd /usr/local/bin/nydusd

Dockerfile

@@ -10,7 +10,7 @@ ARG CNI_VERSION=v1.1.1
 ARG STARGZ_SNAPSHOTTER_VERSION=v0.13.0
 ARG NERDCTL_VERSION=v1.0.0
 ARG DNSNAME_VERSION=v1.3.1
-ARG NYDUS_VERSION=v2.1.0
+ARG NYDUS_VERSION=v2.1.6
 
 ARG ALPINE_VERSION=3.17
 

cmd/buildctl/build.go

@@ -105,6 +105,10 @@ var buildCommand = cli.Command{
 			Name:  "ref-file",
 			Usage: "Write build ref to a file",
 		},
+		cli.StringSliceFlag{
+			Name:  "registry-auth-tlscontext",
+			Usage: "Overwrite TLS configuration when authenticating with registries, e.g. --registry-auth-tlscontext host=https://myserver:2376,ca=/path/to/my/ca.crt,cert=/path/to/my/cert.crt,key=/path/to/my/key.crt",
+		},
 	},
 }
 
@@ -158,7 +162,11 @@ func buildAction(clicontext *cli.Context) error {
 	}
 
 	dockerConfig := config.LoadDefaultConfigFile(os.Stderr)
-	attachable := []session.Attachable{authprovider.NewDockerAuthProvider(dockerConfig)}
+	tlsConfigs, err := build.ParseRegistryAuthTLSContext(clicontext.StringSlice("registry-auth-tlscontext"))
+	if err != nil {
+		return err
+	}
+	attachable := []session.Attachable{authprovider.NewDockerAuthProvider(dockerConfig, tlsConfigs)}
 
 	if ssh := clicontext.StringSlice("ssh"); len(ssh) > 0 {
 		configs, err := build.ParseSSH(ssh)

cmd/buildctl/build/registryauthtlscontext.go

@@ -0,0 +1,84 @@
+package build
+
+import (
+	"encoding/csv"
+	"strings"
+
+	"github.com/moby/buildkit/session/auth/authprovider"
+	"github.com/pkg/errors"
+)
+
+type authTLSContextEntry struct {
+	Host string
+	CA   string
+	Cert string
+	Key  string
+}
+
+func parseRegistryAuthTLSContextCSV(s string) (authTLSContextEntry, error) {
+	authTLSContext := authTLSContextEntry{}
+	csvReader := csv.NewReader(strings.NewReader(s))
+	fields, err := csvReader.Read()
+	if err != nil {
+		return authTLSContext, err
+	}
+	for _, field := range fields {
+		key, value, ok := strings.Cut(field, "=")
+		if !ok {
+			return authTLSContext, errors.Errorf("invalid value %s", field)
+		}
+		key = strings.ToLower(key)
+		switch key {
+		case "host":
+			authTLSContext.Host = value
+		case "ca":
+			authTLSContext.CA = value
+		case "cert":
+			authTLSContext.Cert = value
+		case "key":
+			authTLSContext.Key = value
+		}
+	}
+	if authTLSContext.Host == "" {
+		return authTLSContext, errors.New("--registry-auth-tlscontext requires host=<host>")
+	}
+	if authTLSContext.CA == "" {
+		if authTLSContext.Cert == "" || authTLSContext.Key == "" {
+			return authTLSContext, errors.New("--registry-auth-tlscontext requires ca=<ca> or cert=<cert>,key=<key>")
+		}
+	} else {
+		if (authTLSContext.Cert != "" && authTLSContext.Key == "") || (authTLSContext.Cert == "" && authTLSContext.Key != "") {
+			return authTLSContext, errors.New("--registry-auth-tlscontext requires cert=<cert>,key=<key>")
+		}
+	}
+	return authTLSContext, nil
+}
+
+func ParseRegistryAuthTLSContext(registryAuthTLSContext []string) (map[string]*authprovider.AuthTLSConfig, error) {
+	var tlsContexts []authTLSContextEntry
+	for _, c := range registryAuthTLSContext {
+		authTLSContext, err := parseRegistryAuthTLSContextCSV(c)
+		if err != nil {
+			return nil, err
+		}
+		tlsContexts = append(tlsContexts, authTLSContext)
+	}
+
+	authConfigs := make(map[string]*authprovider.AuthTLSConfig)
+	for _, c := range tlsContexts {
+		_, ok := authConfigs[c.Host]
+		if !ok {
+			authConfigs[c.Host] = &authprovider.AuthTLSConfig{}
+		}
+		if c.CA != "" {
+			authConfigs[c.Host].RootCAs = append(authConfigs[c.Host].RootCAs, c.CA)
+		}
+		if c.Cert != "" && c.Key != "" {
+			authConfigs[c.Host].KeyPairs = append(authConfigs[c.Host].KeyPairs, authprovider.TLSKeyPair{
+				Key:         c.Key,
+				Certificate: c.Cert,
+			})
+		}
+	}
+	return authConfigs, nil
+}

cmd/buildctl/build/registryauthtlscontext_test.go

@@ -0,0 +1,109 @@
+package build
+
+import (
+	"testing"
+
+	"github.com/moby/buildkit/session/auth/authprovider"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseRegistryAuthTLSContext(t *testing.T) {
+	type testCase struct {
+		registryAuthTLSContext []string //--registry-auth-tlscontext
+		expected               map[string]*authprovider.AuthTLSConfig
+		expectedErr            string
+	}
+	testCases := []testCase{
+		{
+			registryAuthTLSContext: []string{
+				"host=tcp://myserver:2376,ca=/home/admin/ca-file,cert=/home/admin/cert-file,key=/home/admin/key-file",
+			},
+			expected: map[string]*authprovider.AuthTLSConfig{
+				"tcp://myserver:2376": {
+					RootCAs: []string{
+						"/home/admin/ca-file",
+					},
+					KeyPairs: []authprovider.TLSKeyPair{
+						{
+							Key:         "/home/admin/key-file",
+							Certificate: "/home/admin/cert-file",
+						},
+					},
+				},
+			},
+		},
+		{
+			registryAuthTLSContext: []string{
+				"host=tcp://myserver:2376,cert=/home/admin/cert-file,key=/home/admin/key-file",
+			},
+			expected: map[string]*authprovider.AuthTLSConfig{
+				"tcp://myserver:2376": {
+					KeyPairs: []authprovider.TLSKeyPair{
+						{
+							Key:         "/home/admin/key-file",
+							Certificate: "/home/admin/cert-file",
+						},
+					},
+				},
+			},
+		},
+		{
+			registryAuthTLSContext: []string{
+				"host=tcp://myserver:2376,ca=/home/admin/ca-file",
+			},
+			expected: map[string]*authprovider.AuthTLSConfig{
+				"tcp://myserver:2376": {
+					RootCAs: []string{
+						"/home/admin/ca-file",
+					},
+				},
+			},
+		},
+		{
+			registryAuthTLSContext: []string{
+				"host=tcp://myserver:2376,ca=/home/admin/ca-file,key=/home/admin/key-file",
+			},
+			expectedErr: "--registry-auth-tlscontext requires cert=<cert>,key=<key>",
+		},
+		{
+			registryAuthTLSContext: []string{
+				"host=tcp://myserver:2376,ca=/home/admin/ca-file,cert=/home/admin/cert-file,key=/home/admin/key-file",
+				"host=https://myserver:2376,ca=/path/to/my/ca.crt,cert=/path/to/my/cert.crt,key=/path/to/my/key.crt",
+			},
+			expected: map[string]*authprovider.AuthTLSConfig{
+				"tcp://myserver:2376": {
+					RootCAs: []string{
+						"/home/admin/ca-file",
+					},
+					KeyPairs: []authprovider.TLSKeyPair{
+						{
+							Key:         "/home/admin/key-file",
+							Certificate: "/home/admin/cert-file",
+						},
+					},
+				},
+				"https://myserver:2376": {
+					RootCAs: []string{
+						"/path/to/my/ca.crt",
+					},
+					KeyPairs: []authprovider.TLSKeyPair{
+						{
+							Key:         "/path/to/my/key.crt",
+							Certificate: "/path/to/my/cert.crt",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		im, err := ParseRegistryAuthTLSContext(tc.registryAuthTLSContext)
+		if tc.expectedErr == "" {
+			require.EqualValues(t, tc.expected, im)
+		} else {
+			require.Error(t, err)
+			require.Contains(t, err.Error(), tc.expectedErr)
+		}
+	}
+}

docs/nydus.md

@@ -16,7 +16,7 @@ go build -tags=nydus -o ./bin/buildkitd ./cmd/buildkitd
 
 ### Building Nydus with BuildKit
 
-Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.1 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd:
+Download `nydus-image` binary from [nydus release page](https://github.com/dragonflyoss/image-service/releases) (require v2.1.6 or higher), then put the `nydus-image` binary path into $PATH or specifying it on `NYDUS_BUILDER` environment variable for buildkitd:
 
 ```
 env NYDUS_BUILDER=/path/to/nydus-image buildkitd ...
@@ -33,7 +33,7 @@ buildctl build ... \
 
 Available options:
 
-- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `5`;
+- `nydus-fs-version`: Specify nydus image filesystem version, possible values: `5`, `6`, default `6`;
 - `nydus-compressor`: Specify nydus image compressor, possible values: `none`, `lz4_block`, `zstd`, default `lz4_block`;
 - `nydus-chunk-dict-image`: Specify nydus chunk dict image reference for data de-duplication;
 

nydus-test/top_images/image_list.txt

@@ -31,7 +31,7 @@ php
 bash
 caddy
 telegraf
-vault
+hashicorp/vault
 couchdb
 eclipse-mosquitto
 cassandra
@@ -43,3 +43,4 @@ kong
 solr
 sentry
 zookeeper
+ghcr.io/dragonflyoss/image-service/pax-uid-test

nydus/nydus.go

@@ -170,10 +170,10 @@ func loadChunkDict(ctx context.Context, registryHosts docker.RegistryHosts, sm *
 	}
 
 	if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] == "" {
-		bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "5"
+		bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] = "6"
 	}
 	if nydusFSVersion == "" {
-		nydusFSVersion = "5"
+		nydusFSVersion = "6"
 	}
 
 	if bootstrapDesc.Annotations[nydusify.LayerAnnotationFSVersion] != nydusFSVersion {

nydus/util/util.go

@@ -15,12 +15,14 @@ type compressorKey struct{}
 type chunkDictDigestKey struct{}
 
 func WithContext(ctx context.Context, fsVersion string, compressor string, chunkDictDigest digest.Digest) context.Context {
-	if fsVersion == "" {
-		fsVersion = "5"
+	if fsVersion != "" {
+		ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion)
+	}
+
+	if compressor != "" {
+		ctx = context.WithValue(ctx, compressorKey{}, compressor)
 	}
 
-	ctx = context.WithValue(ctx, fsVersionKey{}, fsVersion)
-	ctx = context.WithValue(ctx, compressorKey{}, compressor)
 	if chunkDictDigest != "" {
 		ctx = context.WithValue(ctx, chunkDictDigestKey{}, chunkDictDigest.String())
 	}
@@ -29,8 +31,8 @@ func WithContext(ctx context.Context, fsVersion string, compressor string, chunk
 }
 
 func GetContext(ctx context.Context) (string, string, string) {
-	fsVersion := ""
-	compressor := ""
+	fsVersion := "6"
+	compressor := "zstd"
 	chunkDictDigest := ""
 
 	ctxValue := ctx.Value(fsVersionKey{})

session/auth/authprovider/authconfig.go

@@ -0,0 +1,11 @@
+package authprovider
+
+type AuthTLSConfig struct {
+	RootCAs  []string
+	KeyPairs []TLSKeyPair
+}
+
+type TLSKeyPair struct {
+	Key         string
+	Certificate string
+}