Add support for zstd-compressed layer media types in container images

.git-blame-ignore-revs

@@ -0,0 +1,6 @@
+# This file contains a list of commits that should be ignored by git blame
+# Each commit that is purely formatting changes should be added here
+# Usage: git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Run dotnet format to fix code formatting
+0ce0dde866fd5a9eab5f60e2e7f07f9f87e8c8c5

src/Containers/Microsoft.NET.Build.Containers/AuthHandshakeMessageHandler.cs

@@ -283,7 +283,8 @@ internal static (string credU, string credP)? GetDockerCredentialsFromEnvironmen
         if (bearerAuthInfo.Scope is not null)
         {
             parameters["scope"] = bearerAuthInfo.Scope;
-        };
+        }
+        ;
         HttpRequestMessage postMessage = new(HttpMethod.Post, uri)
         {
             Content = new FormUrlEncodedContent(parameters)

src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs

@@ -26,7 +26,7 @@ internal readonly struct BuiltImage
     /// <summary>
     /// Gets image manifest.
     /// </summary>
-    internal required string Manifest { get; init; } 
+    internal required string Manifest { get; init; }
 
     /// <summary>
     /// Gets manifest digest.

src/Containers/Microsoft.NET.Build.Containers/ContainerHelpers.cs

@@ -14,6 +14,7 @@
 using Microsoft.NET.Build.Containers.Resources;
 
 namespace Microsoft.NET.Build.Containers;
+
 public static class ContainerHelpers
 {
     internal const string HostObjectUser = "DOTNET_CONTAINER_REGISTRY_UNAME";

src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs

@@ -43,10 +43,13 @@ public static string PathForDescriptor(Descriptor descriptor)
 
         string extension = descriptor.MediaType switch
         {
-            "application/vnd.docker.image.rootfs.diff.tar.gzip"
-            or "application/vnd.oci.image.layer.v1.tar+gzip"
+            SchemaTypes.DockerLayerGzip
+            or SchemaTypes.OciLayerGzipV1
             or "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
                 => ".tar.gz",
+            SchemaTypes.DockerLayerZstd
+            or SchemaTypes.OciLayerZstdV1
+                => ".tar.zst",
             "application/vnd.docker.image.rootfs.diff.tar"
             or "application/vnd.oci.image.layer.v1.tar"
                 => ".tar",

src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs

@@ -6,7 +6,7 @@ namespace Microsoft.NET.Build.Containers;
 internal sealed class UnableToDownloadFromRepositoryException : Exception
 {
     public UnableToDownloadFromRepositoryException(string repository)
-        : base($"The download of the image from repository { repository } has failed.")
+        : base($"The download of the image from repository {repository} has failed.")
     {
     }
 }

src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs

@@ -2,8 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Text;
-using System.Text.RegularExpressions;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using Microsoft.Extensions.Logging;
 using Microsoft.NET.Build.Containers.Resources;
 

src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs

@@ -217,7 +217,8 @@ internal void AddLayer(Layer l)
         _rootFsLayers.Add(l.Descriptor.UncompressedDigest!);
     }
 
-    internal void SetUser(string user, bool isUserInteraction = false) {
+    internal void SetUser(string user, bool isUserInteraction = false)
+    {
         // we don't let automatic/inferred user settings overwrite an explicit user request
         if (_userHasBeenExplicitlySet && !isUserInteraction)
         {

src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs

@@ -64,7 +64,7 @@ internal static string GenerateImageIndex(BuiltImage[] images, string manifestMe
         // Here we are using ManifestListV2 struct, but we could use ImageIndexV1 struct as well.
         // We are filling the same fields, so we can use the same struct.
         var manifests = new PlatformSpecificManifest[images.Length];
-        
+
         for (int i = 0; i < images.Length; i++)
         {
             manifests[i] = new PlatformSpecificManifest
@@ -93,7 +93,7 @@ internal static string GenerateImageIndex(BuiltImage[] images, string manifestMe
     internal static string GenerateImageIndexWithAnnotations(string manifestMediaType, string manifestDigest, long manifestSize, string repository, string[] tags)
     {
         string containerdImageNamePrefix = repository.Contains('/') ? "docker.io/" : "docker.io/library/";
-        
+
         var manifests = new PlatformSpecificOciManifest[tags.Length];
         for (int i = 0; i < tags.Length; i++)
         {
@@ -103,10 +103,10 @@ internal static string GenerateImageIndexWithAnnotations(string manifestMediaTyp
                 mediaType = manifestMediaType,
                 size = manifestSize,
                 digest = manifestDigest,
-                annotations = new Dictionary<string, string> 
+                annotations = new Dictionary<string, string>
                 {
                     { "io.containerd.image.name", $"{containerdImageNamePrefix}{repository}:{tag}" },
-                    { "org.opencontainers.image.ref.name", tag } 
+                    { "org.opencontainers.image.ref.name", tag }
                 }
             };
         }

src/Containers/Microsoft.NET.Build.Containers/Layer.cs

@@ -201,7 +201,7 @@ static UnixFileMode DetermineFileMode(FileSystemInfo file)
 
         string layerMediaType = manifestMediaType switch
         {
-             // TODO: configurable? gzip always?
+            // TODO: configurable? gzip always?
             SchemaTypes.DockerManifestV2 => SchemaTypes.DockerLayerGzip,
             SchemaTypes.OciManifestV1 => SchemaTypes.OciLayerGzipV1,
             _ => throw new ArgumentException(Resource.FormatString(nameof(Strings.UnrecognizedMediaType), manifestMediaType))

src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ArchiveFileRegistry.cs

@@ -14,7 +14,7 @@ public ArchiveFileRegistry(string archiveOutputPath)
         ArchiveOutputPath = archiveOutputPath;
     }
 
-    internal async Task LoadAsync<T>(T image, SourceImageReference sourceReference, 
+    internal async Task LoadAsync<T>(T image, SourceImageReference sourceReference,
         DestinationImageReference destinationReference, CancellationToken cancellationToken,
         Func<T, SourceImageReference, DestinationImageReference, Stream, CancellationToken, Task> writeStreamFunc)
     {
@@ -25,7 +25,7 @@ internal async Task LoadAsync<T>(T image, SourceImageReference sourceReference,
         // if doesn't end with a file extension, assume it's a directory
         if (!Path.HasExtension(fullPath))
         {
-           fullPath += Path.DirectorySeparatorChar;
+            fullPath += Path.DirectorySeparatorChar;
         }
 
         // pointing to a directory? -> append default name
@@ -50,13 +50,13 @@ internal async Task LoadAsync<T>(T image, SourceImageReference sourceReference,
 
     public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReference,
         DestinationImageReference destinationReference,
-        CancellationToken cancellationToken) 
+        CancellationToken cancellationToken)
         => await LoadAsync(image, sourceReference, destinationReference, cancellationToken,
             DockerCli.WriteImageToStreamAsync);
 
     public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference,
         DestinationImageReference destinationReference,
-        CancellationToken cancellationToken) 
+        CancellationToken cancellationToken)
         => await LoadAsync(multiArchImage, sourceReference, destinationReference, cancellationToken,
             DockerCli.WriteMultiArchOciImageToStreamAsync);
 

src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs

@@ -130,13 +130,13 @@ await writeStreamFunc(image, sourceReference, destinationReference, loadProcess.
         }
     }
 
-    public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) 
+    public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken)
         // For loading to the local registry, we use the Docker format. Two reasons: one - compatibility with previous behavior before oci formatted publishing was available, two - Podman cannot load multi tag oci image tarball.
         => await LoadAsync(image, sourceReference, destinationReference, WriteDockerImageToStreamAsync, cancellationToken);
 
-    public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) 
+    public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken)
         => await LoadAsync(multiArchImage, sourceReference, destinationReference, WriteMultiArchOciImageToStreamAsync, cancellationToken, checkContainerdStore: true);
-    
+
     public async Task<bool> IsAvailableAsync(CancellationToken cancellationToken)
     {
         bool commandPathWasUnknown = _command is null; // avoid running the version command twice.
@@ -539,7 +539,7 @@ private static async Task WriteIndexJsonForMultiArchOciImage(
         var manifestListDigest = DigestUtils.GetDigest(multiArchImage.ImageIndex);
         var manifestListSha = DigestUtils.GetShaFromDigest(manifestListDigest);
         var manifestListPath = $"{_blobsPath}/{manifestListSha}";
-        
+
         using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(multiArchImage.ImageIndex)))
         {
             PaxTarEntry indexEntry = new(TarEntryType.RegularFile, manifestListPath)
@@ -553,10 +553,10 @@ private static async Task WriteIndexJsonForMultiArchOciImage(
         cancellationToken.ThrowIfCancellationRequested();
 
         string indexJson = ImageIndexGenerator.GenerateImageIndexWithAnnotations(
-            multiArchImage.ImageIndexMediaType, 
-            manifestListDigest, 
-            multiArchImage.ImageIndex.Length, 
-            destinationReference.Repository, 
+            multiArchImage.ImageIndexMediaType,
+            manifestListDigest,
+            multiArchImage.ImageIndex.Length,
+            destinationReference.Repository,
             destinationReference.Tags);
 
         using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(indexJson)))

src/Containers/Microsoft.NET.Build.Containers/MultiArchImage.cs

@@ -13,4 +13,4 @@ internal readonly struct MultiArchImage
     internal required string ImageIndexMediaType { get; init; }
 
     internal BuiltImage[]? Images { get; init; }
-}
+}

src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs

@@ -1,12 +1,12 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using NuGet.Packaging;
 using System.Diagnostics;
 using System.Net.Http.Json;
 using System.Text.Json.Nodes;
 using Microsoft.Extensions.Logging;
 using Microsoft.NET.Build.Containers.Resources;
+using NuGet.Packaging;
 using NuGet.RuntimeModel;
 
 namespace Microsoft.NET.Build.Containers;
@@ -301,7 +301,7 @@ private static IReadOnlyDictionary<string, PlatformSpecificOciManifest> GetManif
         // TODO: we _may_ need OS-specific version parsing. Need to do more research on what the field looks like across more manifest lists.
         var versionPart = platform.version?.Split('.') switch
         {
-        [var major, ..] => major,
+            [var major, ..] => major,
             _ => null
         };
         var platformPart = platform.architecture switch
@@ -408,28 +408,34 @@ public async Task<string> DownloadBlobAsync(string repository, Descriptor descri
     {
         cancellationToken.ThrowIfCancellationRequested();
         string localPath = ContentStore.PathForDescriptor(descriptor);
-    
+
         if (File.Exists(localPath))
         {
             // Assume file is up to date and just return it
             return localPath;
         }
-
+
+        // Log when we encounter zstd-compressed layers
+        if (descriptor.MediaType == SchemaTypes.DockerLayerZstd || descriptor.MediaType == SchemaTypes.OciLayerZstdV1)
+        {
+            _logger.LogTrace("Downloading zstd-compressed layer (mediaType: {0}). Layer will be passed through unmodified.", descriptor.MediaType);
+        }
+
         string tempTarballPath = ContentStore.GetTempFile();
-    
+
         int retryCount = 0;
         while (retryCount < MaxDownloadRetries)
         {
             try
             {
                 // No local copy, so download one
                 using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false);
-    
+
                 using (FileStream fs = File.Create(tempTarballPath))
                 {
                     await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
                 }
-    
+
                 // Break the loop if successful
                 break;
             }
@@ -440,16 +446,16 @@ public async Task<string> DownloadBlobAsync(string repository, Descriptor descri
                 {
                     throw new UnableToDownloadFromRepositoryException(repository);
                 }
-    
+
                 _logger.LogTrace("Download attempt {0}/{1} for repository '{2}' failed. Error: {3}", retryCount, MaxDownloadRetries, repository, ex.ToString());
-    
+
                 // Wait before retrying
                 await Task.Delay(_retryDelayProvider(), cancellationToken).ConfigureAwait(false);
             }
         }
-    
+
         File.Move(tempTarballPath, localPath, overwrite: true);
-    
+
         return localPath;
     }
 
@@ -566,7 +572,7 @@ public async Task PushManifestListAsync(
             _logger.LogInformation(Strings.Registry_TagUploadStarted, tag, RegistryName);
             await _registryAPI.Manifest.PutAsync(destinationImageReference.Repository, tag, multiArchImage.ImageIndex, multiArchImage.ImageIndexMediaType, cancellationToken).ConfigureAwait(false);
             _logger.LogInformation(Strings.Registry_TagUploaded, tag, RegistryName);
-        }          
+        }
     }
 
     public Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, CancellationToken cancellationToken)

src/Containers/Microsoft.NET.Build.Containers/Registry/SchemaTypes.cs

@@ -16,4 +16,7 @@ internal class SchemaTypes
 
     internal const string DockerLayerGzip = "application/vnd.docker.image.rootfs.diff.tar.gzip";
     internal const string OciLayerGzipV1 = "application/vnd.oci.image.layer.v1.tar+gzip";
+
+    internal const string DockerLayerZstd = "application/vnd.docker.image.rootfs.diff.tar.zstd";
+    internal const string OciLayerZstdV1 = "application/vnd.oci.image.layer.v1.tar+zstd";
 }

src/Containers/Microsoft.NET.Build.Containers/Tasks/ComputeDotnetBaseImageAndTag.cs

@@ -130,7 +130,7 @@ private bool TargetRuntimeIdentiriersAreValid()
         {
             if (muslRidsCount == TargetRuntimeIdentifiers.Length)
             {
-                IsMuslRid = true;              
+                IsMuslRid = true;
             }
             else
             {
@@ -311,7 +311,8 @@ private bool ComputeRepositoryAndTag([NotNullWhen(true)] out string? repository,
             default:
                 Log.LogError(Resources.Strings.InvalidSdkPrereleaseVersion, channel);
                 return null;
-        };
+        }
+        ;
     }
 
     private bool UserImageIsMicrosoftBaseImage => UserBaseImage?.StartsWith("mcr.microsoft.com/") ?? false;

src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.Interface.cs

@@ -95,4 +95,4 @@ public CreateImageIndex()
 
         TaskResources = Resource.Manager;
     }
-} 
+}

src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.cs

@@ -84,7 +84,7 @@ internal async Task<bool> ExecuteAsync(CancellationToken cancellationToken)
         var telemetry = new Telemetry(sourceImageReference, destinationImageReference, Log);
 
         await ImagePublisher.PublishImageAsync(multiArchImage, sourceImageReference, destinationImageReference, Log, telemetry, cancellationToken)
-            .ConfigureAwait(false); 
+            .ConfigureAwait(false);
 
         return !Log.HasLoggedErrors;
     }
@@ -127,7 +127,7 @@ private BuiltImage[] ParseImages(DestinationImageReferenceKind destinationKind)
                 imageDigest = manifestV2.Config.digest;
                 imageSha = DigestUtils.GetShaFromDigest(imageDigest);
                 layers = manifestV2.Layers;
-            }     
+            }
 
             images[i] = new BuiltImage()
             {
@@ -159,7 +159,7 @@ private BuiltImage[] ParseImages(DestinationImageReferenceKind destinationKind)
         {
             Log.LogError(Strings.ImageConfigMissingArchitecture);
             return (string.Empty, string.Empty);
-        } 
+        }
         var os = configJson["os"]?.ToString();
         if (os is null)
         {

src/Containers/Microsoft.NET.Build.Containers/Telemetry.cs

@@ -113,4 +113,4 @@ public void LogLocalLoadError()
         props.Add("error", "local_load");
         Log.LogTelemetry("sdk/container/publish/error", props);
     }
-}
+}