From e2cbd637b82a153a6756f2af0519e8fe769ee9ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 11 Nov 2024 15:03:02 +0100 Subject: [PATCH 1/2] use new azure-sdk --- azure/azure.go | 85 +++++++++++++------------ azure/package_pool.go | 45 ++++++------- azure/package_pool_test.go | 8 ++- azure/public.go | 127 ++++++++++++++++++++----------------- azure/public_test.go | 49 +++++++------- context/context.go | 1 - deb/list.go | 1 - go.mod | 8 +-- go.sum | 49 ++++++-------- 9 files changed, 188 insertions(+), 185 deletions(-) diff --git a/azure/azure.go b/azure/azure.go index b313f90479..3f12678b1b 100644 --- a/azure/azure.go +++ b/azure/azure.go @@ -5,28 +5,35 @@ package azure import ( "context" "encoding/hex" + "errors" "fmt" "io" - "net/url" + "os" "path/filepath" "time" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/aptly-dev/aptly/aptly" ) func isBlobNotFound(err error) bool { - storageError, ok := err.(azblob.StorageError) - return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == 404 // BlobNotFound + } + return false } type azContext struct { - container azblob.ContainerURL + client *azblob.Client + container string prefix string } func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) { - credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { return nil, err } @@ -35,15 +42,14 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) ( endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName) } - url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container)) + serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil) if err != nil { return nil, err } - containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{})) - result := &azContext{ - container: containerURL, + client: serviceClient, + container: container, prefix: prefix, } @@ -54,10 +60,6 @@ func (az *azContext) blobPath(path string) string { return filepath.Join(az.prefix, path) } -func (az *azContext) blobURL(path string) azblob.BlobURL { - return az.container.NewBlobURL(az.blobPath(path)) -} - func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) { const delimiter = "/" paths = make([]string, 0, 1024) @@ -67,27 +69,33 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p prefix += delimiter } - for marker := (azblob.Marker{}); marker.NotDone(); { - listBlob, err := az.container.ListBlobsFlatSegment( - context.Background(), marker, azblob.ListBlobsSegmentOptions{ - Prefix: prefix, - MaxResults: 1, - Details: azblob.BlobListingDetails{Metadata: true}}) + ctx := context.Background() + maxResults := int32(1) + pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + MaxResults: &maxResults, + Include: azblob.ListBlobsInclude{Metadata: true}, + }) + + // Iterate over each page + for pager.More() { + page, err := pager.NextPage(ctx) if err != nil { return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err) } - marker = listBlob.NextMarker - - for _, blob := range listBlob.Segment.BlobItems { + for _, blob := range page.Segment.BlobItems { if prefix == "" { - paths = append(paths, blob.Name) + paths = append(paths, *blob.Name) } else { - paths = append(paths, blob.Name[len(prefix):]) + name := *blob.Name + paths = append(paths, name[len(prefix):]) } - md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5)) - } + b := *blob + md5 := b.Properties.ContentMD5 + md5s = append(md5s, fmt.Sprintf("%x", md5)) + } if progress != nil { time.Sleep(time.Duration(500) * time.Millisecond) progress.AddBar(1) @@ -97,28 +105,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p return paths, md5s, nil } -func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error { - uploadOptions := azblob.UploadStreamToBlockBlobOptions{ - BufferSize: 4 * 1024 * 1024, - MaxBuffers: 8, +func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error { + uploadOptions := &azblob.UploadFileOptions{ + BlockSize: 4 * 1024 * 1024, + Concurrency: 8, } + path := az.blobPath(blobName) if len(sourceMD5) > 0 { decodedMD5, err := hex.DecodeString(sourceMD5) if err != nil { return err } - uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{ - ContentMD5: decodedMD5, + uploadOptions.HTTPHeaders = &blob.HTTPHeaders{ + BlobContentMD5: decodedMD5, } } - _, err := azblob.UploadStreamToBlockBlob( - context.Background(), - source, - blob.ToBlockBlobURL(), - uploadOptions, - ) + var err error + if file, ok := source.(*os.File); ok { + _, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions) + } return err } diff --git a/azure/package_pool.go b/azure/package_pool.go index 167ab6bedd..ad32a11ffa 100644 --- a/azure/package_pool.go +++ b/azure/package_pool.go @@ -5,7 +5,6 @@ import ( "os" "path/filepath" - "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" "github.com/pkg/errors" @@ -41,10 +40,7 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename) } -func (pool *PackagePool) ensureChecksums( - poolPath string, - checksumStorage aptly.ChecksumStorage, -) (*utils.ChecksumInfo, error) { +func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) { targetChecksums, err := checksumStorage.Get(poolPath) if err != nil { return nil, err @@ -52,8 +48,7 @@ func (pool *PackagePool) ensureChecksums( if targetChecksums == nil { // we don't have checksums stored yet for this file - blob := pool.az.blobURL(poolPath) - download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) + download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil) if err != nil { if isBlobNotFound(err) { return nil, nil @@ -63,7 +58,7 @@ func (pool *PackagePool) ensureChecksums( } targetChecksums = &utils.ChecksumInfo{} - *targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{})) + *targetChecksums, err = utils.ChecksumsForReader(download.Body) if err != nil { return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath) } @@ -92,46 +87,49 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er } func (pool *PackagePool) Size(path string) (int64, error) { - blob := pool.az.blobURL(path) - props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := pool.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(pool.az.container) + blobClient := containerClient.NewBlobClient(path) + + props, err := blobClient.GetProperties(context.TODO(), nil) if err != nil { return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) } - return props.ContentLength(), nil + return *props.ContentLength, nil } func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) { - blob := pool.az.blobURL(path) - temp, err := os.CreateTemp("", "blob-download") if err != nil { - return nil, errors.Wrap(err, "error creating temporary file for blob download") + return nil, errors.Wrapf(err, "error creating tempfile for %s", path) } - defer os.Remove(temp.Name()) - err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{}) + _, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil) if err != nil { - return nil, errors.Wrapf(err, "error downloading blob at %s", path) + return nil, errors.Wrapf(err, "error downloading blob %s", path) } return temp, nil } func (pool *PackagePool) Remove(path string) (int64, error) { - blob := pool.az.blobURL(path) - props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := pool.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(pool.az.container) + blobClient := containerClient.NewBlobClient(path) + + props, err := blobClient.GetProperties(context.TODO(), nil) if err != nil { - return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool) + return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) } - _, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + _, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil) if err != nil { return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool) } - return props.ContentLength(), nil + return *props.ContentLength, nil } func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) { @@ -145,7 +143,6 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } path := pool.buildPoolPath(basename, checksums) - blob := pool.az.blobURL(path) targetChecksums, err := pool.ensureChecksums(path, checksumStorage) if err != nil { return "", err @@ -161,7 +158,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } defer source.Close() - err = pool.az.putFile(blob, source, checksums.MD5) + err = pool.az.putFile(path, source, checksums.MD5) if err != nil { return "", err } diff --git a/azure/package_pool_test.go b/azure/package_pool_test.go index 58949ec638..a32bbff760 100644 --- a/azure/package_pool_test.go +++ b/azure/package_pool_test.go @@ -7,7 +7,7 @@ import ( "path/filepath" "runtime" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/utils" @@ -50,8 +50,10 @@ func (s *PackagePoolSuite) SetUpTest(c *C) { s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint) c.Assert(err, IsNil) - cnt := s.pool.az.container - _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) + publicAccessType := azblob.PublicAccessTypeContainer + _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{ + Access: &publicAccessType, + }) c.Assert(err, IsNil) s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint) diff --git a/azure/public.go b/azure/public.go index 50a8202981..6bda2599be 100644 --- a/azure/public.go +++ b/azure/public.go @@ -3,20 +3,21 @@ package azure import ( "context" "fmt" - "net/http" "os" "path/filepath" "time" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" + "github.com/pborman/uuid" "github.com/pkg/errors" ) // PublishedStorage abstract file system with published files (actually hosted on Azure) type PublishedStorage struct { - container azblob.ContainerURL prefix string az *azContext pathCache map[string]map[string]string @@ -66,7 +67,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err } defer source.Close() - err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5) + err = storage.az.putFile(path, source, sourceMD5) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) } @@ -76,14 +77,15 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err // RemoveDirs removes directory structure under public path func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error { + path = storage.az.blobPath(path) filelist, err := storage.Filelist(path) if err != nil { return err } for _, filename := range filelist { - blob := storage.az.blobURL(filepath.Join(path, filename)) - _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + blob := filepath.Join(path, filename) + _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil) if err != nil { return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err) } @@ -94,8 +96,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // Remove removes single file under public path func (storage *PublishedStorage) Remove(path string) error { - blob := storage.az.blobURL(path) - _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + path = storage.az.blobPath(path) + _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err)) } @@ -114,9 +116,8 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { relFilePath := filepath.Join(publishedRelPath, fileName) - // prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) - // FIXME: check how to integrate publishedPrefix: - poolPath := storage.az.blobPath(fileName) + prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) + poolPath := storage.az.blobPath(prefixRelFilePath) if storage.pathCache == nil { storage.pathCache = make(map[string]map[string]string) @@ -159,7 +160,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, } defer source.Close() - err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5) + err = storage.az.putFile(relFilePath, source, sourceMD5) if err == nil { pathCache[relFilePath] = sourceMD5 } else { @@ -176,57 +177,58 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { } // Internal copy or move implementation -func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error { +func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error { const leaseDuration = 30 + leaseID := uuid.NewRandom().String() - dstBlobURL := storage.az.blobURL(dst) - srcBlobURL := storage.az.blobURL(src) - leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{}) - if err != nil || leaseResp.StatusCode() != http.StatusCreated { - return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL) + serviceClient := storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(storage.az.container) + srcBlobClient := containerClient.NewBlobClient(src) + blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)}) + if err != nil { + return fmt.Errorf("error acquiring lease on source blob %s", src) + } + + _, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil) + if err != nil { + return fmt.Errorf("error acquiring lease on source blob %s", src) } - defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) - srcBlobLeaseID := leaseResp.LeaseID() - - copyResp, err := dstBlobURL.StartCopyFromURL( - context.Background(), - srcBlobURL.URL(), - metadata, - azblob.ModifiedAccessConditions{}, - azblob.BlobAccessConditions{}, - azblob.DefaultAccessTier, - nil) + defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))}) + + dstBlobClient := containerClient.NewBlobClient(dst) + copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{ + Metadata: metadata, + }) + if err != nil { return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err) } - copyStatus := copyResp.CopyStatus() + copyStatus := *copyResp.CopyStatus for { - if copyStatus == azblob.CopyStatusSuccess { + if copyStatus == blob.CopyStatusTypeSuccess { if move { - _, err = srcBlobURL.Delete( - context.Background(), - azblob.DeleteSnapshotsOptionNone, - azblob.BlobAccessConditions{ - LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}, - }) + _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{ + AccessConditions: &blob.AccessConditions{ + LeaseAccessConditions: &blob.LeaseAccessConditions{ + LeaseID: &leaseID, + }, + }, + }) return err } return nil - } else if copyStatus == azblob.CopyStatusPending { + } else if copyStatus == blob.CopyStatusTypePending { time.Sleep(1 * time.Second) - blobPropsResp, err := dstBlobURL.GetProperties( - context.Background(), - azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}}, - azblob.ClientProvidedKeyOptions{}) + getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil) if err != nil { - return fmt.Errorf("error getting destination blob properties %s", dstBlobURL) + return fmt.Errorf("error getting copy progress %s", dst) } - copyStatus = blobPropsResp.CopyStatus() + copyStatus = *getMetadata.CopyStatus - _, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{}) + _, err = blobLeaseClient.RenewLease(context.Background(), nil) if err != nil { - return fmt.Errorf("error renewing source blob lease %s", srcBlobURL) + return fmt.Errorf("error renewing source blob lease %s", src) } } else { return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus) @@ -241,7 +243,9 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { // SymLink creates a copy of src file and adds link information as meta data func (storage *PublishedStorage) SymLink(src string, dst string) error { - return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */) + metadata := make(map[string]*string) + metadata["SymLink"] = &src + return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */) } // HardLink using symlink functionality as hard links do not exist @@ -251,28 +255,33 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) (bool, error) { - blob := storage.az.blobURL(path) - resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(storage.az.container) + blobClient := containerClient.NewBlobClient(path) + _, err := blobClient.GetProperties(context.Background(), nil) if err != nil { if isBlobNotFound(err) { return false, nil } - return false, err - } else if resp.StatusCode() == http.StatusOK { - return true, nil + return false, fmt.Errorf("error checking if blob %s exists: %v", path, err) } - return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) + return true, nil } // ReadLink returns the symbolic link pointed to by path. // This simply reads text file created with SymLink func (storage *PublishedStorage) ReadLink(path string) (string, error) { - blob := storage.az.blobURL(path) - resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(storage.az.container) + blobClient := containerClient.NewBlobClient(path) + props, err := blobClient.GetProperties(context.Background(), nil) if err != nil { - return "", err - } else if resp.StatusCode() != http.StatusOK { - return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) + return "", fmt.Errorf("failed to get blob properties: %v", err) + } + + metadata := props.Metadata + if originalBlob, exists := metadata["original_blob"]; exists { + return *originalBlob, nil } - return resp.NewMetadata()["SymLink"], nil + return "", fmt.Errorf("error reading link %s: %v", path, err) } diff --git a/azure/public_test.go b/azure/public_test.go index 767f2495d7..3acd767f72 100644 --- a/azure/public_test.go +++ b/azure/public_test.go @@ -7,8 +7,11 @@ import ( "io/ioutil" "os" "path/filepath" + "bytes" - "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/utils" . "gopkg.in/check.v1" @@ -66,8 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) { s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint) c.Assert(err, IsNil) - cnt := s.storage.az.container - _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) + publicAccessType := azblob.PublicAccessTypeContainer + _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{ + Access: &publicAccessType, + }) c.Assert(err, IsNil) s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint) @@ -75,41 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) { } func (s *PublishedStorageSuite) TearDownTest(c *C) { - cnt := s.storage.az.container - _, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{}) + _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil) c.Assert(err, IsNil) } func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { - blob := s.storage.az.container.NewBlobURL(path) - resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) + resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil) c.Assert(err, IsNil) - body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3}) - data, err := ioutil.ReadAll(body) + data, err := ioutil.ReadAll(resp.Body) c.Assert(err, IsNil) return data } func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) { - _, err := s.storage.az.container.NewBlobURL(path).GetProperties( - context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + serviceClient := s.storage.az.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(s.storage.az.container) + blobClient := containerClient.NewBlobClient(path) + _, err := blobClient.GetProperties(context.Background(), nil) c.Assert(err, NotNil) - storageError, ok := err.(azblob.StorageError) + + storageError, ok := err.(*azcore.ResponseError) c.Assert(ok, Equals, true) - c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound))) + c.Assert(storageError.StatusCode, Equals, 404) } func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) { hash := md5.Sum(data) - _, err := azblob.UploadBufferToBlockBlob( - context.Background(), - data, - s.storage.az.container.NewBlockBlobURL(path), - azblob.UploadToBlockBlobOptions{ - BlobHTTPHeaders: azblob.BlobHTTPHeaders{ - ContentMD5: hash[:], - }, - }) + uploadOptions := &azblob.UploadStreamOptions{ + HTTPHeaders: &blob.HTTPHeaders{ + BlobContentMD5: hash[:], + }, + } + reader := bytes.NewReader(data) + _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions) c.Assert(err, IsNil) } @@ -330,7 +333,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { // 2nd link from pool, providing wrong path for source file // - // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) + // this test should check that file already exists in Azure and skip upload (which would fail if not skipped) s.prefixedStorage.pathCache = nil err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) c.Check(err, IsNil) diff --git a/context/context.go b/context/context.go index 6e0dfe67f2..e599102d10 100644 --- a/context/context.go +++ b/context/context.go @@ -101,7 +101,6 @@ func (context *AptlyContext) config() *utils.ConfigStructure { configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"} for _, configLocation := range configLocations { - // FIXME: check if exists, check if readable err = utils.LoadConfig(configLocation, &utils.Config) if os.IsPermission(err) || os.IsNotExist(err) { continue diff --git a/deb/list.go b/deb/list.go index ada6fab7f7..4f863ccf1c 100644 --- a/deb/list.go +++ b/deb/list.go @@ -549,7 +549,6 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie // // when follow-all-variants is enabled, we need to try to expand anyway, // as even if dependency is satisfied now, there might be other ways to satisfy dependency - // FIXME: do not search twice if result.Search(dep, false, true) != nil { if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) diff --git a/go.mod b/go.mod index d053f1dff6..b1e030c049 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.22.7 require ( github.com/AlekSi/pointer v1.1.0 - github.com/Azure/azure-storage-blob-go v0.15.0 github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 github.com/awalterschulze/gographviz v2.0.1+incompatible github.com/cavaliergopher/grab/v3 v3.0.1 @@ -43,7 +42,7 @@ require ( ) require ( - github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -89,7 +88,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-ieproxy v0.0.9 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -99,7 +97,7 @@ require ( github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect go.etcd.io/etcd/api/v3 v3.5.15 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect @@ -119,6 +117,8 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 github.com/ProtonMail/go-crypto v1.0.0 github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/config v1.25.1 diff --git a/go.sum b/go.sum index ac476e3183..43c4b545ea 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,17 @@ github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= -github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= -github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= -github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= -github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 h1:jbGlDKdzAZ92NzK65hUP98ri0/r50vVVvmZsFP/nIqo= github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55/go.mod h1:GCzqZQHydohgVLSIqRKZeTt8IGb1Y4NaFfim3H40uUI= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -91,8 +88,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -125,6 +120,8 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -144,7 +141,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -190,9 +186,6 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= -github.com/mattn/go-ieproxy v0.0.9 h1:RvVbLiMv/Hbjf1gRaC2AQyzwbdVhdId7D2vPnXIml4k= -github.com/mattn/go-ieproxy v0.0.9/go.mod h1:eF30/rfdQUO9EnzNIZQr0r9HiLMlZNCpJkHbmMuOAE0= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -228,6 +221,8 @@ github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -245,8 +240,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= @@ -306,8 +301,6 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -322,15 +315,12 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -352,7 +342,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -360,12 +349,10 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From d80b905945171ca16544ac21adefee8093848221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Wed, 13 Nov 2024 13:04:45 +0100 Subject: [PATCH 2/2] run azure tests in docker --- Makefile | 23 +++++++++++++++++++++-- system/Dockerfile | 4 +++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5294db9aad..632d7066ce 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,13 @@ swagger-install: # Install swag @test -f $(BINPATH)/swag || GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest +azurite-start: + azurite & \ + echo $$! > ~/.azurite.pid + +azurite-stop: + kill `cat ~/.azurite.pid` + swagger: swagger-install # Generate swagger docs @PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs @@ -160,10 +167,22 @@ docker-deb: ## Build debian packages in docker container @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 docker-unit-test: ## Run unit tests in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper test + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ + azurite-start \ + AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ + AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ + AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ + test \ + azurite-stop docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests) - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper system-test TEST=$(TEST) + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ + azurite-start \ + AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ + AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ + AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ + system-test TEST=$(TEST) \ + azurite-stop docker-serve: ## Run development server (auto recompiling) on http://localhost:3142 @docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true diff --git a/system/Dockerfile b/system/Dockerfile index b2da26e3d8..f2435244ed 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg b golang/bookworm-backports golang-go/bookworm-backports golang-doc/bookworm-backports golang-src/bookworm-backports \ make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage \ g++ python3-etcd3 python3-plyvel graphviz devscripts sudo dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu \ - binutils-arm-linux-gnueabihf bash-completion zip ruby3.1-dev lintian \ + binutils-arm-linux-gnueabihf bash-completion zip ruby3.1-dev lintian npm \ libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \ gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu && \ apt-get clean && rm -rf /var/lib/apt/lists/* @@ -18,6 +18,7 @@ RUN mkdir /work WORKDIR /work/src RUN chown aptly /work +# install test dependencies RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-db.git RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-pool.git RUN cd /var/lib/aptly; curl -O http://repo.aptly.info/system-tests/etcd.db.xz && xz -d etcd.db.xz @@ -25,3 +26,4 @@ RUN echo "aptly ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/aptly ADD system/t13_etcd/install-etcd.sh /src/ RUN /src/install-etcd.sh RUN rm -rf /src +RUN npm install -g azurite