From d409522c4989498f9e8ac8ae71d8551c7e7851dd Mon Sep 17 00:00:00 2001 From: sandeep kumar Date: Sat, 28 Sep 2024 09:51:44 +0530 Subject: [PATCH 1/9] system test for download dir case 1 --- tests/cli_tests/zboxcli_download_dir_test.go | 780 +++++++++++++++++++ 1 file changed, 780 insertions(+) create mode 100644 tests/cli_tests/zboxcli_download_dir_test.go diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go new file mode 100644 index 0000000000..bf731209f1 --- /dev/null +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -0,0 +1,780 @@ +package cli_tests + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/0chain/system_test/internal/api/util/test" + cliutils "github.com/0chain/system_test/internal/cli/util" + + "github.com/stretchr/testify/require" +) + +const StatusCompletedCBDD = "Status completed callback" + +func TestDownloadDir(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + t.SetSmokeTests("Download Directory Should Work") + t.Parallel() + + // Create a folder to keep all the generated files to be uploaded + err := os.MkdirAll("tmp", os.ModePerm) + require.Nil(t, err) + + // Success Scenarios + t.Run("Download Directory from Root Directory Should Work", func(t *test.SystemTest) { + allocSize := int64(2048) + filesize := int64(256) + remotepath := "/dir1" + + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": allocSize, + "tokens": 9, + }) + + filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + originalFileChecksum := generateChecksum(t, filename) + + // Delete the uploaded file, since we will be downloading it now + err := os.Remove(filename) + require.Nil(t, err) + + output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath + filepath.Base(filename), + "localpath": "tmp/", + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 2) + + require.Contains(t, output[1], StatusCompletedCB) + require.Contains(t, output[1], filepath.Base(filename)) + + downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + require.Equal(t, originalFileChecksum, downloadedFileChecksum) + }) + + // t.RunWithTimeout("Download Directory Concurrently Should Work for two Different Directory", 6*time.Minute, func(t *test.SystemTest) { + // allocSize := int64(4096) + // filesize := int64(1024) + // remoteFilePaths := [2]string{"/dir1/", "/dir2/"} + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // fileNameOfFirstDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[0], filesize) + // fileNameOfSecondDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[1], filesize) + // originalFirstFileChecksum := generateChecksum(t, fileNameOfFirstDirectory) + // originalSecondFileChecksum := generateChecksum(t, fileNameOfSecondDirectory) + + // // deleting uploaded file from /dir1 since we will be downloading it now + // err := os.Remove(fileNameOfFirstDirectory) + // require.Nil(t, err) + + // // deleting uploaded file from /dir2 since we will be downloading it now + // err = os.Remove(fileNameOfSecondDirectory) + // require.Nil(t, err) + + // var outputList [2][]string + // var errorList [2]error + // var wg sync.WaitGroup + + // fileNames := [2]string{fileNameOfFirstDirectory, fileNameOfSecondDirectory} + // for index, fileName := range fileNames { + // wg.Add(1) + // go func(currentFileName string, currentIndex int) { + // defer wg.Done() + // op, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remoteFilePaths[currentIndex] + filepath.Base(currentFileName), + // "localpath": "tmp/", + // }), true) + // errorList[currentIndex] = err + // outputList[currentIndex] = op + // }(fileName, index) + // } + + // wg.Wait() + + // require.Nil(t, errorList[0], strings.Join(outputList[0], "\n")) + // require.Len(t, outputList[0], 2) + + // require.Contains(t, outputList[0][1], StatusCompletedCB) + // require.Contains(t, outputList[0][1], filepath.Base(fileNameOfFirstDirectory)) + // downloadedFileFromFirstDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfFirstDirectory)) + + // require.Equal(t, originalFirstFileChecksum, downloadedFileFromFirstDirectoryChecksum) + // require.Nil(t, errorList[1], strings.Join(outputList[1], "\n")) + // require.Len(t, outputList[1], 2) + + // require.Contains(t, outputList[1][1], StatusCompletedCB) + // require.Contains(t, outputList[1][1], filepath.Base(fileNameOfSecondDirectory)) + // downloadedFileFromSecondDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfSecondDirectory)) + // require.Equal(t, originalSecondFileChecksum, downloadedFileFromSecondDirectoryChecksum) + // }) + + // t.Run("Download Directory from a Directory Should Work", func(t *test.SystemTest) { + // allocSize := int64(2048) + // filesize := int64(256) + // remotepath := "/dir/" + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum := generateChecksum(t, filename) + + // // Delete the uploaded file, since we will be downloading it now + // err := os.Remove(filename) + // require.Nil(t, err) + + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // "localpath": "tmp/", + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.Run("Download Directory from Nested Directory Should Work", func(t *test.SystemTest) { + // allocSize := int64(2048) + // filesize := int64(256) + // remotepath := "/nested/dir/" + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum := generateChecksum(t, filename) + + // // Delete the uploaded file, since we will be downloading it now + // err := os.Remove(filename) + // require.Nil(t, err) + + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // "localpath": "tmp/", + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.RunWithTimeout("Download Entire Directory Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { // todo: slow + // allocSize := int64(2048) + // filesize := int64(256) + // remotepath := "/nested/dir/" + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + + // // Delete the uploaded file, since we will be downloading it now + // err := os.Remove(filename) + // require.Nil(t, err) + + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath, + // "localpath": "tmp/dir", + // }), false) + // require.Error(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // require.Contains(t, output[0], "consensus_not_met") + // }) + + // t.RunWithTimeout("Download Directory From Shared Folder Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { + // var authTicket, filename string + + // filesize := int64(10) + // remotepath := "/" + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share Entire Folder from Another Wallet", func(t *test.SystemTest) { + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "tokens": 9, + // }) + // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath, + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + // }) + + // // Just create a wallet so that we can work further + // createWallet(t) + + // // Download file using auth-ticket: should work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": "tmp/dir", + // "remotepath": "/" + filename, + // }), false) + // require.NotNil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // aggregatedOutput := strings.Join(output, " ") + // require.Contains(t, aggregatedOutput, "consensus_not_met") + // }) + + // t.RunWithTimeout("Download Shared File Should Work", 5*time.Minute, func(t *test.SystemTest) { // todo: too slow + // var authTicket, filename, originalFileChecksum string + + // filesize := int64(10) + // remotepath := "/" + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "tokens": 9, + // }) + // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum = generateChecksum(t, filename) + + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + // }) + + // // Just create a wallet so that we can work further + // err := createWalletAndLockReadTokens(t, configPath) + // require.Nil(t, err) + + // // Download file using auth-ticket: should work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": "tmp/", + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.RunWithTimeout("Download Encrypted Directory Should Work", 5*time.Minute, func(t *test.SystemTest) { + // allocSize := int64(10 * MB) + // filesize := int64(10) + // remotepath := "/" + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // filename := generateRandomTestFileName(t) + // err := createFileWithSize(filename, filesize) + // require.Nil(t, err) + // originalFileChecksum := generateChecksum(t, filename) + + // // Upload parameters + // uploadWithParam(t, configPath, map[string]interface{}{ + // "allocation": allocationID, + // "localpath": filename, + // "remotepath": remotepath + filepath.Base(filename), + // "encrypt": "", + // }) + + // // Delete the uploaded file, since we will be downloading it now + // err = os.Remove(filename) + // require.Nil(t, err) + + // // Downloading encrypted file should work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // "localpath": os.TempDir(), + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + // require.Contains(t, output[len(output)-1], StatusCompletedCB) + // require.Contains(t, output[len(output)-1], filepath.Base(filename)) + // downloadedFileChecksum := generateChecksum(t, strings.TrimSuffix(os.TempDir(), "/")+"/"+filepath.Base(filename)) + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.RunWithTimeout("Download Shared Encrypted Directory Should Work", 5*time.Minute, func(t *test.SystemTest) { //todo: slow + // var authTicket, filename string + + // filesize := int64(10) + // remotepath := "/" + // var allocationID string + + // // create viewer wallet + // viewerWalletName := escapedTestName(t) + "_viewer" + // createWalletForNameAndLockReadTokens(t, configPath, viewerWalletName) + + // viewerWallet, err := getWalletForName(t, configPath, viewerWalletName) + // require.Nil(t, err) + // require.NotNil(t, viewerWallet) + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share File from Another Wallet", func(t *test.SystemTest) { + // allocationID = setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "tokens": 9, + // }) + // filename = generateFileAndUploadWithParam(t, allocationID, remotepath, filesize, map[string]interface{}{ + // "encrypt": "", + // }) + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // "encryptionpublickey": viewerWallet.EncryptionPublicKey, + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + // }) + + // file := "tmp/" + filepath.Base(filename) + + // // Download file using auth-ticket: should work + // output, err := downloadFileForWallet(t, viewerWalletName, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": file, + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[len(output)-1], StatusCompletedCB) + // require.Contains(t, output[len(output)-1], filepath.Base(filename)) + + // os.Remove(file) //nolint + + // // Download file using auth-ticket and lookuphash: should work + // output, err = downloadFileForWallet(t, viewerWalletName, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "lookuphash": GetReferenceLookup(allocationID, remotepath+filepath.Base(filename)), + // "localpath": file, + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[len(output)-1], StatusCompletedCB) + // require.Contains(t, output[len(output)-1], filepath.Base(filename)) + // }) + + // t.RunWithTimeout("Download Directory From Shared Folder by Remotepath Should Work", 5*time.Minute, func(t *test.SystemTest) { + // var authTicket, filename, originalFileChecksum string + + // filesize := int64(10) + // remotepath := "/dir/" + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "tokens": 9, + // }) + // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum = generateChecksum(t, filename) + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath, + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + // }) + + // // Just create a wallet so that we can work further + // err := createWalletAndLockReadTokens(t, configPath) + // require.Nil(t, err) + + // // Download file using auth-ticket: should work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": "tmp/", + // "remotepath": remotepath + filepath.Base(filename), + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.RunWithTimeout("Download Directory From Shared Folder by Lookup Hash Should Work", 5*time.Minute, func(t *test.SystemTest) { + // var authTicket, lookuphash, filename, originalFileChecksum string + + // filesize := int64(10) + // remotepath := "/dir/" + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "tokens": 9, + // }) + // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum = generateChecksum(t, filename) + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath, + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + + // h := sha3.Sum256([]byte(fmt.Sprintf("%s:%s%s", allocationID, remotepath, filepath.Base(filename)))) + // lookuphash = fmt.Sprintf("%x", h) + // require.NotEqual(t, "", lookuphash, "Lookup Hash: ", lookuphash) + // }) + + // // Just create a wallet so that we can work further + // err := createWalletAndLockReadTokens(t, configPath) + // require.Nil(t, err) + + // // Download file using auth-ticket: should work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": "tmp/", + // "lookuphash": lookuphash, + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.RunWithTimeout("Download Shared Directory without Paying Should Not Work", 5*time.Minute, func(t *test.SystemTest) { + // t.Skip() + // var authTicket, filename string + + // filesize := int64(10) + // remotepath := "/" + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "tokens": 9, + // }) + // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + // }) + + // // Just create a wallet so that we can work further + // createWallet(t) + + // // Download file using auth-ticket: shouldn't work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": "tmp/", + // }), false) + // require.NotNil(t, err) + // require.Greater(t, len(output), 0) + // aggregatedOutput := strings.Join(output, " ") + // require.Contains(t, aggregatedOutput, "pre-redeeming read marker") + // }) + + // t.RunWithTimeout("Download Shared Directory by Paying Should Work", 5*time.Minute, func(t *test.SystemTest) { + // var allocationID, authTicket, filename string + + // filesize := int64(10) + // remotepath := "/" + + // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { + // allocationID = setupAllocation(t, configPath, map[string]interface{}{ + // "size": 10 * 1024, + // "lock": 9, + // }) + // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + // require.NotEqual(t, "", filename) + + // // Delete the uploaded file from tmp folder if it exist, + // // since we will be downloading it now + // err := os.RemoveAll("tmp/" + filepath.Base(filename)) + // require.Nil(t, err) + + // shareParam := createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // }) + + // output, err := shareFolderInAllocation(t, configPath, shareParam) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // authTicket, err = extractAuthToken(output[0]) + // require.Nil(t, err, "extract auth token failed") + // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + // }) + + // err = createWalletAndLockReadTokens(t, configPath) + // require.Nil(t, err) + // // Download file using auth-ticket: should work + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "authticket": authTicket, + // "localpath": "tmp/", + // }), false) + + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + // aggregatedOutput := strings.Join(output, " ") + // require.Contains(t, aggregatedOutput, filepath.Base(filename)) + // }) + + // t.Run("Download to Non-Existent Path Should Work", func(t *test.SystemTest) { + // allocSize := int64(2048) + // filesize := int64(256) + // remotepath := "/" + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum := generateChecksum(t, filename) + + // // Delete the uploaded file, since we will be downloading it now + // err := os.Remove(filename) + // require.Nil(t, err) + + // newLocalPath := "tmp/tmp2/" + filepath.Base(filename) + // defer func() { + // os.Remove(newLocalPath) //nolint: errcheck + // }() + + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath + filepath.Base(filename), + // "localpath": newLocalPath, + // }), true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, newLocalPath) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) + + // t.Run("Download without any Parameter Should Fail", func(t *test.SystemTest) { + // createWallet(t) + + // output, err := downloadFile(t, configPath, "", false) + // require.NotNil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + + // require.Equal(t, "Error: remotepath / authticket flag is missing", output[0]) + // }) + + // t.Run("Download from Allocation without other Parameter Should Fail", func(t *test.SystemTest) { + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": 10000, + // "tokens": 9, + // }) + + // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // }), false) + + // require.NotNil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // require.Equal(t, "Error: remotepath / authticket flag is missing", output[0]) + // }) + + // t.RunWithTimeout("Download Moved Directory Should Work", 5*time.Minute, func(t *test.SystemTest) { + // allocSize := int64(2048) + // filesize := int64(256) + // remotepath := "/" + + // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + // "size": allocSize, + // "tokens": 9, + // }) + + // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum := generateChecksum(t, filename) + + // // Delete the uploaded file, since we will be downloading it now + // err := os.Remove(filename) + // require.Nil(t, err) + + // remotepath += filepath.Base(filename) + // destpath := "/child/" + // output, err := moveFile(t, configPath, map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": remotepath, + // "destpath": destpath, + // }, true) + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // require.Equal(t, fmt.Sprintf(remotepath+" moved"), output[0]) + + // defer func() { + // os.Remove("tmp/" + filepath.Base(filename)) //nolint: errcheck + // }() + + // output, err = downloadFile(t, configPath, createParams(map[string]interface{}{ + // "allocation": allocationID, + // "remotepath": destpath + filepath.Base(filename), + // "localpath": "tmp/", + // }), true) + + // require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) + + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) + + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // }) +} + +func downloadDirectory(t *test.SystemTest, cliConfigFilename, param string, retry bool) ([]string, error) { + return downloadDirectoryFromWallet(t, escapedTestName(t), cliConfigFilename, param, retry) +} + +func downloadDirectoryFromWallet(t *test.SystemTest, wallet, cliConfigFilename, param string, retry bool) ([]string, error) { + cliutils.Wait(t, 15*time.Second) // TODO replace with pollers + t.Logf("Downloading file...") + cmd := fmt.Sprintf( + "./zbox downloaddir %s --silent --wallet %s --configDir ./config --config %s", + param, + wallet+"_wallet.json", + cliConfigFilename, + ) + + if retry { + return cliutils.RunCommand(t, cmd, 3, time.Second*2) + } else { + return cliutils.RunCommandWithoutRetry(cmd) + } +} From 9fecaa89090297f573b3e3be356a681333f50533 Mon Sep 17 00:00:00 2001 From: sandeep kumar Date: Sun, 29 Sep 2024 11:33:05 +0530 Subject: [PATCH 2/9] system test for download dir case 1 --- tests/cli_tests/zboxcli_download_dir_test.go | 505 +++++++++---------- 1 file changed, 251 insertions(+), 254 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index bf731209f1..20dcf97b8f 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -37,7 +38,7 @@ func TestDownloadDir(testSetup *testing.T) { }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - originalFileChecksum := generateChecksum(t, filename) + // originalFileChecksum := generateChecksum(t, filename) // Delete the uploaded file, since we will be downloading it now err := os.Remove(filename) @@ -45,322 +46,318 @@ func TestDownloadDir(testSetup *testing.T) { output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ "allocation": allocationID, - "remotepath": remotepath + filepath.Base(filename), + "remotepath": remotepath, "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - require.Len(t, output, 2) + // require.Len(t, output, 2) - require.Contains(t, output[1], StatusCompletedCB) - require.Contains(t, output[1], filepath.Base(filename)) + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) - downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - require.Equal(t, originalFileChecksum, downloadedFileChecksum) + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) }) - // t.RunWithTimeout("Download Directory Concurrently Should Work for two Different Directory", 6*time.Minute, func(t *test.SystemTest) { - // allocSize := int64(4096) - // filesize := int64(1024) - // remoteFilePaths := [2]string{"/dir1/", "/dir2/"} + t.RunWithTimeout("Download Directory Concurrently Should Work for two Different Directory", 6*time.Minute, func(t *test.SystemTest) { + allocSize := int64(4096) + filesize := int64(1024) + remoteFilePaths := [2]string{"/dir1/", "/dir2/"} - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) - - // fileNameOfFirstDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[0], filesize) - // fileNameOfSecondDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[1], filesize) - // originalFirstFileChecksum := generateChecksum(t, fileNameOfFirstDirectory) - // originalSecondFileChecksum := generateChecksum(t, fileNameOfSecondDirectory) + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": allocSize, + "tokens": 9, + }) - // // deleting uploaded file from /dir1 since we will be downloading it now - // err := os.Remove(fileNameOfFirstDirectory) - // require.Nil(t, err) + fileNameOfFirstDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[0], filesize) + fileNameOfSecondDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[1], filesize) + // originalFirstFileChecksum := generateChecksum(t, fileNameOfFirstDirectory) + // originalSecondFileChecksum := generateChecksum(t, fileNameOfSecondDirectory) - // // deleting uploaded file from /dir2 since we will be downloading it now - // err = os.Remove(fileNameOfSecondDirectory) - // require.Nil(t, err) + // deleting uploaded file from /dir1 since we will be downloading it now + err := os.Remove(fileNameOfFirstDirectory) + require.Nil(t, err) - // var outputList [2][]string - // var errorList [2]error - // var wg sync.WaitGroup - - // fileNames := [2]string{fileNameOfFirstDirectory, fileNameOfSecondDirectory} - // for index, fileName := range fileNames { - // wg.Add(1) - // go func(currentFileName string, currentIndex int) { - // defer wg.Done() - // op, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remoteFilePaths[currentIndex] + filepath.Base(currentFileName), - // "localpath": "tmp/", - // }), true) - // errorList[currentIndex] = err - // outputList[currentIndex] = op - // }(fileName, index) - // } - - // wg.Wait() - - // require.Nil(t, errorList[0], strings.Join(outputList[0], "\n")) - // require.Len(t, outputList[0], 2) - - // require.Contains(t, outputList[0][1], StatusCompletedCB) - // require.Contains(t, outputList[0][1], filepath.Base(fileNameOfFirstDirectory)) - // downloadedFileFromFirstDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfFirstDirectory)) - - // require.Equal(t, originalFirstFileChecksum, downloadedFileFromFirstDirectoryChecksum) - // require.Nil(t, errorList[1], strings.Join(outputList[1], "\n")) - // require.Len(t, outputList[1], 2) - - // require.Contains(t, outputList[1][1], StatusCompletedCB) - // require.Contains(t, outputList[1][1], filepath.Base(fileNameOfSecondDirectory)) - // downloadedFileFromSecondDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfSecondDirectory)) - // require.Equal(t, originalSecondFileChecksum, downloadedFileFromSecondDirectoryChecksum) - // }) + // deleting uploaded file from /dir2 since we will be downloading it now + err = os.Remove(fileNameOfSecondDirectory) + require.Nil(t, err) - // t.Run("Download Directory from a Directory Should Work", func(t *test.SystemTest) { - // allocSize := int64(2048) - // filesize := int64(256) - // remotepath := "/dir/" + var outputList [2][]string + var errorList [2]error + var wg sync.WaitGroup + + fileNames := [2]string{fileNameOfFirstDirectory, fileNameOfSecondDirectory} + for index, fileName := range fileNames { + wg.Add(1) + go func(currentFileName string, currentIndex int) { + defer wg.Done() + op, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remoteFilePaths[currentIndex][:len(remoteFilePaths[currentIndex])-1], + "localpath": "tmp/", + }), true) + errorList[currentIndex] = err + outputList[currentIndex] = op + }(fileName, index) + } + + wg.Wait() + + require.Nil(t, errorList[0], strings.Join(outputList[0], "\n")) + // require.Len(t, outputList[0], 2) + + // require.Contains(t, outputList[0][1], StatusCompletedCB) + // require.Contains(t, outputList[0][1], filepath.Base(fileNameOfFirstDirectory)) + // downloadedFileFromFirstDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfFirstDirectory)) + + // require.Equal(t, originalFirstFileChecksum, downloadedFileFromFirstDirectoryChecksum) + require.Nil(t, errorList[1], strings.Join(outputList[1], "\n")) + // require.Len(t, outputList[1], 2) + + // require.Contains(t, outputList[1][1], StatusCompletedCB) + // require.Contains(t, outputList[1][1], filepath.Base(fileNameOfSecondDirectory)) + // downloadedFileFromSecondDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfSecondDirectory)) + // require.Equal(t, originalSecondFileChecksum, downloadedFileFromSecondDirectoryChecksum) + }) - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) + t.Run("Download Directory from a Directory Should Work", func(t *test.SystemTest) { + allocSize := int64(2048) + filesize := int64(256) + remotepath := "/dir/dir1" - // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": allocSize, + "tokens": 9, + }) - // // Delete the uploaded file, since we will be downloading it now - // err := os.Remove(filename) - // require.Nil(t, err) + filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum := generateChecksum(t, filename) - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // "localpath": "tmp/", - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) + // Delete the uploaded file, since we will be downloading it now + err := os.Remove(filename) + require.Nil(t, err) - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) + output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath, + "localpath": "tmp/", + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - // t.Run("Download Directory from Nested Directory Should Work", func(t *test.SystemTest) { - // allocSize := int64(2048) - // filesize := int64(256) - // remotepath := "/nested/dir/" + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + }) - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) + t.Run("Download Directory from Nested Directory Should Work", func(t *test.SystemTest) { + allocSize := int64(2048) + filesize := int64(256) + remotepath := "/nested/dir/" - // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": allocSize, + "tokens": 9, + }) - // // Delete the uploaded file, since we will be downloading it now - // err := os.Remove(filename) - // require.Nil(t, err) + filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + // originalFileChecksum := generateChecksum(t, filename) - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // "localpath": "tmp/", - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) + // Delete the uploaded file, since we will be downloading it now + err := os.Remove(filename) + require.Nil(t, err) - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) + output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath, + "localpath": "tmp/", + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + // require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - // t.RunWithTimeout("Download Entire Directory Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { // todo: slow - // allocSize := int64(2048) - // filesize := int64(256) - // remotepath := "/nested/dir/" + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + }) - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) + t.RunWithTimeout("Download Entire Directory Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { // todo: slow + allocSize := int64(2048) + filesize := int64(256) + remotepath := "/nested/dir/" - // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": allocSize, + "tokens": 9, + }) - // // Delete the uploaded file, since we will be downloading it now - // err := os.Remove(filename) - // require.Nil(t, err) + filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath, - // "localpath": "tmp/dir", - // }), false) - // require.Error(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // require.Contains(t, output[0], "consensus_not_met") - // }) + // Delete the uploaded file, since we will be downloading it now + err := os.Remove(filename) + require.Nil(t, err) - // t.RunWithTimeout("Download Directory From Shared Folder Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { - // var authTicket, filename string + output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath, + "localpath": "tmp/dir", + }), false) + require.Error(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // require.Contains(t, output[0], "consensus_not_met") + }) - // filesize := int64(10) - // remotepath := "/" + t.RunWithTimeout("Download Directory From Shared Folder Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { + var authTicket, filename string - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share Entire Folder from Another Wallet", func(t *test.SystemTest) { - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "tokens": 9, - // }) - // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) + filesize := int64(10) + remotepath := "/dirx" - // require.NotEqual(t, "", filename) + // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + t.Run("Share Entire Folder from Another Wallet", func(t *test.SystemTest) { + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": 10 * 1024, + "tokens": 9, + }) + filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) + require.NotEqual(t, "", filename) - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath, - // }) + // Delete the uploaded file from tmp folder if it exist, + // since we will be downloading it now + err := os.RemoveAll("tmp/" + filepath.Base(filename)) + require.Nil(t, err) - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) + shareParam := createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath, + }) - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - // }) + output, err := shareFolderInAllocation(t, configPath, shareParam) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) - // // Just create a wallet so that we can work further - // createWallet(t) + authTicket, err = extractAuthToken(output[0]) + require.Nil(t, err, "extract auth token failed") + require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + }) - // // Download file using auth-ticket: should work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": "tmp/dir", - // "remotepath": "/" + filename, - // }), false) - // require.NotNil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // aggregatedOutput := strings.Join(output, " ") - // require.Contains(t, aggregatedOutput, "consensus_not_met") - // }) + // Just create a wallet so that we can work further + createWallet(t) - // t.RunWithTimeout("Download Shared File Should Work", 5*time.Minute, func(t *test.SystemTest) { // todo: too slow - // var authTicket, filename, originalFileChecksum string + // Download file using auth-ticket: should work + output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "authticket": authTicket, + "localpath": "tmp/dir", + "remotepath": "/dirx", + }), false) + require.NotNil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 1) + // aggregatedOutput := strings.Join(output, " ") + // require.Contains(t, aggregatedOutput, "consensus_not_met") + }) - // filesize := int64(10) - // remotepath := "/" + t.RunWithTimeout("Download Shared File Should Work", 5*time.Minute, func(t *test.SystemTest) { // todo: too slow + var authTicket, filename string - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "tokens": 9, - // }) - // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum = generateChecksum(t, filename) + filesize := int64(10) + remotepath := "/dirx" - // require.NotEqual(t, "", filename) + // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file + t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": 10 * 1024, + "tokens": 9, + }) + filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) + require.NotEqual(t, "", filename) - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // }) + // Delete the uploaded file from tmp folder if it exist, + // since we will be downloading it now + err := os.RemoveAll("tmp/" + filepath.Base(filename)) + require.Nil(t, err) - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) + shareParam := createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath, + }) - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - // }) + output, err := shareFolderInAllocation(t, configPath, shareParam) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 1) - // // Just create a wallet so that we can work further - // err := createWalletAndLockReadTokens(t, configPath) - // require.Nil(t, err) + authTicket, err = extractAuthToken(output[0]) + require.Nil(t, err, "extract auth token failed") + require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) + }) - // // Download file using auth-ticket: should work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": "tmp/", - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) + // Just create a wallet so that we can work further + err := createWalletAndLockReadTokens(t, configPath) + require.Nil(t, err) - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) + // Download file using auth-ticket: should work + output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ + "authticket": authTicket, + "localpath": "tmp/", + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + // require.Len(t, output, 2) - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) + require.Contains(t, output[1], StatusCompletedCB) + // require.Contains(t, output[1], filepath.Base(filename)) - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) + // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - // t.RunWithTimeout("Download Encrypted Directory Should Work", 5*time.Minute, func(t *test.SystemTest) { - // allocSize := int64(10 * MB) - // filesize := int64(10) - // remotepath := "/" + // require.Equal(t, originalFileChecksum, downloadedFileChecksum) + }) - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) + t.RunWithTimeout("Download Directory having Encrypted files Should Work", 5*time.Minute, func(t *test.SystemTest) { + allocSize := int64(10 * MB) + filesize := int64(10) + remotepath := "/dirx" - // filename := generateRandomTestFileName(t) - // err := createFileWithSize(filename, filesize) - // require.Nil(t, err) - // originalFileChecksum := generateChecksum(t, filename) + allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + "size": allocSize, + "tokens": 9, + }) - // // Upload parameters - // uploadWithParam(t, configPath, map[string]interface{}{ - // "allocation": allocationID, - // "localpath": filename, - // "remotepath": remotepath + filepath.Base(filename), - // "encrypt": "", - // }) + filename := generateRandomTestFileName(t) + err := createFileWithSize(filename, filesize) + require.Nil(t, err) + originalFileChecksum := generateChecksum(t, filename) - // // Delete the uploaded file, since we will be downloading it now - // err = os.Remove(filename) - // require.Nil(t, err) + // Upload parameters + uploadWithParam(t, configPath, map[string]interface{}{ + "allocation": allocationID, + "localpath": filename, + "remotepath": remotepath + filepath.Base(filename), + "encrypt": "", + }) - // // Downloading encrypted file should work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // "localpath": os.TempDir(), - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - // require.Contains(t, output[len(output)-1], StatusCompletedCB) - // require.Contains(t, output[len(output)-1], filepath.Base(filename)) - // downloadedFileChecksum := generateChecksum(t, strings.TrimSuffix(os.TempDir(), "/")+"/"+filepath.Base(filename)) - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) + // Delete the uploaded file, since we will be downloading it now + err = os.Remove(filename) + require.Nil(t, err) - // t.RunWithTimeout("Download Shared Encrypted Directory Should Work", 5*time.Minute, func(t *test.SystemTest) { //todo: slow - // var authTicket, filename string + // Downloading encrypted file should work + output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ + "allocation": allocationID, + "remotepath": remotepath + filepath.Base(filename), + "localpath": os.TempDir(), + }), true) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 2) + require.Contains(t, output[len(output)-1], StatusCompletedCB) + require.Contains(t, output[len(output)-1], filepath.Base(filename)) + downloadedFileChecksum := generateChecksum(t, strings.TrimSuffix(os.TempDir(), "/")+"/"+filepath.Base(filename)) + require.Equal(t, originalFileChecksum, downloadedFileChecksum) + }) // filesize := int64(10) // remotepath := "/" From 19af81eb4d2b2fdfdccc2d9c418c8acf6415fb72 Mon Sep 17 00:00:00 2001 From: sandeep kumar Date: Sun, 29 Sep 2024 11:55:53 +0530 Subject: [PATCH 3/9] system test for download dir case 1 --- tests/cli_tests/zboxcli_download_dir_test.go | 448 ------------------- 1 file changed, 448 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index 20dcf97b8f..09b94f3d99 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -50,14 +50,7 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) }) t.RunWithTimeout("Download Directory Concurrently Should Work for two Different Directory", 6*time.Minute, func(t *test.SystemTest) { @@ -72,8 +65,6 @@ func TestDownloadDir(testSetup *testing.T) { fileNameOfFirstDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[0], filesize) fileNameOfSecondDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[1], filesize) - // originalFirstFileChecksum := generateChecksum(t, fileNameOfFirstDirectory) - // originalSecondFileChecksum := generateChecksum(t, fileNameOfSecondDirectory) // deleting uploaded file from /dir1 since we will be downloading it now err := os.Remove(fileNameOfFirstDirectory) @@ -105,20 +96,9 @@ func TestDownloadDir(testSetup *testing.T) { wg.Wait() require.Nil(t, errorList[0], strings.Join(outputList[0], "\n")) - // require.Len(t, outputList[0], 2) - - // require.Contains(t, outputList[0][1], StatusCompletedCB) - // require.Contains(t, outputList[0][1], filepath.Base(fileNameOfFirstDirectory)) - // downloadedFileFromFirstDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfFirstDirectory)) - // require.Equal(t, originalFirstFileChecksum, downloadedFileFromFirstDirectoryChecksum) require.Nil(t, errorList[1], strings.Join(outputList[1], "\n")) - // require.Len(t, outputList[1], 2) - // require.Contains(t, outputList[1][1], StatusCompletedCB) - // require.Contains(t, outputList[1][1], filepath.Base(fileNameOfSecondDirectory)) - // downloadedFileFromSecondDirectoryChecksum := generateChecksum(t, "tmp/"+filepath.Base(fileNameOfSecondDirectory)) - // require.Equal(t, originalSecondFileChecksum, downloadedFileFromSecondDirectoryChecksum) }) t.Run("Download Directory from a Directory Should Work", func(t *test.SystemTest) { @@ -144,14 +124,7 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) }) t.Run("Download Directory from Nested Directory Should Work", func(t *test.SystemTest) { @@ -177,14 +150,7 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) }) t.RunWithTimeout("Download Entire Directory Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { // todo: slow @@ -209,8 +175,6 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/dir", }), false) require.Error(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // require.Contains(t, output[0], "consensus_not_met") }) t.RunWithTimeout("Download Directory From Shared Folder Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { @@ -258,9 +222,6 @@ func TestDownloadDir(testSetup *testing.T) { "remotepath": "/dirx", }), false) require.NotNil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // aggregatedOutput := strings.Join(output, " ") - // require.Contains(t, aggregatedOutput, "consensus_not_met") }) t.RunWithTimeout("Download Shared File Should Work", 5*time.Minute, func(t *test.SystemTest) { // todo: too slow @@ -308,14 +269,6 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) }) t.RunWithTimeout("Download Directory having Encrypted files Should Work", 5*time.Minute, func(t *test.SystemTest) { @@ -331,7 +284,6 @@ func TestDownloadDir(testSetup *testing.T) { filename := generateRandomTestFileName(t) err := createFileWithSize(filename, filesize) require.Nil(t, err) - originalFileChecksum := generateChecksum(t, filename) // Upload parameters uploadWithParam(t, configPath, map[string]interface{}{ @@ -352,407 +304,7 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": os.TempDir(), }), true) require.Nil(t, err, strings.Join(output, "\n")) - require.Len(t, output, 2) - require.Contains(t, output[len(output)-1], StatusCompletedCB) - require.Contains(t, output[len(output)-1], filepath.Base(filename)) - downloadedFileChecksum := generateChecksum(t, strings.TrimSuffix(os.TempDir(), "/")+"/"+filepath.Base(filename)) - require.Equal(t, originalFileChecksum, downloadedFileChecksum) }) - - // filesize := int64(10) - // remotepath := "/" - // var allocationID string - - // // create viewer wallet - // viewerWalletName := escapedTestName(t) + "_viewer" - // createWalletForNameAndLockReadTokens(t, configPath, viewerWalletName) - - // viewerWallet, err := getWalletForName(t, configPath, viewerWalletName) - // require.Nil(t, err) - // require.NotNil(t, viewerWallet) - - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share File from Another Wallet", func(t *test.SystemTest) { - // allocationID = setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "tokens": 9, - // }) - // filename = generateFileAndUploadWithParam(t, allocationID, remotepath, filesize, map[string]interface{}{ - // "encrypt": "", - // }) - // require.NotEqual(t, "", filename) - - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) - - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // "encryptionpublickey": viewerWallet.EncryptionPublicKey, - // }) - - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - // }) - - // file := "tmp/" + filepath.Base(filename) - - // // Download file using auth-ticket: should work - // output, err := downloadFileForWallet(t, viewerWalletName, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": file, - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[len(output)-1], StatusCompletedCB) - // require.Contains(t, output[len(output)-1], filepath.Base(filename)) - - // os.Remove(file) //nolint - - // // Download file using auth-ticket and lookuphash: should work - // output, err = downloadFileForWallet(t, viewerWalletName, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "lookuphash": GetReferenceLookup(allocationID, remotepath+filepath.Base(filename)), - // "localpath": file, - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[len(output)-1], StatusCompletedCB) - // require.Contains(t, output[len(output)-1], filepath.Base(filename)) - // }) - - // t.RunWithTimeout("Download Directory From Shared Folder by Remotepath Should Work", 5*time.Minute, func(t *test.SystemTest) { - // var authTicket, filename, originalFileChecksum string - - // filesize := int64(10) - // remotepath := "/dir/" - - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "tokens": 9, - // }) - // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum = generateChecksum(t, filename) - // require.NotEqual(t, "", filename) - - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) - - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath, - // }) - - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - // }) - - // // Just create a wallet so that we can work further - // err := createWalletAndLockReadTokens(t, configPath) - // require.Nil(t, err) - - // // Download file using auth-ticket: should work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": "tmp/", - // "remotepath": remotepath + filepath.Base(filename), - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) - - // t.RunWithTimeout("Download Directory From Shared Folder by Lookup Hash Should Work", 5*time.Minute, func(t *test.SystemTest) { - // var authTicket, lookuphash, filename, originalFileChecksum string - - // filesize := int64(10) - // remotepath := "/dir/" - - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "tokens": 9, - // }) - // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum = generateChecksum(t, filename) - // require.NotEqual(t, "", filename) - - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) - - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath, - // }) - - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - - // h := sha3.Sum256([]byte(fmt.Sprintf("%s:%s%s", allocationID, remotepath, filepath.Base(filename)))) - // lookuphash = fmt.Sprintf("%x", h) - // require.NotEqual(t, "", lookuphash, "Lookup Hash: ", lookuphash) - // }) - - // // Just create a wallet so that we can work further - // err := createWalletAndLockReadTokens(t, configPath) - // require.Nil(t, err) - - // // Download file using auth-ticket: should work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": "tmp/", - // "lookuphash": lookuphash, - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) - - // t.RunWithTimeout("Download Shared Directory without Paying Should Not Work", 5*time.Minute, func(t *test.SystemTest) { - // t.Skip() - // var authTicket, filename string - - // filesize := int64(10) - // remotepath := "/" - - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "tokens": 9, - // }) - // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // require.NotEqual(t, "", filename) - - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) - - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // }) - - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - // }) - - // // Just create a wallet so that we can work further - // createWallet(t) - - // // Download file using auth-ticket: shouldn't work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": "tmp/", - // }), false) - // require.NotNil(t, err) - // require.Greater(t, len(output), 0) - // aggregatedOutput := strings.Join(output, " ") - // require.Contains(t, aggregatedOutput, "pre-redeeming read marker") - // }) - - // t.RunWithTimeout("Download Shared Directory by Paying Should Work", 5*time.Minute, func(t *test.SystemTest) { - // var allocationID, authTicket, filename string - - // filesize := int64(10) - // remotepath := "/" - - // // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file - // t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - // allocationID = setupAllocation(t, configPath, map[string]interface{}{ - // "size": 10 * 1024, - // "lock": 9, - // }) - // filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - // require.NotEqual(t, "", filename) - - // // Delete the uploaded file from tmp folder if it exist, - // // since we will be downloading it now - // err := os.RemoveAll("tmp/" + filepath.Base(filename)) - // require.Nil(t, err) - - // shareParam := createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // }) - - // output, err := shareFolderInAllocation(t, configPath, shareParam) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // authTicket, err = extractAuthToken(output[0]) - // require.Nil(t, err, "extract auth token failed") - // require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) - // }) - - // err = createWalletAndLockReadTokens(t, configPath) - // require.Nil(t, err) - // // Download file using auth-ticket: should work - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "authticket": authTicket, - // "localpath": "tmp/", - // }), false) - - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - // aggregatedOutput := strings.Join(output, " ") - // require.Contains(t, aggregatedOutput, filepath.Base(filename)) - // }) - - // t.Run("Download to Non-Existent Path Should Work", func(t *test.SystemTest) { - // allocSize := int64(2048) - // filesize := int64(256) - // remotepath := "/" - - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) - - // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) - - // // Delete the uploaded file, since we will be downloading it now - // err := os.Remove(filename) - // require.Nil(t, err) - - // newLocalPath := "tmp/tmp2/" + filepath.Base(filename) - // defer func() { - // os.Remove(newLocalPath) //nolint: errcheck - // }() - - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath + filepath.Base(filename), - // "localpath": newLocalPath, - // }), true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, newLocalPath) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) - - // t.Run("Download without any Parameter Should Fail", func(t *test.SystemTest) { - // createWallet(t) - - // output, err := downloadFile(t, configPath, "", false) - // require.NotNil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - - // require.Equal(t, "Error: remotepath / authticket flag is missing", output[0]) - // }) - - // t.Run("Download from Allocation without other Parameter Should Fail", func(t *test.SystemTest) { - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": 10000, - // "tokens": 9, - // }) - - // output, err := downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // }), false) - - // require.NotNil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // require.Equal(t, "Error: remotepath / authticket flag is missing", output[0]) - // }) - - // t.RunWithTimeout("Download Moved Directory Should Work", 5*time.Minute, func(t *test.SystemTest) { - // allocSize := int64(2048) - // filesize := int64(256) - // remotepath := "/" - - // allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ - // "size": allocSize, - // "tokens": 9, - // }) - - // filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) - - // // Delete the uploaded file, since we will be downloading it now - // err := os.Remove(filename) - // require.Nil(t, err) - - // remotepath += filepath.Base(filename) - // destpath := "/child/" - // output, err := moveFile(t, configPath, map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": remotepath, - // "destpath": destpath, - // }, true) - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 1) - // require.Equal(t, fmt.Sprintf(remotepath+" moved"), output[0]) - - // defer func() { - // os.Remove("tmp/" + filepath.Base(filename)) //nolint: errcheck - // }() - - // output, err = downloadFile(t, configPath, createParams(map[string]interface{}{ - // "allocation": allocationID, - // "remotepath": destpath + filepath.Base(filename), - // "localpath": "tmp/", - // }), true) - - // require.Nil(t, err, strings.Join(output, "\n")) - // require.Len(t, output, 2) - - // require.Contains(t, output[1], StatusCompletedCB) - // require.Contains(t, output[1], filepath.Base(filename)) - - // downloadedFileChecksum := generateChecksum(t, "tmp/"+filepath.Base(filename)) - - // require.Equal(t, originalFileChecksum, downloadedFileChecksum) - // }) } func downloadDirectory(t *test.SystemTest, cliConfigFilename, param string, retry bool) ([]string, error) { From 4fc1b4001222746b462d653f4f108c3a3b7fe886 Mon Sep 17 00:00:00 2001 From: sandeep kumar Date: Sun, 29 Sep 2024 12:08:05 +0530 Subject: [PATCH 4/9] system test for download dir --- tests/cli_tests/zboxcli_download_dir_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index 09b94f3d99..7e36c1d595 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -50,7 +50,6 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - }) t.RunWithTimeout("Download Directory Concurrently Should Work for two Different Directory", 6*time.Minute, func(t *test.SystemTest) { @@ -94,11 +93,8 @@ func TestDownloadDir(testSetup *testing.T) { } wg.Wait() - require.Nil(t, errorList[0], strings.Join(outputList[0], "\n")) - require.Nil(t, errorList[1], strings.Join(outputList[1], "\n")) - }) t.Run("Download Directory from a Directory Should Work", func(t *test.SystemTest) { @@ -150,7 +146,6 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - }) t.RunWithTimeout("Download Entire Directory Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { // todo: slow @@ -190,9 +185,7 @@ func TestDownloadDir(testSetup *testing.T) { "tokens": 9, }) filename = generateFileAndUpload(t, allocationID, remotepath, filesize) - require.NotEqual(t, "", filename) - // Delete the uploaded file from tmp folder if it exist, // since we will be downloading it now err := os.RemoveAll("tmp/" + filepath.Base(filename)) @@ -202,7 +195,6 @@ func TestDownloadDir(testSetup *testing.T) { "allocation": allocationID, "remotepath": remotepath, }) - output, err := shareFolderInAllocation(t, configPath, shareParam) require.Nil(t, err, strings.Join(output, "\n")) require.Len(t, output, 1) @@ -211,10 +203,8 @@ func TestDownloadDir(testSetup *testing.T) { require.Nil(t, err, "extract auth token failed") require.NotEqual(t, "", authTicket, "Ticket: ", authTicket) }) - // Just create a wallet so that we can work further createWallet(t) - // Download file using auth-ticket: should work output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ "authticket": authTicket, From 18d4235b6939689e93caf428ce40aad0935e8e8e Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 24 Dec 2024 16:48:28 +0530 Subject: [PATCH 5/9] lint fix --- tests/cli_tests/zboxcli_download_dir_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index 7e36c1d595..34fac7f188 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -32,7 +32,7 @@ func TestDownloadDir(testSetup *testing.T) { filesize := int64(256) remotepath := "/dir1" - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": allocSize, "tokens": 9, }) @@ -57,7 +57,7 @@ func TestDownloadDir(testSetup *testing.T) { filesize := int64(1024) remoteFilePaths := [2]string{"/dir1/", "/dir2/"} - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": allocSize, "tokens": 9, }) @@ -102,7 +102,7 @@ func TestDownloadDir(testSetup *testing.T) { filesize := int64(256) remotepath := "/dir/dir1" - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": allocSize, "tokens": 9, }) @@ -128,7 +128,7 @@ func TestDownloadDir(testSetup *testing.T) { filesize := int64(256) remotepath := "/nested/dir/" - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": allocSize, "tokens": 9, }) @@ -153,7 +153,7 @@ func TestDownloadDir(testSetup *testing.T) { filesize := int64(256) remotepath := "/nested/dir/" - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": allocSize, "tokens": 9, }) @@ -180,7 +180,7 @@ func TestDownloadDir(testSetup *testing.T) { // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file t.Run("Share Entire Folder from Another Wallet", func(t *test.SystemTest) { - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": 10 * 1024, "tokens": 9, }) @@ -222,7 +222,7 @@ func TestDownloadDir(testSetup *testing.T) { // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": 10 * 1024, "tokens": 9, }) @@ -250,8 +250,7 @@ func TestDownloadDir(testSetup *testing.T) { }) // Just create a wallet so that we can work further - err := createWalletAndLockReadTokens(t, configPath) - require.Nil(t, err) + createWallet(t) // Download file using auth-ticket: should work output, err := downloadDirectory(t, configPath, createParams(map[string]interface{}{ @@ -266,7 +265,7 @@ func TestDownloadDir(testSetup *testing.T) { filesize := int64(10) remotepath := "/dirx" - allocationID := setupAllocationAndReadLock(t, configPath, map[string]interface{}{ + allocationID := setupAllocation(t, configPath, map[string]interface{}{ "size": allocSize, "tokens": 9, }) From 2432f064fbdfc183bc1817d01b06be7e6bb8a8bb Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 24 Dec 2024 17:04:15 +0530 Subject: [PATCH 6/9] lint fix --- tests/cli_tests/zboxcli_download_dir_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index 34fac7f188..7c9f245f0d 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -38,7 +38,6 @@ func TestDownloadDir(testSetup *testing.T) { }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) // Delete the uploaded file, since we will be downloading it now err := os.Remove(filename) @@ -108,7 +107,6 @@ func TestDownloadDir(testSetup *testing.T) { }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) // Delete the uploaded file, since we will be downloading it now err := os.Remove(filename) @@ -120,7 +118,6 @@ func TestDownloadDir(testSetup *testing.T) { "localpath": "tmp/", }), true) require.Nil(t, err, strings.Join(output, "\n")) - }) t.Run("Download Directory from Nested Directory Should Work", func(t *test.SystemTest) { @@ -134,7 +131,6 @@ func TestDownloadDir(testSetup *testing.T) { }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) - // originalFileChecksum := generateChecksum(t, filename) // Delete the uploaded file, since we will be downloading it now err := os.Remove(filename) From 28c5ce5ddd068221b9d5d9cd965fe39d923d541d Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Tue, 24 Dec 2024 19:10:33 +0530 Subject: [PATCH 7/9] fix --- tests/cli_tests/zboxcli_download_dir_test.go | 24 +++++++------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index 7c9f245f0d..8c41cd7845 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -33,8 +33,7 @@ func TestDownloadDir(testSetup *testing.T) { remotepath := "/dir1" allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - "tokens": 9, + "size": allocSize, }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -57,8 +56,7 @@ func TestDownloadDir(testSetup *testing.T) { remoteFilePaths := [2]string{"/dir1/", "/dir2/"} allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - "tokens": 9, + "size": allocSize, }) fileNameOfFirstDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[0], filesize) @@ -102,8 +100,7 @@ func TestDownloadDir(testSetup *testing.T) { remotepath := "/dir/dir1" allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - "tokens": 9, + "size": allocSize, }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -126,8 +123,7 @@ func TestDownloadDir(testSetup *testing.T) { remotepath := "/nested/dir/" allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - "tokens": 9, + "size": allocSize, }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -150,8 +146,7 @@ func TestDownloadDir(testSetup *testing.T) { remotepath := "/nested/dir/" allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - "tokens": 9, + "size": allocSize, }) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -177,8 +172,7 @@ func TestDownloadDir(testSetup *testing.T) { // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file t.Run("Share Entire Folder from Another Wallet", func(t *test.SystemTest) { allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": 10 * 1024, - "tokens": 9, + "size": 10 * 1024, }) filename = generateFileAndUpload(t, allocationID, remotepath, filesize) require.NotEqual(t, "", filename) @@ -219,8 +213,7 @@ func TestDownloadDir(testSetup *testing.T) { // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": 10 * 1024, - "tokens": 9, + "size": 10 * 1024, }) filename = generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -262,8 +255,7 @@ func TestDownloadDir(testSetup *testing.T) { remotepath := "/dirx" allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - "tokens": 9, + "size": allocSize, }) filename := generateRandomTestFileName(t) From dc9b56588ee5aff791cf3696c9b41f83d6bcc29f Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Fri, 27 Dec 2024 17:23:01 +0530 Subject: [PATCH 8/9] fixeS --- tests/cli_tests/zboxcli_download_dir_test.go | 34 ++++---------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index 8c41cd7845..a45e3fde29 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -28,13 +28,10 @@ func TestDownloadDir(testSetup *testing.T) { // Success Scenarios t.Run("Download Directory from Root Directory Should Work", func(t *test.SystemTest) { - allocSize := int64(2048) filesize := int64(256) remotepath := "/dir1" - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) + allocationID := setupAllocation(t, configPath) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -51,13 +48,10 @@ func TestDownloadDir(testSetup *testing.T) { }) t.RunWithTimeout("Download Directory Concurrently Should Work for two Different Directory", 6*time.Minute, func(t *test.SystemTest) { - allocSize := int64(4096) filesize := int64(1024) remoteFilePaths := [2]string{"/dir1/", "/dir2/"} - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) + allocationID := setupAllocation(t, configPath) fileNameOfFirstDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[0], filesize) fileNameOfSecondDirectory := generateFileAndUpload(t, allocationID, remoteFilePaths[1], filesize) @@ -95,13 +89,10 @@ func TestDownloadDir(testSetup *testing.T) { }) t.Run("Download Directory from a Directory Should Work", func(t *test.SystemTest) { - allocSize := int64(2048) filesize := int64(256) remotepath := "/dir/dir1" - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) + allocationID := setupAllocation(t, configPath) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -118,13 +109,10 @@ func TestDownloadDir(testSetup *testing.T) { }) t.Run("Download Directory from Nested Directory Should Work", func(t *test.SystemTest) { - allocSize := int64(2048) filesize := int64(256) remotepath := "/nested/dir/" - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) + allocationID := setupAllocation(t, configPath) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -141,13 +129,10 @@ func TestDownloadDir(testSetup *testing.T) { }) t.RunWithTimeout("Download Entire Directory Should Work but does not see blobber/issues/588", 3*time.Minute, func(t *test.SystemTest) { // todo: slow - allocSize := int64(2048) filesize := int64(256) remotepath := "/nested/dir/" - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) + allocationID := setupAllocation(t, configPath) filename := generateFileAndUpload(t, allocationID, remotepath, filesize) @@ -171,9 +156,7 @@ func TestDownloadDir(testSetup *testing.T) { // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file t.Run("Share Entire Folder from Another Wallet", func(t *test.SystemTest) { - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": 10 * 1024, - }) + allocationID := setupAllocation(t, configPath) filename = generateFileAndUpload(t, allocationID, remotepath, filesize) require.NotEqual(t, "", filename) // Delete the uploaded file from tmp folder if it exist, @@ -250,13 +233,10 @@ func TestDownloadDir(testSetup *testing.T) { }) t.RunWithTimeout("Download Directory having Encrypted files Should Work", 5*time.Minute, func(t *test.SystemTest) { - allocSize := int64(10 * MB) filesize := int64(10) remotepath := "/dirx" - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": allocSize, - }) + allocationID := setupAllocation(t, configPath) filename := generateRandomTestFileName(t) err := createFileWithSize(filename, filesize) From 077c3d69d4a21d8d7eeafc308e0eaa6a5e389a37 Mon Sep 17 00:00:00 2001 From: smaulik13 Date: Fri, 27 Dec 2024 20:48:34 +0530 Subject: [PATCH 9/9] size fix --- tests/cli_tests/zboxcli_download_dir_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/cli_tests/zboxcli_download_dir_test.go b/tests/cli_tests/zboxcli_download_dir_test.go index a45e3fde29..1fa170c8e1 100644 --- a/tests/cli_tests/zboxcli_download_dir_test.go +++ b/tests/cli_tests/zboxcli_download_dir_test.go @@ -195,9 +195,7 @@ func TestDownloadDir(testSetup *testing.T) { // This test creates a separate wallet and allocates there, test nesting is required to create another wallet json file t.Run("Share Directory from Another Wallet", func(t *test.SystemTest) { - allocationID := setupAllocation(t, configPath, map[string]interface{}{ - "size": 10 * 1024, - }) + allocationID := setupAllocation(t, configPath) filename = generateFileAndUpload(t, allocationID, remotepath, filesize) require.NotEqual(t, "", filename)