feat: backup restore streaming

.env.example

@@ -0,0 +1,4 @@
+DRUID_S3_ACCESS_KEY=
+DRUID_S3_SECRET_KEY=
+DRUID_S3_BUCKET=druid-testing
+DRUID_S3_ENDPOINT=fsn1

.gitignore

@@ -13,4 +13,6 @@ debug**
 
 druid-cli-test
 
-!.docker/**
+!.docker/**
+
+.env

.vscode/launch.json

@@ -11,9 +11,10 @@
             "args": [
                 "backup",
                 "--cwd",
-                "${workspaceFolder}/examples/backup-test",
-                "https://gist.github.com/jlmaccal/ca6b08a307800f80986661075c82c432/raw/dbd5e7a38d0660b7047476de33da142bd79c22ef/test.tgz"
+                "${workspaceFolder}/examples/minecraft",
+                "testtest.tgz"
             ],
+            "envFile": "${workspaceFolder}/.env",
         },
         {
             "name": "Debug Daemon restore",

internal/core/services/snapshot/progress.go

@@ -2,52 +2,27 @@ package services
 
 import (
 	"fmt"
-	"io"
 
 	"github.com/highcard-dev/daemon/internal/utils/logger"
 	"go.uber.org/zap"
 )
 
-type ProgressTracker struct {
-	reader      io.ReadCloser
-	read        int64
-	fileSize    int64
-	lastPercent float64
+type GeneralProgressTracker struct {
+	total int64
+	read  int64
 }
 
-func (pr *ProgressTracker) Read(p []byte) (int, error) {
-	n, err := pr.reader.Read(p)
-	pr.read += int64(n)
-
-	// Calculate current percentage of upload progress
-	currentPercent := (float64(pr.read) * 100) / float64(pr.fileSize)
-
-	// Update progress if we've moved at least 0.1% or it's been more than the update frequency since the last update
-	if currentPercent > pr.lastPercent+0.1 {
-		logger.Log().Info("Snapshot operation progress", zap.String("percentage", fmt.Sprintf("%.1f%%", currentPercent)), zap.String("read", fmt.Sprintf("%d/%d", pr.read, pr.fileSize)))
-		pr.lastPercent = currentPercent
-	}
-
-	// If the upload is finished
-	if pr.read == pr.fileSize {
-		logger.Log().Info("Snapshot operation complete")
+func NewGeneralProgressTracker(total int64) *GeneralProgressTracker {
+	return &GeneralProgressTracker{
+		total: total,
+		read:  0,
 	}
-
-	return n, err
-}
-
-func (pr *ProgressTracker) Close() error {
-	pr.lastPercent = 100
-	return pr.reader.Close()
 }
 
-func (pt *ProgressTracker) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
-	pt.reader = stream
-	pt.fileSize = totalSize
-	logger.Log().Info("Snapshot operation started", zap.String("source", src), zap.String("size", fmt.Sprintf("%d bytes", totalSize)))
-	return pt
+func (pt *GeneralProgressTracker) GetPercent() float64 {
+	return (float64(pt.read) / float64(pt.total)) * 100
 }
-
-func (pt *ProgressTracker) GetPercent() float64 {
-	return pt.lastPercent
+func (pt *GeneralProgressTracker) TrackProgress() {
+	pt.read++
+	logger.Log().Info("Progress", zap.Int64("total", pt.total), zap.Int64("read", pt.read), zap.String("percentage", fmt.Sprintf("%.1f%%", pt.GetPercent())))
 }

internal/core/services/snapshot/progress_reader.go

@@ -0,0 +1,57 @@
+package services
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/highcard-dev/daemon/internal/utils"
+	"github.com/highcard-dev/daemon/internal/utils/logger"
+	"go.uber.org/zap"
+)
+
+type ProgressTracker struct {
+	reader      io.ReadCloser
+	read        int64
+	fileSize    int64
+	lastPercent float64
+}
+
+func (pr *ProgressTracker) Read(p []byte) (int, error) {
+	n, err := pr.reader.Read(p)
+	pr.read += int64(n)
+
+	// Calculate current percentage of upload progress
+	currentPercent := (float64(pr.read) * 100) / float64(pr.fileSize)
+
+	sizeHuman := utils.HumanizeBytes(pr.fileSize)
+	readHuman := utils.HumanizeBytes(pr.read)
+
+	// Update progress if we've moved at least 0.1% or it's been more than the update frequency since the last update
+	if currentPercent > pr.lastPercent+0.1 {
+		logger.Log().Info("Snapshot operation progress", zap.String("percentage", fmt.Sprintf("%.1f%%", currentPercent)), zap.String("read", fmt.Sprintf("%s/%s", readHuman, sizeHuman)))
+		pr.lastPercent = currentPercent
+	}
+
+	// If the upload is finished
+	if pr.read == pr.fileSize {
+		logger.Log().Info("Snapshot operation complete")
+	}
+
+	return n, err
+}
+
+func (pr *ProgressTracker) Close() error {
+	pr.lastPercent = 100
+	return pr.reader.Close()
+}
+
+func (pt *ProgressTracker) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
+	pt.reader = stream
+	pt.fileSize = totalSize
+	logger.Log().Info("Snapshot operation started", zap.String("source", src), zap.String("size", fmt.Sprintf("%d bytes", totalSize)))
+	return pt
+}
+
+func (pt *ProgressTracker) GetPercent() float64 {
+	return pt.lastPercent
+}