diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..6002147 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +DRUID_S3_ACCESS_KEY= +DRUID_S3_SECRET_KEY= +DRUID_S3_BUCKET=druid-testing +DRUID_S3_ENDPOINT=fsn1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7610f8f..ddcecc5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,6 @@ debug** druid-cli-test -!.docker/** \ No newline at end of file +!.docker/** + +.env \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 6443fae..d39948b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,9 +11,10 @@ "args": [ "backup", "--cwd", - "${workspaceFolder}/examples/backup-test", - "https://gist.github.com/jlmaccal/ca6b08a307800f80986661075c82c432/raw/dbd5e7a38d0660b7047476de33da142bd79c22ef/test.tgz" + "${workspaceFolder}/examples/minecraft", + "testtest.tgz" ], + "envFile": "${workspaceFolder}/.env", }, { "name": "Debug Daemon restore", @@ -104,8 +105,6 @@ "--watch-ports", "-p", "9190", - "--max-health-check-startup-timeount", - "10", ], }, { diff --git a/cmd/serve.go b/cmd/serve.go index 7e34e5f..6be6f52 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -370,7 +370,7 @@ func startup(scrollService *services.ScrollService, snapshotService ports.Snapsh func initScroll(scrollService *services.ScrollService, snapshotService ports.SnapshotService, processLauncher *services.ProcedureLauncher, queueManager *services.QueueManager) (bool, error) { - lock, err := scrollService.Bootstrap(ignoreVersionCheck) + lock, err := scrollService.ReloadLock(ignoreVersionCheck) if err != nil { return false, err } @@ -380,19 +380,22 @@ func initScroll(scrollService *services.ScrollService, snapshotService ports.Sna if newScroll { if initSnapshotUrl != "" { - logger.Log().Info("Starting restore process") + logger.Log().Info("Starting restore process from Init Snapshot") err := snapshotService.RestoreSnapshot(scrollService.GetCwd(), initSnapshotUrl, ports.RestoreSnapshotOptions{}) if err != nil { return false, err } + logger.Log().Info("Restore process from Init Snapshot done") + currentScroll, err := scrollService.ReloadScroll() if err != nil { return false, err } + logger.Log().Info("Reloaded scroll after restore", zap.String("Name", currentScroll.Name), zap.Any("Version", currentScroll.Version), zap.String("AppVersion", currentScroll.AppVersion), zap.Any("Ports", currentScroll.Ports)) - lock, err = scrollService.Bootstrap(ignoreVersionCheck) + lock, err = scrollService.ReloadLock(ignoreVersionCheck) if err != nil { return false, err } @@ -403,18 +406,16 @@ func initScroll(scrollService *services.ScrollService, snapshotService ports.Sna if !newScroll { lock.SetStatus(currentScroll.Init, domain.ScrollLockStatusWaiting, nil) } - } - if newScroll { - logger.Log().Info("No lock file found, but init command available. Bootstrapping...") + } - logger.Log().Info("Creating lock and bootstrapping files") - //There is an error here. We need to bootstrap the files before we render out the templates in the bootstrap func above - err := scrollService.CreateLockAndBootstrapFiles() - if err != nil { - return newScroll, err - } + logger.Log().Info("Creating lock and bootstrapping files") + //There is an error here. We need to bootstrap the files before we render out the templates in the bootstrap func above + err := scrollService.CopyingInitFiles() + if err != nil { + return newScroll, err } + } else { logger.Log().Info("Found lock file, bootstrapping done") } diff --git a/internal/core/ports/services_ports.go b/internal/core/ports/services_ports.go index 33df0ed..807e025 100644 --- a/internal/core/ports/services_ports.go +++ b/internal/core/ports/services_ports.go @@ -34,6 +34,8 @@ type ScrollServiceInterface interface { WriteNewScrollLock() *domain.ScrollLock GetLock() (*domain.ScrollLock, error) GetCommand(cmd string) (*domain.CommandInstructionSet, error) + InitFiles(files ...string) error + InitTemplateFiles(files ...string) error } type ProcedureLauchnerInterface interface { @@ -84,7 +86,7 @@ type ProcessMonitorInterface interface { type TemplateRendererInterface interface { RenderTemplate(templatePath string, data interface{}) (string, error) - RenderScrollTemplateFiles(templateFiles []string, data interface{}, ouputPath string) error + RenderScrollTemplateFiles(templateBase string, templateFiles []string, data interface{}, ouputPath string) error } type OciRegistryInterface interface { diff --git a/internal/core/services/scroll_service.go b/internal/core/services/scroll_service.go index 8a9fa5f..280161e 100644 --- a/internal/core/services/scroll_service.go +++ b/internal/core/services/scroll_service.go @@ -3,8 +3,10 @@ package services import ( "errors" "os" + "path" "path/filepath" "regexp" + "slices" "strings" "github.com/highcard-dev/daemon/internal/core/domain" @@ -51,7 +53,7 @@ func (sc *ScrollService) ReloadScroll() (*domain.Scroll, error) { } // Load Scroll and render templates in the cwd -func (sc *ScrollService) Bootstrap(ignoreVersionCheck bool) (*domain.ScrollLock, error) { +func (sc *ScrollService) ReloadLock(ignoreVersionCheck bool) (*domain.ScrollLock, error) { var scroll = sc.scroll @@ -73,73 +75,79 @@ func (sc *ScrollService) Bootstrap(ignoreVersionCheck bool) (*domain.ScrollLock, return lock, nil } -func (sc *ScrollService) CreateLockAndBootstrapFiles() error { - //init-files just get copied over - initPath := strings.TrimRight(sc.GetDir(), "/") + "/init-files" - exist, _ := utils.FileExists(initPath) - if exist { - err := filepath.Walk(initPath, func(path string, f os.FileInfo, err error) error { - strippedPath := strings.TrimPrefix(filepath.Clean(path), filepath.Clean(initPath)) - realPath := filepath.Join(sc.processCwd, strippedPath) - if f.IsDir() { - if strippedPath == "" { - return nil - } - err := os.MkdirAll(realPath, f.Mode()) - if err != nil { - return err - } - } else { - - b, err := os.ReadFile(path) - - if err != nil { - return err - } - - return os.WriteFile(realPath, b, f.Mode()) - } +func (sc *ScrollService) InitFiles(fls ...string) error { + //init-files-template needs to be rendered + files, err := sc.filterFiles("init-files", fls...) + if err != nil { + return err + } + initFileDir := path.Join(sc.GetDir(), "init-files") - return err - }) + for _, file := range files { + basePath := strings.TrimPrefix(file, initFileDir) + dest := path.Join(sc.processCwd, basePath) + + err := utils.CopyFile(file, dest) if err != nil { return err } } + return nil +} + +func (sc *ScrollService) filterFiles(path string, fls ...string) ([]string, error) { //init-files-template needs to be rendered - initPath = strings.TrimRight(sc.GetDir(), "/") + "/init-files-template" - exist, _ = utils.FileExists(initPath) + initPath := strings.TrimRight(sc.GetDir(), "/") + "/" + path + "/" + exist, _ := utils.FileExists(initPath) files := []string{} if exist { err := filepath.Walk(initPath, func(path string, f os.FileInfo, err error) error { - if f.IsDir() { - err := os.MkdirAll(path, f.Mode()) - if err != nil { - return err - } - } else { + basePath := strings.TrimPrefix(path, initPath) + if !f.IsDir() && (slices.Contains(fls, basePath) || len(fls) == 0) { files = append(files, path) } - return nil }) - if len(files) == 0 { - return nil - } - if err != nil { - return err - } - err = sc.templateRenderer.RenderScrollTemplateFiles(files, sc.scroll, sc.processCwd) if err != nil { - return err + return []string{}, err } + + return files, nil } - return nil + return []string{}, nil +} + +func (sc *ScrollService) InitTemplateFiles(fls ...string) error { + + //init-files-template needs to be rendered + files, err := sc.filterFiles("init-files-template", fls...) + if err != nil { + return err + } + if len(files) == 0 { + return nil + } + templateBase := path.Join(sc.GetDir(), "init-files-template") + + for i, file := range files { + basePath := strings.TrimPrefix(file, templateBase) + files[i] = basePath + } + + return sc.templateRenderer.RenderScrollTemplateFiles(templateBase, files, sc.scroll, sc.processCwd) +} + +func (sc *ScrollService) CopyingInitFiles() error { + err := sc.InitFiles() + if err != nil { + return err + } + return sc.InitTemplateFiles() } func (sc *ScrollService) LockExists() bool { @@ -217,7 +225,7 @@ func (s ScrollService) RenderCwdTemplates() error { config := TemplateData{Config: s.GetScrollConfig()} - return s.templateRenderer.RenderScrollTemplateFiles(files, config, "") + return s.templateRenderer.RenderScrollTemplateFiles("", files, config, "") } diff --git a/internal/core/services/snapshot/progress.go b/internal/core/services/snapshot/progress.go index 81425a1..4b6666a 100644 --- a/internal/core/services/snapshot/progress.go +++ b/internal/core/services/snapshot/progress.go @@ -2,52 +2,27 @@ package services import ( "fmt" - "io" "github.com/highcard-dev/daemon/internal/utils/logger" "go.uber.org/zap" ) -type ProgressTracker struct { - reader io.ReadCloser - read int64 - fileSize int64 - lastPercent float64 +type GeneralProgressTracker struct { + total int64 + read int64 } -func (pr *ProgressTracker) Read(p []byte) (int, error) { - n, err := pr.reader.Read(p) - pr.read += int64(n) - - // Calculate current percentage of upload progress - currentPercent := (float64(pr.read) * 100) / float64(pr.fileSize) - - // Update progress if we've moved at least 0.1% or it's been more than the update frequency since the last update - if currentPercent > pr.lastPercent+0.1 { - logger.Log().Info("Snapshot operation progress", zap.String("percentage", fmt.Sprintf("%.1f%%", currentPercent)), zap.String("read", fmt.Sprintf("%d/%d", pr.read, pr.fileSize))) - pr.lastPercent = currentPercent - } - - // If the upload is finished - if pr.read == pr.fileSize { - logger.Log().Info("Snapshot operation complete") +func NewGeneralProgressTracker(total int64) *GeneralProgressTracker { + return &GeneralProgressTracker{ + total: total, + read: 0, } - - return n, err -} - -func (pr *ProgressTracker) Close() error { - pr.lastPercent = 100 - return pr.reader.Close() } -func (pt *ProgressTracker) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser { - pt.reader = stream - pt.fileSize = totalSize - logger.Log().Info("Snapshot operation started", zap.String("source", src), zap.String("size", fmt.Sprintf("%d bytes", totalSize))) - return pt +func (pt *GeneralProgressTracker) GetPercent() float64 { + return (float64(pt.read) / float64(pt.total)) * 100 } - -func (pt *ProgressTracker) GetPercent() float64 { - return pt.lastPercent +func (pt *GeneralProgressTracker) TrackProgress() { + pt.read++ + logger.Log().Info("Progress", zap.Int64("total", pt.total), zap.Int64("read", pt.read), zap.String("percentage", fmt.Sprintf("%.1f%%", pt.GetPercent()))) } diff --git a/internal/core/services/snapshot/progress_reader.go b/internal/core/services/snapshot/progress_reader.go new file mode 100644 index 0000000..8294bb1 --- /dev/null +++ b/internal/core/services/snapshot/progress_reader.go @@ -0,0 +1,57 @@ +package services + +import ( + "fmt" + "io" + + "github.com/highcard-dev/daemon/internal/utils" + "github.com/highcard-dev/daemon/internal/utils/logger" + "go.uber.org/zap" +) + +type ProgressTracker struct { + reader io.ReadCloser + read int64 + fileSize int64 + lastPercent float64 +} + +func (pr *ProgressTracker) Read(p []byte) (int, error) { + n, err := pr.reader.Read(p) + pr.read += int64(n) + + // Calculate current percentage of upload progress + currentPercent := (float64(pr.read) * 100) / float64(pr.fileSize) + + sizeHuman := utils.HumanizeBytes(pr.fileSize) + readHuman := utils.HumanizeBytes(pr.read) + + // Update progress if we've moved at least 0.1% or it's been more than the update frequency since the last update + if currentPercent > pr.lastPercent+0.1 { + logger.Log().Info("Snapshot operation progress", zap.String("percentage", fmt.Sprintf("%.1f%%", currentPercent)), zap.String("read", fmt.Sprintf("%s/%s", readHuman, sizeHuman))) + pr.lastPercent = currentPercent + } + + // If the upload is finished + if pr.read == pr.fileSize { + logger.Log().Info("Snapshot operation complete") + } + + return n, err +} + +func (pr *ProgressTracker) Close() error { + pr.lastPercent = 100 + return pr.reader.Close() +} + +func (pt *ProgressTracker) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser { + pt.reader = stream + pt.fileSize = totalSize + logger.Log().Info("Snapshot operation started", zap.String("source", src), zap.String("size", fmt.Sprintf("%d bytes", totalSize))) + return pt +} + +func (pt *ProgressTracker) GetPercent() float64 { + return pt.lastPercent +} diff --git a/internal/core/services/snapshot/restore_service.go b/internal/core/services/snapshot/restore_service.go index e3e1e9e..aae0f1f 100644 --- a/internal/core/services/snapshot/restore_service.go +++ b/internal/core/services/snapshot/restore_service.go @@ -5,14 +5,12 @@ import ( "compress/gzip" "context" "crypto/tls" - "errors" "fmt" "io" "net/http" "os" "path" "path/filepath" - "strings" "github.com/hashicorp/go-getter" "github.com/highcard-dev/daemon/internal/core/ports" @@ -34,7 +32,7 @@ func NewSnapshotService() *SnapshotService { } } -func (rc *SnapshotService) setActivity(mode ports.SnapshotMode, progressTracker *ProgressTracker) { +func (rc *SnapshotService) setActivity(mode ports.SnapshotMode, progressTracker ports.ProgressTracker) { rc.currentMode = mode rc.currentProgressTracker = progressTracker } @@ -46,98 +44,37 @@ func (rc *SnapshotService) GetCurrentProgressTracker() *ports.ProgressTracker { return &rc.currentProgressTracker } -func (rc *SnapshotService) Snapshot(dir string, destination string, options ports.SnapshotOptions) error { - - var target string - if options.TempDir == "" { - target = filepath.Join(dir, "snapshot.tgz") - } else { - target = filepath.Join(options.TempDir, "snapshot.tgz") - } - - logger.Log().Info("Creating snapshot", zap.String("source", dir), zap.String("destination", target)) - // Define the source URL and destination directory - err := rc.createTarGz(dir, target) - if err != nil { - return err - } - logger.Log().Info("Snapshot created", zap.String("source", dir), zap.String("destination", target)) - - //TODO: upload - if strings.HasPrefix(destination, "http") { - logger.Log().Info("Uploading snapshot", zap.String("source", target), zap.String("destination", destination)) - err = rc.uploadFileUsingPresignedURL(destination, target) +func (rc *SnapshotService) countFilesRec(dir string) (int64, error) { + var fileCount int64 + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { - logger.Log().Error("Error occured while uploading snapshot", zap.Error(err)) return err } - logger.Log().Info("Snapshot uploaded", zap.String("source", target), zap.String("destination", destination)) - } else if options.S3Destination != nil { - logger.Log().Info("Uploading snapshot", zap.String("source", target), zap.String("destination", destination)) - err = rc.uploadFileUsingS3(destination, target, options.S3Destination) - if err != nil { - logger.Log().Error("Error occured while uploading snapshot", zap.Error(err)) - return err + if info.Mode().IsRegular() { + fileCount++ } - logger.Log().Info("Snapshot uploaded", zap.String("source", target), zap.String("destination", destination)) - } else { - return errors.New("destination must be a presigned S3 URL") - } - - return os.Remove(target) -} - -func (rc *SnapshotService) uploadFileUsingS3(objectKey, filePath string, s3Destination *ports.S3Destination) error { - - ctx := context.TODO() - - endpoint := s3Destination.Endpoint - region := s3Destination.Region - if region == "" { - region = "us-east-1" - } - accessKey := s3Destination.AccessKey - secretKey := s3Destination.SecretKey - bucketName := s3Destination.Bucket - - // Load AWS config with custom S3-compatible settings - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), - Secure: true, + return nil }) if err != nil { - return fmt.Errorf("Failed to create S3 client: %v", err) + return 0, err } + return fileCount, nil +} - // Open the file - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - // Get the file size - fileInfo, err := file.Stat() - if err != nil { - return fmt.Errorf("failed to get file info: %w", err) - } - fileSize := fileInfo.Size() +func (rc *SnapshotService) Snapshot(dir string, destination string, options ports.SnapshotOptions) error { - // Wrap the file reader in the ProgressReader with an update frequency of 1 second - progressReader := &ProgressTracker{ - reader: file, - fileSize: fileSize, - } - rc.setActivity(ports.SnapshotModeSnapshot, progressReader) - defer rc.setActivity(ports.SnapshotModeNoop, nil) + totalFiles := int64(0) + totalFiles, _ = rc.countFilesRec(dir) + progessTracker := NewGeneralProgressTracker(totalFiles) - contentType := "application/octet-stream" - _, err = minioClient.PutObject(ctx, bucketName, objectKey, progressReader, fileSize, minio.PutObjectOptions{ContentType: contentType}) - if err != nil { - return fmt.Errorf("failed to upload file: %v", err) + rc.setActivity(ports.SnapshotModeSnapshot, progessTracker) + defer rc.setActivity(ports.SnapshotModeNoop, progessTracker) + //check if rootPath exists + if _, err := os.Stat(dir); os.IsNotExist(err) { + return fmt.Errorf("source path does not exist: %s", dir) } - return nil + return rc.uploadS3(dir, destination, options.S3Destination, progessTracker) } func (rc *SnapshotService) RestoreSnapshot(dir string, source string, options ports.RestoreSnapshotOptions) error { @@ -192,71 +129,86 @@ func (rc *SnapshotService) RestoreSnapshot(dir string, source string, options po return nil } -func (rc *SnapshotService) createTarGz(rootPath, target string) error { - // Create the target .tgz file - tgzFile, err := os.Create(target) - if err != nil { - return err - } - defer tgzFile.Close() +func (rc *SnapshotService) uploadS3(rootPath, objectKey string, s3Destination *ports.S3Destination, progessTracker *GeneralProgressTracker) error { - // Create a gzip writer - gzipWriter := gzip.NewWriter(tgzFile) - defer gzipWriter.Close() + pipeReader, pipeWriter := io.Pipe() - // Create a tar writer - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() + go func() { + defer pipeWriter.Close() - // Walk through the source directory - return filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + // Create a gzip writer + gzipWriter := gzip.NewWriter(pipeWriter) + defer gzipWriter.Close() - // Skip the target file - if absTarget, err := filepath.Abs(target); err != nil { - return err - } else if absFile, err := filepath.Abs(path); err != nil { - return err - } else if absFile == absTarget { - return nil - } + // Create a tar writer + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() - linkName := "" - if info.Mode()&os.ModeSymlink == os.ModeSymlink { - linkName, err = os.Readlink(path) + // Walk through the source directory + filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - } - hdr, err := tar.FileInfoHeader(info, linkName) - if err != nil { - return err - } + linkName := "" + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + linkName, err = os.Readlink(path) + if err != nil { + return err + } + } - hdr.Name, _ = filepath.Rel(rootPath, path) + hdr, err := tar.FileInfoHeader(info, linkName) + if err != nil { + return err + } - if err := tarWriter.WriteHeader(hdr); err != nil { - return err - } + hdr.Name, _ = filepath.Rel(rootPath, path) - if info.Mode().IsRegular() { - file, err := os.Open(path) - if err != nil { + if err := tarWriter.WriteHeader(hdr); err != nil { return err } - defer file.Close() - _, err = io.Copy(tarWriter, file) - return err - } + if info.Mode().IsRegular() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() - return nil + _, err = io.Copy(tarWriter, file) + progessTracker.TrackProgress() + return err + } + + return nil + }) + }() + + endpoint := s3Destination.Endpoint + region := s3Destination.Region + if region == "" { + region = "us-east-1" + } + accessKey := s3Destination.AccessKey + secretKey := s3Destination.SecretKey + bucketName := s3Destination.Bucket + + // Load AWS config with custom S3-compatible settings + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: true, }) + if err != nil { + return fmt.Errorf("failed to create S3 client: %v", err) + } + + contentType := "application/octet-stream" + _, err = minioClient.PutObject(context.TODO(), bucketName, objectKey, pipeReader, -1, minio.PutObjectOptions{ContentType: contentType}) + return err } +// Todo: refactor this to do streaming upload func (rc *SnapshotService) uploadFileUsingPresignedURL(presignedURL, filePath string) error { // Open the file file, err := os.Open(filePath) diff --git a/internal/core/services/template_renderer.go b/internal/core/services/template_renderer.go index a0c8cc5..4ed3c48 100644 --- a/internal/core/services/template_renderer.go +++ b/internal/core/services/template_renderer.go @@ -4,6 +4,7 @@ import ( "bytes" "html/template" "os" + "path" "path/filepath" "strings" @@ -32,23 +33,26 @@ func (tr *TemplateRenderer) RenderTemplate(templatePath string, data interface{} return tpl.String(), err } -func (tr *TemplateRenderer) RenderScrollTemplateFiles(templateFiles []string, data any, outputDir string) error { - - // Parse the template files - templates, err := template.New("scroll_template").Funcs(sprig.TxtFuncMap()).ParseFiles(templateFiles...) - if err != nil { - return err - } - +func (tr *TemplateRenderer) RenderScrollTemplateFiles(templateBase string, templateFiles []string, data any, outputDir string) error { + tpl := template.New("scroll_template").Funcs(sprig.TxtFuncMap()) for _, templateFile := range templateFiles { + // Parse the template files + templates, err := tpl.ParseFiles(path.Join(templateBase, templateFile)) + if err != nil { + return err + } // Remove the "template" suffix from the file name outputFileName := strings.TrimSuffix(templateFile, ".scroll_template") if outputDir != "" { - filename := filepath.Base(outputFileName) - // Prepend the output directory if specified - outputFileName = filepath.Join(outputDir, filename) + outputFileName = filepath.Join(outputDir, outputFileName) + } + + //ensure the output directory exists + outputDirPath := filepath.Dir(outputFileName) + if err := os.MkdirAll(outputDirPath, os.ModePerm); err != nil { + return err } // Create a new file for the rendered output diff --git a/internal/utils/fs.go b/internal/utils/fs.go index 9d42dc8..6d7231d 100644 --- a/internal/utils/fs.go +++ b/internal/utils/fs.go @@ -1,6 +1,7 @@ package utils import ( + "io" "os" "path/filepath" ) @@ -53,3 +54,43 @@ func MoveContents(src, dest string) error { } return nil } + +func CopyFile(src, dest string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + // Get source file info for permissions + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + + //ensure the destination directory exists + destDir := filepath.Dir(dest) + if err := os.MkdirAll(destDir, os.ModePerm); err != nil { + return err + } + + // Create destination file with same permissions + destFile, err := os.Create(dest) + if err != nil { + return err + } + defer destFile.Close() + + // Copy file contents + if _, err := io.Copy(destFile, srcFile); err != nil { + return err + } + + // Sync to ensure data is written + if err := destFile.Sync(); err != nil { + return err + } + + // Set same permissions as source + return os.Chmod(dest, srcInfo.Mode()) +} diff --git a/internal/utils/misc.go b/internal/utils/misc.go index 5e95deb..0630dc3 100644 --- a/internal/utils/misc.go +++ b/internal/utils/misc.go @@ -2,6 +2,7 @@ package utils import ( "errors" + "fmt" "strings" ) @@ -40,3 +41,25 @@ func InterfaceToStringSlice(data interface{}) ([]string, error) { } return instructions, nil } + +func HumanizeBytes(bytes int64) string { + const ( + KB = 1024 + MB = KB * 1024 + GB = MB * 1024 + TB = GB * 1024 + ) + + switch { + case bytes < KB: + return fmt.Sprintf("%dB", bytes) + case bytes < MB: + return fmt.Sprintf("%.2fKB", float64(bytes)/KB) + case bytes < GB: + return fmt.Sprintf("%.2fMB", float64(bytes)/MB) + case bytes < TB: + return fmt.Sprintf("%.2fGB", float64(bytes)/GB) + default: + return fmt.Sprintf("%.2fTB", float64(bytes)/TB) + } +} diff --git a/test/integration/example_test.go b/test/integration/example_test.go index fafebc2..19f8bb3 100644 --- a/test/integration/example_test.go +++ b/test/integration/example_test.go @@ -122,7 +122,7 @@ func TestExamples(t *testing.T) { go queueManager.Work() scrollService.WriteNewScrollLock() - scrollService.Bootstrap(false) + scrollService.ReloadLock(false) err = queueManager.AddAndRememberItem("start") diff --git a/test/mock/services.go b/test/mock/services.go index 52d4108..8371608 100644 --- a/test/mock/services.go +++ b/test/mock/services.go @@ -215,6 +215,42 @@ func (mr *MockScrollServiceInterfaceMockRecorder) GetScrollConfigRawYaml() *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScrollConfigRawYaml", reflect.TypeOf((*MockScrollServiceInterface)(nil).GetScrollConfigRawYaml)) } +// InitFiles mocks base method. +func (m *MockScrollServiceInterface) InitFiles(files ...string) error { + m.ctrl.T.Helper() + varargs := []any{} + for _, a := range files { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "InitFiles", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitFiles indicates an expected call of InitFiles. +func (mr *MockScrollServiceInterfaceMockRecorder) InitFiles(files ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitFiles", reflect.TypeOf((*MockScrollServiceInterface)(nil).InitFiles), files...) +} + +// InitTemplateFiles mocks base method. +func (m *MockScrollServiceInterface) InitTemplateFiles(files ...string) error { + m.ctrl.T.Helper() + varargs := []any{} + for _, a := range files { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "InitTemplateFiles", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitTemplateFiles indicates an expected call of InitTemplateFiles. +func (mr *MockScrollServiceInterfaceMockRecorder) InitTemplateFiles(files ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitTemplateFiles", reflect.TypeOf((*MockScrollServiceInterface)(nil).InitTemplateFiles), files...) +} + // WriteNewScrollLock mocks base method. func (m *MockScrollServiceInterface) WriteNewScrollLock() *domain.ScrollLock { m.ctrl.T.Helper() @@ -770,17 +806,17 @@ func (m *MockTemplateRendererInterface) EXPECT() *MockTemplateRendererInterfaceM } // RenderScrollTemplateFiles mocks base method. -func (m *MockTemplateRendererInterface) RenderScrollTemplateFiles(templateFiles []string, data any, ouputPath string) error { +func (m *MockTemplateRendererInterface) RenderScrollTemplateFiles(templateBase string, templateFiles []string, data any, ouputPath string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RenderScrollTemplateFiles", templateFiles, data, ouputPath) + ret := m.ctrl.Call(m, "RenderScrollTemplateFiles", templateBase, templateFiles, data, ouputPath) ret0, _ := ret[0].(error) return ret0 } // RenderScrollTemplateFiles indicates an expected call of RenderScrollTemplateFiles. -func (mr *MockTemplateRendererInterfaceMockRecorder) RenderScrollTemplateFiles(templateFiles, data, ouputPath any) *gomock.Call { +func (mr *MockTemplateRendererInterfaceMockRecorder) RenderScrollTemplateFiles(templateBase, templateFiles, data, ouputPath any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenderScrollTemplateFiles", reflect.TypeOf((*MockTemplateRendererInterface)(nil).RenderScrollTemplateFiles), templateFiles, data, ouputPath) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RenderScrollTemplateFiles", reflect.TypeOf((*MockTemplateRendererInterface)(nil).RenderScrollTemplateFiles), templateBase, templateFiles, data, ouputPath) } // RenderTemplate mocks base method. @@ -1158,6 +1194,20 @@ func (m *MockColdStarterHandlerInterface) EXPECT() *MockColdStarterHandlerInterf return m.recorder } +// Close mocks base method. +func (m *MockColdStarterHandlerInterface) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockColdStarterHandlerInterfaceMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockColdStarterHandlerInterface)(nil).Close)) +} + // GetHandler mocks base method. func (m *MockColdStarterHandlerInterface) GetHandler(funcs map[string]func(...string)) (ports.ColdStarterPacketHandlerInterface, error) { m.ctrl.T.Helper() @@ -1185,32 +1235,32 @@ func (mr *MockColdStarterHandlerInterfaceMockRecorder) SetFinishedAt(finishedAt return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFinishedAt", reflect.TypeOf((*MockColdStarterHandlerInterface)(nil).SetFinishedAt), finishedAt) } -// MockColdStarterServerInterface is a mock of ColdStarterServerInterface interface. -type MockColdStarterServerInterface struct { +// MockColdStarterPacketHandlerInterface is a mock of ColdStarterPacketHandlerInterface interface. +type MockColdStarterPacketHandlerInterface struct { ctrl *gomock.Controller - recorder *MockColdStarterServerInterfaceMockRecorder + recorder *MockColdStarterPacketHandlerInterfaceMockRecorder isgomock struct{} } -// MockColdStarterServerInterfaceMockRecorder is the mock recorder for MockColdStarterServerInterface. -type MockColdStarterServerInterfaceMockRecorder struct { - mock *MockColdStarterServerInterface +// MockColdStarterPacketHandlerInterfaceMockRecorder is the mock recorder for MockColdStarterPacketHandlerInterface. +type MockColdStarterPacketHandlerInterfaceMockRecorder struct { + mock *MockColdStarterPacketHandlerInterface } -// NewMockColdStarterServerInterface creates a new mock instance. -func NewMockColdStarterServerInterface(ctrl *gomock.Controller) *MockColdStarterServerInterface { - mock := &MockColdStarterServerInterface{ctrl: ctrl} - mock.recorder = &MockColdStarterServerInterfaceMockRecorder{mock} +// NewMockColdStarterPacketHandlerInterface creates a new mock instance. +func NewMockColdStarterPacketHandlerInterface(ctrl *gomock.Controller) *MockColdStarterPacketHandlerInterface { + mock := &MockColdStarterPacketHandlerInterface{ctrl: ctrl} + mock.recorder = &MockColdStarterPacketHandlerInterfaceMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockColdStarterServerInterface) EXPECT() *MockColdStarterServerInterfaceMockRecorder { +func (m *MockColdStarterPacketHandlerInterface) EXPECT() *MockColdStarterPacketHandlerInterfaceMockRecorder { return m.recorder } // Handle mocks base method. -func (m *MockColdStarterServerInterface) Handle(data []byte, funcs map[string]func(...string)) error { +func (m *MockColdStarterPacketHandlerInterface) Handle(data []byte, funcs map[string]func(...string)) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Handle", data, funcs) ret0, _ := ret[0].(error) @@ -1218,9 +1268,9 @@ func (m *MockColdStarterServerInterface) Handle(data []byte, funcs map[string]fu } // Handle indicates an expected call of Handle. -func (mr *MockColdStarterServerInterfaceMockRecorder) Handle(data, funcs any) *gomock.Call { +func (mr *MockColdStarterPacketHandlerInterfaceMockRecorder) Handle(data, funcs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handle", reflect.TypeOf((*MockColdStarterServerInterface)(nil).Handle), data, funcs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Handle", reflect.TypeOf((*MockColdStarterPacketHandlerInterface)(nil).Handle), data, funcs) } // MockColdStarterInterface is a mock of ColdStarterInterface interface. @@ -1260,15 +1310,27 @@ func (mr *MockColdStarterInterfaceMockRecorder) Finish(arg0 any) *gomock.Call { } // Stop mocks base method. -func (m *MockColdStarterInterface) Stop(arg0 uint) { +func (m *MockColdStarterInterface) Stop() { m.ctrl.T.Helper() - m.ctrl.Call(m, "Stop", arg0) + m.ctrl.Call(m, "Stop") } // Stop indicates an expected call of Stop. -func (mr *MockColdStarterInterfaceMockRecorder) Stop(arg0 any) *gomock.Call { +func (mr *MockColdStarterInterfaceMockRecorder) Stop() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockColdStarterInterface)(nil).Stop), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockColdStarterInterface)(nil).Stop)) +} + +// StopWithDeplay mocks base method. +func (m *MockColdStarterInterface) StopWithDeplay(arg0 uint) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "StopWithDeplay", arg0) +} + +// StopWithDeplay indicates an expected call of StopWithDeplay. +func (mr *MockColdStarterInterfaceMockRecorder) StopWithDeplay(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopWithDeplay", reflect.TypeOf((*MockColdStarterInterface)(nil).StopWithDeplay), arg0) } // MockProgressTracker is a mock of ProgressTracker interface. @@ -1347,6 +1409,20 @@ func (mr *MockSnapshotServiceMockRecorder) GetCurrentMode() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentMode", reflect.TypeOf((*MockSnapshotService)(nil).GetCurrentMode)) } +// GetCurrentProgressTracker mocks base method. +func (m *MockSnapshotService) GetCurrentProgressTracker() *ports.ProgressTracker { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCurrentProgressTracker") + ret0, _ := ret[0].(*ports.ProgressTracker) + return ret0 +} + +// GetCurrentProgressTracker indicates an expected call of GetCurrentProgressTracker. +func (mr *MockSnapshotServiceMockRecorder) GetCurrentProgressTracker() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCurrentProgressTracker", reflect.TypeOf((*MockSnapshotService)(nil).GetCurrentProgressTracker)) +} + // GetProgressTracker mocks base method. func (m *MockSnapshotService) GetProgressTracker() *ports.ProgressTracker { m.ctrl.T.Helper() @@ -1388,3 +1464,55 @@ func (mr *MockSnapshotServiceMockRecorder) Snapshot(dir, destination, options an mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Snapshot", reflect.TypeOf((*MockSnapshotService)(nil).Snapshot), dir, destination, options) } + +// MockColdStarterServerInterface is a mock of ColdStarterServerInterface interface. +type MockColdStarterServerInterface struct { + ctrl *gomock.Controller + recorder *MockColdStarterServerInterfaceMockRecorder + isgomock struct{} +} + +// MockColdStarterServerInterfaceMockRecorder is the mock recorder for MockColdStarterServerInterface. +type MockColdStarterServerInterfaceMockRecorder struct { + mock *MockColdStarterServerInterface +} + +// NewMockColdStarterServerInterface creates a new mock instance. +func NewMockColdStarterServerInterface(ctrl *gomock.Controller) *MockColdStarterServerInterface { + mock := &MockColdStarterServerInterface{ctrl: ctrl} + mock.recorder = &MockColdStarterServerInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockColdStarterServerInterface) EXPECT() *MockColdStarterServerInterfaceMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockColdStarterServerInterface) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockColdStarterServerInterfaceMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockColdStarterServerInterface)(nil).Close)) +} + +// Start mocks base method. +func (m *MockColdStarterServerInterface) Start(port int, onFinish func()) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", port, onFinish) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *MockColdStarterServerInterfaceMockRecorder) Start(port, onFinish any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockColdStarterServerInterface)(nil).Start), port, onFinish) +}