feat: allways bootstrap on init snapshot

.env.example

@@ -0,0 +1,4 @@
+DRUID_S3_ACCESS_KEY=
+DRUID_S3_SECRET_KEY=
+DRUID_S3_BUCKET=druid-testing
+DRUID_S3_ENDPOINT=fsn1

.gitignore

@@ -13,4 +13,6 @@ debug**
 
 druid-cli-test
 
-!.docker/**
+!.docker/**
+
+.env

.vscode/launch.json

@@ -11,9 +11,10 @@
             "args": [
                 "backup",
                 "--cwd",
-                "${workspaceFolder}/examples/backup-test",
-                "https://gist.github.com/jlmaccal/ca6b08a307800f80986661075c82c432/raw/dbd5e7a38d0660b7047476de33da142bd79c22ef/test.tgz"
+                "${workspaceFolder}/examples/minecraft",
+                "testtest.tgz"
             ],
+            "envFile": "${workspaceFolder}/.env",
         },
         {
             "name": "Debug Daemon restore",
@@ -104,8 +105,6 @@
                 "--watch-ports",
                 "-p",
                 "9190",
-                "--max-health-check-startup-timeount",
-                "10",
             ],
         },
         {

cmd/serve.go

@@ -370,7 +370,7 @@ func startup(scrollService *services.ScrollService, snapshotService ports.Snapsh
 
 func initScroll(scrollService *services.ScrollService, snapshotService ports.SnapshotService, processLauncher *services.ProcedureLauncher, queueManager *services.QueueManager) (bool, error) {
 
-	lock, err := scrollService.Bootstrap(ignoreVersionCheck)
+	lock, err := scrollService.ReloadLock(ignoreVersionCheck)
 	if err != nil {
 		return false, err
 	}
@@ -380,19 +380,22 @@ func initScroll(scrollService *services.ScrollService, snapshotService ports.Sna
 	if newScroll {
 
 		if initSnapshotUrl != "" {
-			logger.Log().Info("Starting restore process")
+			logger.Log().Info("Starting restore process from Init Snapshot")
 			err := snapshotService.RestoreSnapshot(scrollService.GetCwd(), initSnapshotUrl, ports.RestoreSnapshotOptions{})
 
 			if err != nil {
 				return false, err
 			}
 
+			logger.Log().Info("Restore process from Init Snapshot done")
+
 			currentScroll, err := scrollService.ReloadScroll()
 			if err != nil {
 				return false, err
 			}
+			logger.Log().Info("Reloaded scroll after restore", zap.String("Name", currentScroll.Name), zap.Any("Version", currentScroll.Version), zap.String("AppVersion", currentScroll.AppVersion), zap.Any("Ports", currentScroll.Ports))
 
-			lock, err = scrollService.Bootstrap(ignoreVersionCheck)
+			lock, err = scrollService.ReloadLock(ignoreVersionCheck)
 			if err != nil {
 				return false, err
 			}
@@ -403,18 +406,16 @@ func initScroll(scrollService *services.ScrollService, snapshotService ports.Sna
 			if !newScroll {
 				lock.SetStatus(currentScroll.Init, domain.ScrollLockStatusWaiting, nil)
 			}
-		}
 
-		if newScroll {
-			logger.Log().Info("No lock file found, but init command available. Bootstrapping...")
+		}
 
-			logger.Log().Info("Creating lock and bootstrapping files")
-			//There is an error here. We need to bootstrap the files before we render out the templates in the bootstrap func above
-			err := scrollService.CreateLockAndBootstrapFiles()
-			if err != nil {
-				return newScroll, err
-			}
+		logger.Log().Info("Creating lock and bootstrapping files")
+		//There is an error here. We need to bootstrap the files before we render out the templates in the bootstrap func above
+		err := scrollService.CopyingInitFiles()
+		if err != nil {
+			return newScroll, err
 		}
+
 	} else {
 		logger.Log().Info("Found lock file, bootstrapping done")
 	}

internal/core/ports/services_ports.go

@@ -34,6 +34,8 @@ type ScrollServiceInterface interface {
 	WriteNewScrollLock() *domain.ScrollLock
 	GetLock() (*domain.ScrollLock, error)
 	GetCommand(cmd string) (*domain.CommandInstructionSet, error)
+	InitFiles(files ...string) error
+	InitTemplateFiles(files ...string) error
 }
 
 type ProcedureLauchnerInterface interface {
@@ -84,7 +86,7 @@ type ProcessMonitorInterface interface {
 
 type TemplateRendererInterface interface {
 	RenderTemplate(templatePath string, data interface{}) (string, error)
-	RenderScrollTemplateFiles(templateFiles []string, data interface{}, ouputPath string) error
+	RenderScrollTemplateFiles(templateBase string, templateFiles []string, data interface{}, ouputPath string) error
 }
 
 type OciRegistryInterface interface {

internal/core/services/scroll_service.go

@@ -3,8 +3,10 @@ package services
 import (
 	"errors"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 
 	"github.com/highcard-dev/daemon/internal/core/domain"
@@ -51,7 +53,7 @@ func (sc *ScrollService) ReloadScroll() (*domain.Scroll, error) {
 }
 
 // Load Scroll and render templates in the cwd
-func (sc *ScrollService) Bootstrap(ignoreVersionCheck bool) (*domain.ScrollLock, error) {
+func (sc *ScrollService) ReloadLock(ignoreVersionCheck bool) (*domain.ScrollLock, error) {
 
 	var scroll = sc.scroll
 
@@ -73,73 +75,79 @@ func (sc *ScrollService) Bootstrap(ignoreVersionCheck bool) (*domain.ScrollLock,
 	return lock, nil
 
 }
-func (sc *ScrollService) CreateLockAndBootstrapFiles() error {
 
-	//init-files just get copied over
-	initPath := strings.TrimRight(sc.GetDir(), "/") + "/init-files"
-	exist, _ := utils.FileExists(initPath)
-	if exist {
-		err := filepath.Walk(initPath, func(path string, f os.FileInfo, err error) error {
-			strippedPath := strings.TrimPrefix(filepath.Clean(path), filepath.Clean(initPath))
-			realPath := filepath.Join(sc.processCwd, strippedPath)
-			if f.IsDir() {
-				if strippedPath == "" {
-					return nil
-				}
-				err := os.MkdirAll(realPath, f.Mode())
-				if err != nil {
-					return err
-				}
-			} else {
-
-				b, err := os.ReadFile(path)
-
-				if err != nil {
-					return err
-				}
-
-				return os.WriteFile(realPath, b, f.Mode())
-			}
+func (sc *ScrollService) InitFiles(fls ...string) error {
+	//init-files-template needs to be rendered
+	files, err := sc.filterFiles("init-files", fls...)
+	if err != nil {
+		return err
+	}
+	initFileDir := path.Join(sc.GetDir(), "init-files")
 
-			return err
-		})
+	for _, file := range files {
+		basePath := strings.TrimPrefix(file, initFileDir)
+		dest := path.Join(sc.processCwd, basePath)
+
+		err := utils.CopyFile(file, dest)
 		if err != nil {
 			return err
 		}
 	}
+	return nil
+}
+
+func (sc *ScrollService) filterFiles(path string, fls ...string) ([]string, error) {
 	//init-files-template needs to be rendered
-	initPath = strings.TrimRight(sc.GetDir(), "/") + "/init-files-template"
-	exist, _ = utils.FileExists(initPath)
+	initPath := strings.TrimRight(sc.GetDir(), "/") + "/" + path + "/"
+	exist, _ := utils.FileExists(initPath)
 
 	files := []string{}
 
 	if exist {
 		err := filepath.Walk(initPath, func(path string, f os.FileInfo, err error) error {
-			if f.IsDir() {
-				err := os.MkdirAll(path, f.Mode())
-				if err != nil {
-					return err
-				}
-			} else {
+			basePath := strings.TrimPrefix(path, initPath)
+			if !f.IsDir() && (slices.Contains(fls, basePath) || len(fls) == 0) {
 				files = append(files, path)
 			}
-
 			return nil
 		})
-		if len(files) == 0 {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
 
-		err = sc.templateRenderer.RenderScrollTemplateFiles(files, sc.scroll, sc.processCwd)
 		if err != nil {
-			return err
+			return []string{}, err
 		}
+
+		return files, nil
 	}
 
-	return nil
+	return []string{}, nil
+}
+
+func (sc *ScrollService) InitTemplateFiles(fls ...string) error {
+
+	//init-files-template needs to be rendered
+	files, err := sc.filterFiles("init-files-template", fls...)
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		return nil
+	}
+	templateBase := path.Join(sc.GetDir(), "init-files-template")
+
+	for i, file := range files {
+		basePath := strings.TrimPrefix(file, templateBase)
+		files[i] = basePath
+	}
+
+	return sc.templateRenderer.RenderScrollTemplateFiles(templateBase, files, sc.scroll, sc.processCwd)
+}
+
+func (sc *ScrollService) CopyingInitFiles() error {
+	err := sc.InitFiles()
+	if err != nil {
+		return err
+	}
+	return sc.InitTemplateFiles()
 }
 
 func (sc *ScrollService) LockExists() bool {
@@ -217,7 +225,7 @@ func (s ScrollService) RenderCwdTemplates() error {
 
 	config := TemplateData{Config: s.GetScrollConfig()}
 
-	return s.templateRenderer.RenderScrollTemplateFiles(files, config, "")
+	return s.templateRenderer.RenderScrollTemplateFiles("", files, config, "")
 
 }
 

internal/core/services/snapshot/progress.go

@@ -2,52 +2,27 @@ package services
 
 import (
 	"fmt"
-	"io"
 
 	"github.com/highcard-dev/daemon/internal/utils/logger"
 	"go.uber.org/zap"
 )
 
-type ProgressTracker struct {
-	reader      io.ReadCloser
-	read        int64
-	fileSize    int64
-	lastPercent float64
+type GeneralProgressTracker struct {
+	total int64
+	read  int64
 }
 
-func (pr *ProgressTracker) Read(p []byte) (int, error) {
-	n, err := pr.reader.Read(p)
-	pr.read += int64(n)
-
-	// Calculate current percentage of upload progress
-	currentPercent := (float64(pr.read) * 100) / float64(pr.fileSize)
-
-	// Update progress if we've moved at least 0.1% or it's been more than the update frequency since the last update
-	if currentPercent > pr.lastPercent+0.1 {
-		logger.Log().Info("Snapshot operation progress", zap.String("percentage", fmt.Sprintf("%.1f%%", currentPercent)), zap.String("read", fmt.Sprintf("%d/%d", pr.read, pr.fileSize)))
-		pr.lastPercent = currentPercent
-	}
-
-	// If the upload is finished
-	if pr.read == pr.fileSize {
-		logger.Log().Info("Snapshot operation complete")
+func NewGeneralProgressTracker(total int64) *GeneralProgressTracker {
+	return &GeneralProgressTracker{
+		total: total,
+		read:  0,
 	}
-
-	return n, err
-}
-
-func (pr *ProgressTracker) Close() error {
-	pr.lastPercent = 100
-	return pr.reader.Close()
 }
 
-func (pt *ProgressTracker) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
-	pt.reader = stream
-	pt.fileSize = totalSize
-	logger.Log().Info("Snapshot operation started", zap.String("source", src), zap.String("size", fmt.Sprintf("%d bytes", totalSize)))
-	return pt
+func (pt *GeneralProgressTracker) GetPercent() float64 {
+	return (float64(pt.read) / float64(pt.total)) * 100
 }
-
-func (pt *ProgressTracker) GetPercent() float64 {
-	return pt.lastPercent
+func (pt *GeneralProgressTracker) TrackProgress() {
+	pt.read++
+	logger.Log().Info("Progress", zap.Int64("total", pt.total), zap.Int64("read", pt.read), zap.String("percentage", fmt.Sprintf("%.1f%%", pt.GetPercent())))
 }

internal/core/services/snapshot/progress_reader.go

@@ -0,0 +1,57 @@
+package services
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/highcard-dev/daemon/internal/utils"
+	"github.com/highcard-dev/daemon/internal/utils/logger"
+	"go.uber.org/zap"
+)
+
+type ProgressTracker struct {
+	reader      io.ReadCloser
+	read        int64
+	fileSize    int64
+	lastPercent float64
+}
+
+func (pr *ProgressTracker) Read(p []byte) (int, error) {
+	n, err := pr.reader.Read(p)
+	pr.read += int64(n)
+
+	// Calculate current percentage of upload progress
+	currentPercent := (float64(pr.read) * 100) / float64(pr.fileSize)
+
+	sizeHuman := utils.HumanizeBytes(pr.fileSize)
+	readHuman := utils.HumanizeBytes(pr.read)
+
+	// Update progress if we've moved at least 0.1% or it's been more than the update frequency since the last update
+	if currentPercent > pr.lastPercent+0.1 {
+		logger.Log().Info("Snapshot operation progress", zap.String("percentage", fmt.Sprintf("%.1f%%", currentPercent)), zap.String("read", fmt.Sprintf("%s/%s", readHuman, sizeHuman)))
+		pr.lastPercent = currentPercent
+	}
+
+	// If the upload is finished
+	if pr.read == pr.fileSize {
+		logger.Log().Info("Snapshot operation complete")
+	}
+
+	return n, err
+}
+
+func (pr *ProgressTracker) Close() error {
+	pr.lastPercent = 100
+	return pr.reader.Close()
+}
+
+func (pt *ProgressTracker) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
+	pt.reader = stream
+	pt.fileSize = totalSize
+	logger.Log().Info("Snapshot operation started", zap.String("source", src), zap.String("size", fmt.Sprintf("%d bytes", totalSize)))
+	return pt
+}
+
+func (pt *ProgressTracker) GetPercent() float64 {
+	return pt.lastPercent
+}