Flow-based MetaLocker ledger

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.22
+          go-version: 1.23
 
       - name: Update packages list
         run: sudo apt-get update
@@ -63,7 +63,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.22
+          go-version: 1.23
       - uses: actions/cache@v3
         with:
           path: |
@@ -75,7 +75,7 @@ jobs:
       - uses: zencargo/github-action-go-mod-tidy@v1
         with:
           path: .
-          go-version: 1.22
+          go-version: 1.23
 
   test:
     runs-on: ubuntu-latest
@@ -89,7 +89,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.22
+          go-version: 1.23
       - name: Set up Cache
         uses: actions/cache@v3
         with:

.gitignore

@@ -33,3 +33,6 @@ vendor
 /coverage.txt
 
 dist/
+
+# local Flow emulator files
+/ledger/onflow/flowdb

cmd/lockerd/modules.go

@@ -22,6 +22,7 @@ import (
 	_ "github.com/piprate/metalocker/index/bolt"
 
 	_ "github.com/piprate/metalocker/ledger/local"
+	_ "github.com/piprate/metalocker/ledger/onflow"
 
 	_ "github.com/piprate/metalocker/storage/memory"
 

cmd/metalo/actions/accounts.go

@@ -40,7 +40,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-func createPersona(name, idType string, firstBlock int64) (*account.Identity, error) {
+func createPersona(name, idType string, firstBlock uint64) (*account.Identity, error) {
 	did, err := model.GenerateDID()
 	if err != nil {
 		return nil, err

cmd/metalo/actions/admin.go

@@ -105,6 +105,29 @@ func ImportBackendData(c *cli.Context) error {
 	return nil
 }
 
+func ImportLedger(c *cli.Context) error {
+	if c.Args().Len() != 1 {
+		return cli.Exit("please specify the path to data folder", InvalidParameter)
+	}
+	folderPath := utils.AbsPathify(c.Args().Get(0))
+
+	configFilePath := c.String("config")
+	if configFilePath == "" {
+		return cli.Exit("please specify the path to MetaLocker configuration file using --config flag", InvalidParameter)
+	}
+
+	importOperations := c.Bool("import-operations")
+	preserveBlocks := c.Bool("preserve-blocks")
+	waitForConfirmation := c.Bool("wait")
+
+	err := operations.ImportLedger(c.Context, folderPath, configFilePath, importOperations, preserveBlocks, waitForConfirmation)
+	if err != nil {
+		log.Err(err).Msg("Ledger import failed")
+		return cli.Exit(err, OperationFailed)
+	}
+	return nil
+}
+
 func UpdateAccountState(c *cli.Context) error {
 	if c.Args().Len() != 1 {
 		fmt.Print("Please specify the account ID (email or DID) to block.\n\n")

cmd/metalo/actions/definitions.go

@@ -642,10 +642,19 @@ var (
 			Usage:  "import MetaLocker ledger data from the given directory",
 			Action: ImportLedger,
 			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name:  "config",
+					Value: "",
+					Usage: "path to MetaLocker configuration file",
+				},
 				&cli.BoolFlag{
 					Name:  "import-operations",
 					Usage: "import operations (be careful if swapping ledgers!)",
 				},
+				&cli.BoolFlag{
+					Name:  "preserve-blocks",
+					Usage: "import blocks while preserving their composition and numbering (if possible)",
+				},
 				&cli.BoolFlag{
 					Name:  "wait",
 					Usage: "If specified, wait until each block is published on the ledger",

cmd/metalo/actions/ledger.go

@@ -40,30 +40,3 @@ func ExportLedger(c *cli.Context) error {
 	}
 	return err
 }
-
-func ImportLedger(c *cli.Context) error {
-	if c.Args().Len() != 1 {
-		fmt.Print("Please specify the path to file or folder.\n\n")
-		return cli.Exit("please specify the path to file or folder", InvalidParameter)
-	}
-
-	importOperations := c.Bool("import-operations")
-	waitForConfirmation := c.Bool("wait")
-
-	dw, err := LoadRemoteDataWallet(c, false)
-	if err != nil {
-		return err
-	}
-
-	ns, err := dw.Services().NotificationService()
-	if err != nil {
-		return err
-	}
-
-	err = operations.ImportLedger(c.Context, dw.Services().Ledger(), dw.Services().OffChainStorage(), ns, c.Args().Get(0), importOperations, waitForConfirmation)
-	if err != nil {
-		log.Err(err).Msg("Ledger import failed")
-		return cli.Exit(err, OperationFailed)
-	}
-	return nil
-}

cmd/metalo/modules.go

@@ -22,6 +22,9 @@ import (
 
 	_ "github.com/piprate/metalocker/storage/memory"
 	_ "github.com/piprate/metalocker/storage/rdb"
+
+	_ "github.com/piprate/metalocker/ledger/local"
+	_ "github.com/piprate/metalocker/ledger/onflow"
 )
 
 func init() {

cmd/metalo/operations/export_ledger.go

@@ -35,7 +35,7 @@ func ExportLedger(ctx context.Context, ledger model.Ledger, offChainStorage mode
 	if err != nil {
 		return err
 	}
-	log.Info().Int64("number", tb.Number).Msg("Top block")
+	log.Info().Uint64("number", tb.Number).Msg("Top block")
 
 	currentBlock := gb.Number
 	blockBatchSize := 10
@@ -83,9 +83,9 @@ func ExportLedger(ctx context.Context, ledger model.Ledger, offChainStorage mode
 }
 
 func SaveBlock(ctx context.Context, ledger model.Ledger, offChainStorage model.OffChainStorage, basePath string, b *model.Block) error {
-	log.Info().Int64("number", b.Number).Msg("Saving block")
+	log.Info().Uint64("number", b.Number).Msg("Saving block")
 
-	dest := path.Join(basePath, utils.Int64ToString(b.Number))
+	dest := path.Join(basePath, utils.Uint64ToString(b.Number))
 	err := os.MkdirAll(dest, 0o700)
 	if err != nil {
 		return err
@@ -124,7 +124,7 @@ func SaveBlock(ctx context.Context, ledger model.Ledger, offChainStorage model.O
 
 		recs = append(recs, rec)
 
-		if rec.Operation == model.OpTypeLease {
+		if rec.Operation == model.OpTypeLease && rec.Status == model.StatusPublished {
 			opRecBytes, err := offChainStorage.GetOperation(ctx, rec.OperationAddress)
 			if err != nil {
 				return err