diff --git a/Makefile b/Makefile index 2ddf9453a..5bc7bd24f 100644 --- a/Makefile +++ b/Makefile @@ -295,13 +295,13 @@ test-lid: cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests devnet/up: - rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d + docker compose -f ./docker/devnet/docker-compose.yaml up -d devnet/%: docker compose -f ./docker/devnet/docker-compose.yaml up --build $* -d devnet/down: - docker compose -f ./docker/devnet/docker-compose.yaml down --rmi=local && sleep 2 && rm -rf ./docker/devnet/data + docker compose -f ./docker/devnet/docker-compose.yaml down --rmi=local process?=/bin/bash devnet/exec: diff --git a/cmd/boost/deal_cidgravity_cmd.go b/cmd/boost/deal_cidgravity_cmd.go new file mode 100644 index 000000000..b73df87c9 --- /dev/null +++ b/cmd/boost/deal_cidgravity_cmd.go @@ -0,0 +1,500 @@ +package main + +import ( + "strings" + + bcli "github.com/filecoin-project/boost/cli" + clinode "github.com/filecoin-project/boost/cli/node" + "github.com/filecoin-project/boost/cmd" + "github.com/filecoin-project/boost/storagemarket/types" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/boost/storagemarket/types/legacytypes/network" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + chain_types "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + "github.com/multiformats/go-multiaddr" + "github.com/urfave/cli/v2" +) + +var dealCIDgravityFlags = []cli.Flag{ + &cli.StringFlag{ + Name: "label", + Usage: "label to be specified in the proposal (default: root CID)", + Value: "", + }, + &cli.StringFlag{ + Name: "provider", + Usage: "storage provider on-chain address", + Required: true, + }, + &cli.StringFlag{ + Name: "commp", + Usage: "commp of the CAR file", + Required: true, + }, + &cli.Uint64Flag{ + Name: "piece-size", + Usage: "size of the CAR file as a padded piece", + Required: true, + }, + &cli.Uint64Flag{ + Name: "car-size", + Usage: "size of the CAR file", + Required: true, + }, + &cli.StringFlag{ + Name: "payload-cid", + Usage: "root CID of the CAR file", + Required: true, + }, + &cli.IntFlag{ + Name: "start-epoch", + Usage: "start epoch by when the deal should be proved by provider on-chain", + DefaultText: "current chain head + 2 days", + }, + &cli.IntFlag{ + Name: "duration", + Usage: "duration of the deal in epochs", + Value: 518400, // default is 2880 * 180 == 180 days + }, + &cli.IntFlag{ + Name: "provider-collateral", + Usage: "deal collateral that storage miner must put in escrow; if empty, the min collateral for the given piece size will be used", + }, + &cli.Int64Flag{ + Name: "storage-price", + Usage: "storage price in attoFIL per epoch per GiB", + Value: 1, + }, + &cli.BoolFlag{ + Name: "verified", + Usage: "whether the deal funds should come from verified client data-cap", + Value: true, + }, +} + +type CheckStatus string + +const ( + Available CheckStatus = "available" + Unavailable CheckStatus = "unavailable" + InternalError CheckStatus = "internal_error" + Unknown CheckStatus = "unknown" +) + +type dealCidGravityResponse struct { + Status CheckStatus `json:"status"` + Reason string `json:"reason"` + Message string `json:"message"` + + Multiaddresses []multiaddr.Multiaddr `json:"multiaddresses"` + PeerId peer.ID `json:"peerId"` + + GetAskPricePerGib string `json:"getAskPricePerGib"` + GetAskVerifiedPricePerGib string `json:"getAskVerifiedPricePerGib"` + GetAskMinPieceSize string `json:"getAskMinPieceSize"` + GetAskMaxPieceSize string `json:"getAskMaxPieceSize"` + GetAskSectorSize string `json:"getAskSectorSize"` + + DealProtocolsSupported protocol.ID `json:"dealProtocolsSupported"` +} + +var dealCidGravityCmd = &cli.Command{ + Name: "cidgravity-deal", + Usage: "Make an Keep Alive deal with Boost using CIDgravity proposal format (offline deal)", + Flags: dealCIDgravityFlags, + Before: before, + Action: func(cctx *cli.Context) error { + return dealCidGravityCmdAction(cctx) + }, +} + +func dealCidGravityCmdAction(cctx *cli.Context) error { + ctx := bcli.ReqContext(cctx) + + n, err := clinode.Setup(cctx.String(cmd.FlagRepo.Name)) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_API_GATEWAY", + Message: err.Error(), + }) + } + + api, closer, err := lcli.GetGatewayAPIV1(cctx) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_API_GATEWAY", + Message: err.Error(), + }) + } + defer closer() + + walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet")) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_INVALID_WALLET", + Message: err.Error(), + }) + } + + log.Debugw("selected wallet", "wallet", walletAddr) + + maddr, err := address.NewFromString(cctx.String("provider")) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_INVALID_PROVIDER", + Message: err.Error(), + }) + } + + addrInfo, sectorSize, reason, err := cmd.GetAddrInfoCid(ctx, api, maddr) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: reason, + Message: err.Error(), + Multiaddresses: []multiaddr.Multiaddr{}, + }) + } + + log.Debugw("found storage provider", "id", addrInfo.ID, "multiaddrs", addrInfo.Addrs, "addr", maddr) + + if err := n.Host.Connect(ctx, *addrInfo); err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_CONNECT_MINER_PEER_ID", + Message: err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + }) + } + + x, err := n.Host.Peerstore().FirstSupportedProtocol(addrInfo.ID, DealProtocolv120) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_DEAL_PROTOCOL_UNSUPPORTED", + Message: err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + if len(x) == 0 { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_NO_MATCHING_DEAL_PROTOCOL_SUPPORTED", + Message: "boost client cannot make a deal with storage provider because it does not support protocol version 1.2", + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + dealUuid := uuid.New() + + commP := cctx.String("commp") + pieceCid, err := cid.Parse(commP) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_INVALID_PARAM_COMMP", + Message: "unable to parse commP : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + pieceSize := cctx.Uint64("piece-size") + if pieceSize == 0 { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_INVALID_PARAM_PIECE_SIZE", + Message: "must provide piece-size parameter for CAR url", + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + payloadCidStr := cctx.String("payload-cid") + rootCid, err := cid.Parse(payloadCidStr) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_INVALID_PARAM_PAYLOAD_CID", + Message: "unable to parse payload cid : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + carFileSize := cctx.Uint64("car-size") + if carFileSize == 0 { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_INVALID_PARAM_CAR_SIZE", + Message: "size of car file cannot be 0", + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + transfer := types.Transfer{ + Size: carFileSize, + } + + var providerCollateral abi.TokenAmount + if cctx.IsSet("provider-collateral") { + providerCollateral = abi.NewTokenAmount(cctx.Int64("provider-collateral")) + } else { + bounds, err := api.StateDealProviderCollateralBounds(ctx, abi.PaddedPieceSize(pieceSize), cctx.Bool("verified"), chain_types.EmptyTSK) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_API_GATEWAY_CANNOT_RETRIEVE_MINIMUM_COLLATERAL", + Message: "node error getting collateral bounds : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + providerCollateral = big.Div(big.Mul(bounds.Min, big.NewInt(6)), big.NewInt(5)) // add 20% + } + + var startEpoch abi.ChainEpoch + if cctx.IsSet("start-epoch") { + startEpoch = abi.ChainEpoch(cctx.Int("start-epoch")) + } else { + tipset, err := api.ChainHead(ctx) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_API_GATEWAY_CANNOT_RETRIEVE_CURRENT_CHAIN_HEAD", + Message: "unable to get chain head : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + head := tipset.Height() + + log.Debugw("current block height", "number", head) + + startEpoch = head + abi.ChainEpoch(5760) // head + 2 days + } + + // Create a deal proposal to storage provider using deal protocol v1.2.0 format + label := cctx.String("label") + + // Send a get ask request for the final check + log.Debugw("about to send a get ask request to", "address", maddr.String()) + + streamGetAsk, err := n.Host.NewStream(ctx, addrInfo.ID, AskProtocolID) + + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_GET_ASK", + Message: "failed to open stream to address : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + defer streamGetAsk.Close() + var resp network.AskResponse + + askRequest := network.AskRequest{ + Miner: maddr, + } + + if err := doRpc(ctx, streamGetAsk, &askRequest, &resp); err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_GET_ASK", + Message: "error while send get ask request rpc: " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + ask := resp.Ask.Ask + + log.Debugw("found get ask information", + "Price per GiB", chain_types.FIL(ask.Price), + "Verified Price per GiB", chain_types.FIL(ask.VerifiedPrice), + "Min Piece size", chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + "Max Piece size", chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + "Sector size", sectorSize.ShortString()) + + // To work with CIDgravity, every price must be 0 + if !ask.Price.Equals(abi.NewTokenAmount(0)) || !ask.VerifiedPrice.Equals(abi.NewTokenAmount(0)) { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_GET_ASK_PRICES_NOT_SET_TO_ZERO", + Message: "get-ask price has to be set to 0", + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + // To work with CIDgravity, the min size must be set >= 256B and max size must be set to {sectorSize} + if ask.MinPieceSize > 256 || ask.MaxPieceSize != abi.PaddedPieceSize(*sectorSize) { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_GET_ASK_SIZES_NOT_PROPERLY_SET", + Message: "get-ask accepting size must be min<=256B and max=" + sectorSize.ShortString(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + // Build the proposal + dealProposal, err := dealProposal(ctx, label, n, walletAddr, rootCid, abi.PaddedPieceSize(pieceSize), pieceCid, maddr, startEpoch, cctx.Int("duration"), cctx.Bool("verified"), providerCollateral, abi.NewTokenAmount(cctx.Int64("storage-price"))) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: InternalError, + Reason: "ERR_SEND_PROPOSAL", + Message: "unable to create a deal proposal : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + // Generate the final proposal + dealParams := types.DealParams{ + DealUUID: dealUuid, + ClientDealProposal: *dealProposal, + DealDataRoot: rootCid, + IsOffline: true, + Transfer: transfer, + RemoveUnsealedCopy: true, + SkipIPNIAnnounce: false, + } + + log.Debugw("about to submit deal proposal", "uuid", dealUuid.String()) + + streamSendProposal, err := n.Host.NewStream(ctx, addrInfo.ID, DealProtocolv120) + + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_SEND_PROPOSAL", + Message: "failed to open stream to peer : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + defer streamSendProposal.Close() + + var respDealResponse types.DealResponse + if err := doRpc(ctx, streamSendProposal, &dealParams, &respDealResponse); err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_SEND_PROPOSAL", + Message: "error while send proposal rpc : " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + // For CIDgravity miner-status-check the proposal will be rejected every time + if !respDealResponse.Accepted { + + // So if the message contains the return code from miner-status-check service, we shouldn't consider as error + if strings.Contains(respDealResponse.Message, "CIDgravity miner status check successful") { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Available, + Reason: "DIAGNOSIS_SUCCESS", + Message: "deal proposal successfully sent and received", + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + // Otherwise, return ERR_CIDGRAVITY_CONNECTOR_MISCONFIGURED + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_CIDGRAVITY_CONNECTOR_MISCONFIGURED", + Message: respDealResponse.Message, + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) + } + + // In our case, this block will never be reached (because all proposals will be rejected) + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unknown, + Reason: "SENT", + Message: "this case should normally never happend : unknown", + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + GetAskPricePerGib: chain_types.FIL(ask.Price).String(), + GetAskVerifiedPricePerGib: chain_types.FIL(ask.VerifiedPrice).String(), + GetAskMinPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MinPieceSize))), + GetAskMaxPieceSize: chain_types.SizeStr(chain_types.NewInt(uint64(ask.MaxPieceSize))), + GetAskSectorSize: sectorSize.ShortString(), + }) +} \ No newline at end of file diff --git a/cmd/boost/deal_cmd.go b/cmd/boost/deal_cmd.go index 17ab2c971..dea8b5965 100644 --- a/cmd/boost/deal_cmd.go +++ b/cmd/boost/deal_cmd.go @@ -6,6 +6,10 @@ import ( "errors" "fmt" "strings" + "golang.org/x/xerrors" + "net/http" + "io" + "bytes" bcli "github.com/filecoin-project/boost/cli" clinode "github.com/filecoin-project/boost/cli/node" @@ -27,6 +31,24 @@ import ( ) const DealProtocolv120 = "/fil/storage/mk/1.2.0" +const cidGravityEncryptLabelUrl = "https://service.cidgravity.com/private/v1/get-erc20-encoded-label" + +type cidGravityEncryptLabelPayload struct { + Currency string `json:"currency"` + PieceCID string `json:"pieceCID"` + Price string `json:"price"` +} + +type cidGravityEncryptLabelResponse struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + + Result struct { + EncodedLabel string `json:"encodedLabel"` + } `json:"result"` +} var dealFlags = []cli.Flag{ &cli.StringFlag{ @@ -68,7 +90,7 @@ var dealFlags = []cli.Flag{ }, &cli.Int64Flag{ Name: "storage-price", - Usage: "storage price in attoFIL per epoch per GiB", + Usage: "storage price in attoFIL per epoch per GiB (can be in USDFC/TiB/30d if flag erc20-deal is set)", Value: 1, }, &cli.BoolFlag{ @@ -90,6 +112,16 @@ var dealFlags = []cli.Flag{ Usage: "indicates that deal index should not be announced to the IPNI(Network Indexer)", Value: false, }, + &cli.BoolFlag{ + Name: "erc20-deal", + Usage: "whether the deal should be an ERC20 CIDgravity deal in USDFC/TiB/30d (cidgravity-token flag is required)", + Value: false, + }, + &cli.StringFlag{ + Name: "cidgravity-token", + Usage: "your CIDgravity token to send API requests", + Required: false, + }, } var dealCmd = &cli.Command{ @@ -130,6 +162,10 @@ var offlineDealCmd = &cli.Command{ func dealCmdAction(cctx *cli.Context, isOnline bool) error { ctx := bcli.ReqContext(cctx) + if cctx.Bool("erc20-deal") && cctx.String("cidgravity-token") == "" { + return fmt.Errorf("cidgravity-token must be provided to send ERC20 deals") + } + n, err := clinode.Setup(cctx.String(cmd.FlagRepo.Name)) if err != nil { return err @@ -260,8 +296,32 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error { startEpoch = head + abi.ChainEpoch(5760) // head + 2 days } + // If flag erc20-deal is set to true, we need to encrypt the label with an API call to CIDgravity services + // and use this encrypted label in the proposal otherwise use empty label to use rootCID. + label := "" + + if cctx.Bool("erc20-deal") { + log.Debugw("about to send an API call to CIDgravity to get encrypted label") + + storagePrice := abi.NewTokenAmount(cctx.Int64("storage-price")).String() + encryptedLabel, err := getEncryptedLabel(pieceCid.String(), storagePrice, cctx.String("cidgravity-token")) + if err != nil { + return cmd.PrintJson(dealCidGravityResponse{ + Status: Unavailable, + Reason: "ERR_ENCRYPTED_LABEL", + Message: "failed to get encrypted label: " + err.Error(), + Multiaddresses: addrInfo.Addrs, + PeerId: addrInfo.ID, + DealProtocolsSupported: x, + }) + } + + log.Debugw("retrieved encrypted label from CIDgravity", "encryptedLabel", encryptedLabel) + label = *encryptedLabel + } + // Create a deal proposal to storage provider using deal protocol v1.2.0 format - dealProposal, err := dealProposal(ctx, n, walletAddr, rootCid, abi.PaddedPieceSize(pieceSize), pieceCid, maddr, startEpoch, cctx.Int("duration"), cctx.Bool("verified"), providerCollateral, abi.NewTokenAmount(cctx.Int64("storage-price"))) + dealProposal, err := dealProposal(ctx, label, n, walletAddr, rootCid, abi.PaddedPieceSize(pieceSize), pieceCid, maddr, startEpoch, cctx.Int("duration"), cctx.Bool("verified"), providerCollateral, abi.NewTokenAmount(cctx.Int64("storage-price"))) if err != nil { return fmt.Errorf("failed to create a deal proposal: %w", err) } @@ -333,22 +393,41 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error { return nil } -func dealProposal(ctx context.Context, n *clinode.Node, clientAddr address.Address, rootCid cid.Cid, pieceSize abi.PaddedPieceSize, pieceCid cid.Cid, minerAddr address.Address, startEpoch abi.ChainEpoch, duration int, verified bool, providerCollateral abi.TokenAmount, storagePrice abi.TokenAmount) (*market.ClientDealProposal, error) { +func dealProposal(ctx context.Context, label string, n *clinode.Node, clientAddr address.Address, rootCid cid.Cid, pieceSize abi.PaddedPieceSize, pieceCid cid.Cid, minerAddr address.Address, startEpoch abi.ChainEpoch, duration int, verified bool, providerCollateral abi.TokenAmount, storagePrice abi.TokenAmount) (*market.ClientDealProposal, error) { endEpoch := startEpoch + abi.ChainEpoch(duration) // deal proposal expects total storage price for deal per epoch, therefore we // multiply pieceSize * storagePrice (which is set per epoch per GiB) and divide by 2^30 storagePricePerEpochForDeal := big.Div(big.Mul(big.NewInt(int64(pieceSize)), storagePrice), big.NewInt(int64(1<<30))) - l, err := market.NewLabelFromString(rootCid.String()) - if err != nil { - return nil, err + // If param --label is empty, use root CID + // If not, put the value in Label field (format use for CIDgravity keep-alive service) + var labelForProposal market.DealLabel + + if label != "" { + customLabel, err := market.NewLabelFromString(label) + + if err != nil { + return nil, err + } + + labelForProposal = customLabel + + } else { + l, err := market.NewLabelFromString(rootCid.String()) + + if err != nil { + return nil, err + } + + labelForProposal = l } + proposal := market.DealProposal{ PieceCID: pieceCid, PieceSize: pieceSize, VerifiedDeal: verified, Client: clientAddr, Provider: minerAddr, - Label: l, + Label: labelForProposal, StartEpoch: startEpoch, EndEpoch: endEpoch, StoragePricePerEpoch: storagePricePerEpochForDeal, @@ -394,3 +473,61 @@ func doRpc(ctx context.Context, s inet.Stream, req interface{}, resp interface{} return ctx.Err() } } + +func getEncryptedLabel(pieceCID, storagePrice, cidgravityToken string) (*string, error) { + data := cidGravityEncryptLabelPayload{ + Currency: "usdfc", + PieceCID: pieceCID, + Price: storagePrice, + } + + payload, err := json.Marshal(data) + if err != nil { + return nil, xerrors.Errorf("error encoding payload: %w", err) + } + + // creating http client + client := &http.Client{} + + // creating request + req, err := http.NewRequest("POST", cidGravityEncryptLabelUrl, bytes.NewBuffer(payload)) + if err != nil { + return nil, xerrors.Errorf("error creating request: %w", err) + } + + // set authentication header + req.Header.Set("X-API-KEY", cidgravityToken) + + // execute the request + resp, err := client.Do(req) + if err != nil { + return nil, xerrors.Errorf("error making request: %w", err) + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + log.Errorf("error closing response body: %w", err) + } + }(resp.Body) + + body := new(bytes.Buffer) + _, err = body.ReadFrom(resp.Body) + if err != nil { + return nil, xerrors.Errorf("error reading response body: %w", err) + } + + response := cidGravityEncryptLabelResponse{} + err = json.Unmarshal(body.Bytes(), &response) + if err != nil { + return nil, xerrors.Errorf("error parsing response body: %w", err) + } + + log.Debugw("got response from CIDgravity to get encrypted label", "response", response) + + if resp.StatusCode != http.StatusOK { + return nil, xerrors.Errorf("error from CIDgravity: %s (code: %s)", response.Error.Message, response.Error.Code) + } + + return &response.Result.EncodedLabel, nil +} \ No newline at end of file diff --git a/cmd/boost/main.go b/cmd/boost/main.go index 2fdc45d29..d5efc4310 100644 --- a/cmd/boost/main.go +++ b/cmd/boost/main.go @@ -35,6 +35,7 @@ func main() { Commands: []*cli.Command{ initCmd, dealCmd, + dealCidGravityCmd, dealStatusCmd, retrieveCmd, offlineDealCmd, diff --git a/cmd/util.go b/cmd/util.go index ef6a2835b..7ddc36452 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/libp2p/go-libp2p/core/peer" @@ -39,6 +40,35 @@ func GetAddrInfo(ctx context.Context, api api.Gateway, maddr address.Address) (* }, nil } +func GetAddrInfoCid(ctx context.Context, api api.Gateway, maddr address.Address) (*peer.AddrInfo, *abi.SectorSize, string, error) { + minfo, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return nil, nil, "", err + } + if minfo.PeerId == nil { + return nil, nil, "ERR_NO_PEER_ID_SET_ON_CHAIN", fmt.Errorf("storage provider %s has no peer ID set on-chain", maddr) + } + + var maddrs []multiaddr.Multiaddr + + if len(minfo.Multiaddrs) == 0 { + return nil, nil, "ERR_NO_MULTI_ADDRESS_SET_ON_CHAIN", fmt.Errorf("storage provider %s has no multiaddrs set on-chain", maddr) + } + + for _, mma := range minfo.Multiaddrs { + ma, err := multiaddr.NewMultiaddrBytes(mma) + if err != nil { + return nil, nil, "ERR_INVALID_MULTI_ADDRESS_IN_MINER_INFO", fmt.Errorf("storage provider %s had invalid multiaddrs in their info: %w", maddr, err) + } + maddrs = append(maddrs, ma) + } + + return &peer.AddrInfo{ + ID: *minfo.PeerId, + Addrs: maddrs, + }, &minfo.SectorSize, "", nil +} + func PrintJson(obj interface{}) error { resJson, err := json.MarshalIndent(obj, "", " ") if err != nil { diff --git a/docker/devnet/Dockerfile.source b/docker/devnet/Dockerfile.source index 5664ff2d5..e209a01e2 100644 --- a/docker/devnet/Dockerfile.source +++ b/docker/devnet/Dockerfile.source @@ -13,9 +13,10 @@ COPY gql /src/gql RUN npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react&& \ npm run --prefix react build ######################################################################################### -FROM golang:1.24-bullseye AS builder +FROM golang:1.24.7-bookworm AS builder RUN apt update && apt install -y \ + nodejs \ build-essential \ bzr pkg-config \ clang \ diff --git a/docker/devnet/docker-compose.yaml b/docker/devnet/docker-compose.yaml index 67ec20ae9..d636ff720 100644 --- a/docker/devnet/docker-compose.yaml +++ b/docker/devnet/docker-compose.yaml @@ -17,8 +17,8 @@ services: image: ${LOTUS_IMAGE} init: true ports: - - "1234:1234" - - "9090:9090" + - "2234:1234" # host 2234 -> container 1234 + - "9091:9090" # host 9091 -> container 9090 environment: - LOTUS_FEVM_ENABLEETHRPC=true - LOTUS_API_LISTENADDRESS=/dns/lotus/tcp/1234/http @@ -26,8 +26,8 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/lotus:/var/lib/lotus:rw - - ./data/genesis:/var/lib/genesis:rw + - ./docker/devnet/data/lotus:/var/lib/lotus:rw + - ./docker/devnet/data/genesis:/var/lib/genesis:rw - ${FIL_PROOFS_PARAMETER_CACHE}:/var/tmp/filecoin-proof-parameters:rw lotus-miner: @@ -35,7 +35,7 @@ services: image: ${LOTUS_MINER_IMAGE} init: true ports: - - "2345:2345" + - "3345:2345" # host 3345 -> container 2345 environment: - LOTUS_API_LISTENADDRESS=/dns/lotus-miner/tcp/2345/http - LOTUS_API_REMOTELISTENADDRESS=lotus-miner:2345 @@ -46,9 +46,9 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/lotus-miner:/var/lib/lotus-miner:rw - - ./data/lotus:/var/lib/lotus:ro - - ./data/genesis:/var/lib/genesis:ro + - ./docker/devnet/data/lotus-miner:/var/lib/lotus-miner:rw + - ./docker/devnet/data/lotus:/var/lib/lotus:ro + - ./docker/devnet/data/genesis:/var/lib/genesis:ro - ${FIL_PROOFS_PARAMETER_CACHE}:/var/tmp/filecoin-proof-parameters:rw boost: @@ -56,9 +56,9 @@ services: image: ${BOOST_IMAGE} init: true ports: - - "8080:8080" - - "1288:1288" # For the /metrics endpoint - - "50000:50000" # Exposed libp2p port + - "18080:8080" # host 18080 -> container 8080 + - "21288:1288" # host 21288 -> container 1288 (/metrics) + - "50000:50000" # libp2p environment: - LOTUS_API_LISTENADDRESS=/dns/boost/tcp/1288/http - LOTUS_PATH=/var/lib/lotus @@ -69,18 +69,18 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/boostd-data:/var/lib/boostd-data:rw - - ./data/boost:/var/lib/boost:rw - - ./data/lotus:/var/lib/lotus:ro - - ./data/lotus-miner:/var/lib/lotus-miner:ro - - ./data/sample:/app/public:rw + - ./docker/devnet/data/boostd-data:/var/lib/boostd-data:rw + - ./docker/devnet/data/boost:/var/lib/boost:rw + - ./docker/devnet/data/lotus:/var/lib/lotus:ro + - ./docker/devnet/data/lotus-miner:/var/lib/lotus-miner:ro + - ./docker/devnet/data/sample:/app/public:rw booster-http: container_name: booster-http image: ${BOOSTER_HTTP_IMAGE} init: true ports: - - "7777:7777" + - "8777:7777" # host 8777 -> container 7777 environment: - BOOST_PATH=/var/lib/boost - LID_API_INFO=http://boost:8044 @@ -89,16 +89,16 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/boost:/var/lib/boost:ro - - ./data/lotus:/var/lib/lotus:ro - - ./data/lotus-miner:/var/lib/lotus-miner:ro + - ./docker/devnet/data/boost:/var/lib/boost:ro + - ./docker/devnet/data/lotus:/var/lib/lotus:ro + - ./docker/devnet/data/lotus-miner:/var/lib/lotus-miner:ro booster-bitswap: container_name: booster-bitswap image: ${BOOSTER_BITSWAP_IMAGE} init: true ports: - - "8888:8888" + - "9888:8888" # host 9888 -> container 8888 environment: - BOOSTER_BITSWAP_REPO=/var/lib/booster-bitswap - BOOST_PATH=/var/lib/boost @@ -108,10 +108,10 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/booster-bitswap:/var/lib/booster-bitswap:rw - - ./data/boost:/var/lib/boost:ro - - ./data/lotus:/var/lib/lotus:ro - - ./data/lotus-miner:/var/lib/lotus-miner:ro + - ./docker/devnet/data/booster-bitswap:/var/lib/booster-bitswap:rw + - ./docker/devnet/data/boost:/var/lib/boost:ro + - ./docker/devnet/data/lotus:/var/lib/lotus:ro + - ./docker/devnet/data/lotus-miner:/var/lib/lotus-miner:ro demo-http-server: container_name: demo-http-server @@ -120,7 +120,7 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/sample:/usr/share/nginx/html:ro + - ./docker/devnet/data/sample:/usr/share/nginx/html:ro yugabytedb: container_name: yugabytedb @@ -133,5 +133,5 @@ services: restart: unless-stopped logging: *default-logging volumes: - - ./data/yugabytedb-data:/root/var/data - - ./data/yugabytedb-logs:/root/var/logs + - ./docker/devnet/data/yugabytedb-data:/root/var/data + - ./docker/devnet/data/yugabytedb-logs:/root/var/logs