diff --git a/cmd/export_assets.go b/cmd/export_assets.go index 47f43f53..3a082418 100644 --- a/cmd/export_assets.go +++ b/cmd/export_assets.go @@ -42,7 +42,7 @@ var assetsCmd = &cobra.Command{ totalNumBytes := 0 var transformedAssets []transform.SchemaParquet for _, transformInput := range paymentOps { - transformed, err := transform.TransformAsset(transformInput.Operation, transformInput.OperationIndex, transformInput.TransactionIndex, transformInput.LedgerSeqNum, transformInput.LedgerCloseMeta) + transformed, err := transform.TransformAsset(transformInput.Operation, transformInput.OperationIndex, transformInput.TransactionIndex, transformInput.LedgerSeqNum, transformInput.LedgerCloseMeta, env.NetworkPassphrase) if err != nil { txIndex := transformInput.TransactionIndex cmdLogger.LogError(fmt.Errorf("could not extract asset from operation %d in transaction %d in ledger %d: ", transformInput.OperationIndex, txIndex, transformInput.LedgerSeqNum)) diff --git a/cmd/export_ledger_entry_changes.go b/cmd/export_ledger_entry_changes.go index d3c6badd..14a3528c 100644 --- a/cmd/export_ledger_entry_changes.go +++ b/cmd/export_ledger_entry_changes.go @@ -167,7 +167,7 @@ be exported.`, continue } for i, change := range changes.Changes { - balance, err := transform.TransformClaimableBalance(change, changes.LedgerHeaders[i]) + balance, err := transform.TransformClaimableBalance(change, changes.LedgerHeaders[i], env.NetworkPassphrase) if err != nil { entry, _, _, _ := utils.ExtractEntryFromChange(change) cmdLogger.LogError(fmt.Errorf("error transforming balance entry last updated at %d: %s", entry.LastModifiedLedgerSeq, err)) @@ -180,7 +180,7 @@ be exported.`, continue } for i, change := range changes.Changes { - offer, err := transform.TransformOffer(change, changes.LedgerHeaders[i]) + offer, err := transform.TransformOffer(change, changes.LedgerHeaders[i], env.NetworkPassphrase) if err != nil { entry, _, _, _ := utils.ExtractEntryFromChange(change) cmdLogger.LogError(fmt.Errorf("error transforming offer entry last updated at %d: %s", entry.LastModifiedLedgerSeq, err)) @@ -193,7 +193,7 @@ be exported.`, continue } for i, change := range changes.Changes { - trust, err := transform.TransformTrustline(change, changes.LedgerHeaders[i]) + trust, err := transform.TransformTrustline(change, changes.LedgerHeaders[i], env.NetworkPassphrase) if err != nil { entry, _, _, _ := utils.ExtractEntryFromChange(change) cmdLogger.LogError(fmt.Errorf("error transforming trustline entry last updated at %d: %s", entry.LastModifiedLedgerSeq, err)) @@ -206,7 +206,7 @@ be exported.`, continue } for i, change := range changes.Changes { - pool, err := transform.TransformPool(change, changes.LedgerHeaders[i]) + pool, err := transform.TransformPool(change, changes.LedgerHeaders[i], env.NetworkPassphrase) if err != nil { entry, _, _, _ := utils.ExtractEntryFromChange(change) cmdLogger.LogError(fmt.Errorf("error transforming liquidity pool entry last updated at %d: %s", entry.LastModifiedLedgerSeq, err)) diff --git a/cmd/export_trades.go b/cmd/export_trades.go index 5c8101de..78918a95 100644 --- a/cmd/export_trades.go +++ b/cmd/export_trades.go @@ -35,7 +35,7 @@ var tradesCmd = &cobra.Command{ totalNumBytes := 0 var transformedTrades []transform.SchemaParquet for _, tradeInput := range trades { - trades, err := transform.TransformTrade(tradeInput.OperationIndex, tradeInput.OperationHistoryID, tradeInput.Transaction, tradeInput.CloseTime) + trades, err := transform.TransformTrade(tradeInput.OperationIndex, tradeInput.OperationHistoryID, tradeInput.Transaction, tradeInput.CloseTime, env.NetworkPassphrase) if err != nil { parsedID := toid.Parse(tradeInput.OperationHistoryID) cmdLogger.LogError(fmt.Errorf("from ledger %d, transaction %d, operation %d: %v", parsedID.LedgerSequence, parsedID.TransactionOrder, parsedID.OperationOrder, err)) diff --git a/internal/input/orderbooks.go b/internal/input/orderbooks.go index d5239911..d438967c 100644 --- a/internal/input/orderbooks.go +++ b/internal/input/orderbooks.go @@ -34,9 +34,9 @@ type OrderbookParser struct { } // convertOffer converts an offer to its normalized form and adds it to the AllConvertedOffers -func (o *OrderbookParser) convertOffer(allConvertedOffers []transform.NormalizedOfferOutput, index int, offer ingest.Change, seq uint32, wg *sync.WaitGroup) { +func (o *OrderbookParser) convertOffer(allConvertedOffers []transform.NormalizedOfferOutput, index int, offer ingest.Change, seq uint32, wg *sync.WaitGroup, passphrase string) { defer wg.Done() - transformed, err := transform.TransformOfferNormalized(offer, seq) + transformed, err := transform.TransformOfferNormalized(offer, seq, passphrase) if err != nil { errorMsg := fmt.Errorf("error json marshalling offer #%d in ledger sequence number #%d: %s", index, seq, err) o.Logger.LogError(errorMsg) @@ -59,12 +59,12 @@ func NewOrderbookParser(logger *utils.EtlLogger) OrderbookParser { } } -func (o *OrderbookParser) parseOrderbook(orderbook []ingest.Change, seq uint32) { +func (o *OrderbookParser) parseOrderbook(orderbook []ingest.Change, seq uint32, passphrase string) { var group sync.WaitGroup allConverted := make([]transform.NormalizedOfferOutput, len(orderbook)) for i, v := range orderbook { group.Add(1) - go o.convertOffer(allConverted, i, v, seq, &group) + go o.convertOffer(allConverted, i, v, seq, &group, passphrase) } group.Wait() @@ -237,7 +237,7 @@ func StreamOrderbooks(core *ledgerbackend.CaptiveStellarCore, start, end, batchS } // ReceiveParsedOrderbooks reads a batch from the orderbookChannel, parses it using an orderbook parser, and returns the parser. -func ReceiveParsedOrderbooks(orderbookChannel chan OrderbookBatch, logger *utils.EtlLogger) *OrderbookParser { +func ReceiveParsedOrderbooks(orderbookChannel chan OrderbookBatch, logger *utils.EtlLogger, passphrase string) *OrderbookParser { batchParser := NewOrderbookParser(logger) batchRead := false for { @@ -250,7 +250,7 @@ func ReceiveParsedOrderbooks(orderbookChannel chan OrderbookBatch, logger *utils } for seq, orderbook := range batch.Orderbooks { - batchParser.parseOrderbook(orderbook, seq) + batchParser.parseOrderbook(orderbook, seq, passphrase) } batchRead = true diff --git a/internal/transform/asset.go b/internal/transform/asset.go index 1e0e3b81..50d383bc 100644 --- a/internal/transform/asset.go +++ b/internal/transform/asset.go @@ -7,11 +7,12 @@ import ( "github.com/stellar/stellar-etl/v2/internal/toid" "github.com/stellar/stellar-etl/v2/internal/utils" + "github.com/stellar/go/strkey" "github.com/stellar/go/xdr" ) // TransformAsset converts an asset from a payment operation into a form suitable for BigQuery -func TransformAsset(operation xdr.Operation, operationIndex int32, transactionIndex int32, ledgerSeq int32, lcm xdr.LedgerCloseMeta) (AssetOutput, error) { +func TransformAsset(operation xdr.Operation, operationIndex int32, transactionIndex int32, ledgerSeq int32, lcm xdr.LedgerCloseMeta, passphrase string) (AssetOutput, error) { operationID := toid.New(ledgerSeq, int32(transactionIndex), operationIndex).ToInt64() opType := operation.Body.Type @@ -37,7 +38,7 @@ func TransformAsset(operation xdr.Operation, operationIndex int32, transactionIn } - outputAsset, err := transformSingleAsset(asset) + outputAsset, err := transformSingleAsset(asset, passphrase) if err != nil { return AssetOutput{}, fmt.Errorf("%s (id %d)", err.Error(), operationID) } @@ -52,7 +53,7 @@ func TransformAsset(operation xdr.Operation, operationIndex int32, transactionIn return outputAsset, nil } -func transformSingleAsset(asset xdr.Asset) (AssetOutput, error) { +func transformSingleAsset(asset xdr.Asset, passphrase string) (AssetOutput, error) { var outputAssetType, outputAssetCode, outputAssetIssuer string err := asset.Extract(&outputAssetType, &outputAssetCode, &outputAssetIssuer) if err != nil { @@ -61,11 +62,22 @@ func transformSingleAsset(asset xdr.Asset) (AssetOutput, error) { farmAssetID := FarmHashAsset(outputAssetCode, outputAssetIssuer, outputAssetType) + contractIdByte, err := asset.ContractID(passphrase) + if err != nil { + return AssetOutput{}, err + } + + contractId, err := strkey.Encode(strkey.VersionByteContract, contractIdByte[:]) + if err != nil { + return AssetOutput{}, err + } + return AssetOutput{ AssetCode: outputAssetCode, AssetIssuer: outputAssetIssuer, AssetType: outputAssetType, AssetID: farmAssetID, + ContractId: contractId, }, nil } diff --git a/internal/transform/asset_test.go b/internal/transform/asset_test.go index 56ac0da7..d1ea26d7 100644 --- a/internal/transform/asset_test.go +++ b/internal/transform/asset_test.go @@ -18,7 +18,8 @@ func TestTransformAsset(t *testing.T) { index int32 txnIndex int32 // transaction xdr.TransactionEnvelope - lcm xdr.LedgerCloseMeta + lcm xdr.LedgerCloseMeta + passphrase string } type transformTest struct { @@ -49,17 +50,19 @@ func TestTransformAsset(t *testing.T) { for i, op := range hardCodedInputTransaction.Envelope.Operations() { tests = append(tests, transformTest{ input: assetInput{ - operation: op, - index: int32(i), - txnIndex: int32(i), - lcm: genericLedgerCloseMeta}, + operation: op, + index: int32(i), + txnIndex: int32(i), + lcm: genericLedgerCloseMeta, + passphrase: "test passphrase", + }, wantOutput: hardCodedOutputArray[i], wantErr: nil, }) } for _, test := range tests { - actualOutput, actualError := TransformAsset(test.input.operation, test.input.index, test.input.txnIndex, 0, test.input.lcm) + actualOutput, actualError := TransformAsset(test.input.operation, test.input.index, test.input.txnIndex, 0, test.input.lcm, test.input.passphrase) assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } @@ -110,6 +113,7 @@ func makeAssetTestOutput() (transformedAssets []AssetOutput) { AssetID: -8205667356306085451, ClosedAt: time.Date(1970, time.January, 1, 0, 0, 10, 0, time.UTC), LedgerSequence: 2, + ContractId: "CCJCSP2CMLAWALLQ4ZBPML2EPYNN7AIC5COPC43PSDR6HZ3ZCVXUNL5M", }, { AssetCode: "", @@ -118,6 +122,7 @@ func makeAssetTestOutput() (transformedAssets []AssetOutput) { AssetID: -5706705804583548011, ClosedAt: time.Date(1970, time.January, 1, 0, 0, 10, 0, time.UTC), LedgerSequence: 2, + ContractId: "CCCVYPOBCE4ZKFBTNRI465A7N2AT3YMLEWXEY5LN76O6NYCHYXYPVXY7", }, } return diff --git a/internal/transform/claimable_balance.go b/internal/transform/claimable_balance.go index c5256b4d..83f79f76 100644 --- a/internal/transform/claimable_balance.go +++ b/internal/transform/claimable_balance.go @@ -21,7 +21,7 @@ func transformClaimants(claimants []xdr.Claimant) []Claimant { } // TransformClaimableBalance converts a claimable balance from the history archive ingestion system into a form suitable for BigQuery -func TransformClaimableBalance(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry) (ClaimableBalanceOutput, error) { +func TransformClaimableBalance(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry, passphrase string) (ClaimableBalanceOutput, error) { ledgerEntry, changeType, outputDeleted, err := utils.ExtractEntryFromChange(ledgerChange) if err != nil { return ClaimableBalanceOutput{}, err @@ -40,7 +40,7 @@ func TransformClaimableBalance(ledgerChange ingest.Change, header xdr.LedgerHead balanceIDStrkey := balanceEntry.BalanceId.MustEncodeToStrkey() outputFlags := uint32(balanceEntry.Flags()) - outputAsset, err := transformSingleAsset(balanceEntry.Asset) + outputAsset, err := transformSingleAsset(balanceEntry.Asset, passphrase) if err != nil { return ClaimableBalanceOutput{}, err } @@ -72,6 +72,7 @@ func TransformClaimableBalance(ledgerChange ingest.Change, header xdr.LedgerHead ClosedAt: closedAt, LedgerSequence: uint32(ledgerSequence), BalanceIDStrkey: balanceIDStrkey, + ContractId: outputAsset.ContractId, } return transformed, nil } diff --git a/internal/transform/claimable_balance_test.go b/internal/transform/claimable_balance_test.go index e2a58bad..8aeea53f 100644 --- a/internal/transform/claimable_balance_test.go +++ b/internal/transform/claimable_balance_test.go @@ -43,7 +43,7 @@ func TestTransformClaimableBalance(t *testing.T) { LedgerSeq: 10, }, } - actualOutput, actualError := TransformClaimableBalance(test.input.ingest, header) + actualOutput, actualError := TransformClaimableBalance(test.input.ingest, header, "test passphrase") assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } @@ -128,5 +128,6 @@ func makeClaimableBalanceTestOutput() ClaimableBalanceOutput { LedgerSequence: 10, ClosedAt: time.Date(1970, time.January, 1, 0, 16, 40, 0, time.UTC), BalanceIDStrkey: "BAAACAQDAQCQMBYIBEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACPGI", + ContractId: "CAVDLQWFTGGUWJ4DDFOAQBZYVMJUXJDJGTCFP5NLL4JWH2ITE2VWCUML", } } diff --git a/internal/transform/effects.go b/internal/transform/effects.go index b0f64bb5..f1b1a32c 100644 --- a/internal/transform/effects.go +++ b/internal/transform/effects.go @@ -433,7 +433,7 @@ func (e *effectsWrapper) addPaymentEffects() { op := e.operation.operation.Body.MustPaymentOp() details := map[string]interface{}{"amount": amount.String(op.Amount)} - addAssetDetails(details, op.Asset, "") + addAssetDetails(details, op.Asset, "", e.operation.network) e.addMuxed( &op.Destination, @@ -453,7 +453,7 @@ func (e *effectsWrapper) pathPaymentStrictReceiveEffects() error { source := e.operation.SourceAccount() details := map[string]interface{}{"amount": amount.String(op.DestAmount)} - addAssetDetails(details, op.DestAsset, "") + addAssetDetails(details, op.DestAsset, "", e.operation.network) e.addMuxed( &op.Destination, @@ -463,7 +463,7 @@ func (e *effectsWrapper) pathPaymentStrictReceiveEffects() error { result := e.operation.OperationResult().MustPathPaymentStrictReceiveResult() details = map[string]interface{}{"amount": amount.String(result.SendAmount())} - addAssetDetails(details, op.SendAsset, "") + addAssetDetails(details, op.SendAsset, "", e.operation.network) e.addMuxed( source, @@ -481,11 +481,11 @@ func (e *effectsWrapper) addPathPaymentStrictSendEffects() error { result := e.operation.OperationResult().MustPathPaymentStrictSendResult() details := map[string]interface{}{"amount": amount.String(result.DestAmount())} - addAssetDetails(details, op.DestAsset, "") + addAssetDetails(details, op.DestAsset, "", e.operation.network) e.addMuxed(&op.Destination, EffectAccountCredited, details) details = map[string]interface{}{"amount": amount.String(op.SendAmount)} - addAssetDetails(details, op.SendAsset, "") + addAssetDetails(details, op.SendAsset, "", e.operation.network) e.addMuxed(source, EffectAccountDebited, details) return e.addIngestTradeEffects(*source, resultSuccess.Offers, true) @@ -687,7 +687,7 @@ func (e *effectsWrapper) addChangeTrustEffects() error { return err } } else { - addAssetDetails(details, op.Line.ToAsset(), "") + addAssetDetails(details, op.Line.ToAsset(), "", e.operation.network) } e.addMuxed(source, effect, details) @@ -704,7 +704,7 @@ func (e *effectsWrapper) addAllowTrustEffects() error { details := map[string]interface{}{ "trustor": op.Trustor.Address(), } - addAssetDetails(details, asset, "") + addAssetDetails(details, asset, "", e.operation.network) switch { case xdr.TrustLineFlags(op.Authorize).IsAuthorized(): @@ -851,7 +851,7 @@ func (e *effectsWrapper) addCreateClaimableBalanceEffects(changes []ingest.Chang details := map[string]interface{}{ "amount": amount.String(cb.Amount), } - addAssetDetails(details, cb.Asset, "") + addAssetDetails(details, cb.Asset, "", e.operation.network) e.addMuxed( source, EffectAccountDebited, @@ -955,7 +955,7 @@ func (e *effectsWrapper) addClaimClaimableBalanceEffects(changes []ingest.Change details = map[string]interface{}{ "amount": amount.String(cBalance.Amount), } - addAssetDetails(details, cBalance.Asset, "") + addAssetDetails(details, cBalance.Asset, "", e.operation.network) e.addMuxed( source, EffectAccountCredited, @@ -984,7 +984,7 @@ func (e *effectsWrapper) addIngestTradeEffects(buyer xdr.MuxedAccount, claims [] func (e *effectsWrapper) addClaimTradeEffects(buyer xdr.MuxedAccount, claim xdr.ClaimAtom, isPathPayment bool) { seller := claim.SellerId() - bd, sd := tradeDetails(buyer, seller, claim) + bd, sd := tradeDetails(buyer, seller, claim, e.operation.network) tradeEffects := []EffectType{ EffectTrade, @@ -1039,7 +1039,7 @@ func (e *effectsWrapper) addClawbackEffects() error { "amount": amount.String(op.Amount), } source := e.operation.SourceAccount() - addAssetDetails(details, op.Asset, "") + addAssetDetails(details, op.Asset, "", e.operation.network) // The funds will be burned, but even with that, we generated an account credited effect e.addMuxed( @@ -1078,7 +1078,7 @@ func (e *effectsWrapper) addClawbackClaimableBalanceEffects(changes []ingest.Cha if c.Type == xdr.LedgerEntryTypeClaimableBalance && c.Post == nil && c.Pre != nil { cb := c.Pre.Data.ClaimableBalance details = map[string]interface{}{"amount": amount.String(cb.Amount)} - addAssetDetails(details, cb.Asset, "") + addAssetDetails(details, cb.Asset, "", e.operation.network) e.addMuxed( source, EffectAccountCredited, @@ -1107,7 +1107,7 @@ func (e *effectsWrapper) addTrustLineFlagsEffect( details := map[string]interface{}{ "trustor": trustor.Address(), } - addAssetDetails(details, asset, "") + addAssetDetails(details, asset, "", e.operation.network) var flagDetailsAdded bool if setFlags != nil { @@ -1226,15 +1226,15 @@ func setAuthFlagDetails(flagDetails map[string]interface{}, flags xdr.AccountFla } } -func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimAtom) (bd map[string]interface{}, sd map[string]interface{}) { +func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimAtom, passphrase string) (bd map[string]interface{}, sd map[string]interface{}) { bd = map[string]interface{}{ "offer_id": claim.OfferId(), "seller": seller.Address(), "bought_amount": amount.String(claim.AmountSold()), "sold_amount": amount.String(claim.AmountBought()), } - addAssetDetails(bd, claim.AssetSold(), "bought_") - addAssetDetails(bd, claim.AssetBought(), "sold_") + addAssetDetails(bd, claim.AssetSold(), "bought_", passphrase) + addAssetDetails(bd, claim.AssetBought(), "sold_", passphrase) sd = map[string]interface{}{ "offer_id": claim.OfferId(), @@ -1242,8 +1242,8 @@ func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimA "sold_amount": amount.String(claim.AmountSold()), } addAccountAndMuxedAccountDetails(sd, buyer, "seller") - addAssetDetails(sd, claim.AssetBought(), "bought_") - addAssetDetails(sd, claim.AssetSold(), "sold_") + addAssetDetails(sd, claim.AssetBought(), "bought_", passphrase) + addAssetDetails(sd, claim.AssetSold(), "sold_", passphrase) return } @@ -1332,7 +1332,7 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev } details := make(map[string]interface{}, 4) - addAssetDetails(details, evt.GetAsset(), "") + addAssetDetails(details, evt.GetAsset(), "", e.operation.network) // // Note: We ignore effects that involve contracts (until the day we have diff --git a/internal/transform/liquidity_pool.go b/internal/transform/liquidity_pool.go index fee8b0b4..28a52339 100644 --- a/internal/transform/liquidity_pool.go +++ b/internal/transform/liquidity_pool.go @@ -10,7 +10,7 @@ import ( ) // TransformPool converts an liquidity pool ledger change entry into a form suitable for BigQuery -func TransformPool(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry) (PoolOutput, error) { +func TransformPool(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry, passphrase string) (PoolOutput, error) { ledgerEntry, changeType, outputDeleted, err := utils.ExtractEntryFromChange(ledgerChange) if err != nil { return PoolOutput{}, err @@ -36,19 +36,15 @@ func TransformPool(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEnt return PoolOutput{}, fmt.Errorf("unknown liquidity pool type: %d", lp.Body.Type) } - var assetAType, assetACode, assetAIssuer string - err = cp.Params.AssetA.Extract(&assetAType, &assetACode, &assetAIssuer) + assetAOutput, err := transformSingleAsset(cp.Params.AssetA, passphrase) if err != nil { return PoolOutput{}, err } - assetAID := FarmHashAsset(assetACode, assetAIssuer, assetAType) - var assetBType, assetBCode, assetBIssuer string - err = cp.Params.AssetB.Extract(&assetBType, &assetBCode, &assetBIssuer) + assetBOutput, err := transformSingleAsset(cp.Params.AssetB, passphrase) if err != nil { return PoolOutput{}, err } - assetBID := FarmHashAsset(assetBCode, assetBIssuer, assetBType) closedAt, err := utils.TimePointToUTCTimeStamp(header.Header.ScpValue.CloseTime) if err != nil { @@ -69,15 +65,15 @@ func TransformPool(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEnt PoolFee: uint32(cp.Params.Fee), TrustlineCount: uint64(cp.PoolSharesTrustLineCount), PoolShareCount: utils.ConvertStroopValueToReal(cp.TotalPoolShares), - AssetAType: assetAType, - AssetACode: assetACode, - AssetAIssuer: assetAIssuer, - AssetAID: assetAID, + AssetAType: assetAOutput.AssetType, + AssetACode: assetAOutput.AssetCode, + AssetAIssuer: assetAOutput.AssetIssuer, + AssetAID: assetAOutput.AssetID, AssetAReserve: utils.ConvertStroopValueToReal(cp.ReserveA), - AssetBType: assetBType, - AssetBCode: assetBCode, - AssetBIssuer: assetBIssuer, - AssetBID: assetBID, + AssetBType: assetBOutput.AssetType, + AssetBCode: assetBOutput.AssetCode, + AssetBIssuer: assetBOutput.AssetIssuer, + AssetBID: assetBOutput.AssetID, AssetBReserve: utils.ConvertStroopValueToReal(cp.ReserveB), LastModifiedLedger: uint32(ledgerEntry.LastModifiedLedgerSeq), LedgerEntryChange: uint32(changeType), @@ -85,6 +81,8 @@ func TransformPool(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEnt ClosedAt: closedAt, LedgerSequence: uint32(ledgerSequence), PoolIDStrkey: poolIDStrkey, + AssetAContractId: assetAOutput.ContractId, + AssetBContractId: assetBOutput.ContractId, } return transformedPool, nil } diff --git a/internal/transform/liquidity_pool_test.go b/internal/transform/liquidity_pool_test.go index 7343113b..0a236aa9 100644 --- a/internal/transform/liquidity_pool_test.go +++ b/internal/transform/liquidity_pool_test.go @@ -56,7 +56,7 @@ func TestTransformPool(t *testing.T) { LedgerSeq: 10, }, } - actualOutput, actualError := TransformPool(test.input.ingest, header) + actualOutput, actualError := TransformPool(test.input.ingest, header, "test passphrase") assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } @@ -117,5 +117,7 @@ func makePoolTestOutput() PoolOutput { LedgerSequence: 10, ClosedAt: time.Date(1970, time.January, 1, 0, 16, 40, 0, time.UTC), PoolIDStrkey: "LALS2QYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC2X", + AssetAContractId: "CCCVYPOBCE4ZKFBTNRI465A7N2AT3YMLEWXEY5LN76O6NYCHYXYPVXY7", + AssetBContractId: "CBJ7HQBIYV65AQ236O43L3LEOUU3AJEXJQD7SCUFCHVMSLC3JXDTUG6Y", } } diff --git a/internal/transform/offer.go b/internal/transform/offer.go index 1f6e6cd5..d5efc0e6 100644 --- a/internal/transform/offer.go +++ b/internal/transform/offer.go @@ -10,7 +10,7 @@ import ( ) // TransformOffer converts an account from the history archive ingestion system into a form suitable for BigQuery -func TransformOffer(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry) (OfferOutput, error) { +func TransformOffer(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry, passphrase string) (OfferOutput, error) { ledgerEntry, changeType, outputDeleted, err := utils.ExtractEntryFromChange(ledgerChange) if err != nil { return OfferOutput{}, err @@ -31,12 +31,12 @@ func TransformOffer(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEn return OfferOutput{}, fmt.Errorf("offerID is negative (%d) for offer from account: %s", outputOfferID, outputSellerID) } - outputSellingAsset, err := transformSingleAsset(offerEntry.Selling) + outputSellingAsset, err := transformSingleAsset(offerEntry.Selling, passphrase) if err != nil { return OfferOutput{}, err } - outputBuyingAsset, err := transformSingleAsset(offerEntry.Buying) + outputBuyingAsset, err := transformSingleAsset(offerEntry.Buying, passphrase) if err != nil { return OfferOutput{}, err } @@ -77,27 +77,29 @@ func TransformOffer(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEn ledgerSequence := header.Header.LedgerSeq transformedOffer := OfferOutput{ - SellerID: outputSellerID, - OfferID: outputOfferID, - SellingAssetType: outputSellingAsset.AssetType, - SellingAssetCode: outputSellingAsset.AssetCode, - SellingAssetIssuer: outputSellingAsset.AssetIssuer, - SellingAssetID: outputSellingAsset.AssetID, - BuyingAssetType: outputBuyingAsset.AssetType, - BuyingAssetCode: outputBuyingAsset.AssetCode, - BuyingAssetIssuer: outputBuyingAsset.AssetIssuer, - BuyingAssetID: outputBuyingAsset.AssetID, - Amount: utils.ConvertStroopValueToReal(outputAmount), - PriceN: outputPriceN, - PriceD: outputPriceD, - Price: outputPrice, - Flags: outputFlags, - LastModifiedLedger: outputLastModifiedLedger, - LedgerEntryChange: uint32(changeType), - Deleted: outputDeleted, - Sponsor: ledgerEntrySponsorToNullString(ledgerEntry), - ClosedAt: closedAt, - LedgerSequence: uint32(ledgerSequence), + SellerID: outputSellerID, + OfferID: outputOfferID, + SellingAssetType: outputSellingAsset.AssetType, + SellingAssetCode: outputSellingAsset.AssetCode, + SellingAssetIssuer: outputSellingAsset.AssetIssuer, + SellingAssetID: outputSellingAsset.AssetID, + BuyingAssetType: outputBuyingAsset.AssetType, + BuyingAssetCode: outputBuyingAsset.AssetCode, + BuyingAssetIssuer: outputBuyingAsset.AssetIssuer, + BuyingAssetID: outputBuyingAsset.AssetID, + Amount: utils.ConvertStroopValueToReal(outputAmount), + PriceN: outputPriceN, + PriceD: outputPriceD, + Price: outputPrice, + Flags: outputFlags, + LastModifiedLedger: outputLastModifiedLedger, + LedgerEntryChange: uint32(changeType), + Deleted: outputDeleted, + Sponsor: ledgerEntrySponsorToNullString(ledgerEntry), + ClosedAt: closedAt, + LedgerSequence: uint32(ledgerSequence), + SellingAssetContractId: outputSellingAsset.ContractId, + BuyingAssetContractId: outputBuyingAsset.ContractId, } return transformedOffer, nil } diff --git a/internal/transform/offer_normalized.go b/internal/transform/offer_normalized.go index 69f1500f..e9ddb4c6 100644 --- a/internal/transform/offer_normalized.go +++ b/internal/transform/offer_normalized.go @@ -13,10 +13,10 @@ import ( ) // TransformOfferNormalized converts an offer into a normalized form, allowing it to be stored as part of the historical orderbook dataset -func TransformOfferNormalized(ledgerChange ingest.Change, ledgerSeq uint32) (NormalizedOfferOutput, error) { +func TransformOfferNormalized(ledgerChange ingest.Change, ledgerSeq uint32, passphrase string) (NormalizedOfferOutput, error) { var header xdr.LedgerHeaderHistoryEntry - transformed, err := TransformOffer(ledgerChange, header) + transformed, err := TransformOffer(ledgerChange, header, passphrase) if err != nil { return NormalizedOfferOutput{}, err } diff --git a/internal/transform/offer_normalized_test.go b/internal/transform/offer_normalized_test.go index c52b1520..a39cc052 100644 --- a/internal/transform/offer_normalized_test.go +++ b/internal/transform/offer_normalized_test.go @@ -12,8 +12,9 @@ import ( func TestTransformOfferNormalized(t *testing.T) { type testInput struct { - change ingest.Change - ledger uint32 + change ingest.Change + ledger uint32 + passphrase string } type transformTest struct { input testInput @@ -44,19 +45,19 @@ func TestTransformOfferNormalized(t *testing.T) { }, }, Post: nil, - }, 100}, + }, 100, "test passphrase"}, wantOutput: NormalizedOfferOutput{}, wantErr: fmt.Errorf("offer 0 is deleted"), }, { - input: testInput{hardCodedInput, 100}, + input: testInput{hardCodedInput, 100, "test passphrase"}, wantOutput: hardCodedOutput, wantErr: nil, }, } for _, test := range tests { - actualOutput, actualError := TransformOfferNormalized(test.input.change, test.input.ledger) + actualOutput, actualError := TransformOfferNormalized(test.input.change, test.input.ledger, test.input.passphrase) assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } diff --git a/internal/transform/offer_test.go b/internal/transform/offer_test.go index fd7c2daa..62d0f400 100644 --- a/internal/transform/offer_test.go +++ b/internal/transform/offer_test.go @@ -106,7 +106,7 @@ func TestTransformOffer(t *testing.T) { LedgerSeq: 10, }, } - actualOutput, actualError := TransformOffer(test.input.ingest, header) + actualOutput, actualError := TransformOffer(test.input.ingest, header, "test passphrase") assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } @@ -162,26 +162,28 @@ func makeOfferTestInput() (ledgerChange ingest.Change, err error) { func makeOfferTestOutput() OfferOutput { return OfferOutput{ - SellerID: testAccount1Address, - OfferID: 260678439, - SellingAssetType: "native", - SellingAssetCode: "", - SellingAssetIssuer: "", - SellingAssetID: -5706705804583548011, - BuyingAssetType: "credit_alphanum4", - BuyingAssetCode: "ETH", - BuyingAssetIssuer: testAccount3Address, - BuyingAssetID: 4476940172956910889, - Amount: 262.8450327, - PriceN: 920936891, - PriceD: 1790879058, - Price: 0.5142373444404865, - Flags: 2, - LastModifiedLedger: 30715263, - LedgerEntryChange: 2, - Deleted: true, - Sponsor: null.StringFrom(testAccount3Address), - LedgerSequence: 10, - ClosedAt: time.Date(1970, time.January, 1, 0, 16, 40, 0, time.UTC), + SellerID: testAccount1Address, + OfferID: 260678439, + SellingAssetType: "native", + SellingAssetCode: "", + SellingAssetIssuer: "", + SellingAssetID: -5706705804583548011, + BuyingAssetType: "credit_alphanum4", + BuyingAssetCode: "ETH", + BuyingAssetIssuer: testAccount3Address, + BuyingAssetID: 4476940172956910889, + Amount: 262.8450327, + PriceN: 920936891, + PriceD: 1790879058, + Price: 0.5142373444404865, + Flags: 2, + LastModifiedLedger: 30715263, + LedgerEntryChange: 2, + Deleted: true, + Sponsor: null.StringFrom(testAccount3Address), + LedgerSequence: 10, + ClosedAt: time.Date(1970, time.January, 1, 0, 16, 40, 0, time.UTC), + SellingAssetContractId: "CCCVYPOBCE4ZKFBTNRI465A7N2AT3YMLEWXEY5LN76O6NYCHYXYPVXY7", + BuyingAssetContractId: "CDTDW4NDEV7UNDX4P35Q2OCAP2EJCBFPIWQLNGXV2LNTOPRQ6AMFNGQ3", } } diff --git a/internal/transform/operation.go b/internal/transform/operation.go index edde2609..4c2b2a22 100644 --- a/internal/transform/operation.go +++ b/internal/transform/operation.go @@ -364,7 +364,7 @@ func formatPrefix(p string) string { return p } -func addAssetDetailsToOperationDetails(result map[string]interface{}, asset xdr.Asset, prefix string) error { +func addAssetDetailsToOperationDetails(result map[string]interface{}, asset xdr.Asset, prefix string, passphrase string) error { var assetType, code, issuer string err := asset.Extract(&assetType, &code, &issuer) if err != nil { @@ -372,8 +372,14 @@ func addAssetDetailsToOperationDetails(result map[string]interface{}, asset xdr. } prefix = formatPrefix(prefix) - result[prefix+"asset_type"] = assetType + outputAsset, err := transformSingleAsset(asset, passphrase) + if err != nil { + return err + } + result[prefix+"asset_contract_id"] = outputAsset.ContractId + + result[prefix+"asset_type"] = assetType if asset.Type == xdr.AssetTypeAssetTypeNative { result[prefix+"asset_id"] = int64(-5706705804583548011) return nil @@ -509,7 +515,7 @@ func addLedgerKeyToDetails(result map[string]interface{}, ledgerKey xdr.LedgerKe return nil } -func transformPath(initialPath []xdr.Asset) []Path { +func transformPath(initialPath []xdr.Asset, passphrase string) []Path { if len(initialPath) == 0 { return nil } @@ -521,10 +527,21 @@ func transformPath(initialPath []xdr.Asset) []Path { return nil } + contractIdByte, err := pathAsset.ContractID(passphrase) + if err != nil { + return nil + } + + contractId, err := strkey.Encode(strkey.VersionByteContract, contractIdByte[:]) + if err != nil { + return nil + } + path = append(path, Path{ AssetType: assetType, AssetIssuer: issuer, AssetCode: code, + ContractId: contractId, }) } return path @@ -612,7 +629,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT return details, err } details["amount"] = utils.ConvertStroopValueToReal(op.Amount) - if err := addAssetDetailsToOperationDetails(details, op.Asset, ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Asset, "", network); err != nil { return details, err } @@ -631,10 +648,10 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT details["amount"] = utils.ConvertStroopValueToReal(op.DestAmount) details["source_amount"] = amount.String(0) details["source_max"] = utils.ConvertStroopValueToReal(op.SendMax) - if err := addAssetDetailsToOperationDetails(details, op.DestAsset, ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.DestAsset, "", network); err != nil { return details, err } - if err := addAssetDetailsToOperationDetails(details, op.SendAsset, "source"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.SendAsset, "source", network); err != nil { return details, err } @@ -655,7 +672,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT details["source_amount"] = utils.ConvertStroopValueToReal(result.SendAmount()) } - details["path"] = transformPath(op.Path) + details["path"] = transformPath(op.Path, network) case xdr.OperationTypePathPaymentStrictSend: op, ok := operation.Body.GetPathPaymentStrictSendOp() @@ -672,10 +689,10 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT details["amount"] = amount.String(0) details["source_amount"] = utils.ConvertStroopValueToReal(op.SendAmount) details["destination_min"] = amount.String(op.DestMin) - if err := addAssetDetailsToOperationDetails(details, op.DestAsset, ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.DestAsset, "", network); err != nil { return details, err } - if err := addAssetDetailsToOperationDetails(details, op.SendAsset, "source"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.SendAsset, "source", network); err != nil { return details, err } @@ -696,7 +713,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT details["amount"] = utils.ConvertStroopValueToReal(result.DestAmount()) } - details["path"] = transformPath(op.Path) + details["path"] = transformPath(op.Path, network) case xdr.OperationTypeManageBuyOffer: op, ok := operation.Body.GetManageBuyOfferOp() @@ -710,10 +727,10 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT return details, err } - if err := addAssetDetailsToOperationDetails(details, op.Buying, "buying"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Buying, "buying", network); err != nil { return details, err } - if err := addAssetDetailsToOperationDetails(details, op.Selling, "selling"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Selling, "selling", network); err != nil { return details, err } @@ -729,10 +746,10 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT return details, err } - if err := addAssetDetailsToOperationDetails(details, op.Buying, "buying"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Buying, "buying", network); err != nil { return details, err } - if err := addAssetDetailsToOperationDetails(details, op.Selling, "selling"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Selling, "selling", network); err != nil { return details, err } @@ -747,10 +764,10 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT return details, err } - if err := addAssetDetailsToOperationDetails(details, op.Buying, "buying"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Buying, "buying", network); err != nil { return details, err } - if err := addAssetDetailsToOperationDetails(details, op.Selling, "selling"); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Selling, "selling", network); err != nil { return details, err } @@ -808,7 +825,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT return details, err } } else { - if err := addAssetDetailsToOperationDetails(details, op.Line.ToAsset(), ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Line.ToAsset(), "", network); err != nil { return details, err } details["trustee"] = details["asset_issuer"] @@ -825,7 +842,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT return details, fmt.Errorf("could not access AllowTrust info for this operation (index %d)", operationIndex) } - if err := addAssetDetailsToOperationDetails(details, op.Asset.ToAsset(sourceAccount.ToAccountId()), ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Asset.ToAsset(sourceAccount.ToAccountId()), "", network); err != nil { return details, err } if err := addAccountAndMuxedAccountDetails(details, sourceAccount, "trustee"); err != nil { @@ -923,7 +940,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT case xdr.OperationTypeClawback: op := operation.Body.MustClawbackOp() - if err := addAssetDetailsToOperationDetails(details, op.Asset, ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Asset, "", network); err != nil { return details, err } if err := addAccountAndMuxedAccountDetails(details, op.From, "from"); err != nil { @@ -943,7 +960,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT case xdr.OperationTypeSetTrustLineFlags: op := operation.Body.MustSetTrustLineFlagsOp() details["trustor"] = op.Trustor.Address() - if err := addAssetDetailsToOperationDetails(details, op.Asset, ""); err != nil { + if err := addAssetDetailsToOperationDetails(details, op.Asset, "", network); err != nil { return details, err } if op.SetFlags > 0 { @@ -984,7 +1001,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT } // Process ReserveA Details - if err := addAssetDetailsToOperationDetails(details, assetA, "reserve_a"); err != nil { + if err := addAssetDetailsToOperationDetails(details, assetA, "reserve_a", network); err != nil { return details, err } details["reserve_a_max_amount"] = utils.ConvertStroopValueToReal(op.MaxAmountA) @@ -995,7 +1012,7 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT details["reserve_a_deposit_amount"] = depositA //Process ReserveB Details - if err := addAssetDetailsToOperationDetails(details, assetB, "reserve_b"); err != nil { + if err := addAssetDetailsToOperationDetails(details, assetB, "reserve_b", network); err != nil { return details, err } details["reserve_b_max_amount"] = utils.ConvertStroopValueToReal(op.MaxAmountB) @@ -1045,14 +1062,14 @@ func extractOperationDetails(operation xdr.Operation, transaction ingest.LedgerT receivedA, receivedB = -delta.ReserveA, -delta.ReserveB } // Process AssetA Details - if err := addAssetDetailsToOperationDetails(details, assetA, "reserve_a"); err != nil { + if err := addAssetDetailsToOperationDetails(details, assetA, "reserve_a", network); err != nil { return details, err } details["reserve_a_min_amount"] = utils.ConvertStroopValueToReal(op.MinAmountA) details["reserve_a_withdraw_amount"] = utils.ConvertStroopValueToReal(receivedA) // Process AssetB Details - if err := addAssetDetailsToOperationDetails(details, assetB, "reserve_b"); err != nil { + if err := addAssetDetailsToOperationDetails(details, assetB, "reserve_b", network); err != nil { return details, err } details["reserve_b_min_amount"] = utils.ConvertStroopValueToReal(op.MinAmountB) @@ -1375,7 +1392,7 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, addAccountAndMuxedAccountDetails(details, *source, "from") addAccountAndMuxedAccountDetails(details, op.Destination, "to") details["amount"] = amount.String(op.Amount) - addAssetDetails(details, op.Asset, "") + addAssetDetails(details, op.Asset, "", operation.network) case xdr.OperationTypePathPaymentStrictReceive: op := operation.operation.Body.MustPathPaymentStrictReceiveOp() addAccountAndMuxedAccountDetails(details, *source, "from") @@ -1384,8 +1401,8 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, details["amount"] = amount.String(op.DestAmount) details["source_amount"] = amount.String(0) details["source_max"] = amount.String(op.SendMax) - addAssetDetails(details, op.DestAsset, "") - addAssetDetails(details, op.SendAsset, "source_") + addAssetDetails(details, op.DestAsset, "", operation.network) + addAssetDetails(details, op.SendAsset, "source_", operation.network) if operation.transaction.Result.Successful() { result := operation.OperationResult().MustPathPaymentStrictReceiveResult() @@ -1395,7 +1412,7 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, var path = make([]map[string]interface{}, len(op.Path)) for i := range op.Path { path[i] = make(map[string]interface{}) - addAssetDetails(path[i], op.Path[i], "") + addAssetDetails(path[i], op.Path[i], "", operation.network) } details["path"] = path @@ -1407,8 +1424,8 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, details["amount"] = amount.String(0) details["source_amount"] = amount.String(op.SendAmount) details["destination_min"] = amount.String(op.DestMin) - addAssetDetails(details, op.DestAsset, "") - addAssetDetails(details, op.SendAsset, "source_") + addAssetDetails(details, op.DestAsset, "", operation.network) + addAssetDetails(details, op.SendAsset, "source_", operation.network) if operation.transaction.Result.Successful() { result := operation.OperationResult().MustPathPaymentStrictSendResult() @@ -1418,7 +1435,7 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, var path = make([]map[string]interface{}, len(op.Path)) for i := range op.Path { path[i] = make(map[string]interface{}) - addAssetDetails(path[i], op.Path[i], "") + addAssetDetails(path[i], op.Path[i], "", operation.network) } details["path"] = path case xdr.OperationTypeManageBuyOffer: @@ -1430,8 +1447,8 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, "n": op.Price.N, "d": op.Price.D, } - addAssetDetails(details, op.Buying, "buying_") - addAssetDetails(details, op.Selling, "selling_") + addAssetDetails(details, op.Buying, "buying_", operation.network) + addAssetDetails(details, op.Selling, "selling_", operation.network) case xdr.OperationTypeManageSellOffer: op := operation.operation.Body.MustManageSellOfferOp() details["offer_id"] = op.OfferId @@ -1441,8 +1458,8 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, "n": op.Price.N, "d": op.Price.D, } - addAssetDetails(details, op.Buying, "buying_") - addAssetDetails(details, op.Selling, "selling_") + addAssetDetails(details, op.Buying, "buying_", operation.network) + addAssetDetails(details, op.Selling, "selling_", operation.network) case xdr.OperationTypeCreatePassiveSellOffer: op := operation.operation.Body.MustCreatePassiveSellOfferOp() details["amount"] = amount.String(op.Amount) @@ -1451,8 +1468,8 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, "n": op.Price.N, "d": op.Price.D, } - addAssetDetails(details, op.Buying, "buying_") - addAssetDetails(details, op.Selling, "selling_") + addAssetDetails(details, op.Buying, "buying_", operation.network) + addAssetDetails(details, op.Selling, "selling_", operation.network) case xdr.OperationTypeSetOptions: op := operation.operation.Body.MustSetOptionsOp() @@ -1499,14 +1516,14 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, return nil, err } } else { - addAssetDetails(details, op.Line.ToAsset(), "") + addAssetDetails(details, op.Line.ToAsset(), "", operation.network) details["trustee"] = details["asset_issuer"] } addAccountAndMuxedAccountDetails(details, *source, "trustor") details["limit"] = amount.String(op.Limit) case xdr.OperationTypeAllowTrust: op := operation.operation.Body.MustAllowTrustOp() - addAssetDetails(details, op.Asset.ToAsset(source.ToAccountId()), "") + addAssetDetails(details, op.Asset.ToAsset(source.ToAccountId()), "", operation.network) addAccountAndMuxedAccountDetails(details, *source, "trustee") details["trustor"] = op.Trustor.Address() details["authorize"] = xdr.TrustLineFlags(op.Authorize).IsAuthorized() @@ -1578,7 +1595,7 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, } case xdr.OperationTypeClawback: op := operation.operation.Body.MustClawbackOp() - addAssetDetails(details, op.Asset, "") + addAssetDetails(details, op.Asset, "", operation.network) addAccountAndMuxedAccountDetails(details, op.From, "from") details["amount"] = amount.String(op.Amount) case xdr.OperationTypeClawbackClaimableBalance: @@ -1592,7 +1609,7 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, case xdr.OperationTypeSetTrustLineFlags: op := operation.operation.Body.MustSetTrustLineFlagsOp() details["trustor"] = op.Trustor.Address() - addAssetDetails(details, op.Asset, "") + addAssetDetails(details, op.Asset, "", operation.network) if op.SetFlags > 0 { addTrustLineFlagDetails(details, xdr.TrustLineFlags(op.SetFlags), "set") } @@ -1922,16 +1939,16 @@ func (operation *transactionOperationWrapper) parseAssetBalanceChangesFromContra switch sacEvent.GetType() { case contractevents.EventTypeTransfer: transferEvt := sacEvent.(*contractevents.TransferEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(transferEvt.From, transferEvt.To, transferEvt.Amount, transferEvt.Asset, "transfer")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(transferEvt.From, transferEvt.To, transferEvt.Amount, transferEvt.Asset, "transfer", operation.network)) case contractevents.EventTypeMint: mintEvt := sacEvent.(*contractevents.MintEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry("", mintEvt.To, mintEvt.Amount, mintEvt.Asset, "mint")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry("", mintEvt.To, mintEvt.Amount, mintEvt.Asset, "mint", operation.network)) case contractevents.EventTypeClawback: clawbackEvt := sacEvent.(*contractevents.ClawbackEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(clawbackEvt.From, "", clawbackEvt.Amount, clawbackEvt.Asset, "clawback")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(clawbackEvt.From, "", clawbackEvt.Amount, clawbackEvt.Asset, "clawback", operation.network)) case contractevents.EventTypeBurn: burnEvt := sacEvent.(*contractevents.BurnEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(burnEvt.From, "", burnEvt.Amount, burnEvt.Asset, "burn")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(burnEvt.From, "", burnEvt.Amount, burnEvt.Asset, "burn", operation.network)) } } } @@ -1957,16 +1974,16 @@ func parseAssetBalanceChangesFromContractEvents(transaction ingest.LedgerTransac switch sacEvent.GetType() { case contractevents.EventTypeTransfer: transferEvt := sacEvent.(*contractevents.TransferEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(transferEvt.From, transferEvt.To, transferEvt.Amount, transferEvt.Asset, "transfer")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(transferEvt.From, transferEvt.To, transferEvt.Amount, transferEvt.Asset, "transfer", network)) case contractevents.EventTypeMint: mintEvt := sacEvent.(*contractevents.MintEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry("", mintEvt.To, mintEvt.Amount, mintEvt.Asset, "mint")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry("", mintEvt.To, mintEvt.Amount, mintEvt.Asset, "mint", network)) case contractevents.EventTypeClawback: clawbackEvt := sacEvent.(*contractevents.ClawbackEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(clawbackEvt.From, "", clawbackEvt.Amount, clawbackEvt.Asset, "clawback")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(clawbackEvt.From, "", clawbackEvt.Amount, clawbackEvt.Asset, "clawback", network)) case contractevents.EventTypeBurn: burnEvt := sacEvent.(*contractevents.BurnEvent) - balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(burnEvt.From, "", burnEvt.Amount, burnEvt.Asset, "burn")) + balanceChanges = append(balanceChanges, createSACBalanceChangeEntry(burnEvt.From, "", burnEvt.Amount, burnEvt.Asset, "burn", network)) } } } @@ -1981,7 +1998,7 @@ func parseAssetBalanceChangesFromContractEvents(transaction ingest.LedgerTransac // changeType - the type of source sac event that triggered this change // // return - a balance changed record expressed as map of key/value's -func createSACBalanceChangeEntry(fromAccount string, toAccount string, amountChanged xdr.Int128Parts, asset xdr.Asset, changeType string) map[string]interface{} { +func createSACBalanceChangeEntry(fromAccount string, toAccount string, amountChanged xdr.Int128Parts, asset xdr.Asset, changeType string, passphrase string) map[string]interface{} { balanceChange := map[string]interface{}{} if fromAccount != "" { @@ -1993,12 +2010,12 @@ func createSACBalanceChangeEntry(fromAccount string, toAccount string, amountCha balanceChange["type"] = changeType balanceChange["amount"] = amount.String128(amountChanged) - addAssetDetails(balanceChange, asset, "") + addAssetDetails(balanceChange, asset, "", passphrase) return balanceChange } // addAssetDetails sets the details for `a` on `result` using keys with `prefix` -func addAssetDetails(result map[string]interface{}, a xdr.Asset, prefix string) error { +func addAssetDetails(result map[string]interface{}, a xdr.Asset, prefix string, passphrase string) error { var ( assetType string code string @@ -2009,8 +2026,14 @@ func addAssetDetails(result map[string]interface{}, a xdr.Asset, prefix string) err = errors.Wrap(err, "xdr.Asset.Extract error") return err } - result[prefix+"asset_type"] = assetType + outputAsset, err := transformSingleAsset(a, passphrase) + if err != nil { + return err + } + result[prefix+"asset_contract_id"] = outputAsset.ContractId + + result[prefix+"asset_type"] = assetType if a.Type == xdr.AssetTypeAssetTypeNative { return nil } diff --git a/internal/transform/operation_test.go b/internal/transform/operation_test.go index 8e7c6b28..c2e89e9a 100644 --- a/internal/transform/operation_test.go +++ b/internal/transform/operation_test.go @@ -1047,26 +1047,28 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4098, OperationDetails: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "amount": 35.0, - "asset_code": "USDT", - "asset_type": "credit_alphanum4", - "asset_issuer": hardCodedDestAccountAddress, - "asset_id": int64(-8205667356306085451), + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "amount": 35.0, + "asset_code": "USDT", + "asset_type": "credit_alphanum4", + "asset_issuer": hardCodedDestAccountAddress, + "asset_id": int64(-8205667356306085451), + "asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "PaymentResultCodePaymentSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "amount": 35.0, - "asset_code": "USDT", - "asset_type": "credit_alphanum4", - "asset_issuer": hardCodedDestAccountAddress, - "asset_id": int64(-8205667356306085451), + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "amount": 35.0, + "asset_code": "USDT", + "asset_type": "credit_alphanum4", + "asset_issuer": hardCodedDestAccountAddress, + "asset_id": int64(-8205667356306085451), + "asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, }, { @@ -1076,22 +1078,24 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4099, OperationDetails: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "amount": 35.0, - "asset_type": "native", - "asset_id": int64(-5706705804583548011), + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "amount": 35.0, + "asset_type": "native", + "asset_id": int64(-5706705804583548011), + "asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "PaymentResultCodePaymentSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "amount": 35.0, - "asset_type": "native", - "asset_id": int64(-5706705804583548011), + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "amount": 35.0, + "asset_type": "native", + "asset_id": int64(-5706705804583548011), + "asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, }, { @@ -1101,32 +1105,36 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4100, OperationDetails: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "source_amount": 894.6764349, - "source_max": 895.14959, - "amount": 895.14959, - "source_asset_type": "native", - "source_asset_id": int64(-5706705804583548011), - "asset_type": "native", - "asset_id": int64(-5706705804583548011), - "path": []Path{usdtAssetPath}, + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "source_amount": 894.6764349, + "source_max": 895.14959, + "amount": 895.14959, + "source_asset_type": "native", + "source_asset_id": int64(-5706705804583548011), + "asset_type": "native", + "asset_id": int64(-5706705804583548011), + "path": []Path{usdtAssetPath}, + "source_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "PathPaymentStrictReceiveResultCodePathPaymentStrictReceiveSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "source_amount": 894.6764349, - "source_max": 895.14959, - "amount": 895.14959, - "source_asset_type": "native", - "source_asset_id": int64(-5706705804583548011), - "asset_type": "native", - "asset_id": int64(-5706705804583548011), - "path": []Path{usdtAssetPath}, + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "source_amount": 894.6764349, + "source_max": 895.14959, + "amount": 895.14959, + "source_asset_type": "native", + "source_asset_id": int64(-5706705804583548011), + "asset_type": "native", + "asset_id": int64(-5706705804583548011), + "path": []Path{usdtAssetPath}, + "asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "source_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, }, { @@ -1143,12 +1151,14 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 128523, Denominator: 250000, }, - "selling_asset_code": "USDT", - "selling_asset_type": "credit_alphanum4", - "selling_asset_issuer": hardCodedDestAccountAddress, - "selling_asset_id": int64(-8205667356306085451), - "buying_asset_type": "native", - "buying_asset_id": int64(-5706705804583548011), + "selling_asset_code": "USDT", + "selling_asset_type": "credit_alphanum4", + "selling_asset_issuer": hardCodedDestAccountAddress, + "selling_asset_id": int64(-8205667356306085451), + "buying_asset_type": "native", + "buying_asset_id": int64(-5706705804583548011), + "selling_asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", + "buying_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", @@ -1162,12 +1172,14 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 128523, Denominator: 250000, }, - "selling_asset_code": "USDT", - "selling_asset_type": "credit_alphanum4", - "selling_asset_issuer": hardCodedDestAccountAddress, - "selling_asset_id": int64(-8205667356306085451), - "buying_asset_type": "native", - "buying_asset_id": int64(-5706705804583548011), + "selling_asset_code": "USDT", + "selling_asset_type": "credit_alphanum4", + "selling_asset_issuer": hardCodedDestAccountAddress, + "selling_asset_id": int64(-8205667356306085451), + "buying_asset_type": "native", + "buying_asset_id": int64(-5706705804583548011), + "selling_asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", + "buying_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, }, { @@ -1183,12 +1195,14 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 99583200, Denominator: 1257990000, }, - "buying_asset_code": "USDT", - "buying_asset_type": "credit_alphanum4", - "buying_asset_issuer": hardCodedDestAccountAddress, - "buying_asset_id": int64(-8205667356306085451), - "selling_asset_type": "native", - "selling_asset_id": int64(-5706705804583548011), + "buying_asset_code": "USDT", + "buying_asset_type": "credit_alphanum4", + "buying_asset_issuer": hardCodedDestAccountAddress, + "buying_asset_id": int64(-8205667356306085451), + "selling_asset_type": "native", + "selling_asset_id": int64(-5706705804583548011), + "selling_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "buying_asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", @@ -1201,12 +1215,14 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 99583200, Denominator: 1257990000, }, - "buying_asset_code": "USDT", - "buying_asset_type": "credit_alphanum4", - "buying_asset_issuer": hardCodedDestAccountAddress, - "buying_asset_id": int64(-8205667356306085451), - "selling_asset_type": "native", - "selling_asset_id": int64(-5706705804583548011), + "buying_asset_code": "USDT", + "buying_asset_type": "credit_alphanum4", + "buying_asset_issuer": hardCodedDestAccountAddress, + "buying_asset_id": int64(-8205667356306085451), + "selling_asset_type": "native", + "selling_asset_id": int64(-5706705804583548011), + "selling_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "buying_asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, }, { @@ -1255,26 +1271,28 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4104, OperationDetails: map[string]interface{}{ - "trustor": hardCodedSourceAccountAddress, - "trustee": hardCodedDestAccountAddress, - "limit": 50000000000.0, - "asset_code": "USSD", - "asset_type": "credit_alphanum4", - "asset_issuer": hardCodedDestAccountAddress, - "asset_id": int64(6690054458235693884), + "trustor": hardCodedSourceAccountAddress, + "trustee": hardCodedDestAccountAddress, + "limit": 50000000000.0, + "asset_code": "USSD", + "asset_type": "credit_alphanum4", + "asset_issuer": hardCodedDestAccountAddress, + "asset_id": int64(6690054458235693884), + "asset_contract_id": "CCNSWASPU6GUUH2EGMJLT7TQ7MKG2OTWA5JG5SBSLH2P4BNYOISABXCD", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "ChangeTrustResultCodeChangeTrustSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "trustor": hardCodedSourceAccountAddress, - "trustee": hardCodedDestAccountAddress, - "limit": 50000000000.0, - "asset_code": "USSD", - "asset_type": "credit_alphanum4", - "asset_issuer": hardCodedDestAccountAddress, - "asset_id": int64(6690054458235693884), + "trustor": hardCodedSourceAccountAddress, + "trustee": hardCodedDestAccountAddress, + "limit": 50000000000.0, + "asset_code": "USSD", + "asset_type": "credit_alphanum4", + "asset_issuer": hardCodedDestAccountAddress, + "asset_id": int64(6690054458235693884), + "asset_contract_id": "CCNSWASPU6GUUH2EGMJLT7TQ7MKG2OTWA5JG5SBSLH2P4BNYOISABXCD", }, }, { @@ -1309,26 +1327,28 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4106, OperationDetails: map[string]interface{}{ - "trustee": hardCodedSourceAccountAddress, - "trustor": hardCodedDestAccountAddress, - "authorize": true, - "asset_code": "USDT", - "asset_type": "credit_alphanum4", - "asset_issuer": hardCodedSourceAccountAddress, - "asset_id": int64(8485542065083974675), + "trustee": hardCodedSourceAccountAddress, + "trustor": hardCodedDestAccountAddress, + "authorize": true, + "asset_code": "USDT", + "asset_type": "credit_alphanum4", + "asset_issuer": hardCodedSourceAccountAddress, + "asset_id": int64(8485542065083974675), + "asset_contract_id": "CAVKFBJY7OADSTGOHTXE5BPQXCTWMSGETZQF5ZMP7ULWAY2KWPXPYCGF", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "AllowTrustResultCodeAllowTrustSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "trustee": hardCodedSourceAccountAddress, - "trustor": hardCodedDestAccountAddress, - "authorize": true, - "asset_code": "USDT", - "asset_type": "credit_alphanum4", - "asset_issuer": hardCodedSourceAccountAddress, - "asset_id": int64(8485542065083974675), + "trustee": hardCodedSourceAccountAddress, + "trustor": hardCodedDestAccountAddress, + "authorize": true, + "asset_code": "USDT", + "asset_type": "credit_alphanum4", + "asset_issuer": hardCodedSourceAccountAddress, + "asset_id": int64(8485542065083974675), + "asset_contract_id": "CAVKFBJY7OADSTGOHTXE5BPQXCTWMSGETZQF5ZMP7ULWAY2KWPXPYCGF", }, }, { @@ -1412,13 +1432,15 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 635863285, Denominator: 1818402817, }, - "selling_asset_code": "USDT", - "selling_asset_type": "credit_alphanum4", - "selling_asset_issuer": hardCodedDestAccountAddress, - "selling_asset_id": int64(-8205667356306085451), - "buying_asset_type": "native", - "buying_asset_id": int64(-5706705804583548011), - "offer_id": int64(100), + "selling_asset_code": "USDT", + "selling_asset_type": "credit_alphanum4", + "selling_asset_issuer": hardCodedDestAccountAddress, + "selling_asset_id": int64(-8205667356306085451), + "buying_asset_type": "native", + "buying_asset_id": int64(-5706705804583548011), + "offer_id": int64(100), + "selling_asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", + "buying_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", @@ -1431,13 +1453,15 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 635863285, Denominator: 1818402817, }, - "selling_asset_code": "USDT", - "selling_asset_type": "credit_alphanum4", - "selling_asset_issuer": hardCodedDestAccountAddress, - "selling_asset_id": int64(-8205667356306085451), - "buying_asset_type": "native", - "buying_asset_id": int64(-5706705804583548011), - "offer_id": int64(100), + "selling_asset_code": "USDT", + "selling_asset_type": "credit_alphanum4", + "selling_asset_issuer": hardCodedDestAccountAddress, + "selling_asset_id": int64(-8205667356306085451), + "buying_asset_type": "native", + "buying_asset_id": int64(-5706705804583548011), + "offer_id": int64(100), + "selling_asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", + "buying_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, }, { @@ -1447,32 +1471,36 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4112, OperationDetails: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "source_amount": 0.1598182, - "destination_min": "428.0460538", - "amount": 433.4043858, - "path": []Path{usdtAssetPath}, - "source_asset_type": "native", - "source_asset_id": int64(-5706705804583548011), - "asset_type": "native", - "asset_id": int64(-5706705804583548011), + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "source_amount": 0.1598182, + "destination_min": "428.0460538", + "amount": 433.4043858, + "path": []Path{usdtAssetPath}, + "source_asset_type": "native", + "source_asset_id": int64(-5706705804583548011), + "asset_type": "native", + "asset_id": int64(-5706705804583548011), + "asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "source_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "PathPaymentStrictSendResultCodePathPaymentStrictSendSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "from": hardCodedSourceAccountAddress, - "to": hardCodedDestAccountAddress, - "source_amount": 0.1598182, - "destination_min": "428.0460538", - "amount": 433.4043858, - "path": []Path{usdtAssetPath}, - "source_asset_type": "native", - "source_asset_id": int64(-5706705804583548011), - "asset_type": "native", - "asset_id": int64(-5706705804583548011), + "from": hardCodedSourceAccountAddress, + "to": hardCodedDestAccountAddress, + "source_amount": 0.1598182, + "destination_min": "428.0460538", + "amount": 433.4043858, + "path": []Path{usdtAssetPath}, + "source_asset_type": "native", + "source_asset_id": int64(-5706705804583548011), + "asset_type": "native", + "asset_id": int64(-5706705804583548011), + "asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "source_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", }, }, { @@ -1670,24 +1698,26 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4123, OperationDetails: map[string]interface{}{ - "from": hardCodedDestAccountAddress, - "amount": 0.1598182, - "asset_code": "USDT", - "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", - "asset_type": "credit_alphanum4", - "asset_id": int64(-8205667356306085451), + "from": hardCodedDestAccountAddress, + "amount": 0.1598182, + "asset_code": "USDT", + "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", + "asset_type": "credit_alphanum4", + "asset_id": int64(-8205667356306085451), + "asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "ClawbackResultCodeClawbackSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "from": hardCodedDestAccountAddress, - "amount": 0.1598182, - "asset_code": "USDT", - "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", - "asset_type": "credit_alphanum4", - "asset_id": int64(-8205667356306085451), + "from": hardCodedDestAccountAddress, + "amount": 0.1598182, + "asset_code": "USDT", + "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", + "asset_type": "credit_alphanum4", + "asset_id": int64(-8205667356306085451), + "asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, }, { @@ -1716,30 +1746,32 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4125, OperationDetails: map[string]interface{}{ - "asset_code": "USDT", - "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", - "asset_type": "credit_alphanum4", - "asset_id": int64(-8205667356306085451), - "trustor": testAccount4Address, - "clear_flags": []int32{1, 2}, - "clear_flags_s": []string{"authorized", "authorized_to_maintain_liabilities"}, - "set_flags": []int32{4}, - "set_flags_s": []string{"clawback_enabled"}, + "asset_code": "USDT", + "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", + "asset_type": "credit_alphanum4", + "asset_id": int64(-8205667356306085451), + "trustor": testAccount4Address, + "clear_flags": []int32{1, 2}, + "clear_flags_s": []string{"authorized", "authorized_to_maintain_liabilities"}, + "set_flags": []int32{4}, + "set_flags_s": []string{"clawback_enabled"}, + "asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "SetTrustLineFlagsResultCodeSetTrustLineFlagsSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "asset_code": "USDT", - "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", - "asset_type": "credit_alphanum4", - "asset_id": int64(-8205667356306085451), - "trustor": testAccount4Address, - "clear_flags": []int32{1, 2}, - "clear_flags_s": []string{"authorized", "authorized_to_maintain_liabilities"}, - "set_flags": []int32{4}, - "set_flags_s": []string{"clawback_enabled"}, + "asset_code": "USDT", + "asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", + "asset_type": "credit_alphanum4", + "asset_id": int64(-8205667356306085451), + "trustor": testAccount4Address, + "clear_flags": []int32{1, 2}, + "clear_flags_s": []string{"authorized", "authorized_to_maintain_liabilities"}, + "set_flags": []int32{4}, + "set_flags_s": []string{"clawback_enabled"}, + "asset_contract_id": "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", }, }, { @@ -1771,7 +1803,9 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 1, Denominator: 1000000, }, - "shares_received": 0.0000002, + "shares_received": 0.0000002, + "reserve_a_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "reserve_b_asset_contract_id": "CCNSWASPU6GUUH2EGMJLT7TQ7MKG2OTWA5JG5SBSLH2P4BNYOISABXCD", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", @@ -1800,7 +1834,9 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { Numerator: 1, Denominator: 1000000, }, - "shares_received": 0.0000002, + "shares_received": 0.0000002, + "reserve_a_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "reserve_b_asset_contract_id": "CCNSWASPU6GUUH2EGMJLT7TQ7MKG2OTWA5JG5SBSLH2P4BNYOISABXCD", }, }, { @@ -1810,38 +1846,42 @@ func makeOperationTestOutputs() (transformedOperations []OperationOutput) { TransactionID: 4096, OperationID: 4127, OperationDetails: map[string]interface{}{ - "liquidity_pool_id": "0102030405060708090000000000000000000000000000000000000000000000", - "liquidity_pool_id_strkey": "LAAQEAYEAUDAOCAJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATUC", - "reserve_a_asset_type": "native", - "reserve_a_asset_id": int64(-5706705804583548011), - "reserve_a_min_amount": 0.0000001, - "reserve_a_withdraw_amount": -0.0001, - "reserve_b_asset_type": "credit_alphanum4", - "reserve_b_asset_code": "USSD", - "reserve_b_asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", - "reserve_b_asset_id": int64(6690054458235693884), - "reserve_b_withdraw_amount": -0.00001, - "reserve_b_min_amount": 0.0000001, - "shares": 0.0000004, + "liquidity_pool_id": "0102030405060708090000000000000000000000000000000000000000000000", + "liquidity_pool_id_strkey": "LAAQEAYEAUDAOCAJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATUC", + "reserve_a_asset_type": "native", + "reserve_a_asset_id": int64(-5706705804583548011), + "reserve_a_min_amount": 0.0000001, + "reserve_a_withdraw_amount": -0.0001, + "reserve_b_asset_type": "credit_alphanum4", + "reserve_b_asset_code": "USSD", + "reserve_b_asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", + "reserve_b_asset_id": int64(6690054458235693884), + "reserve_b_withdraw_amount": -0.00001, + "reserve_b_min_amount": 0.0000001, + "shares": 0.0000004, + "reserve_a_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "reserve_b_asset_contract_id": "CCNSWASPU6GUUH2EGMJLT7TQ7MKG2OTWA5JG5SBSLH2P4BNYOISABXCD", }, ClosedAt: hardCodedLedgerClose, OperationResultCode: "OperationResultCodeOpInner", OperationTraceCode: "LiquidityPoolWithdrawResultCodeLiquidityPoolWithdrawSuccess", LedgerSequence: 0, OperationDetailsJSON: map[string]interface{}{ - "liquidity_pool_id": "0102030405060708090000000000000000000000000000000000000000000000", - "liquidity_pool_id_strkey": "LAAQEAYEAUDAOCAJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATUC", - "reserve_a_asset_type": "native", - "reserve_a_asset_id": int64(-5706705804583548011), - "reserve_a_min_amount": 0.0000001, - "reserve_a_withdraw_amount": -0.0001, - "reserve_b_asset_type": "credit_alphanum4", - "reserve_b_asset_code": "USSD", - "reserve_b_asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", - "reserve_b_asset_id": int64(6690054458235693884), - "reserve_b_withdraw_amount": -0.00001, - "reserve_b_min_amount": 0.0000001, - "shares": 0.0000004, + "liquidity_pool_id": "0102030405060708090000000000000000000000000000000000000000000000", + "liquidity_pool_id_strkey": "LAAQEAYEAUDAOCAJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAATUC", + "reserve_a_asset_type": "native", + "reserve_a_asset_id": int64(-5706705804583548011), + "reserve_a_min_amount": 0.0000001, + "reserve_a_withdraw_amount": -0.0001, + "reserve_b_asset_type": "credit_alphanum4", + "reserve_b_asset_code": "USSD", + "reserve_b_asset_issuer": "GBVVRXLMNCJQW3IDDXC3X6XCH35B5Q7QXNMMFPENSOGUPQO7WO7HGZPA", + "reserve_b_asset_id": int64(6690054458235693884), + "reserve_b_withdraw_amount": -0.00001, + "reserve_b_min_amount": 0.0000001, + "shares": 0.0000004, + "reserve_a_asset_contract_id": "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", + "reserve_b_asset_contract_id": "CCNSWASPU6GUUH2EGMJLT7TQ7MKG2OTWA5JG5SBSLH2P4BNYOISABXCD", }, }, OperationOutput{ diff --git a/internal/transform/parquet_converter.go b/internal/transform/parquet_converter.go index 1b863aa4..8679c05a 100644 --- a/internal/transform/parquet_converter.go +++ b/internal/transform/parquet_converter.go @@ -182,6 +182,8 @@ func (po PoolOutput) ToParquet() interface{} { Deleted: po.Deleted, ClosedAt: po.ClosedAt.UnixMilli(), LedgerSequence: int64(po.LedgerSequence), + AssetAContractId: po.AssetAContractId, + AssetBContractId: po.AssetBContractId, } } @@ -193,6 +195,7 @@ func (ao AssetOutput) ToParquet() interface{} { AssetID: ao.AssetID, ClosedAt: ao.ClosedAt.UnixMilli(), LedgerSequence: int64(ao.LedgerSequence), + ContractId: ao.ContractId, } } @@ -216,32 +219,35 @@ func (to TrustlineOutput) ToParquet() interface{} { Deleted: to.Deleted, ClosedAt: to.ClosedAt.UnixMilli(), LedgerSequence: int64(to.LedgerSequence), + ContractId: to.ContractId, } } func (oo OfferOutput) ToParquet() interface{} { return OfferOutputParquet{ - SellerID: oo.SellerID, - OfferID: oo.OfferID, - SellingAssetType: oo.SellingAssetType, - SellingAssetCode: oo.SellingAssetCode, - SellingAssetIssuer: oo.SellingAssetIssuer, - SellingAssetID: oo.SellingAssetID, - BuyingAssetType: oo.BuyingAssetType, - BuyingAssetCode: oo.BuyingAssetCode, - BuyingAssetIssuer: oo.BuyingAssetIssuer, - BuyingAssetID: oo.BuyingAssetID, - Amount: oo.Amount, - PriceN: oo.PriceN, - PriceD: oo.PriceD, - Price: oo.Price, - Flags: int64(oo.Flags), - LastModifiedLedger: int64(oo.LastModifiedLedger), - LedgerEntryChange: int64(oo.LedgerEntryChange), - Deleted: oo.Deleted, - Sponsor: oo.Sponsor.String, - ClosedAt: oo.ClosedAt.UnixMilli(), - LedgerSequence: int64(oo.LedgerSequence), + SellerID: oo.SellerID, + OfferID: oo.OfferID, + SellingAssetType: oo.SellingAssetType, + SellingAssetCode: oo.SellingAssetCode, + SellingAssetIssuer: oo.SellingAssetIssuer, + SellingAssetID: oo.SellingAssetID, + BuyingAssetType: oo.BuyingAssetType, + BuyingAssetCode: oo.BuyingAssetCode, + BuyingAssetIssuer: oo.BuyingAssetIssuer, + BuyingAssetID: oo.BuyingAssetID, + Amount: oo.Amount, + PriceN: oo.PriceN, + PriceD: oo.PriceD, + Price: oo.Price, + Flags: int64(oo.Flags), + LastModifiedLedger: int64(oo.LastModifiedLedger), + LedgerEntryChange: int64(oo.LedgerEntryChange), + Deleted: oo.Deleted, + Sponsor: oo.Sponsor.String, + ClosedAt: oo.ClosedAt.UnixMilli(), + LedgerSequence: int64(oo.LedgerSequence), + SellingAssetContractId: oo.SellingAssetContractId, + BuyingAssetContractId: oo.BuyingAssetContractId, } } @@ -271,6 +277,8 @@ func (to TradeOutput) ToParquet() interface{} { TradeType: to.TradeType, RoundingSlippage: to.RoundingSlippage.Int64, SellerIsExact: to.SellerIsExact.Bool, + SellingAssetContractId: to.SellingAssetContractId, + BuyingAssetContractId: to.BuyingAssetContractId, } } diff --git a/internal/transform/schema.go b/internal/transform/schema.go index 89e20ef0..9f960651 100644 --- a/internal/transform/schema.go +++ b/internal/transform/schema.go @@ -166,6 +166,7 @@ type ClaimableBalanceOutput struct { ClosedAt time.Time `json:"closed_at"` LedgerSequence uint32 `json:"ledger_sequence"` BalanceIDStrkey string `json:"balance_id_strkey"` + ContractId string `json:"contract_id"` } // Claimants @@ -185,19 +186,23 @@ type Path struct { AssetCode string `json:"asset_code"` AssetIssuer string `json:"asset_issuer"` AssetType string `json:"asset_type"` + ContractId string `json:"contract_id"` } +// TODO: confirm if this struct is not used // LiquidityPoolAsset represents the asset pairs in a liquidity pool -type LiquidityPoolAsset struct { - AssetAType string - AssetACode string - AssetAIssuer string - AssetAAmount float64 - AssetBType string - AssetBCode string - AssetBIssuer string - AssetBAmount float64 -} +//type LiquidityPoolAsset struct { +// AssetAType string +// AssetACode string +// AssetAIssuer string +// AssetAAmount float64 +// AssetBType string +// AssetBCode string +// AssetBIssuer string +// AssetBAmount float64 +// AssetAContractId string +// AssetBContractId string +//} // PoolOutput is a representation of a liquidity pool that aligns with the Bigquery table liquidity_pools type PoolOutput struct { @@ -222,6 +227,8 @@ type PoolOutput struct { ClosedAt time.Time `json:"closed_at"` LedgerSequence uint32 `json:"ledger_sequence"` PoolIDStrkey string `json:"liquidity_pool_id_strkey"` + AssetAContractId string `json:"asset_a_contract_id"` + AssetBContractId string `json:"asset_b_contract_id"` } // AssetOutput is a representation of an asset that aligns with the BigQuery table history_assets @@ -232,6 +239,7 @@ type AssetOutput struct { AssetID int64 `json:"asset_id"` ClosedAt time.Time `json:"closed_at"` LedgerSequence uint32 `json:"ledger_sequence"` + ContractId string `json:"contract_id"` } // TrustlineOutput is a representation of a trustline that aligns with the BigQuery table trust_lines @@ -255,31 +263,34 @@ type TrustlineOutput struct { ClosedAt time.Time `json:"closed_at"` LedgerSequence uint32 `json:"ledger_sequence"` LiquidityPoolIDStrkey string `json:"liquidity_pool_id_strkey"` + ContractId string `json:"contract_id"` } // OfferOutput is a representation of an offer that aligns with the BigQuery table offers type OfferOutput struct { - SellerID string `json:"seller_id"` // Account address of the seller - OfferID int64 `json:"offer_id"` - SellingAssetType string `json:"selling_asset_type"` - SellingAssetCode string `json:"selling_asset_code"` - SellingAssetIssuer string `json:"selling_asset_issuer"` - SellingAssetID int64 `json:"selling_asset_id"` - BuyingAssetType string `json:"buying_asset_type"` - BuyingAssetCode string `json:"buying_asset_code"` - BuyingAssetIssuer string `json:"buying_asset_issuer"` - BuyingAssetID int64 `json:"buying_asset_id"` - Amount float64 `json:"amount"` - PriceN int32 `json:"pricen"` - PriceD int32 `json:"priced"` - Price float64 `json:"price"` - Flags uint32 `json:"flags"` - LastModifiedLedger uint32 `json:"last_modified_ledger"` - LedgerEntryChange uint32 `json:"ledger_entry_change"` - Deleted bool `json:"deleted"` - Sponsor null.String `json:"sponsor"` - ClosedAt time.Time `json:"closed_at"` - LedgerSequence uint32 `json:"ledger_sequence"` + SellerID string `json:"seller_id"` // Account address of the seller + OfferID int64 `json:"offer_id"` + SellingAssetType string `json:"selling_asset_type"` + SellingAssetCode string `json:"selling_asset_code"` + SellingAssetIssuer string `json:"selling_asset_issuer"` + SellingAssetID int64 `json:"selling_asset_id"` + BuyingAssetType string `json:"buying_asset_type"` + BuyingAssetCode string `json:"buying_asset_code"` + BuyingAssetIssuer string `json:"buying_asset_issuer"` + BuyingAssetID int64 `json:"buying_asset_id"` + Amount float64 `json:"amount"` + PriceN int32 `json:"pricen"` + PriceD int32 `json:"priced"` + Price float64 `json:"price"` + Flags uint32 `json:"flags"` + LastModifiedLedger uint32 `json:"last_modified_ledger"` + LedgerEntryChange uint32 `json:"ledger_entry_change"` + Deleted bool `json:"deleted"` + Sponsor null.String `json:"sponsor"` + ClosedAt time.Time `json:"closed_at"` + LedgerSequence uint32 `json:"ledger_sequence"` + SellingAssetContractId string `json:"selling_asset_contract_id"` + BuyingAssetContractId string `json:"buying_asset_contract_id"` } // TradeOutput is a representation of a trade that aligns with the BigQuery table history_trades @@ -309,6 +320,8 @@ type TradeOutput struct { RoundingSlippage null.Int `json:"rounding_slippage"` SellerIsExact null.Bool `json:"seller_is_exact"` SellingLiquidityPoolIDStrkey null.String `json:"selling_liquidity_pool_id_strkey"` + SellingAssetContractId string `json:"selling_asset_contract_id"` + BuyingAssetContractId string `json:"buying_asset_contract_id"` } // DimAccount is a representation of an account that aligns with the BigQuery table dim_accounts diff --git a/internal/transform/schema_parquet.go b/internal/transform/schema_parquet.go index 829ab047..4fbaede8 100644 --- a/internal/transform/schema_parquet.go +++ b/internal/transform/schema_parquet.go @@ -156,6 +156,8 @@ type PoolOutputParquet struct { Deleted bool `parquet:"name=deleted, type=BOOLEAN"` ClosedAt int64 `parquet:"name=closed_at, type=INT64, convertedtype=TIMESTAMP_MILLIS"` LedgerSequence int64 `parquet:"name=ledger_sequence, type=INT64, convertedtype=UINT_64"` + AssetAContractId string `parquet:"name=asset_a_contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + AssetBContractId string `parquet:"name=asset_b_contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` } // AssetOutputParquet is a representation of an asset that aligns with the BigQuery table history_assets @@ -166,6 +168,7 @@ type AssetOutputParquet struct { AssetID int64 `parquet:"name=asset_id, type=INT64"` ClosedAt int64 `parquet:"name=closed_at, type=INT64, convertedtype=TIMESTAMP_MILLIS"` LedgerSequence int64 `parquet:"name=ledger_sequence, type=INT64, convertedtype=UINT_64"` + ContractId string `parquet:"name=contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` } // TrustlineOutputParquet is a representation of a trustline that aligns with the BigQuery table trust_lines @@ -188,31 +191,34 @@ type TrustlineOutputParquet struct { Deleted bool `parquet:"name=deleted, type=BOOLEAN"` ClosedAt int64 `parquet:"name=closed_at, type=INT64, convertedtype=TIMESTAMP_MILLIS"` LedgerSequence int64 `parquet:"name=ledger_sequence, type=INT64, convertedtype=UINT_64"` + ContractId string `parquet:"name=contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` } // OfferOutputParquet is a representation of an offer that aligns with the BigQuery table offers type OfferOutputParquet struct { - SellerID string `parquet:"name=seller_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - OfferID int64 `parquet:"name=offer_id, type=INT64"` - SellingAssetType string `parquet:"name=selling_asset_type, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - SellingAssetCode string `parquet:"name=selling_asset_code, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - SellingAssetIssuer string `parquet:"name=selling_asset_issuer, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - SellingAssetID int64 `parquet:"name=selling_asset_id, type=INT64"` - BuyingAssetType string `parquet:"name=buying_asset_type, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - BuyingAssetCode string `parquet:"name=buying_asset_code, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - BuyingAssetIssuer string `parquet:"name=buying_asset_issuer, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - BuyingAssetID int64 `parquet:"name=buying_asset_id, type=INT64"` - Amount float64 `parquet:"name=amount, type=DOUBLE"` - PriceN int32 `parquet:"name=pricen, type=INT32"` - PriceD int32 `parquet:"name=priced, type=INT32"` - Price float64 `parquet:"name=price, type=DOUBLE"` - Flags int64 `parquet:"name=flags, type=INT64, convertedtype=UINT_64"` - LastModifiedLedger int64 `parquet:"name=last_modified_ledger, type=INT64, convertedtype=UINT_64"` - LedgerEntryChange int64 `parquet:"name=ledger_entry_change, type=INT64, convertedtype=UINT_64"` - Deleted bool `parquet:"name=deleted, type=BOOLEAN"` - Sponsor string `parquet:"name=sponsor, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - ClosedAt int64 `parquet:"name=closed_at, type=INT64, convertedtype=TIMESTAMP_MILLIS"` - LedgerSequence int64 `parquet:"name=ledger_sequence, type=INT64, convertedtype=UINT_64"` + SellerID string `parquet:"name=seller_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + OfferID int64 `parquet:"name=offer_id, type=INT64"` + SellingAssetType string `parquet:"name=selling_asset_type, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + SellingAssetCode string `parquet:"name=selling_asset_code, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + SellingAssetIssuer string `parquet:"name=selling_asset_issuer, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + SellingAssetID int64 `parquet:"name=selling_asset_id, type=INT64"` + BuyingAssetType string `parquet:"name=buying_asset_type, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + BuyingAssetCode string `parquet:"name=buying_asset_code, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + BuyingAssetIssuer string `parquet:"name=buying_asset_issuer, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + BuyingAssetID int64 `parquet:"name=buying_asset_id, type=INT64"` + Amount float64 `parquet:"name=amount, type=DOUBLE"` + PriceN int32 `parquet:"name=pricen, type=INT32"` + PriceD int32 `parquet:"name=priced, type=INT32"` + Price float64 `parquet:"name=price, type=DOUBLE"` + Flags int64 `parquet:"name=flags, type=INT64, convertedtype=UINT_64"` + LastModifiedLedger int64 `parquet:"name=last_modified_ledger, type=INT64, convertedtype=UINT_64"` + LedgerEntryChange int64 `parquet:"name=ledger_entry_change, type=INT64, convertedtype=UINT_64"` + Deleted bool `parquet:"name=deleted, type=BOOLEAN"` + Sponsor string `parquet:"name=sponsor, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + ClosedAt int64 `parquet:"name=closed_at, type=INT64, convertedtype=TIMESTAMP_MILLIS"` + LedgerSequence int64 `parquet:"name=ledger_sequence, type=INT64, convertedtype=UINT_64"` + SellingAssetContractId string `parquet:"name=selling_asset_contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + BuyingAssetContractId string `parquet:"name=buying_asset_contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` } // TradeOutputParquet is a representation of a trade that aligns with the BigQuery table history_trades @@ -241,6 +247,8 @@ type TradeOutputParquet struct { TradeType int32 `parquet:"name=trade_type, type=INT32"` RoundingSlippage int64 `parquet:"name=rounding_slippage, type=INT64"` SellerIsExact bool `parquet:"name=seller_is_exact, type=BOOLEAN"` + SellingAssetContractId string `parquet:"name=selling_asset_contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + BuyingAssetContractId string `parquet:"name=buying_asset_contract_id, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` } // EffectOutputParquet is a representation of an operation that aligns with the BigQuery table history_effects diff --git a/internal/transform/test_variables_test.go b/internal/transform/test_variables_test.go index 63b8a124..c662b1d5 100644 --- a/internal/transform/test_variables_test.go +++ b/internal/transform/test_variables_test.go @@ -174,6 +174,7 @@ var usdtAssetPath = Path{ AssetType: "credit_alphanum4", AssetCode: "USDT", AssetIssuer: testAccount4Address, + ContractId: "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", } var ethAsset = xdr.Asset{ diff --git a/internal/transform/trade.go b/internal/transform/trade.go index fd9e860a..a12bb1bb 100644 --- a/internal/transform/trade.go +++ b/internal/transform/trade.go @@ -18,7 +18,7 @@ import ( ) // TransformTrade converts a relevant operation from the history archive ingestion system into a form suitable for BigQuery -func TransformTrade(operationIndex int32, operationID int64, transaction ingest.LedgerTransaction, ledgerCloseTime time.Time) ([]TradeOutput, error) { +func TransformTrade(operationIndex int32, operationID int64, transaction ingest.LedgerTransaction, ledgerCloseTime time.Time, passphrase string) ([]TradeOutput, error) { operationResults, ok := transaction.Result.OperationResults() if !ok { return []TradeOutput{}, fmt.Errorf("could not get any results from this transaction") @@ -42,24 +42,20 @@ func TransformTrade(operationIndex int32, operationID int64, transaction ingest. outputOrder := int32(claimOrder) outputLedgerClosedAt := ledgerCloseTime - var outputSellingAssetType, outputSellingAssetCode, outputSellingAssetIssuer string - err = claimOffer.AssetSold().Extract(&outputSellingAssetType, &outputSellingAssetCode, &outputSellingAssetIssuer) + sellingAssetOutput, err := transformSingleAsset(claimOffer.AssetSold(), passphrase) if err != nil { return []TradeOutput{}, err } - outputSellingAssetID := FarmHashAsset(outputSellingAssetCode, outputSellingAssetIssuer, outputSellingAssetType) outputSellingAmount := claimOffer.AmountSold() if outputSellingAmount < 0 { return []TradeOutput{}, fmt.Errorf("amount sold is negative (%d) for operation at index %d", outputSellingAmount, operationIndex) } - var outputBuyingAssetType, outputBuyingAssetCode, outputBuyingAssetIssuer string - err = claimOffer.AssetBought().Extract(&outputBuyingAssetType, &outputBuyingAssetCode, &outputBuyingAssetIssuer) + buyingAssetOutput, err := transformSingleAsset(claimOffer.AssetBought(), passphrase) if err != nil { return []TradeOutput{}, err } - outputBuyingAssetID := FarmHashAsset(outputBuyingAssetCode, outputBuyingAssetIssuer, outputBuyingAssetType) outputBuyingAmount := int64(claimOffer.AmountBought()) if outputBuyingAmount < 0 { @@ -132,16 +128,16 @@ func TransformTrade(operationIndex int32, operationID int64, transaction ingest. Order: outputOrder, LedgerClosedAt: outputLedgerClosedAt, SellingAccountAddress: outputSellingAccountAddress, - SellingAssetType: outputSellingAssetType, - SellingAssetCode: outputSellingAssetCode, - SellingAssetIssuer: outputSellingAssetIssuer, - SellingAssetID: outputSellingAssetID, + SellingAssetType: sellingAssetOutput.AssetType, + SellingAssetCode: sellingAssetOutput.AssetCode, + SellingAssetIssuer: sellingAssetOutput.AssetIssuer, + SellingAssetID: sellingAssetOutput.AssetID, SellingAmount: utils.ConvertStroopValueToReal(outputSellingAmount), BuyingAccountAddress: outputBuyingAccountAddress, - BuyingAssetType: outputBuyingAssetType, - BuyingAssetCode: outputBuyingAssetCode, - BuyingAssetIssuer: outputBuyingAssetIssuer, - BuyingAssetID: outputBuyingAssetID, + BuyingAssetType: buyingAssetOutput.AssetType, + BuyingAssetCode: buyingAssetOutput.AssetCode, + BuyingAssetIssuer: buyingAssetOutput.AssetIssuer, + BuyingAssetID: buyingAssetOutput.AssetID, BuyingAmount: utils.ConvertStroopValueToReal(xdr.Int64(outputBuyingAmount)), PriceN: outputPriceN, PriceD: outputPriceD, @@ -154,6 +150,8 @@ func TransformTrade(operationIndex int32, operationID int64, transaction ingest. RoundingSlippage: roundingSlippageBips, SellerIsExact: sellerIsExact, SellingLiquidityPoolIDStrkey: liquidityPoolIDStrkey, + SellingAssetContractId: sellingAssetOutput.ContractId, + BuyingAssetContractId: buyingAssetOutput.ContractId, } transformedTrades = append(transformedTrades, trade) diff --git a/internal/transform/trade_test.go b/internal/transform/trade_test.go index e9a2a19a..0e92dca5 100644 --- a/internal/transform/trade_test.go +++ b/internal/transform/trade_test.go @@ -19,6 +19,7 @@ func TestTransformTrade(t *testing.T) { index int32 transaction ingest.LedgerTransaction closeTime time.Time + passphrase string } type transformTest struct { input tradeInput @@ -169,7 +170,7 @@ func TestTransformTrade(t *testing.T) { } for _, test := range tests { - actualOutput, actualError := TransformTrade(test.input.index, 100, test.input.transaction, test.input.closeTime) + actualOutput, actualError := TransformTrade(test.input.index, 100, test.input.transaction, test.input.closeTime, test.input.passphrase) assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } @@ -719,48 +720,52 @@ func makeTradeTestInput() (inputTransaction ingest.LedgerTransaction) { func makeTradeTestOutput() [][]TradeOutput { offerOneOutput := TradeOutput{ - Order: 0, - LedgerClosedAt: genericCloseTime, - SellingAccountAddress: testAccount1Address, - SellingAssetCode: "ETH", - SellingAssetIssuer: testAccount3Address, - SellingAssetType: "credit_alphanum4", - SellingAssetID: 4476940172956910889, - SellingAmount: 13300347 * 0.0000001, - BuyingAccountAddress: testAccount3Address, - BuyingAssetCode: "USDT", - BuyingAssetIssuer: testAccount4Address, - BuyingAssetType: "credit_alphanum4", - BuyingAssetID: -8205667356306085451, - BuyingAmount: 12634 * 0.0000001, - PriceN: 12634, - PriceD: 13300347, - SellingOfferID: null.IntFrom(97684906), - BuyingOfferID: null.IntFrom(4611686018427388005), - HistoryOperationID: 101, - TradeType: 1, + Order: 0, + LedgerClosedAt: genericCloseTime, + SellingAccountAddress: testAccount1Address, + SellingAssetCode: "ETH", + SellingAssetIssuer: testAccount3Address, + SellingAssetType: "credit_alphanum4", + SellingAssetID: 4476940172956910889, + SellingAmount: 13300347 * 0.0000001, + BuyingAccountAddress: testAccount3Address, + BuyingAssetCode: "USDT", + BuyingAssetIssuer: testAccount4Address, + BuyingAssetType: "credit_alphanum4", + BuyingAssetID: -8205667356306085451, + BuyingAmount: 12634 * 0.0000001, + PriceN: 12634, + PriceD: 13300347, + SellingOfferID: null.IntFrom(97684906), + BuyingOfferID: null.IntFrom(4611686018427388005), + HistoryOperationID: 101, + TradeType: 1, + SellingAssetContractId: "CB3BEFUUYP7WRI7LIUDQWD5KTXZXD7OAWK52RTULOXCIY5BE46VVVZAF", + BuyingAssetContractId: "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", } offerTwoOutput := TradeOutput{ - Order: 0, - LedgerClosedAt: genericCloseTime, - SellingAccountAddress: testAccount3Address, - SellingAssetCode: "USDT", - SellingAssetIssuer: testAccount4Address, - SellingAssetType: "credit_alphanum4", - SellingAssetID: -8205667356306085451, - SellingAmount: 500 * 0.0000001, - BuyingAccountAddress: testAccount3Address, - BuyingAssetCode: "", - BuyingAssetIssuer: "", - BuyingAssetType: "native", - BuyingAssetID: -5706705804583548011, - BuyingAmount: 20 * 0.0000001, - PriceN: 25, - PriceD: 1, - SellingOfferID: null.IntFrom(86106895), - BuyingOfferID: null.IntFrom(4611686018427388005), - HistoryOperationID: 101, - TradeType: 1, + Order: 0, + LedgerClosedAt: genericCloseTime, + SellingAccountAddress: testAccount3Address, + SellingAssetCode: "USDT", + SellingAssetIssuer: testAccount4Address, + SellingAssetType: "credit_alphanum4", + SellingAssetID: -8205667356306085451, + SellingAmount: 500 * 0.0000001, + BuyingAccountAddress: testAccount3Address, + BuyingAssetCode: "", + BuyingAssetIssuer: "", + BuyingAssetType: "native", + BuyingAssetID: -5706705804583548011, + BuyingAmount: 20 * 0.0000001, + PriceN: 25, + PriceD: 1, + SellingOfferID: null.IntFrom(86106895), + BuyingOfferID: null.IntFrom(4611686018427388005), + HistoryOperationID: 101, + TradeType: 1, + SellingAssetContractId: "CA374MGQ6KEYR55VBTK6LDM76PQA5QJQWFOAF4EBBNF7M77YZP5FAHJN", + BuyingAssetContractId: "CB2HF7THKN3V25XQ23URFTQYZTN5ZKZTCY4CEQDVPFFSWREE4DX7C2FI", } lPOneOutput := TradeOutput{ @@ -787,6 +792,8 @@ func makeTradeTestOutput() [][]TradeOutput { RoundingSlippage: null.IntFrom(0), SellerIsExact: null.BoolFrom(false), SellingLiquidityPoolIDStrkey: null.StringFrom("LACAKBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGOE"), + SellingAssetContractId: "CB72Y2ITZOX6WAL3BGPWC5PY6J23LP7HY5VDBSYWRRL26XTTJXVEJ7FG", + BuyingAssetContractId: "CBYGYI4BBA5JGMLTJK4GPKXUGVGGJLWB76US3VUWMHZZ26FYRUPBCHWD", } lPTwoOutput := TradeOutput{ @@ -813,6 +820,8 @@ func makeTradeTestOutput() [][]TradeOutput { RoundingSlippage: null.IntFrom(9223372036854775807), SellerIsExact: null.BoolFrom(true), SellingLiquidityPoolIDStrkey: null.StringFrom("LAAQEAYEAUDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABUTF"), + SellingAssetContractId: "CCR5HNZO2UWNN75EERXCXRDTD34NP4Z4HAIHPG4ML2CHCDATMTIJDRBZ", + BuyingAssetContractId: "CA3CL5YLHO5C4M2OEKAPMMURG64FPAFYUMDFCBDENQQELPW5KNGB74DH", } onePriceIsAmount := offerOneOutput diff --git a/internal/transform/trustline.go b/internal/transform/trustline.go index 68f34acd..d1491c7e 100644 --- a/internal/transform/trustline.go +++ b/internal/transform/trustline.go @@ -15,7 +15,7 @@ import ( ) // TransformTrustline converts a trustline from the history archive ingestion system into a form suitable for BigQuery -func TransformTrustline(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry) (TrustlineOutput, error) { +func TransformTrustline(ledgerChange ingest.Change, header xdr.LedgerHeaderHistoryEntry, passphrase string) (TrustlineOutput, error) { ledgerEntry, changeType, outputDeleted, err := utils.ExtractEntryFromChange(ledgerChange) if err != nil { return TrustlineOutput{}, err @@ -31,7 +31,9 @@ func TransformTrustline(ledgerChange ingest.Change, header xdr.LedgerHeaderHisto return TrustlineOutput{}, err } - var assetType, outputAssetCode, outputAssetIssuer, poolID, poolIDStrkey string + // Using separate asset variables instead of the AssetOutput struct to make the final TrustlineOutput struct + // construction easiser to populate. + var assetType, outputAssetCode, outputAssetIssuer, poolID, poolIDStrkey, contractId string asset := trustEntry.Asset @@ -49,9 +51,25 @@ func TransformTrustline(ledgerChange ingest.Change, header xdr.LedgerHeaderHisto poolID = PoolIDToString(trustEntry.Asset.MustLiquidityPoolId()) assetType = "pool_share" } else { + // TODO: We could write a wrapper for extracting and converting xdr.TrustlineAsset into xdr.Asset + // but we can do that when we see the need to do this conversion in multiple places instead of only + // in the trustline transform. if err = asset.Extract(&assetType, &outputAssetCode, &outputAssetIssuer); err != nil { return TrustlineOutput{}, errors.Wrap(err, fmt.Sprintf("could not parse asset for trustline with account %s", outputAccountID)) } + + nonPoolAsset := xdr.Asset{ + Type: asset.Type, + AlphaNum4: asset.AlphaNum4, + AlphaNum12: asset.AlphaNum12, + } + + assetOutput, err := transformSingleAsset(nonPoolAsset, passphrase) + if err != nil { + return TrustlineOutput{}, err + } + + contractId = assetOutput.ContractId } outputAssetID := FarmHashAsset(outputAssetCode, outputAssetIssuer, asset.Type.String()) @@ -85,6 +103,7 @@ func TransformTrustline(ledgerChange ingest.Change, header xdr.LedgerHeaderHisto ClosedAt: closedAt, LedgerSequence: uint32(ledgerSequence), LiquidityPoolIDStrkey: poolIDStrkey, + ContractId: contractId, } return transformedTrustline, nil diff --git a/internal/transform/trustline_test.go b/internal/transform/trustline_test.go index dad0190a..15947fa7 100644 --- a/internal/transform/trustline_test.go +++ b/internal/transform/trustline_test.go @@ -59,7 +59,7 @@ func TestTransformTrustline(t *testing.T) { LedgerSeq: 10, }, } - actualOutput, actualError := TransformTrustline(test.input.ingest, header) + actualOutput, actualError := TransformTrustline(test.input.ingest, header, "test passphrase") assert.Equal(t, test.wantErr, actualError) assert.Equal(t, test.wantOutput, actualOutput) } @@ -145,6 +145,7 @@ func makeTrustlineTestOutput() []TrustlineOutput { Deleted: false, LedgerSequence: 10, ClosedAt: time.Date(1970, time.January, 1, 0, 16, 40, 0, time.UTC), + ContractId: "CDTDW4NDEV7UNDX4P35Q2OCAP2EJCBFPIWQLNGXV2LNTOPRQ6AMFNGQ3", }, { LedgerKey: "AAAAAQAAAAAcR0GXGO76pFs4y38vJVAanjnLg4emNun7zAx0pHcDGAAAAAMBAwQFBwkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",