From cb596bfd3d6c0a08dfe2a49ef53c390fabe0cd6a Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 25 Mar 2020 10:49:33 +0300 Subject: [PATCH 01/18] WIP --- commands/export.go | 154 +++++++++++++++++------------------- commands/fast_replay.go | 105 ++++++++++++++++++++++++ config/main.go | 7 +- db/main.go | 28 ------- db/tx_history_row.go | 3 +- es/ledger_header.go | 24 ++++++ es/ledger_serializer_xdr.go | 106 +++++++++++++++++++++++++ es/transaction.go | 45 +++++++++++ go.mod | 6 +- go.sum | 8 +- main.go | 9 ++- stellar-core.cfg | 63 +++++++++++++++ stellar/main.go | 140 ++++++++++++++++++++++++++++++++ support/main.go | 32 ++++++++ 14 files changed, 610 insertions(+), 120 deletions(-) create mode 100644 commands/fast_replay.go create mode 100644 es/ledger_serializer_xdr.go create mode 100644 stellar-core.cfg create mode 100644 stellar/main.go create mode 100644 support/main.go diff --git a/commands/export.go b/commands/export.go index 571647f..77cc255 100644 --- a/commands/export.go +++ b/commands/export.go @@ -6,20 +6,13 @@ import ( "math/rand" "time" - progressbar "github.com/schollz/progressbar/v2" - - "github.com/astroband/astrologer/config" - "github.com/astroband/astrologer/db" "github.com/astroband/astrologer/es" -) - -var ( - bar *progressbar.ProgressBar + "github.com/astroband/astrologer/stellar" ) // ExportCommandConfig represents configuration options for `export` CLI command type ExportCommandConfig struct { - Start config.NumberWithSign + Start int Count int RetryCount int DryRun bool @@ -29,7 +22,6 @@ type ExportCommandConfig struct { // ExportCommand represents the `export` CLI command type ExportCommand struct { ES es.Adapter - DB db.Adapter Config ExportCommandConfig firstLedger int @@ -38,9 +30,10 @@ type ExportCommand struct { // Execute starts the export process func (cmd *ExportCommand) Execute() { - cmd.firstLedger, cmd.lastLedger = cmd.getRange() + total := cmd.Config.Count - total := cmd.DB.LedgerHeaderRowCount(cmd.firstLedger, cmd.lastLedger) + cmd.firstLedger = cmd.Config.Start + cmd.lastLedger = cmd.firstLedger + cmd.Config.Count if total == 0 { log.Fatal("Nothing to export within given range!", cmd.firstLedger, cmd.lastLedger) @@ -48,45 +41,59 @@ func (cmd *ExportCommand) Execute() { log.Println("Exporting ledgers from", cmd.firstLedger, "to", cmd.lastLedger, "total", total) - createBar(total) + var b bytes.Buffer + + for meta := range stellar.StreamLedgers(cmd.firstLedger, cmd.lastLedger) { + seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) + + if seq < cmd.firstLedger || seq > cmd.lastLedger { + continue + } + + log.Println(seq) - for i := 0; i < cmd.blockCount(total); i++ { - i := i - pool.Submit(func() { cmd.exportBlock(i) }) + es.SerializeLedgerFromHistory(meta, &b) } + log.Println("DONE") - pool.StopWait() - finishBar() -} + log.Println(b.Len()) + indexed := cmd.ES.BulkInsert(&b) -func (cmd *ExportCommand) exportBlock(i int) { - var b bytes.Buffer + if !indexed { + log.Fatal("Cannot bulk insert") + } - rows := cmd.DB.LedgerHeaderRowFetchBatch(i, cmd.firstLedger, cmd.Config.BatchSize) + // for i := 0; i < cmd.blockCount(total); i++ { + // i := i + // pool.Submit(func() { cmd.exportBlock(i) }) + // } - for n := 0; n < len(rows); n++ { - txs := cmd.DB.TxHistoryRowForSeq(rows[n].LedgerSeq) - fees := cmd.DB.TxFeeHistoryRowsForRows(txs) + // pool.StopWait() +} - err := es.SerializeLedger(rows[n], txs, fees, &b) +// func (cmd *ExportCommand) exportBlock(i int) { +// var b bytes.Buffer - if err != nil { - log.Fatalf("Failed to ingest ledger %d: %v\n", rows[n].LedgerSeq, err) - } +// rows := cmd.DB.LedgerHeaderRowFetchBatch(i, cmd.firstLedger, cmd.Config.BatchSize) - if !*config.Verbose { - bar.Add(1) - } - } +// for n := 0; n < len(rows); n++ { +// txs := cmd.DB.TxHistoryRowForSeq(rows[n].LedgerSeq) +// fees := cmd.DB.TxFeeHistoryRowsForRows(txs) +// es.SerializeLedger(rows[n], txs, fees, &b) - if *config.Verbose { - log.Println(b.String()) - } +// if !*config.Verbose { +// bar.Add(1) +// } +// } - if !cmd.Config.DryRun { - cmd.ES.IndexWithRetries(&b, cmd.Config.RetryCount) - } -} +// if *config.Verbose { +// log.Println(b.String()) +// } + +// if !cmd.Config.DryRun { +// cmd.ES.IndexWithRetries(&b, cmd.Config.RetryCount) +// } +// } func (cmd *ExportCommand) index(b *bytes.Buffer, retry int) { indexed := cmd.ES.BulkInsert(b) @@ -104,49 +111,30 @@ func (cmd *ExportCommand) index(b *bytes.Buffer, retry int) { } // Parses range of export command -func (cmd *ExportCommand) getRange() (first int, last int) { - firstLedger := cmd.DB.LedgerHeaderFirstRow() - lastLedger := cmd.DB.LedgerHeaderLastRow() - - if cmd.Config.Start.Explicit { - if cmd.Config.Start.Value < 0 { - first = lastLedger.LedgerSeq + cmd.Config.Start.Value + 1 - } else if config.Start.Value > 0 { - first = firstLedger.LedgerSeq + cmd.Config.Start.Value - } - } else if cmd.Config.Start.Value != 0 { - first = cmd.Config.Start.Value - } else { - first = firstLedger.LedgerSeq - } - - if cmd.Config.Count == 0 { - last = lastLedger.LedgerSeq - } else { - last = first + cmd.Config.Count - 1 - } - - return first, last -} - -func createBar(count int) { - bar = progressbar.NewOptions( - count, - progressbar.OptionEnableColorCodes(false), - progressbar.OptionShowCount(), - progressbar.OptionThrottle(500*time.Millisecond), - progressbar.OptionSetRenderBlankState(true), - progressbar.OptionSetWidth(100), - ) - - bar.RenderBlank() -} - -func finishBar() { - if !*config.Verbose { - bar.Finish() - } -} +// func (cmd *ExportCommand) getRange() (first int, last int) { +// firstLedger := cmd.DB.LedgerHeaderFirstRow() +// lastLedger := cmd.DB.LedgerHeaderLastRow() + +// if cmd.Config.Start.Explicit { +// if cmd.Config.Start.Value < 0 { +// first = lastLedger.LedgerSeq + cmd.Config.Start.Value + 1 +// } else if config.Start.Value > 0 { +// first = firstLedger.LedgerSeq + cmd.Config.Start.Value +// } +// } else if cmd.Config.Start.Value != 0 { +// first = cmd.Config.Start.Value +// } else { +// first = firstLedger.LedgerSeq +// } + +// if cmd.Config.Count == 0 { +// last = lastLedger.LedgerSeq +// } else { +// last = first + cmd.Config.Count - 1 +// } + +// return first, last +// } func (cmd *ExportCommand) blockCount(count int) (blocks int) { blocks = count / cmd.Config.BatchSize diff --git a/commands/fast_replay.go b/commands/fast_replay.go new file mode 100644 index 0000000..79b480d --- /dev/null +++ b/commands/fast_replay.go @@ -0,0 +1,105 @@ +package commands + +import ( + "bufio" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "log" + "os" + "os/exec" + "syscall" + + "github.com/astroband/astrologer/es" +) + +type FastReplayCommandConfig struct { + UpTo int + Count int +} + +type FastReplayCommand struct { + ES es.Adapter + Config FastReplayCommandConfig +} + +// Execute starts the export process +func (cmd *FastReplayCommand) Execute() { + log.Printf("Fast replay %d ledgers up to %d\n", cmd.Config.Count, cmd.Config.UpTo) + + pipeFile := "astrologer_meta_stream" + + os.Remove(pipeFile) + err := syscall.Mkfifo(pipeFile, 0666) + + if err != nil { + log.Fatal("Make named pipe file error:", err) + } + + stellarCoreInstance := exec.Command( + "stellar-core", + "catchup", + fmt.Sprintf("%d/%d", cmd.Config.UpTo, cmd.Config.Count), + "--replay-in-memory", + "--conf", + "./stellar-core.cfg", + ) + + // stdout, err := stellarCoreInstance.StdoutPipe() + + // if err != nil { + // log.Fatal(err) + // } + + go func() { + if err := stellarCoreInstance.Start(); err != nil { + log.Fatal(err) + } + // scanner := bufio.NewScanner(stdout) + + // for scanner.Scan() { + // log.Println(scanner.Text()) + // } + + // if err := scanner.Err(); err != nil { + // log.Fatal(err) + // } + }() + + file, err := os.OpenFile(pipeFile, os.O_RDONLY, os.ModeNamedPipe) + + if err != nil { + log.Fatal("Open named pipe file error:", err) + } + + reader := bufio.NewReader(file) + sizeBytes := make([]byte, 4) + + for { + bytesRead, err := io.ReadFull(reader, sizeBytes) + + if err != nil { + log.Fatal(err) + } + + if bytesRead == 0 { + break + } + + sizeBytes[0] &= 0x7f + size := binary.BigEndian.Uint32(sizeBytes) + log.Printf("Gonna read %d bytes\n", size) + + data := make([]byte, size) + + _, err = io.ReadFull(reader, data) + + if err == nil { + sEnc := base64.StdEncoding.EncodeToString(data) + log.Printf("META: %s\n\n", sEnc) + } else { + log.Fatal("Error reading from pipe:", err) + } + } +} diff --git a/config/main.go b/config/main.go index 860f1f4..ffccc4a 100644 --- a/config/main.go +++ b/config/main.go @@ -53,6 +53,7 @@ var ( createIndexCommand = kingpin.Command("create-index", "Create ES indexes") exportCommand = kingpin.Command("export", "Run export") ingestCommand = kingpin.Command("ingest", "Start real time ingestion") + fastReplayCommand = kingpin.Command("fast-replay", "Experiment with using stellar-core fast in-memory replay catchup") _ = kingpin.Command("stats", "Print database ledger statistics") _ = kingpin.Command("es-stats", "Print ES ranges stats") @@ -91,8 +92,7 @@ var ( Default("25"). Int() - // Start ledger to start with - Start = NumberWithSignParse(exportCommand.Arg("start", "Ledger to start indexing, +100 means offset 100 from the first")) + Start = exportCommand.Arg("start", "Ledger to start indexing, +100 means offset 100 from the first").Default("0").Int() // Count ledgers Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() @@ -108,4 +108,7 @@ var ( // ForceRecreateIndexes Allows indexes to be deleted before creation ForceRecreateIndexes = createIndexCommand.Flag("force", "Delete indexes before creation").Bool() + + FastReplayUpTo = fastReplayCommand.Arg("upto", "Ledger to start indexing").Int() + FastReplayCount = fastReplayCommand.Arg("count", "Ledgers count to catchup").Int() ) diff --git a/db/main.go b/db/main.go index 813ca61..9a797e8 100644 --- a/db/main.go +++ b/db/main.go @@ -1,41 +1,13 @@ package db import ( - "bytes" "log" "net/url" - "unicode/utf8" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" // Postgres driver ) -// Copy paste from Horizon -func utf8Scrub(in string) string { - - // First check validity using the stdlib, returning if the string is already - // valid - if utf8.ValidString(in) { - return in - } - - left := []byte(in) - var result bytes.Buffer - - for len(left) > 0 { - r, n := utf8.DecodeRune(left) - - _, err := result.WriteRune(r) - if err != nil { - panic(err) - } - - left = left[n:] - } - - return result.String() -} - // Adapter defines the interface to work with ledger database type Adapter interface { LedgerHeaderRowCount(first int, last int) int diff --git a/db/tx_history_row.go b/db/tx_history_row.go index 1f31655..2b18fe3 100644 --- a/db/tx_history_row.go +++ b/db/tx_history_row.go @@ -6,6 +6,7 @@ import ( "log" "strings" + "github.com/astroband/astrologer/support" "github.com/guregu/null" "github.com/stellar/go/xdr" ) @@ -44,7 +45,7 @@ func (tx *TxHistoryRow) MemoValue() null.String { case xdr.MemoTypeMemoNone: value, valid = "", false case xdr.MemoTypeMemoText: - scrubbed := utf8Scrub(memo.MustText()) + scrubbed := support.Utf8Scrub(tx.Envelope.Tx.Memo.MustText()) notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") value, valid = notnull, true case xdr.MemoTypeMemoId: diff --git a/es/ledger_header.go b/es/ledger_header.go index d8534b0..737bf1e 100644 --- a/es/ledger_header.go +++ b/es/ledger_header.go @@ -1,9 +1,11 @@ package es import ( + "encoding/hex" "time" "github.com/astroband/astrologer/db" + "github.com/stellar/go/xdr" ) // LedgerHeader represents json-serializable struct for LedgerHeader to index @@ -48,6 +50,28 @@ func NewLedgerHeader(row *db.LedgerHeaderRow) *LedgerHeader { } } +func NewLedgerHeaderFromHistory(historyEntry xdr.LedgerHeaderHistoryEntry) *LedgerHeader { + header := historyEntry.Header + seq := int(header.LedgerSeq) + + return &LedgerHeader{ + Hash: hex.EncodeToString(historyEntry.Hash[:]), + PrevHash: hex.EncodeToString(header.PreviousLedgerHash[:]), + BucketListHash: hex.EncodeToString(header.BucketListHash[:]), + Seq: seq, + PagingToken: PagingToken{LedgerSeq: seq}, + CloseTime: time.Unix(int64(header.ScpValue.CloseTime), 0), + Version: int(header.LedgerVersion), + TotalCoins: int(header.TotalCoins), + FeePool: int(header.FeePool), + InflationSeq: int(header.InflationSeq), + IDPool: int(header.IdPool), + BaseFee: int(header.BaseFee), + BaseReserve: int(header.BaseReserve), + MaxTxSetSize: int(header.MaxTxSetSize), + } +} + // DocID returns es id (seq number in this case) func (h *LedgerHeader) DocID() *string { s := h.PagingToken.String() diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go new file mode 100644 index 0000000..ab8bcf7 --- /dev/null +++ b/es/ledger_serializer_xdr.go @@ -0,0 +1,106 @@ +package es + +import ( + "bytes" + + "github.com/stellar/go/xdr" +) + +type ledgerSerializerXDR struct { + ledgerHeader *LedgerHeader + + buffer *bytes.Buffer +} + +// SerializeLedger serializes ledger data into ES bulk index data +func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { + serializer := &ledgerSerializerXDR{ + ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), + buffer: buffer, + } + + serializer.serialize() +} + +func (s *ledgerSerializerXDR) serialize() { + SerializeForBulk(s.ledgerHeader, s.buffer) + + // for i, tx := range s.transactions { + // transaction := NewTransactionFromXDR(tx, s.txResults[i], s.ledger.Seq, i, s.ledger.CloseTime) + // SerializeForBulk(transaction, s.buffer) + + // if transaction.Successful { + // changes := s.feeRows[transaction.Index-1].Changes + // s.serializeBalances(changes, transaction, nil, BalanceSourceFee) + // } + + // s.serializeOperations(transactionRow, transaction) + // } +} + +// func (s *ledgerSerializerXDR) serializeOperations(transactionRow db.TxHistoryRow, transaction *Transaction) { +// effectsCount := 0 +// xdrs := transactionRow.Operations() + +// for index, xdr := range xdrs { +// result := transactionRow.ResultFor(index) +// operation := ProduceOperation(transaction, &xdr, result, index+1) +// SerializeForBulk(operation, s.buffer) + +// if transaction.Successful { +// metas := transactionRow.MetasFor(index) +// if metas != nil { +// effectsCount = s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) +// } + +// s.serializeTrades(result, transaction, operation, effectsCount) + +// h := ProduceSignerHistory(operation) +// if h != nil { +// SerializeForBulk(h, s.buffer) +// } +// } +// } +// } + +// func (s *ledgerSerializerXDR) serializeBalances(changes xdr.LedgerEntryChanges, transaction *Transaction, operation *Operation, source BalanceSource) int { +// if len(changes) == 0 { +// return 0 +// } + +// pagingToken := PagingToken{ +// LedgerSeq: s.ledger.Seq, +// TransactionOrder: transaction.Index, +// } + +// if operation != nil { +// pagingToken.OperationOrder = operation.Index +// } + +// balances := ProduceBalances(changes, s.ledger.CloseTime, source, pagingToken) + +// if len(balances) > 0 { +// for _, balance := range balances { +// SerializeForBulk(balance, s.buffer) +// } +// } + +// return len(balances) +// } + +// func (s *ledgerSerializerXDR) serializeTrades(result *xdr.OperationResult, transaction *Transaction, operation *Operation, startIndex int) int { +// pagingToken := PagingToken{ +// LedgerSeq: s.ledger.Seq, +// TransactionOrder: transaction.Index, +// OperationOrder: operation.Index, +// } + +// trades := ProduceTrades(result, operation, s.ledger.CloseTime, pagingToken, startIndex) +// if len(trades) > 0 { +// for _, trade := range trades { +// SerializeForBulk(&trade, s.buffer) +// } +// } + +// return len(trades) +// } diff --git a/es/transaction.go b/es/transaction.go index 88ee007..dc4caec 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -1,9 +1,12 @@ package es import ( + "encoding/hex" "time" "github.com/astroband/astrologer/db" + "github.com/astroband/astrologer/stellar" + "github.com/astroband/astrologer/util" "github.com/stellar/go/xdr" ) @@ -91,6 +94,48 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T return transaction, nil } +func NewTransactionFromXDR(txXDR xdr.Transaction, txResult xdr.TransactionResultPair, seq, index int, t time.Time) (*Transaction, error) { + resultCode := txResult.Result.Result.Code + + binTx, err := txXDR.MarshalBinary() + + if err != nil { + return nil, err + } + + tx := &Transaction{ + ID: hex.EncodeToString(binTx), + Index: index, + Seq: seq, + PagingToken: PagingToken{LedgerSeq: seq, TransactionOrder: index}, + Fee: int(txXDR.Fee), + FeeCharged: int(txResult.Result.FeeCharged), + OperationCount: len(txXDR.Operations), + CloseTime: t, + Successful: resultCode == xdr.TransactionResultCodeTxSuccess, + ResultCode: int(resultCode), + SourceAccountID: txXDR.SourceAccount.Address(), + } + + if txXDR.Memo.Type != xdr.MemoTypeMemoNone { + value := stellar.MemoValue(txXDR.Memo) + + tx.Memo = &Memo{ + Type: int(txXDR.Memo.Type), + Value: value.String, + } + } + + if txXDR.TimeBounds != nil { + tx.TimeBounds = &TimeBounds{ + MinTime: int64(txXDR.TimeBounds.MinTime), + MaxTime: int64(txXDR.TimeBounds.MaxTime), + } + } + + return tx, nil +} + // DocID return es transaction id (tx id in this case) func (tx *Transaction) DocID() *string { return &tx.ID diff --git a/go.mod b/go.mod index 7f54791..7b1ef5c 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,15 @@ require ( github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/elastic/go-elasticsearch/v7 v7.7.0 github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896 // indirect - github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6 + github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753 github.com/guregu/null v4.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 - github.com/lib/pq v1.5.2 + github.com/lib/pq v1.7.0 github.com/mattn/go-runewidth v0.0.9 // indirect github.com/olekukonko/tablewriter v0.0.4 github.com/pkg/errors v0.9.1 // indirect github.com/schollz/progressbar/v2 v2.15.0 - github.com/stellar/go v0.0.0-20200526231405-08ec13c54232 + github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5 github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) diff --git a/go.sum b/go.sum index c549f44..77c5268 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ github.com/gammazero/workerpool v0.0.0-20190515092108-567ff334f003 h1:9FFqhJ//pG github.com/gammazero/workerpool v0.0.0-20190515092108-567ff334f003/go.mod h1:lU9Kl5oGNQzGAc+k1OcuqtXCPBqnz8zrxoxEwqc9mRc= github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6 h1:1Cy/haf7XO4OyrkGid0Wq5CMluIErbvDptVAt8UTy38= github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= +github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753 h1:oSQ61LxZkz3Z4La0O5cbyVDvLWEfbNgiD43cSPdjPQQ= +github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= github.com/getsentry/raven-go v0.0.0-20160805001729-c9d3cc542ad1/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v3.1.5+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= @@ -118,6 +120,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw= github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= +github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.5.4/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739/go.mod h1:zUx1mhth20V3VKgL5jbd1BSQcW4Fy6Qs4PZvQwRFwzM= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -195,7 +199,8 @@ github.com/stellar/go v0.0.0-20200515153832-820becd8083b/go.mod h1:kfZjGLxoc7P6Q github.com/stellar/go v0.0.0-20200526231405-08ec13c54232 h1:HweN0TbrsTmdp58L+I1ozu2JSzn+w6fhaGSiLohEtxI= github.com/stellar/go v0.0.0-20200526231405-08ec13c54232/go.mod h1:Q4JDgZhGw7Ytu1924PzNkwr2GsLptrzAjwC/icuXnLU= github.com/stellar/go v0.0.0-20200528062442-f08b35a3f034 h1:YbQySRDlm+qU2+khzooq+bCEteADDprTrqh2YMNehqI= -github.com/stellar/go v0.0.0-20200617134134-abd5c5f8ad8f h1:vARqQGSAvss4qUaK+/7vJOLoNAOrod9+LYiqVVUxtrI= +github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5 h1:3ggUT5Mx9wmzFMCVW7eLiYLqaY2p9nj5Cb6XpnGuHF0= +github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5/go.mod h1:nZnKe/dfWbtFypSihxxX2UfMzMgeWgI1HvoK4jLh13I= github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e h1:n/hfey8pO+RYMoGXyvyzuw5pdO8IFDoyAL/g5OiCesY= github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e/go.mod h1:gpOLVzy6TVYTQ3LvHSN9RJC700FkhFCpSE82u37aNRM= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 h1:K9H+A+eWe8ZlnpNha+pXbEK+jtIluQp/2dKxkK8k7OE= @@ -263,6 +268,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go index cb0b966..8cd6956 100644 --- a/main.go +++ b/main.go @@ -24,7 +24,6 @@ func main() { config := cmd.CreateIndexCommandConfig{Force: *cfg.ForceRecreateIndexes} command = &cmd.CreateIndexCommand{ES: esClient, Config: config} case "export": - dbClient := db.Connect(*cfg.DatabaseURL) config := cmd.ExportCommandConfig{ Start: *cfg.Start, Count: *cfg.Count, @@ -32,12 +31,18 @@ func main() { RetryCount: *cfg.Retries, BatchSize: *cfg.BatchSize, } - command = &cmd.ExportCommand{ES: esClient, DB: dbClient, Config: config} + command = &cmd.ExportCommand{ES: esClient, Config: config} case "ingest": dbClient := db.Connect(*cfg.DatabaseURL) command = &cmd.IngestCommand{ES: esClient, DB: dbClient} case "es-stats": command = &cmd.EsStatsCommand{ES: esClient} + case "fast-replay": + config := cmd.FastReplayCommandConfig{ + UpTo: *cfg.FastReplayUpTo, + Count: *cfg.FastReplayCount, + } + command = &cmd.FastReplayCommand{ES: esClient, Config: config} } command.Execute() diff --git a/stellar-core.cfg b/stellar-core.cfg new file mode 100644 index 0000000..8680669 --- /dev/null +++ b/stellar-core.cfg @@ -0,0 +1,63 @@ +LOG_FILE_PATH="" +BUCKET_DIR_PATH="/var/lib/stellar/buckets" +DATABASE="postgresql://dbname=stellar user=charlie host=localhost connect_timeout=5" +HTTP_PORT=11626 +PUBLIC_HTTP_PORT=true +HTTP_MAX_CLIENT=128 + +NODE_NAMES=[ +"GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH sdf_1", +"GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK sdf_2", +"GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ sdf_3", +"GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE satoshipay_de", +"GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT satoshipay_sg", +"GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY satoshipay_us" +] +NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" +PEER_PORT=11625 +KNOWN_PEERS=[ +"core-live-a.stellar.org", +"core-live-b.stellar.org", +"core-live-c.stellar.org", +"stellar-de-fra.satoshipay.io", +"stellar-sg-sin.satoshipay.io", +"stellar-us-iowa.satoshipay.io", +"stellar.256kw.com", +"stellar.smartlands.io", +"ohio-1.stellar.stellarport.io", +"ohio-2.stellar.stellarport.io", +"stellar1.tempo.eu.com" +] +NODE_SEED="SB6W7PNMIGC7AF47D3UV5XLNKE6SVIZ24UX2RTHEGXJD2I5FKHMIYN5H" +MAX_CONCURRENT_SUBPROCESSES=2 +AUTOMATIC_MAINTENANCE_PERIOD=0 +METADATA_OUTPUT_STREAM="astrologer_meta_stream" + +[HISTORY.satoshipay_de] +get="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}" + +[HISTORY.satoshipay_sg] +get="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}" + +[HISTORY.satoshipay_us] +get="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}" + +[HISTORY.sdf_1] +get="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" + +[HISTORY.sdf_2] +get="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" + +[HISTORY.sdf_3] +get="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" + +[QUORUM_SET] + +VALIDATORS=[ +"$sdf_1", +"$sdf_2", +"$sdf_3", +"$satoshipay_de", +"$satoshipay_sg", +"$satoshipay_us" +] diff --git a/stellar/main.go b/stellar/main.go new file mode 100644 index 0000000..3905521 --- /dev/null +++ b/stellar/main.go @@ -0,0 +1,140 @@ +package stellar + +import ( + "bufio" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "log" + "os" + "os/exec" + "strings" + "syscall" + + "github.com/astroband/astrologer/support" + "github.com/guregu/null" + "github.com/stellar/go/xdr" +) + +func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { + const pipeFile = "astrologer_meta_stream" + os.Remove(pipeFile) + ch := make(chan xdr.LedgerCloseMeta) + + err := syscall.Mkfifo(pipeFile, 0666) + + if err != nil { + log.Fatal("Make named pipe file error:", err) + } + + stellarCoreInstance := exec.Command( + "stellar-core", + "catchup", + fmt.Sprintf("%d/%d", lastLedger, lastLedger-firstLedger+1), + "--replay-in-memory", + "--conf", + "./stellar-core.cfg", + ) + + // stdout, err := stellarCoreInstance.StdoutPipe() + + // if err != nil { + // log.Fatal(err) + // } + + go func() { + if err := stellarCoreInstance.Start(); err != nil { + log.Fatal(err) + } + // scanner := bufio.NewScanner(stdout) + + // for scanner.Scan() { + // log.Println(scanner.Text()) + // } + + // if err := scanner.Err(); err != nil { + // log.Fatal(err) + // } + }() + + pipe, err := os.OpenFile(pipeFile, os.O_RDONLY, os.ModeNamedPipe) + + if err != nil { + log.Fatal("Open named pipe file error:", err) + } + + reader := bufio.NewReader(pipe) + sizeBytes := make([]byte, 4) + + go func() { + defer close(ch) + + for { + bytesRead, err := io.ReadFull(reader, sizeBytes) + + if err != nil { + log.Fatal(err) + } + + if bytesRead == 0 { + break + } + + sizeBytes[0] &= 0x7f + size := binary.BigEndian.Uint32(sizeBytes) + + data := make([]byte, size) + + _, err = io.ReadFull(reader, data) + + if err == nil { + var meta xdr.LedgerCloseMeta + + meta.UnmarshalBinary(data) + ch <- meta + } else { + log.Fatal("Error reading from pipe:", err) + } + } + + err = pipe.Close() + + if err != nil { + log.Println(err) + } + }() + + return ch +} + +func MemoValue(memo xdr.Memo) null.String { + var ( + value string + valid bool + ) + switch memo.Type { + case xdr.MemoTypeMemoNone: + value, valid = "", false + case xdr.MemoTypeMemoText: + scrubbed := support.Utf8Scrub(memo.MustText()) + notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") + value, valid = notnull, true + case xdr.MemoTypeMemoId: + value, valid = fmt.Sprintf("%d", memo.MustId()), true + case xdr.MemoTypeMemoHash: + hash := memo.MustHash() + value, valid = + base64.StdEncoding.EncodeToString(hash[:]), + true + case xdr.MemoTypeMemoReturn: + hash := memo.MustRetHash() + value, valid = + base64.StdEncoding.EncodeToString(hash[:]), + true + default: + panic(fmt.Errorf("invalid memo type: %v", memo.Type)) + } + + return null.NewString(value, valid) +} diff --git a/support/main.go b/support/main.go new file mode 100644 index 0000000..b6a48b5 --- /dev/null +++ b/support/main.go @@ -0,0 +1,32 @@ +package support + +import ( + "bytes" + "unicode/utf8" +) + +// Copy paste from Horizon +func Utf8Scrub(in string) string { + + // First check validity using the stdlib, returning if the string is already + // valid + if utf8.ValidString(in) { + return in + } + + left := []byte(in) + var result bytes.Buffer + + for len(left) > 0 { + r, n := utf8.DecodeRune(left) + + _, err := result.WriteRune(r) + if err != nil { + panic(err) + } + + left = left[n:] + } + + return result.String() +} From cfcd7678bbaaf8ba20a2f10e0a5ee794a7de8640 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 25 Mar 2020 15:01:31 +0300 Subject: [PATCH 02/18] Ingest ledgers and transactions --- commands/export.go | 12 +++---- es/adapter.go | 17 +++++++--- es/ledger_serializer_xdr.go | 62 ++++++++++++++++++++++++++++--------- es/main.go | 2 +- es/transaction.go | 52 +++++++++++++++++++------------ stellar/main.go | 9 +++--- 6 files changed, 105 insertions(+), 49 deletions(-) diff --git a/commands/export.go b/commands/export.go index 77cc255..bb03d03 100644 --- a/commands/export.go +++ b/commands/export.go @@ -54,13 +54,11 @@ func (cmd *ExportCommand) Execute() { es.SerializeLedgerFromHistory(meta, &b) } - log.Println("DONE") - log.Println(b.Len()) - indexed := cmd.ES.BulkInsert(&b) + err := cmd.ES.BulkInsert(&b) - if !indexed { - log.Fatal("Cannot bulk insert") + if err != nil { + log.Fatal("Cannot bulk insert", err) } // for i := 0; i < cmd.blockCount(total); i++ { @@ -96,9 +94,9 @@ func (cmd *ExportCommand) Execute() { // } func (cmd *ExportCommand) index(b *bytes.Buffer, retry int) { - indexed := cmd.ES.BulkInsert(b) + err := cmd.ES.BulkInsert(b) - if !indexed { + if err != nil { if retry > cmd.Config.RetryCount { log.Fatal("Retries for bulk failed, aborting") } diff --git a/es/adapter.go b/es/adapter.go index 1919aa8..0355e9b 100644 --- a/es/adapter.go +++ b/es/adapter.go @@ -3,6 +3,7 @@ package es import ( "bytes" "encoding/json" + "errors" "log" "math/rand" "net/http" @@ -106,14 +107,22 @@ func (es *Client) LedgerSeqRangeQuery(ranges []map[string]interface{}) map[strin } // BulkInsert sends the payload to ES using bulk operation -func (es *Client) BulkInsert(payload *bytes.Buffer) (success bool) { +func (es *Client) BulkInsert(payload *bytes.Buffer) error { res, err := es.rawClient.Bulk(bytes.NewReader(payload.Bytes())) if res != nil { defer res.Body.Close() } - return err == nil && (res == nil || !res.IsError()) + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil } // LedgerCountInRange counts number of ledgers from the given range persisted into ES @@ -183,9 +192,9 @@ func (es *Client) GetLedgerSeqsInRange(min, max int) (seqs []int) { // IndexWithRetries performs a bulk insert into ES cluster with retries on failures func (es *Client) IndexWithRetries(payload *bytes.Buffer, retryCount int) { - isIndexed := es.BulkInsert(payload) + err := es.BulkInsert(payload) - if !isIndexed { + if err != nil { if retryCount-1 == 0 { log.Fatal("Retries for bulk failed, aborting") } diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go index ab8bcf7..4c42025 100644 --- a/es/ledger_serializer_xdr.go +++ b/es/ledger_serializer_xdr.go @@ -2,21 +2,37 @@ package es import ( "bytes" + "log" "github.com/stellar/go/xdr" ) type ledgerSerializerXDR struct { - ledgerHeader *LedgerHeader + ledgerHeader *LedgerHeader + transactions []xdr.Transaction + transactionResults []xdr.TransactionResultPair buffer *bytes.Buffer } // SerializeLedger serializes ledger data into ES bulk index data func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { + transactions := make([]xdr.Transaction, len(meta.V0.TxSet.Txs)) + transactionResults := make([]xdr.TransactionResultPair, len(meta.V0.TxProcessing)) + + for i, txe := range meta.V0.TxSet.Txs { + transactions[i] = txe.Tx + } + + for i, txp := range meta.V0.TxProcessing { + transactionResults[i] = txp.Result + } + serializer := &ledgerSerializerXDR{ - ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), - buffer: buffer, + ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), + transactions: transactions, + transactionResults: transactionResults, + buffer: buffer, } serializer.serialize() @@ -25,17 +41,35 @@ func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) func (s *ledgerSerializerXDR) serialize() { SerializeForBulk(s.ledgerHeader, s.buffer) - // for i, tx := range s.transactions { - // transaction := NewTransactionFromXDR(tx, s.txResults[i], s.ledger.Seq, i, s.ledger.CloseTime) - // SerializeForBulk(transaction, s.buffer) - - // if transaction.Successful { - // changes := s.feeRows[transaction.Index-1].Changes - // s.serializeBalances(changes, transaction, nil, BalanceSourceFee) - // } - - // s.serializeOperations(transactionRow, transaction) - // } + for i, tx := range s.transactions { + transaction, err := NewTransactionFromXDR( + &transactionData{ + xdr: tx, + result: s.transactionResults[i], + index: i + 1, + ledgerSeq: s.ledgerHeader.Seq, + closeTime: s.ledgerHeader.CloseTime, + }, + ) + + if err != nil { + log.Fatal(err) + } + + SerializeForBulk(transaction, s.buffer) + + // if transaction.Successful { + // changes := s.feeRows[transaction.Index-1].Changes + // s.serializeBalances(changes, transaction, nil, BalanceSourceFee) + // } + + // if transaction.Successful { + // changes := s.feeRows[transaction.Index-1].Changes + // s.serializeBalances(changes, transaction, nil, BalanceSourceFee) + // } + + // s.serializeOperations(transactionRow, transaction) + } } // func (s *ledgerSerializerXDR) serializeOperations(transactionRow db.TxHistoryRow, transaction *Transaction) { diff --git a/es/main.go b/es/main.go index 7e7c43e..7b0f289 100644 --- a/es/main.go +++ b/es/main.go @@ -22,7 +22,7 @@ type Adapter interface { IndexExists(name IndexName) bool CreateIndex(name IndexName, body IndexDefinition) DeleteIndex(name IndexName) - BulkInsert(payload *bytes.Buffer) (success bool) + BulkInsert(payload *bytes.Buffer) error IndexWithRetries(payload *bytes.Buffer, retriesCount int) } diff --git a/es/transaction.go b/es/transaction.go index dc4caec..6d1f96e 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -7,9 +7,22 @@ import ( "github.com/astroband/astrologer/db" "github.com/astroband/astrologer/stellar" "github.com/astroband/astrologer/util" + "github.com/stellar/go/network" "github.com/stellar/go/xdr" ) +// transactionData represents all necessary pieces we need +// from stellar-core to ingest single transaction +type transactionData struct { + ledgerSeq int + // ledger close time + closeTime time.Time + // index of transaction in ledger + index int + xdr xdr.Transaction + result xdr.TransactionResultPair +} + // Transaction represents ES-serializable transaction type Transaction struct { ID string `json:"id"` @@ -61,8 +74,8 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T CloseTime: t, Successful: success, ResultCode: int(result.Code), - OperationCount: len(envelope.Operations()), SourceAccountID: sourceAccountAddress, + OperationCount: len(envelope.Operations()), } if envelope.IsFeeBump() { @@ -94,42 +107,43 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T return transaction, nil } -func NewTransactionFromXDR(txXDR xdr.Transaction, txResult xdr.TransactionResultPair, seq, index int, t time.Time) (*Transaction, error) { - resultCode := txResult.Result.Result.Code +func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { + resultCode := data.result.Result.Result.Code - binTx, err := txXDR.MarshalBinary() + //FIXME remove hardcoded network passphrase + txHash, err := network.HashTransaction(&data.xdr, "Public Global Stellar Network ; September 2015") if err != nil { return nil, err } tx := &Transaction{ - ID: hex.EncodeToString(binTx), - Index: index, - Seq: seq, - PagingToken: PagingToken{LedgerSeq: seq, TransactionOrder: index}, - Fee: int(txXDR.Fee), - FeeCharged: int(txResult.Result.FeeCharged), - OperationCount: len(txXDR.Operations), - CloseTime: t, + ID: hex.EncodeToString(txHash[:]), + Index: data.index, + Seq: data.ledgerSeq, + PagingToken: PagingToken{LedgerSeq: data.ledgerSeq, TransactionOrder: data.index}, + Fee: int(data.xdr.Fee), + FeeCharged: int(data.result.Result.FeeCharged), + OperationCount: len(data.xdr.Operations), + CloseTime: data.closeTime, Successful: resultCode == xdr.TransactionResultCodeTxSuccess, ResultCode: int(resultCode), - SourceAccountID: txXDR.SourceAccount.Address(), + SourceAccountID: data.xdr.SourceAccount.Address(), } - if txXDR.Memo.Type != xdr.MemoTypeMemoNone { - value := stellar.MemoValue(txXDR.Memo) + if data.xdr.Memo.Type != xdr.MemoTypeMemoNone { + value := stellar.MemoValue(data.xdr.Memo) tx.Memo = &Memo{ - Type: int(txXDR.Memo.Type), + Type: int(data.xdr.Memo.Type), Value: value.String, } } - if txXDR.TimeBounds != nil { + if data.xdr.TimeBounds != nil { tx.TimeBounds = &TimeBounds{ - MinTime: int64(txXDR.TimeBounds.MinTime), - MaxTime: int64(txXDR.TimeBounds.MaxTime), + MinTime: int64(data.xdr.TimeBounds.MinTime), + MaxTime: int64(data.xdr.TimeBounds.MaxTime), } } diff --git a/stellar/main.go b/stellar/main.go index 3905521..af68cb4 100644 --- a/stellar/main.go +++ b/stellar/main.go @@ -73,14 +73,15 @@ func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { for { bytesRead, err := io.ReadFull(reader, sizeBytes) - if err != nil { - log.Fatal(err) - } - + // No more to read if bytesRead == 0 { break } + if err != nil { + log.Fatal("Error on reading from pipe", err) + } + sizeBytes[0] &= 0x7f size := binary.BigEndian.Uint32(sizeBytes) From 290e3128d588eddde9750d9d88af87ffb09299f6 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 25 Mar 2020 15:31:29 +0300 Subject: [PATCH 03/18] Ingest balances --- es/ledger_serializer_xdr.go | 55 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go index 4c42025..282a66c 100644 --- a/es/ledger_serializer_xdr.go +++ b/es/ledger_serializer_xdr.go @@ -11,6 +11,7 @@ type ledgerSerializerXDR struct { ledgerHeader *LedgerHeader transactions []xdr.Transaction transactionResults []xdr.TransactionResultPair + changes []xdr.LedgerEntryChanges buffer *bytes.Buffer } @@ -19,6 +20,7 @@ type ledgerSerializerXDR struct { func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { transactions := make([]xdr.Transaction, len(meta.V0.TxSet.Txs)) transactionResults := make([]xdr.TransactionResultPair, len(meta.V0.TxProcessing)) + changes := make([]xdr.LedgerEntryChanges, len(meta.V0.TxProcessing)) for i, txe := range meta.V0.TxSet.Txs { transactions[i] = txe.Tx @@ -26,12 +28,14 @@ func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) for i, txp := range meta.V0.TxProcessing { transactionResults[i] = txp.Result + changes[i] = txp.FeeProcessing } serializer := &ledgerSerializerXDR{ ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), transactions: transactions, transactionResults: transactionResults, + changes: changes, buffer: buffer, } @@ -58,15 +62,10 @@ func (s *ledgerSerializerXDR) serialize() { SerializeForBulk(transaction, s.buffer) - // if transaction.Successful { - // changes := s.feeRows[transaction.Index-1].Changes - // s.serializeBalances(changes, transaction, nil, BalanceSourceFee) - // } - - // if transaction.Successful { - // changes := s.feeRows[transaction.Index-1].Changes - // s.serializeBalances(changes, transaction, nil, BalanceSourceFee) - // } + if transaction.Successful { + changes := s.changes[i] + s.serializeBalances(changes, transaction, nil, BalanceSourceFee) + } // s.serializeOperations(transactionRow, transaction) } @@ -97,30 +96,30 @@ func (s *ledgerSerializerXDR) serialize() { // } // } -// func (s *ledgerSerializerXDR) serializeBalances(changes xdr.LedgerEntryChanges, transaction *Transaction, operation *Operation, source BalanceSource) int { -// if len(changes) == 0 { -// return 0 -// } +func (s *ledgerSerializerXDR) serializeBalances(changes xdr.LedgerEntryChanges, transaction *Transaction, operation *Operation, source BalanceSource) int { + if len(changes) == 0 { + return 0 + } -// pagingToken := PagingToken{ -// LedgerSeq: s.ledger.Seq, -// TransactionOrder: transaction.Index, -// } + pagingToken := PagingToken{ + LedgerSeq: s.ledgerHeader.Seq, + TransactionOrder: transaction.Index, + } -// if operation != nil { -// pagingToken.OperationOrder = operation.Index -// } + if operation != nil { + pagingToken.OperationOrder = operation.Index + } -// balances := ProduceBalances(changes, s.ledger.CloseTime, source, pagingToken) + balances := ProduceBalances(changes, s.ledgerHeader.CloseTime, source, pagingToken) -// if len(balances) > 0 { -// for _, balance := range balances { -// SerializeForBulk(balance, s.buffer) -// } -// } + if len(balances) > 0 { + for _, balance := range balances { + SerializeForBulk(balance, s.buffer) + } + } -// return len(balances) -// } + return len(balances) +} // func (s *ledgerSerializerXDR) serializeTrades(result *xdr.OperationResult, transaction *Transaction, operation *Operation, startIndex int) int { // pagingToken := PagingToken{ From dee8b4bfd138967ee80fe73ffa1490cd48c37402 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 25 Mar 2020 22:17:18 +0300 Subject: [PATCH 04/18] Ingest operations and signers --- es/ledger_serializer_xdr.go | 49 ++++++++++++++++++++----------------- stellar/main.go | 14 +++++++++++ 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go index 282a66c..9704640 100644 --- a/es/ledger_serializer_xdr.go +++ b/es/ledger_serializer_xdr.go @@ -4,6 +4,7 @@ import ( "bytes" "log" + "github.com/astroband/astrologer/stellar" "github.com/stellar/go/xdr" ) @@ -11,6 +12,7 @@ type ledgerSerializerXDR struct { ledgerHeader *LedgerHeader transactions []xdr.Transaction transactionResults []xdr.TransactionResultPair + transactionMetas []xdr.TransactionMeta changes []xdr.LedgerEntryChanges buffer *bytes.Buffer @@ -21,6 +23,7 @@ func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) transactions := make([]xdr.Transaction, len(meta.V0.TxSet.Txs)) transactionResults := make([]xdr.TransactionResultPair, len(meta.V0.TxProcessing)) changes := make([]xdr.LedgerEntryChanges, len(meta.V0.TxProcessing)) + transactionMetas := make([]xdr.TransactionMeta, len(meta.V0.TxProcessing)) for i, txe := range meta.V0.TxSet.Txs { transactions[i] = txe.Tx @@ -29,12 +32,14 @@ func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) for i, txp := range meta.V0.TxProcessing { transactionResults[i] = txp.Result changes[i] = txp.FeeProcessing + transactionMetas[i] = txp.TxApplyProcessing } serializer := &ledgerSerializerXDR{ ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), transactions: transactions, transactionResults: transactionResults, + transactionMetas: transactionMetas, changes: changes, buffer: buffer, } @@ -71,30 +76,30 @@ func (s *ledgerSerializerXDR) serialize() { } } -// func (s *ledgerSerializerXDR) serializeOperations(transactionRow db.TxHistoryRow, transaction *Transaction) { -// effectsCount := 0 -// xdrs := transactionRow.Operations() +func (s *ledgerSerializerXDR) serializeOperations(operations []xdr.Operation, operationResults *[]xdr.OperationResult, transaction *Transaction) { + // effectsCount := 0 -// for index, xdr := range xdrs { -// result := transactionRow.ResultFor(index) -// operation := ProduceOperation(transaction, &xdr, result, index+1) -// SerializeForBulk(operation, s.buffer) + for index, operation := range operations { + result := (*operationResults)[index] + operation := ProduceOperation(transaction, &operation, &result, index+1) + SerializeForBulk(operation, s.buffer) -// if transaction.Successful { -// metas := transactionRow.MetasFor(index) -// if metas != nil { -// effectsCount = s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) -// } - -// s.serializeTrades(result, transaction, operation, effectsCount) - -// h := ProduceSignerHistory(operation) -// if h != nil { -// SerializeForBulk(h, s.buffer) -// } -// } -// } -// } + if transaction.Successful { + metas := stellar.OperationMeta(s.transactionMetas[transaction.Index], index) + if metas != nil { + // effectsCount = s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) + s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) + } + + // s.serializeTrades(result, transaction, operation, effectsCount) + + h := ProduceSignerHistory(operation) + if h != nil { + SerializeForBulk(h, s.buffer) + } + } + } +} func (s *ledgerSerializerXDR) serializeBalances(changes xdr.LedgerEntryChanges, transaction *Transaction, operation *Operation, source BalanceSource) int { if len(changes) == 0 { diff --git a/stellar/main.go b/stellar/main.go index af68cb4..9203af9 100644 --- a/stellar/main.go +++ b/stellar/main.go @@ -139,3 +139,17 @@ func MemoValue(memo xdr.Memo) null.String { return null.NewString(value, valid) } + +func OperationMeta(txMeta xdr.TransactionMeta, opIndex int) (result *xdr.OperationMeta) { + if v1, ok := txMeta.GetV1(); ok { + ops := v1.Operations + return &ops[opIndex] + } + + ops, ok := txMeta.GetOperations() + if !ok { + return nil + } + + return &ops[opIndex] +} From ef1de8c5c1b5c8d3352117fb5c3275cda5d6afbe Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Fri, 27 Mar 2020 16:32:30 +0300 Subject: [PATCH 05/18] [WIP] Try to ingest in batches --- .gitignore | 2 + commands/export.go | 116 ++++++-------------- es/adapter.go | 4 +- es/main.go | 2 +- go.sum | 2 + stellar-core.cfg => stellar-core/pubnet.cfg | 2 +- stellar/main.go | 43 ++++---- 7 files changed, 65 insertions(+), 106 deletions(-) rename stellar-core.cfg => stellar-core/pubnet.cfg (97%) diff --git a/.gitignore b/.gitignore index 9f3758e..6d85c82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /astrologer vendor dist + +stellar-core/buckets diff --git a/commands/export.go b/commands/export.go index bb03d03..769447a 100644 --- a/commands/export.go +++ b/commands/export.go @@ -2,9 +2,7 @@ package commands import ( "bytes" - "log" - "math/rand" - "time" + log "github.com/sirupsen/logrus" "github.com/astroband/astrologer/es" "github.com/astroband/astrologer/stellar" @@ -39,100 +37,52 @@ func (cmd *ExportCommand) Execute() { log.Fatal("Nothing to export within given range!", cmd.firstLedger, cmd.lastLedger) } - log.Println("Exporting ledgers from", cmd.firstLedger, "to", cmd.lastLedger, "total", total) + log.Infof("Exporting ledgers from %d to %d. Total: %d ledgers\n", cmd.firstLedger, cmd.lastLedger, total) + log.Infof("Will insert %d batches %d ledgers each\n", cmd.blockCount(total), cmd.Config.BatchSize) - var b bytes.Buffer + for i := 0; i < cmd.blockCount(total); i++ { + var b bytes.Buffer + ledgerCounter := 0 + batchNum := i + 1 - for meta := range stellar.StreamLedgers(cmd.firstLedger, cmd.lastLedger) { - seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) + for meta := range stellar.StreamLedgers(cmd.firstLedger, cmd.lastLedger) { + seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) - if seq < cmd.firstLedger || seq > cmd.lastLedger { - continue - } - - log.Println(seq) - - es.SerializeLedgerFromHistory(meta, &b) - } + if seq < cmd.firstLedger || seq > cmd.lastLedger { + continue + } - err := cmd.ES.BulkInsert(&b) + ledgerCounter += 1 - if err != nil { - log.Fatal("Cannot bulk insert", err) - } - - // for i := 0; i < cmd.blockCount(total); i++ { - // i := i - // pool.Submit(func() { cmd.exportBlock(i) }) - // } - - // pool.StopWait() -} + log.Println(seq) -// func (cmd *ExportCommand) exportBlock(i int) { -// var b bytes.Buffer + es.SerializeLedgerFromHistory(meta, &b) -// rows := cmd.DB.LedgerHeaderRowFetchBatch(i, cmd.firstLedger, cmd.Config.BatchSize) + log.Printf("Ledger %d of %d in batch %d\n", ledgerCounter, cmd.Config.BatchSize, batchNum) -// for n := 0; n < len(rows); n++ { -// txs := cmd.DB.TxHistoryRowForSeq(rows[n].LedgerSeq) -// fees := cmd.DB.TxFeeHistoryRowsForRows(txs) -// es.SerializeLedger(rows[n], txs, fees, &b) - -// if !*config.Verbose { -// bar.Add(1) -// } -// } - -// if *config.Verbose { -// log.Println(b.String()) -// } - -// if !cmd.Config.DryRun { -// cmd.ES.IndexWithRetries(&b, cmd.Config.RetryCount) -// } -// } - -func (cmd *ExportCommand) index(b *bytes.Buffer, retry int) { - err := cmd.ES.BulkInsert(b) + if ledgerCounter == cmd.Config.BatchSize { + break + } + } - if err != nil { - if retry > cmd.Config.RetryCount { - log.Fatal("Retries for bulk failed, aborting") + if cmd.Config.DryRun { + continue } - delay := time.Duration((rand.Intn(10) + 5)) - time.Sleep(delay * time.Second) + pool.Submit(func() { + log.Printf("Gonna bulk insert %d bytes\n", b.Len()) + err := cmd.ES.BulkInsert(b) - cmd.index(b, retry+1) + if err != nil { + log.Fatal("Cannot bulk insert", err) + } else { + log.Printf("Batch %d successfully inserted\n", batchNum) + } + }) } -} -// Parses range of export command -// func (cmd *ExportCommand) getRange() (first int, last int) { -// firstLedger := cmd.DB.LedgerHeaderFirstRow() -// lastLedger := cmd.DB.LedgerHeaderLastRow() - -// if cmd.Config.Start.Explicit { -// if cmd.Config.Start.Value < 0 { -// first = lastLedger.LedgerSeq + cmd.Config.Start.Value + 1 -// } else if config.Start.Value > 0 { -// first = firstLedger.LedgerSeq + cmd.Config.Start.Value -// } -// } else if cmd.Config.Start.Value != 0 { -// first = cmd.Config.Start.Value -// } else { -// first = firstLedger.LedgerSeq -// } - -// if cmd.Config.Count == 0 { -// last = lastLedger.LedgerSeq -// } else { -// last = first + cmd.Config.Count - 1 -// } - -// return first, last -// } + pool.StopWait() +} func (cmd *ExportCommand) blockCount(count int) (blocks int) { blocks = count / cmd.Config.BatchSize diff --git a/es/adapter.go b/es/adapter.go index 0355e9b..5dc007c 100644 --- a/es/adapter.go +++ b/es/adapter.go @@ -107,7 +107,7 @@ func (es *Client) LedgerSeqRangeQuery(ranges []map[string]interface{}) map[strin } // BulkInsert sends the payload to ES using bulk operation -func (es *Client) BulkInsert(payload *bytes.Buffer) error { +func (es *Client) BulkInsert(payload bytes.Buffer) error { res, err := es.rawClient.Bulk(bytes.NewReader(payload.Bytes())) if res != nil { @@ -192,7 +192,7 @@ func (es *Client) GetLedgerSeqsInRange(min, max int) (seqs []int) { // IndexWithRetries performs a bulk insert into ES cluster with retries on failures func (es *Client) IndexWithRetries(payload *bytes.Buffer, retryCount int) { - err := es.BulkInsert(payload) + err := es.BulkInsert(*payload) if err != nil { if retryCount-1 == 0 { diff --git a/es/main.go b/es/main.go index 7b0f289..68c7d54 100644 --- a/es/main.go +++ b/es/main.go @@ -22,7 +22,7 @@ type Adapter interface { IndexExists(name IndexName) bool CreateIndex(name IndexName, body IndexDefinition) DeleteIndex(name IndexName) - BulkInsert(payload *bytes.Buffer) error + BulkInsert(payload bytes.Buffer) error IndexWithRetries(payload *bytes.Buffer, retriesCount int) } diff --git a/go.sum b/go.sum index 77c5268..ee32858 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,7 @@ github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2/go.mod h1:8z github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/sirupsen/logrus v1.0.6-0.20180530095059-070c81def33f/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -230,6 +231,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/stellar-core.cfg b/stellar-core/pubnet.cfg similarity index 97% rename from stellar-core.cfg rename to stellar-core/pubnet.cfg index 8680669..f3f7675 100644 --- a/stellar-core.cfg +++ b/stellar-core/pubnet.cfg @@ -1,5 +1,5 @@ LOG_FILE_PATH="" -BUCKET_DIR_PATH="/var/lib/stellar/buckets" +BUCKET_DIR_PATH="./stellar-core/buckets" DATABASE="postgresql://dbname=stellar user=charlie host=localhost connect_timeout=5" HTTP_PORT=11626 PUBLIC_HTTP_PORT=true diff --git a/stellar/main.go b/stellar/main.go index 9203af9..a72d358 100644 --- a/stellar/main.go +++ b/stellar/main.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "fmt" "io" - "log" "os" "os/exec" "strings" @@ -14,6 +13,7 @@ import ( "github.com/astroband/astrologer/support" "github.com/guregu/null" + log "github.com/sirupsen/logrus" "github.com/stellar/go/xdr" ) @@ -34,28 +34,29 @@ func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { fmt.Sprintf("%d/%d", lastLedger, lastLedger-firstLedger+1), "--replay-in-memory", "--conf", - "./stellar-core.cfg", + "./stellar-core/pubnet.cfg", ) + stellarCoreLogger := log.WithFields(log.Fields{"process": "stellar-core"}) + stellarCoreStdOut, err := stellarCoreInstance.StdoutPipe() - // stdout, err := stellarCoreInstance.StdoutPipe() - - // if err != nil { - // log.Fatal(err) - // } + if err != nil { + log.Fatal(err) + } go func() { if err := stellarCoreInstance.Start(); err != nil { - log.Fatal(err) + stellarCoreLogger.Fatal(err) } - // scanner := bufio.NewScanner(stdout) - // for scanner.Scan() { - // log.Println(scanner.Text()) - // } + scanner := bufio.NewScanner(stellarCoreStdOut) + + for scanner.Scan() { + stellarCoreLogger.Println(scanner.Text()) + } - // if err := scanner.Err(); err != nil { - // log.Fatal(err) - // } + if err := scanner.Err(); err != nil { + stellarCoreLogger.Fatal(err) + } }() pipe, err := os.OpenFile(pipeFile, os.O_RDONLY, os.ModeNamedPipe) @@ -67,6 +68,8 @@ func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { reader := bufio.NewReader(pipe) sizeBytes := make([]byte, 4) + readerLogger := log.WithFields(log.Fields{"process": "reader"}) + go func() { defer close(ch) @@ -75,11 +78,12 @@ func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { // No more to read if bytesRead == 0 { + readerLogger.Info("Reached the end of the stream") break } if err != nil { - log.Fatal("Error on reading from pipe", err) + readerLogger.Fatal("Failed to read from pipe", err) } sizeBytes[0] &= 0x7f @@ -91,18 +95,19 @@ func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { if err == nil { var meta xdr.LedgerCloseMeta - meta.UnmarshalBinary(data) + + readerLogger.Info("Writing to the channel...") ch <- meta } else { - log.Fatal("Error reading from pipe:", err) + readerLogger.Fatal("Error reading from pipe:", err) } } err = pipe.Close() if err != nil { - log.Println(err) + readerLogger.Println(err) } }() From 10f9f9e286de07e8c8d922a247a125a869d2bde0 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 1 Apr 2020 18:08:52 +0300 Subject: [PATCH 06/18] Fix channel reading --- commands/export.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/commands/export.go b/commands/export.go index 769447a..47af4ae 100644 --- a/commands/export.go +++ b/commands/export.go @@ -40,12 +40,14 @@ func (cmd *ExportCommand) Execute() { log.Infof("Exporting ledgers from %d to %d. Total: %d ledgers\n", cmd.firstLedger, cmd.lastLedger, total) log.Infof("Will insert %d batches %d ledgers each\n", cmd.blockCount(total), cmd.Config.BatchSize) + channel := stellar.StreamLedgers(cmd.firstLedger, cmd.lastLedger) + for i := 0; i < cmd.blockCount(total); i++ { var b bytes.Buffer ledgerCounter := 0 batchNum := i + 1 - for meta := range stellar.StreamLedgers(cmd.firstLedger, cmd.lastLedger) { + for meta := range channel { seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) if seq < cmd.firstLedger || seq > cmd.lastLedger { From 7d8784154c057f92a7f5d335153d5585db22e13c Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 17 Jun 2020 14:08:11 +0300 Subject: [PATCH 07/18] [WIP] Protocol 13 support --- db/tx_history_row.go | 2 +- es/ledger_serializer_xdr.go | 7 ++++++- es/transaction.go | 5 ++++- go.mod | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/db/tx_history_row.go b/db/tx_history_row.go index 2b18fe3..25146c8 100644 --- a/db/tx_history_row.go +++ b/db/tx_history_row.go @@ -45,7 +45,7 @@ func (tx *TxHistoryRow) MemoValue() null.String { case xdr.MemoTypeMemoNone: value, valid = "", false case xdr.MemoTypeMemoText: - scrubbed := support.Utf8Scrub(tx.Envelope.Tx.Memo.MustText()) + scrubbed := support.Utf8Scrub(memo.MustText()) notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") value, valid = notnull, true case xdr.MemoTypeMemoId: diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go index 9704640..5d18ea3 100644 --- a/es/ledger_serializer_xdr.go +++ b/es/ledger_serializer_xdr.go @@ -26,7 +26,12 @@ func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) transactionMetas := make([]xdr.TransactionMeta, len(meta.V0.TxProcessing)) for i, txe := range meta.V0.TxSet.Txs { - transactions[i] = txe.Tx + switch txe.Type { + case xdr.EnvelopeTypeEnvelopeTypeTxV0: + transactions[i] = txe.V0.Tx + case xdr.EnvelopeTypeEnvelopeTypeTx: + transactions[i] = txe.V1.Tx + } } for i, txp := range meta.V0.TxProcessing { diff --git a/es/transaction.go b/es/transaction.go index 6d1f96e..7eb6782 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -111,7 +111,10 @@ func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { resultCode := data.result.Result.Result.Code //FIXME remove hardcoded network passphrase - txHash, err := network.HashTransaction(&data.xdr, "Public Global Stellar Network ; September 2015") + txHash, err := network.HashTransaction( + data.xdr, + "Public Global Stellar Network ; September 2015", + ) if err != nil { return nil, err diff --git a/go.mod b/go.mod index 7b1ef5c..168326c 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/olekukonko/tablewriter v0.0.4 github.com/pkg/errors v0.9.1 // indirect github.com/schollz/progressbar/v2 v2.15.0 + github.com/sirupsen/logrus v1.2.0 github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5 github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 From 44695df7d5eaadd43ef9c98f08e80c0cadcc31f8 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Wed, 15 Jul 2020 15:22:17 +0300 Subject: [PATCH 08/18] Handle Protocol 13 --- es/ledger_serializer_xdr.go | 87 +++++++++++++++++++----------- es/transaction.go | 102 +++++++++++++++++++++++++++--------- 2 files changed, 134 insertions(+), 55 deletions(-) diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go index 5d18ea3..e708672 100644 --- a/es/ledger_serializer_xdr.go +++ b/es/ledger_serializer_xdr.go @@ -8,9 +8,22 @@ import ( "github.com/stellar/go/xdr" ) +type txVersion int + +const ( + v0 = iota + v1 = iota +) + +type txSetMember struct { + v txVersion + xdrV0 *xdr.TransactionV0 + xdrV1 *xdr.Transaction +} + type ledgerSerializerXDR struct { ledgerHeader *LedgerHeader - transactions []xdr.Transaction + txSet []txSetMember transactionResults []xdr.TransactionResultPair transactionMetas []xdr.TransactionMeta changes []xdr.LedgerEntryChanges @@ -20,33 +33,30 @@ type ledgerSerializerXDR struct { // SerializeLedger serializes ledger data into ES bulk index data func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { - transactions := make([]xdr.Transaction, len(meta.V0.TxSet.Txs)) - transactionResults := make([]xdr.TransactionResultPair, len(meta.V0.TxProcessing)) - changes := make([]xdr.LedgerEntryChanges, len(meta.V0.TxProcessing)) - transactionMetas := make([]xdr.TransactionMeta, len(meta.V0.TxProcessing)) + serializer := &ledgerSerializerXDR{ + txSet: make([]txSetMember, len(meta.V0.TxSet.Txs)), + ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), + transactionResults: make([]xdr.TransactionResultPair, len(meta.V0.TxProcessing)), + changes: make([]xdr.LedgerEntryChanges, len(meta.V0.TxProcessing)), + transactionMetas: make([]xdr.TransactionMeta, len(meta.V0.TxProcessing)), + buffer: buffer, + } for i, txe := range meta.V0.TxSet.Txs { switch txe.Type { case xdr.EnvelopeTypeEnvelopeTypeTxV0: - transactions[i] = txe.V0.Tx + serializer.txSet[i] = txSetMember{v: v0, xdrV0: &txe.V0.Tx} case xdr.EnvelopeTypeEnvelopeTypeTx: - transactions[i] = txe.V1.Tx + serializer.txSet[i] = txSetMember{v: v1, xdrV1: &txe.V1.Tx} + default: + log.Fatal("Unknown type") } } for i, txp := range meta.V0.TxProcessing { - transactionResults[i] = txp.Result - changes[i] = txp.FeeProcessing - transactionMetas[i] = txp.TxApplyProcessing - } - - serializer := &ledgerSerializerXDR{ - ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), - transactions: transactions, - transactionResults: transactionResults, - transactionMetas: transactionMetas, - changes: changes, - buffer: buffer, + serializer.transactionResults[i] = txp.Result + serializer.changes[i] = txp.FeeProcessing + serializer.transactionMetas[i] = txp.TxApplyProcessing } serializer.serialize() @@ -55,16 +65,24 @@ func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) func (s *ledgerSerializerXDR) serialize() { SerializeForBulk(s.ledgerHeader, s.buffer) - for i, tx := range s.transactions { - transaction, err := NewTransactionFromXDR( - &transactionData{ - xdr: tx, - result: s.transactionResults[i], - index: i + 1, - ledgerSeq: s.ledgerHeader.Seq, - closeTime: s.ledgerHeader.CloseTime, - }, - ) + for i, tx := range s.txSet { + txData := transactionData{ + result: s.transactionResults[i], + index: i + 1, + ledgerSeq: s.ledgerHeader.Seq, + closeTime: s.ledgerHeader.CloseTime, + } + + switch tx.v { + case v0: + txData.v = v0 + txData.xdrV0 = tx.xdrV0 + case v1: + txData.v = v1 + txData.xdrV1 = tx.xdrV1 + } + + transaction, err := NewTransactionFromXDR(&txData) if err != nil { log.Fatal(err) @@ -81,12 +99,17 @@ func (s *ledgerSerializerXDR) serialize() { } } -func (s *ledgerSerializerXDR) serializeOperations(operations []xdr.Operation, operationResults *[]xdr.OperationResult, transaction *Transaction) { +func (s *ledgerSerializerXDR) serializeOperations(operations []xdr.Operation, operationResults *[]xdr.OperationResult, transaction *Transaction) error { // effectsCount := 0 for index, operation := range operations { result := (*operationResults)[index] - operation := ProduceOperation(transaction, &operation, &result, index+1) + operation, err := ProduceOperation(transaction, &operation, &result, index+1) + + if err != nil { + return err + } + SerializeForBulk(operation, s.buffer) if transaction.Successful { @@ -104,6 +127,8 @@ func (s *ledgerSerializerXDR) serializeOperations(operations []xdr.Operation, op } } } + + return nil } func (s *ledgerSerializerXDR) serializeBalances(changes xdr.LedgerEntryChanges, transaction *Transaction, operation *Operation, source BalanceSource) int { diff --git a/es/transaction.go b/es/transaction.go index 7eb6782..ec59b57 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -19,7 +19,9 @@ type transactionData struct { closeTime time.Time // index of transaction in ledger index int - xdr xdr.Transaction + v txVersion + xdrV0 *xdr.TransactionV0 + xdrV1 *xdr.Transaction result xdr.TransactionResultPair } @@ -110,12 +112,78 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { resultCode := data.result.Result.Result.Code - //FIXME remove hardcoded network passphrase - txHash, err := network.HashTransaction( - data.xdr, - "Public Global Stellar Network ; September 2015", + var ( + err error + txHash [32]byte + sourceAccountID string + maxFee int + operationsCount int + memo Memo + timeBounds TimeBounds ) + switch data.v { + case v0: + //FIXME remove hardcoded network passphrase + txHash, err = network.HashTransactionV0( + *data.xdrV0, + "Public Global Stellar Network ; September 2015", + ) + if err != nil { + return nil, err + } + + sourceAccountID, err = util.EncodeEd25519(data.xdrV0.SourceAccountEd25519) + maxFee = int(data.xdrV0.Fee) + operationsCount = len(data.xdrV0.Operations) + + if data.xdrV0.Memo.Type != xdr.MemoTypeMemoNone { + value := stellar.MemoValue(data.xdrV0.Memo) + + memo = Memo{ + Type: int(data.xdrV0.Memo.Type), + Value: value.String, + } + } + + if data.xdrV0.TimeBounds != nil { + timeBounds = TimeBounds{ + MinTime: int64(data.xdrV0.TimeBounds.MinTime), + MaxTime: int64(data.xdrV0.TimeBounds.MaxTime), + } + } + + case v1: + //FIXME remove hardcoded network passphrase + txHash, err = network.HashTransaction( + *data.xdrV1, + "Public Global Stellar Network ; September 2015", + ) + if err != nil { + return nil, err + } + + sourceAccountID, err = util.EncodeMuxedAccount(data.xdrV1.SourceAccount) + maxFee = int(data.xdrV1.Fee) + operationsCount = len(data.xdrV1.Operations) + + if data.xdrV1.Memo.Type != xdr.MemoTypeMemoNone { + value := stellar.MemoValue(data.xdrV1.Memo) + + memo = Memo{ + Type: int(data.xdrV1.Memo.Type), + Value: value.String, + } + } + + if data.xdrV1.TimeBounds != nil { + timeBounds = TimeBounds{ + MinTime: int64(data.xdrV1.TimeBounds.MinTime), + MaxTime: int64(data.xdrV1.TimeBounds.MaxTime), + } + } + } + if err != nil { return nil, err } @@ -125,29 +193,15 @@ func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { Index: data.index, Seq: data.ledgerSeq, PagingToken: PagingToken{LedgerSeq: data.ledgerSeq, TransactionOrder: data.index}, - Fee: int(data.xdr.Fee), + MaxFee: maxFee, FeeCharged: int(data.result.Result.FeeCharged), - OperationCount: len(data.xdr.Operations), + OperationCount: operationsCount, CloseTime: data.closeTime, Successful: resultCode == xdr.TransactionResultCodeTxSuccess, ResultCode: int(resultCode), - SourceAccountID: data.xdr.SourceAccount.Address(), - } - - if data.xdr.Memo.Type != xdr.MemoTypeMemoNone { - value := stellar.MemoValue(data.xdr.Memo) - - tx.Memo = &Memo{ - Type: int(data.xdr.Memo.Type), - Value: value.String, - } - } - - if data.xdr.TimeBounds != nil { - tx.TimeBounds = &TimeBounds{ - MinTime: int64(data.xdr.TimeBounds.MinTime), - MaxTime: int64(data.xdr.TimeBounds.MaxTime), - } + SourceAccountID: sourceAccountID, + Memo: &memo, + TimeBounds: &timeBounds, } return tx, nil From 6dfa5ce96438c1e366fbd5f3a5b9f504a1a3874a Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Mon, 20 Jul 2020 12:40:07 +0300 Subject: [PATCH 09/18] Use Horizon captive ledger backend --- commands/export.go | 113 +++++++++++++++++-------- db/tx_history_row.go | 41 +-------- es/adapter.go | 6 +- es/ledger_header.go | 1 + es/ledger_serializer_xdr.go | 4 +- es/main.go | 2 +- es/transaction.go | 8 +- go.mod | 21 +++-- go.sum | 55 +++++++++++++ stellar/main.go | 160 ------------------------------------ support/main.go | 51 ++++++++++++ 11 files changed, 211 insertions(+), 251 deletions(-) delete mode 100644 stellar/main.go diff --git a/commands/export.go b/commands/export.go index 47af4ae..731ee5a 100644 --- a/commands/export.go +++ b/commands/export.go @@ -5,7 +5,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/astroband/astrologer/es" - "github.com/astroband/astrologer/stellar" + lb "github.com/stellar/go/exp/ingest/ledgerbackend" ) // ExportCommandConfig represents configuration options for `export` CLI command @@ -22,16 +22,16 @@ type ExportCommand struct { ES es.Adapter Config ExportCommandConfig - firstLedger int - lastLedger int + firstLedger uint32 + lastLedger uint32 } // Execute starts the export process func (cmd *ExportCommand) Execute() { total := cmd.Config.Count - cmd.firstLedger = cmd.Config.Start - cmd.lastLedger = cmd.firstLedger + cmd.Config.Count + cmd.firstLedger = uint32(cmd.Config.Start) + cmd.lastLedger = cmd.firstLedger + uint32(cmd.Config.Count) - 1 if total == 0 { log.Fatal("Nothing to export within given range!", cmd.firstLedger, cmd.lastLedger) @@ -40,48 +40,91 @@ func (cmd *ExportCommand) Execute() { log.Infof("Exporting ledgers from %d to %d. Total: %d ledgers\n", cmd.firstLedger, cmd.lastLedger, total) log.Infof("Will insert %d batches %d ledgers each\n", cmd.blockCount(total), cmd.Config.BatchSize) - channel := stellar.StreamLedgers(cmd.firstLedger, cmd.lastLedger) + ledgerBackend := lb.NewCaptive( + "stellar-core", + "Public Global Stellar Network ; September 2015", + []string{ + "https://history.stellar.org/prd/core-live/core_live_001", + "https://history.stellar.org/prd/core-live/core_live_002", + "https://history.stellar.org/prd/core-live/core_live_003", + }, + ) - for i := 0; i < cmd.blockCount(total); i++ { - var b bytes.Buffer - ledgerCounter := 0 - batchNum := i + 1 + err := ledgerBackend.PrepareRange(cmd.firstLedger, cmd.lastLedger) - for meta := range channel { - seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) + if err != nil { + log.Fatal(err) + } - if seq < cmd.firstLedger || seq > cmd.lastLedger { - continue - } + var batchBuffer bytes.Buffer - ledgerCounter += 1 + for ledgerSeq := cmd.firstLedger; ledgerSeq <= cmd.lastLedger; ledgerSeq++ { + _, meta, err := ledgerBackend.GetLedger(ledgerSeq) - log.Println(seq) + if err != nil { + // FIXME skip instead of failing + log.Fatal(err) + } - es.SerializeLedgerFromHistory(meta, &b) + es.SerializeLedgerFromHistory(meta, &batchBuffer) - log.Printf("Ledger %d of %d in batch %d\n", ledgerCounter, cmd.Config.BatchSize, batchNum) + if (ledgerSeq-cmd.firstLedger+1)%uint32(cmd.Config.BatchSize) == 0 || ledgerSeq == cmd.lastLedger { + payload := batchBuffer.String() + pool.Submit(func() { + log.Printf("Gonna bulk insert %d bytes\n", len(payload)) + err := cmd.ES.BulkInsert(payload) - if ledgerCounter == cmd.Config.BatchSize { - break - } - } + if err != nil { + log.Fatal("Cannot bulk insert", err) + } else { + log.Printf("Batch successfully inserted\n") + } + }) - if cmd.Config.DryRun { - continue + batchBuffer.Reset() } + } - pool.Submit(func() { - log.Printf("Gonna bulk insert %d bytes\n", b.Len()) - err := cmd.ES.BulkInsert(b) + // for i := 0; i < cmd.blockCount(total); i++ { + // var b bytes.Buffer + // ledgerCounter := 0 + // batchNum := i + 1 - if err != nil { - log.Fatal("Cannot bulk insert", err) - } else { - log.Printf("Batch %d successfully inserted\n", batchNum) - } - }) - } + // for meta := range channel { + // seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) + + // if seq < cmd.firstLedger || seq > cmd.lastLedger { + // continue + // } + + // ledgerCounter += 1 + + // log.Println(seq) + + // es.SerializeLedgerFromHistory(meta, &b) + + // log.Printf("Ledger %d of %d in batch %d\n", ledgerCounter, cmd.Config.BatchSize, batchNum) + + // if ledgerCounter == cmd.Config.BatchSize { + // break + // } + // } + + // if cmd.Config.DryRun { + // continue + // } + + // pool.Submit(func() { + // log.Printf("Gonna bulk insert %d bytes\n", b.Len()) + // err := cmd.ES.BulkInsert(b) + + // if err != nil { + // log.Fatal("Cannot bulk insert", err) + // } else { + // log.Printf("Batch %d successfully inserted\n", batchNum) + // } + // }) + // } pool.StopWait() } diff --git a/db/tx_history_row.go b/db/tx_history_row.go index 25146c8..07c01cc 100644 --- a/db/tx_history_row.go +++ b/db/tx_history_row.go @@ -1,14 +1,9 @@ package db import ( - "encoding/base64" "fmt" - "log" - "strings" - - "github.com/astroband/astrologer/support" - "github.com/guregu/null" "github.com/stellar/go/xdr" + "log" ) // TxHistoryRow represents row of txhistory table @@ -33,40 +28,6 @@ func (db *Client) TxHistoryRowForSeq(seq int) []TxHistoryRow { return txs } -// MemoValue Returns clean memo value, this is copy paste from horizon internal package -func (tx *TxHistoryRow) MemoValue() null.String { - var ( - value string - valid bool - memo = tx.Envelope.Memo() - ) - - switch memo.Type { - case xdr.MemoTypeMemoNone: - value, valid = "", false - case xdr.MemoTypeMemoText: - scrubbed := support.Utf8Scrub(memo.MustText()) - notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") - value, valid = notnull, true - case xdr.MemoTypeMemoId: - value, valid = fmt.Sprintf("%d", memo.MustId()), true - case xdr.MemoTypeMemoHash: - hash := memo.MustHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - case xdr.MemoTypeMemoReturn: - hash := memo.MustRetHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - default: - panic(fmt.Errorf("invalid memo type: %v", memo.Type)) - } - - return null.NewString(value, valid) -} - // Operations returns operations array func (tx *TxHistoryRow) Operations() (error, []xdr.Operation) { switch tx.Envelope.Type { diff --git a/es/adapter.go b/es/adapter.go index 5dc007c..e03b11f 100644 --- a/es/adapter.go +++ b/es/adapter.go @@ -107,8 +107,8 @@ func (es *Client) LedgerSeqRangeQuery(ranges []map[string]interface{}) map[strin } // BulkInsert sends the payload to ES using bulk operation -func (es *Client) BulkInsert(payload bytes.Buffer) error { - res, err := es.rawClient.Bulk(bytes.NewReader(payload.Bytes())) +func (es *Client) BulkInsert(payload string) error { + res, err := es.rawClient.Bulk(strings.NewReader(payload)) if res != nil { defer res.Body.Close() @@ -192,7 +192,7 @@ func (es *Client) GetLedgerSeqsInRange(min, max int) (seqs []int) { // IndexWithRetries performs a bulk insert into ES cluster with retries on failures func (es *Client) IndexWithRetries(payload *bytes.Buffer, retryCount int) { - err := es.BulkInsert(*payload) + err := es.BulkInsert(payload.String()) if err != nil { if retryCount-1 == 0 { diff --git a/es/ledger_header.go b/es/ledger_header.go index 737bf1e..9cb7e5e 100644 --- a/es/ledger_header.go +++ b/es/ledger_header.go @@ -55,6 +55,7 @@ func NewLedgerHeaderFromHistory(historyEntry xdr.LedgerHeaderHistoryEntry) *Ledg seq := int(header.LedgerSeq) return &LedgerHeader{ + ID: seq, Hash: hex.EncodeToString(historyEntry.Hash[:]), PrevHash: hex.EncodeToString(header.PreviousLedgerHash[:]), BucketListHash: hex.EncodeToString(header.BucketListHash[:]), diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go index e708672..6422ee9 100644 --- a/es/ledger_serializer_xdr.go +++ b/es/ledger_serializer_xdr.go @@ -4,7 +4,7 @@ import ( "bytes" "log" - "github.com/astroband/astrologer/stellar" + "github.com/astroband/astrologer/support" "github.com/stellar/go/xdr" ) @@ -113,7 +113,7 @@ func (s *ledgerSerializerXDR) serializeOperations(operations []xdr.Operation, op SerializeForBulk(operation, s.buffer) if transaction.Successful { - metas := stellar.OperationMeta(s.transactionMetas[transaction.Index], index) + metas := support.OperationMeta(s.transactionMetas[transaction.Index], index) if metas != nil { // effectsCount = s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) diff --git a/es/main.go b/es/main.go index 68c7d54..2e5ed14 100644 --- a/es/main.go +++ b/es/main.go @@ -22,7 +22,7 @@ type Adapter interface { IndexExists(name IndexName) bool CreateIndex(name IndexName, body IndexDefinition) DeleteIndex(name IndexName) - BulkInsert(payload bytes.Buffer) error + BulkInsert(payload string) error IndexWithRetries(payload *bytes.Buffer, retriesCount int) } diff --git a/es/transaction.go b/es/transaction.go index ec59b57..6efe30d 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -5,7 +5,7 @@ import ( "time" "github.com/astroband/astrologer/db" - "github.com/astroband/astrologer/stellar" + "github.com/astroband/astrologer/support" "github.com/astroband/astrologer/util" "github.com/stellar/go/network" "github.com/stellar/go/xdr" @@ -95,7 +95,7 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T if envelope.Memo().Type != xdr.MemoTypeMemoNone { transaction.Memo = &Memo{ Type: int(envelope.Memo().Type), - Value: row.MemoValue().String, + Value: support.MemoValue(row.Envelope.Memo()).String, } } @@ -138,7 +138,7 @@ func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { operationsCount = len(data.xdrV0.Operations) if data.xdrV0.Memo.Type != xdr.MemoTypeMemoNone { - value := stellar.MemoValue(data.xdrV0.Memo) + value := support.MemoValue(data.xdrV0.Memo) memo = Memo{ Type: int(data.xdrV0.Memo.Type), @@ -168,7 +168,7 @@ func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { operationsCount = len(data.xdrV1.Operations) if data.xdrV1.Memo.Type != xdr.MemoTypeMemoNone { - value := stellar.MemoValue(data.xdrV1.Memo) + value := support.MemoValue(data.xdrV1.Memo) memo = Memo{ Type: int(data.xdrV1.Memo.Type), diff --git a/go.mod b/go.mod index 168326c..6697776 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,29 @@ module github.com/astroband/astrologer go 1.12 require ( + github.com/Masterminds/squirrel v1.4.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/elastic/go-elasticsearch/v7 v7.7.0 + github.com/aws/aws-sdk-go v1.33.7 // indirect + github.com/elastic/go-elasticsearch/v7 v7.8.0 github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896 // indirect - github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753 + github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae + github.com/go-errors/errors v1.1.1 // indirect github.com/guregu/null v4.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.7.0 github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mndrix/ps v0.0.0-20170330174427-18e65badd6ab // indirect + github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077 // indirect github.com/olekukonko/tablewriter v0.0.4 - github.com/pkg/errors v0.9.1 // indirect github.com/schollz/progressbar/v2 v2.15.0 - github.com/sirupsen/logrus v1.2.0 - github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5 - github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 // indirect + github.com/sirupsen/logrus v1.6.0 + github.com/stellar/go v0.0.0-20200719182404-6934026f350b + github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/testify v1.6.1 // indirect + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 + gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect ) diff --git a/go.sum b/go.sum index ee32858..602738d 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,12 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT firebase.google.com/go v3.12.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= github.com/BurntSushi/toml v0.2.1-0.20160717150709-99064174e013/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 h1:PPfYWScYacO3Q6JMCLkyh6Ea2Q/REDTMgmiTAeiV8Jg= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= +github.com/Masterminds/squirrel v1.4.0 h1:he5i/EXixZxrBUWcxzDYMiju9WZ3ld/l7QBNuo/eN3w= +github.com/Masterminds/squirrel v1.4.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= @@ -19,7 +24,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/asaskevich/govalidator v0.0.0-20180319081651-7d2e70ef918f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.6.11-0.20170104181648-8649d278323e/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= +github.com/aws/aws-sdk-go v1.25.25 h1:j3HLOqcDWjNox1DyvJRs+kVQF42Ghtv6oL6cVBfXS3U= github.com/aws/aws-sdk-go v1.25.25/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.33.7 h1:vOozL5hmWHHriRviVTQnUwz8l05RS0rehmEFymI+/x8= +github.com/aws/aws-sdk-go v1.33.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,6 +41,8 @@ github.com/elastic/go-elasticsearch/v7 v7.2.0 h1:9gdDHRMHjM+rPV5lLWiG8vkWn8OCgQH github.com/elastic/go-elasticsearch/v7 v7.2.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-elasticsearch/v7 v7.7.0 h1:oQBx/S3RiaH0/kiP0scYSay9xgSmVAYJpuqEf+e9GZg= github.com/elastic/go-elasticsearch/v7 v7.7.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/elastic/go-elasticsearch/v7 v7.8.0 h1:M9D55OK13IEgg51Jb57mZgseag1AsncwAUn4C6j1vlc= +github.com/elastic/go-elasticsearch/v7 v7.8.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/inject v0.0.0-20161006174721-cc1aa653e50f/go.mod h1:oO8UHw+fDHjDsk4CTy/E96WDzFUYozAtBAaGNoVL0+c= @@ -53,16 +63,22 @@ github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6 h1:1Cy/haf7XO github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753 h1:oSQ61LxZkz3Z4La0O5cbyVDvLWEfbNgiD43cSPdjPQQ= github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= +github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae h1:kNWKQA116SMe5ydvryvVlSV8qVfuv+NS8cO74LacufM= +github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= github.com/getsentry/raven-go v0.0.0-20160805001729-c9d3cc542ad1/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v3.1.5+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-errors/errors v0.0.0-20150906023321-a41850380601 h1:jxTbmDuqQUTI6MscgbqB39vtxGfr2fi61nYIcFQUnlE= github.com/go-errors/errors v0.0.0-20150906023321-a41850380601/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-ini/ini v1.23.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/packr v1.12.1/go.mod h1:H2dZhQFqHeZwr/5A/uGQkBp7xYuMGuzXFeKhYdcz5No= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -97,7 +113,10 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -107,13 +126,21 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v0.0.0-20161106143436-e3b7981a12dd/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.0.0-20150520163514-e6ac2fc51e89 h1:Smt4CPhAnATQEGlX/nyqGETX4Tj8bg/7shBT5gH8d7s= github.com/kr/pretty v0.0.0-20150520163514-e6ac2fc51e89/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA= github.com/kr/text v0.0.0-20150520163712-e373e137fafd h1:ohpc+F5FseC/PPrR1wz2WcZxOc4kplnJ39pGbFlj/eY= github.com/kr/text v0.0.0-20150520163712-e373e137fafd/go.mod h1:sjUstKUATFIcff4qlB53Kml0wQPtJVc/3fWrmuUmcfA= +github.com/lann/builder v0.0.0-20140829050551-c603884a2c1f h1:GYBg1t6ujjhgyYsiO9i0qwbnUZzTiPVLCA/QUkD7ECQ= github.com/lann/builder v0.0.0-20140829050551-c603884a2c1f/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -138,7 +165,9 @@ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2Em github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mndrix/ps v0.0.0-20131111202200-33ddf69629c1 h1:kCroTjOY+wyp+iHA2lZOV5aJ6WfBVjGnW8bCYmXmLPo= github.com/mndrix/ps v0.0.0-20131111202200-33ddf69629c1/go.mod h1:dHgTaDInzkAqJv67VaX1IkK449M2UoBY68CZeI/bNCU= +github.com/mndrix/ps v0.0.0-20170330174427-18e65badd6ab/go.mod h1:dHgTaDInzkAqJv67VaX1IkK449M2UoBY68CZeI/bNCU= github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077 h1:A804awGqaW7i61y8KnbtHmh3scqbNuTJqcycq3u5ZAU= @@ -180,12 +209,16 @@ github.com/schollz/progressbar/v2 v2.13.3-0.20190604152910-b6d7f92524db/go.mod h github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8= github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI= github.com/sebest/xff v0.0.0-20150611211316-7a36e3a787b5/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= +github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 h1:S4OC0+OBKz6mJnzuHioeEat74PuQ4Sgvbf8eus695sc= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2/go.mod h1:8zLRYR5npGjaOXgPSKat5+oOh+UHd8OdbS18iqX9F6Y= github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/sirupsen/logrus v1.0.6-0.20180530095059-070c81def33f/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= @@ -202,19 +235,26 @@ github.com/stellar/go v0.0.0-20200526231405-08ec13c54232/go.mod h1:Q4JDgZhGw7Ytu github.com/stellar/go v0.0.0-20200528062442-f08b35a3f034 h1:YbQySRDlm+qU2+khzooq+bCEteADDprTrqh2YMNehqI= github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5 h1:3ggUT5Mx9wmzFMCVW7eLiYLqaY2p9nj5Cb6XpnGuHF0= github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5/go.mod h1:nZnKe/dfWbtFypSihxxX2UfMzMgeWgI1HvoK4jLh13I= +github.com/stellar/go v0.0.0-20200719182404-6934026f350b h1:BIM4A/5MhgfBabSv+MIimHwdeQlamXFD0zlhkOXjvmg= +github.com/stellar/go v0.0.0-20200719182404-6934026f350b/go.mod h1:LDU2cdYFQbYGt7UtgciOw8g5DXdc93r6N3e1mdHWems= github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e h1:n/hfey8pO+RYMoGXyvyzuw5pdO8IFDoyAL/g5OiCesY= github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e/go.mod h1:gpOLVzy6TVYTQ3LvHSN9RJC700FkhFCpSE82u37aNRM= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 h1:K9H+A+eWe8ZlnpNha+pXbEK+jtIluQp/2dKxkK8k7OE= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible/go.mod h1:7CJ23pXirXBJq45DqvO6clzTEGM/l1SfKrgrzLry8b4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tyler-smith/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v0.0.0-20170109085056-0a7f0a797cd6/go.mod h1:+g/po7GqyG5E+1CNgquiIxJnsXEi5vwFn5weFujbO78= @@ -233,6 +273,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -250,6 +292,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -265,11 +309,16 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -306,5 +355,11 @@ gopkg.in/tylerb/graceful.v1 v1.2.13/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/stellar/main.go b/stellar/main.go deleted file mode 100644 index a72d358..0000000 --- a/stellar/main.go +++ /dev/null @@ -1,160 +0,0 @@ -package stellar - -import ( - "bufio" - "encoding/base64" - "encoding/binary" - "fmt" - "io" - "os" - "os/exec" - "strings" - "syscall" - - "github.com/astroband/astrologer/support" - "github.com/guregu/null" - log "github.com/sirupsen/logrus" - "github.com/stellar/go/xdr" -) - -func StreamLedgers(firstLedger, lastLedger int) chan xdr.LedgerCloseMeta { - const pipeFile = "astrologer_meta_stream" - os.Remove(pipeFile) - ch := make(chan xdr.LedgerCloseMeta) - - err := syscall.Mkfifo(pipeFile, 0666) - - if err != nil { - log.Fatal("Make named pipe file error:", err) - } - - stellarCoreInstance := exec.Command( - "stellar-core", - "catchup", - fmt.Sprintf("%d/%d", lastLedger, lastLedger-firstLedger+1), - "--replay-in-memory", - "--conf", - "./stellar-core/pubnet.cfg", - ) - stellarCoreLogger := log.WithFields(log.Fields{"process": "stellar-core"}) - stellarCoreStdOut, err := stellarCoreInstance.StdoutPipe() - - if err != nil { - log.Fatal(err) - } - - go func() { - if err := stellarCoreInstance.Start(); err != nil { - stellarCoreLogger.Fatal(err) - } - - scanner := bufio.NewScanner(stellarCoreStdOut) - - for scanner.Scan() { - stellarCoreLogger.Println(scanner.Text()) - } - - if err := scanner.Err(); err != nil { - stellarCoreLogger.Fatal(err) - } - }() - - pipe, err := os.OpenFile(pipeFile, os.O_RDONLY, os.ModeNamedPipe) - - if err != nil { - log.Fatal("Open named pipe file error:", err) - } - - reader := bufio.NewReader(pipe) - sizeBytes := make([]byte, 4) - - readerLogger := log.WithFields(log.Fields{"process": "reader"}) - - go func() { - defer close(ch) - - for { - bytesRead, err := io.ReadFull(reader, sizeBytes) - - // No more to read - if bytesRead == 0 { - readerLogger.Info("Reached the end of the stream") - break - } - - if err != nil { - readerLogger.Fatal("Failed to read from pipe", err) - } - - sizeBytes[0] &= 0x7f - size := binary.BigEndian.Uint32(sizeBytes) - - data := make([]byte, size) - - _, err = io.ReadFull(reader, data) - - if err == nil { - var meta xdr.LedgerCloseMeta - meta.UnmarshalBinary(data) - - readerLogger.Info("Writing to the channel...") - ch <- meta - } else { - readerLogger.Fatal("Error reading from pipe:", err) - } - } - - err = pipe.Close() - - if err != nil { - readerLogger.Println(err) - } - }() - - return ch -} - -func MemoValue(memo xdr.Memo) null.String { - var ( - value string - valid bool - ) - switch memo.Type { - case xdr.MemoTypeMemoNone: - value, valid = "", false - case xdr.MemoTypeMemoText: - scrubbed := support.Utf8Scrub(memo.MustText()) - notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") - value, valid = notnull, true - case xdr.MemoTypeMemoId: - value, valid = fmt.Sprintf("%d", memo.MustId()), true - case xdr.MemoTypeMemoHash: - hash := memo.MustHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - case xdr.MemoTypeMemoReturn: - hash := memo.MustRetHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - default: - panic(fmt.Errorf("invalid memo type: %v", memo.Type)) - } - - return null.NewString(value, valid) -} - -func OperationMeta(txMeta xdr.TransactionMeta, opIndex int) (result *xdr.OperationMeta) { - if v1, ok := txMeta.GetV1(); ok { - ops := v1.Operations - return &ops[opIndex] - } - - ops, ok := txMeta.GetOperations() - if !ok { - return nil - } - - return &ops[opIndex] -} diff --git a/support/main.go b/support/main.go index b6a48b5..bb7f9b2 100644 --- a/support/main.go +++ b/support/main.go @@ -2,9 +2,60 @@ package support import ( "bytes" + "encoding/base64" + "fmt" + "strings" "unicode/utf8" + + "github.com/guregu/null" + "github.com/stellar/go/xdr" ) +func MemoValue(memo xdr.Memo) null.String { + var ( + value string + valid bool + ) + switch memo.Type { + case xdr.MemoTypeMemoNone: + value, valid = "", false + case xdr.MemoTypeMemoText: + scrubbed := Utf8Scrub(memo.MustText()) + notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") + value, valid = notnull, true + case xdr.MemoTypeMemoId: + value, valid = fmt.Sprintf("%d", memo.MustId()), true + case xdr.MemoTypeMemoHash: + hash := memo.MustHash() + value, valid = + base64.StdEncoding.EncodeToString(hash[:]), + true + case xdr.MemoTypeMemoReturn: + hash := memo.MustRetHash() + value, valid = + base64.StdEncoding.EncodeToString(hash[:]), + true + default: + panic(fmt.Errorf("invalid memo type: %v", memo.Type)) + } + + return null.NewString(value, valid) +} + +func OperationMeta(txMeta xdr.TransactionMeta, opIndex int) *xdr.OperationMeta { + if v1, ok := txMeta.GetV1(); ok { + ops := v1.Operations + return &ops[opIndex] + } + + ops, ok := txMeta.GetOperations() + if !ok { + return nil + } + + return &ops[opIndex] +} + // Copy paste from Horizon func Utf8Scrub(in string) string { From 8720d9e76988ee6d02302f52faef03c4c5194b93 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Mon, 20 Jul 2020 17:31:20 +0300 Subject: [PATCH 10/18] Add network passphrase config --- commands/export.go | 38 +++++++++++++++++++++++++++----------- config/main.go | 7 ++++++- es/ledger_header.go | 5 +++-- main.go | 11 ++++++----- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/commands/export.go b/commands/export.go index 731ee5a..a411b8b 100644 --- a/commands/export.go +++ b/commands/export.go @@ -10,11 +10,12 @@ import ( // ExportCommandConfig represents configuration options for `export` CLI command type ExportCommandConfig struct { - Start int - Count int - RetryCount int - DryRun bool - BatchSize int + Start int + Count int + RetryCount int + DryRun bool + BatchSize int + NetworkPassphrase string } // ExportCommand represents the `export` CLI command @@ -42,12 +43,8 @@ func (cmd *ExportCommand) Execute() { ledgerBackend := lb.NewCaptive( "stellar-core", - "Public Global Stellar Network ; September 2015", - []string{ - "https://history.stellar.org/prd/core-live/core_live_001", - "https://history.stellar.org/prd/core-live/core_live_002", - "https://history.stellar.org/prd/core-live/core_live_003", - }, + cmd.Config.NetworkPassphrase, + getHistoryURLs(cmd.Config.NetworkPassphrase), ) err := ledgerBackend.PrepareRange(cmd.firstLedger, cmd.lastLedger) @@ -138,3 +135,22 @@ func (cmd *ExportCommand) blockCount(count int) (blocks int) { return blocks } + +func getHistoryURLs(networkPassphrase string) []string { + switch networkPassphrase { + case "Public Global Stellar Network ; September 2015": + return []string{ + "https://history.stellar.org/prd/core-live/core_live_001", + "https://history.stellar.org/prd/core-live/core_live_002", + "https://history.stellar.org/prd/core-live/core_live_003", + } + case "Test SDF Network ; September 2015": + return []string{ + "http://history.stellar.org/prd/core-testnet/core_testnet_001", + "http://history.stellar.org/prd/core-testnet/core_testnet_002", + "http://history.stellar.org/prd/core-testnet/core_testnet_003", + } + default: + return []string{} + } +} diff --git a/config/main.go b/config/main.go index ffccc4a..9ed2468 100644 --- a/config/main.go +++ b/config/main.go @@ -95,7 +95,12 @@ var ( Start = exportCommand.Arg("start", "Ledger to start indexing, +100 means offset 100 from the first").Default("0").Int() // Count ledgers - Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() + Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() + NetworkPassphrase = exportCommand. + Flag("network-passphrase", "Network passphrase to use"). + Default("Test SDF Network ; September 2015"). + OverrideDefaultFromEnvar("NETWORK_PASSPHRASE"). + String() // StartIngest ledger to start with ingesting StartIngest = ingestCommand.Arg("start", "Ledger to start ingesting").Int() diff --git a/es/ledger_header.go b/es/ledger_header.go index 9cb7e5e..9292c2f 100644 --- a/es/ledger_header.go +++ b/es/ledger_header.go @@ -53,14 +53,15 @@ func NewLedgerHeader(row *db.LedgerHeaderRow) *LedgerHeader { func NewLedgerHeaderFromHistory(historyEntry xdr.LedgerHeaderHistoryEntry) *LedgerHeader { header := historyEntry.Header seq := int(header.LedgerSeq) + pagingToken := PagingToken{LedgerSeq: seq} return &LedgerHeader{ - ID: seq, + ID: pagingToken.String(), Hash: hex.EncodeToString(historyEntry.Hash[:]), PrevHash: hex.EncodeToString(header.PreviousLedgerHash[:]), BucketListHash: hex.EncodeToString(header.BucketListHash[:]), Seq: seq, - PagingToken: PagingToken{LedgerSeq: seq}, + PagingToken: pagingToken, CloseTime: time.Unix(int64(header.ScpValue.CloseTime), 0), Version: int(header.LedgerVersion), TotalCoins: int(header.TotalCoins), diff --git a/main.go b/main.go index 8cd6956..5dccef2 100644 --- a/main.go +++ b/main.go @@ -25,11 +25,12 @@ func main() { command = &cmd.CreateIndexCommand{ES: esClient, Config: config} case "export": config := cmd.ExportCommandConfig{ - Start: *cfg.Start, - Count: *cfg.Count, - DryRun: *cfg.ExportDryRun, - RetryCount: *cfg.Retries, - BatchSize: *cfg.BatchSize, + Start: *cfg.Start, + Count: *cfg.Count, + DryRun: *cfg.ExportDryRun, + RetryCount: *cfg.Retries, + BatchSize: *cfg.BatchSize, + NetworkPassphrase: *cfg.NetworkPassphrase, } command = &cmd.ExportCommand{ES: esClient, Config: config} case "ingest": From ebc9cb3351bc3093bf84ceca40e35ee14fddbb37 Mon Sep 17 00:00:00 2001 From: Sergey Nebolsin Date: Thu, 30 Jul 2020 17:49:49 +0300 Subject: [PATCH 11/18] go mod tidy --- go.mod | 3 -- go.sum | 89 +++++++++++++++++++++------------------------------------- 2 files changed, 32 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index 6697776..fe0215f 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,7 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.7.0 github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mndrix/ps v0.0.0-20170330174427-18e65badd6ab // indirect - github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077 // indirect github.com/olekukonko/tablewriter v0.0.4 - github.com/schollz/progressbar/v2 v2.15.0 github.com/sirupsen/logrus v1.6.0 github.com/stellar/go v0.0.0-20200719182404-6934026f350b github.com/stretchr/objx v0.3.0 // indirect diff --git a/go.sum b/go.sum index 602738d..de08dbb 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c/go.mod h1:1vhO7Mn/ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= firebase.google.com/go v3.12.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= -github.com/BurntSushi/toml v0.2.1-0.20160717150709-99064174e013/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 h1:PPfYWScYacO3Q6JMCLkyh6Ea2Q/REDTMgmiTAeiV8Jg= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= @@ -12,6 +11,7 @@ github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+q github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -23,7 +23,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/asaskevich/govalidator v0.0.0-20180319081651-7d2e70ef918f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.6.11-0.20170104181648-8649d278323e/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.25.25 h1:j3HLOqcDWjNox1DyvJRs+kVQF42Ghtv6oL6cVBfXS3U= github.com/aws/aws-sdk-go v1.25.25/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.33.7 h1:vOozL5hmWHHriRviVTQnUwz8l05RS0rehmEFymI+/x8= @@ -37,94 +36,84 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-elasticsearch/v7 v7.2.0 h1:9gdDHRMHjM+rPV5lLWiG8vkWn8OCgQHoGwr1pINn8aE= -github.com/elastic/go-elasticsearch/v7 v7.2.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/go-elasticsearch/v7 v7.7.0 h1:oQBx/S3RiaH0/kiP0scYSay9xgSmVAYJpuqEf+e9GZg= -github.com/elastic/go-elasticsearch/v7 v7.7.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-elasticsearch/v7 v7.8.0 h1:M9D55OK13IEgg51Jb57mZgseag1AsncwAUn4C6j1vlc= github.com/elastic/go-elasticsearch/v7 v7.8.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/inject v0.0.0-20161006174721-cc1aa653e50f/go.mod h1:oO8UHw+fDHjDsk4CTy/E96WDzFUYozAtBAaGNoVL0+c= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/structtag v0.0.0-20150214074306-217e25fb9691/go.mod h1:sKLL1iua/0etWfo/nPCmyz+v2XDMXy+Ho53W7RAuZNY= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gammazero/deque v0.0.0-20190515085511-50f5fc498a27 h1:eqgrF1fSzfhRQet8I3ws3MrGPLS7z2n1emuetE3yr2w= -github.com/gammazero/deque v0.0.0-20190515085511-50f5fc498a27/go.mod h1:GeIq9qoE43YdGnDXURnmKTnGg15pQz4mYkXSTChbneI= github.com/gammazero/deque v0.0.0-20200227231300-1e9af0e52b46/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w= github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896 h1:p5u6mxVWDmHzLSxFbtMmlSTL9svFPkBXMvTEIfvEXzY= github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w= -github.com/gammazero/workerpool v0.0.0-20190515092108-567ff334f003 h1:9FFqhJ//pGQuuogGF9kRZqtlmrc+BREVC0+dk42Q5VY= -github.com/gammazero/workerpool v0.0.0-20190515092108-567ff334f003/go.mod h1:lU9Kl5oGNQzGAc+k1OcuqtXCPBqnz8zrxoxEwqc9mRc= -github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6 h1:1Cy/haf7XO4OyrkGid0Wq5CMluIErbvDptVAt8UTy38= -github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= -github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753 h1:oSQ61LxZkz3Z4La0O5cbyVDvLWEfbNgiD43cSPdjPQQ= -github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae h1:kNWKQA116SMe5ydvryvVlSV8qVfuv+NS8cO74LacufM= github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= +github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955 h1:gmtGRvSexPU4B1T/yYo0sLOKzER1YT+b4kPxPpm0Ty4= github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= github.com/getsentry/raven-go v0.0.0-20160805001729-c9d3cc542ad1/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/go-chi/chi v3.1.5+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY= github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v0.0.0-20150906023321-a41850380601 h1:jxTbmDuqQUTI6MscgbqB39vtxGfr2fi61nYIcFQUnlE= github.com/go-errors/errors v0.0.0-20150906023321-a41850380601/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-ini/ini v1.23.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/packr v1.12.1/go.mod h1:H2dZhQFqHeZwr/5A/uGQkBp7xYuMGuzXFeKhYdcz5No= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 h1:oERTZ1buOUYlpmKaqlO5fYmz8cZ1rYu5DieJzF4ZVmU= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY= github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/graph-gophers/graphql-go v0.0.0-20190225005345-3e8838d4614c/go.mod h1:uJhtPXrcJLqyi0H5IuMFh+fgW+8cMMakK3Txrbk/WJE= github.com/guregu/null v2.1.3-0.20151024101046-79c5bd36b615+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= -github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM= -github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31 h1:Aw95BEvxJ3K6o9GGv5ppCd1P8hkeIeEJ30FO+OhOJpM= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v0.0.0-20161106143436-e3b7981a12dd h1:vQ0EEfHpdFUtNRj1ri25MUq5jb3Vma+kKhLyjeUTVow= github.com/klauspost/compress v0.0.0-20161106143436-e3b7981a12dd/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc h1:WW8B7p7QBnFlqRVv/k6ro/S8Z7tCnYjJHcQNScx9YVs= github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -145,35 +134,28 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw= -github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.5.4/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 h1:ykXz+pRRTibcSjG1yRhpdSHInF8yZY/mfn+Rz2Nd1rE= github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739/go.mod h1:zUx1mhth20V3VKgL5jbd1BSQcW4Fy6Qs4PZvQwRFwzM= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mndrix/ps v0.0.0-20131111202200-33ddf69629c1 h1:kCroTjOY+wyp+iHA2lZOV5aJ6WfBVjGnW8bCYmXmLPo= github.com/mndrix/ps v0.0.0-20131111202200-33ddf69629c1/go.mod h1:dHgTaDInzkAqJv67VaX1IkK449M2UoBY68CZeI/bNCU= -github.com/mndrix/ps v0.0.0-20170330174427-18e65badd6ab/go.mod h1:dHgTaDInzkAqJv67VaX1IkK449M2UoBY68CZeI/bNCU= +github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db h1:eZgFHVkk9uOTaOQLC6tgjkzdp7Ays8eEVecBcfHZlJQ= github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077 h1:A804awGqaW7i61y8KnbtHmh3scqbNuTJqcycq3u5ZAU= -github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077/go.mod h1:sZZi9x5aHXGZ/RRp7Ne5rkvtDxZb7pd7vgVA+gmE35A= -github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -199,46 +181,32 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rcrowley/go-metrics v0.0.0-20160113235030-51425a2415d2/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rubenv/sql-migrate v0.0.0-20190717103323-87ce952f7079/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= -github.com/schollz/progressbar/v2 v2.13.3-0.20190604152910-b6d7f92524db h1:skK2aM0X0H72NfIqzqBGytur9ymS2pnF/qFrtleDuvU= -github.com/schollz/progressbar/v2 v2.13.3-0.20190604152910-b6d7f92524db/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= -github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8= -github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI= github.com/sebest/xff v0.0.0-20150611211316-7a36e3a787b5/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 h1:S4OC0+OBKz6mJnzuHioeEat74PuQ4Sgvbf8eus695sc= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2/go.mod h1:8zLRYR5npGjaOXgPSKat5+oOh+UHd8OdbS18iqX9F6Y= +github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca h1:oR/RycYTFTVXzND5r4FdsvbnBn0HJXSVeNAnwaTXRwk= github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/sirupsen/logrus v1.0.6-0.20180530095059-070c81def33f/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cobra v0.0.0-20160830174925-9c28e4bbd74e/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20161005214240-4bd69631f475/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v0.0.0-20150621231900-db7ff930a189/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/stellar/go v0.0.0-20191016082916-2f64a5e02f78 h1:fIdPEXodE+DAGOioOVd0lT2mqsxzWqrqVL4TqbgmfQg= -github.com/stellar/go v0.0.0-20191016082916-2f64a5e02f78/go.mod h1:U4zGmzxsNw5Kr0Z8f0clhHVFOIfi80iVh8MBp7y8pY0= -github.com/stellar/go v0.0.0-20200515153832-820becd8083b h1:bXG6jOM0MGSyR4euTMYZPazBbtUZbt/ZzphYN1vZ04I= -github.com/stellar/go v0.0.0-20200515153832-820becd8083b/go.mod h1:kfZjGLxoc7P6QAk7BOrXL1qVk7gJV3+OSk4FCf0dXsM= -github.com/stellar/go v0.0.0-20200526231405-08ec13c54232 h1:HweN0TbrsTmdp58L+I1ozu2JSzn+w6fhaGSiLohEtxI= -github.com/stellar/go v0.0.0-20200526231405-08ec13c54232/go.mod h1:Q4JDgZhGw7Ytu1924PzNkwr2GsLptrzAjwC/icuXnLU= -github.com/stellar/go v0.0.0-20200528062442-f08b35a3f034 h1:YbQySRDlm+qU2+khzooq+bCEteADDprTrqh2YMNehqI= -github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5 h1:3ggUT5Mx9wmzFMCVW7eLiYLqaY2p9nj5Cb6XpnGuHF0= -github.com/stellar/go v0.0.0-20200616201928-2e9029aa9df5/go.mod h1:nZnKe/dfWbtFypSihxxX2UfMzMgeWgI1HvoK4jLh13I= github.com/stellar/go v0.0.0-20200719182404-6934026f350b h1:BIM4A/5MhgfBabSv+MIimHwdeQlamXFD0zlhkOXjvmg= github.com/stellar/go v0.0.0-20200719182404-6934026f350b/go.mod h1:LDU2cdYFQbYGt7UtgciOw8g5DXdc93r6N3e1mdHWems= -github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e h1:n/hfey8pO+RYMoGXyvyzuw5pdO8IFDoyAL/g5OiCesY= -github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e/go.mod h1:gpOLVzy6TVYTQ3LvHSN9RJC700FkhFCpSE82u37aNRM= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 h1:K9H+A+eWe8ZlnpNha+pXbEK+jtIluQp/2dKxkK8k7OE= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible/go.mod h1:7CJ23pXirXBJq45DqvO6clzTEGM/l1SfKrgrzLry8b4= @@ -256,21 +224,29 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tyler-smith/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v0.0.0-20170109085056-0a7f0a797cd6 h1:s0IDmR1jFyWvOK7jVIuAsmHQaGkXUuTas8NXFUOwuAI= github.com/valyala/fasthttp v0.0.0-20170109085056-0a7f0a797cd6/go.mod h1:+g/po7GqyG5E+1CNgquiIxJnsXEi5vwFn5weFujbO78= +github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 h1:KM4T3G70MiR+JtqplcYkNVoNz7pDwYaBxWBXQK804So= github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c h1:XZWnr3bsDQWAZg4Ne+cPoXRPILrNlPNQfxBuwLl43is= github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce h1:cVSRGH8cOveJNwFEEZLXtB+XMnRqKLjUP6V/ZFYQCXI= github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb h1:06WAhQa+mYv7BiOk13B/ywyTlkoE/S7uu6TBKU6FHnE= github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M= github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce h1:888GrqRxabUce7lj4OaoShPxodm3kXOMpSa85wdYzfY= github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= @@ -337,7 +313,6 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -345,8 +320,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 h1:r5ptJ1tBxVAeqw4CrYWhXIMr0SybY3CDHuIbCg5CFVw= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0/go.mod h1:WtiW9ZA1LdaWqtQRo1VbIL/v4XZ8NDta+O/kSpGgVek= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= From e0f3662f637b2caa4128070ac5df6ece86a48ed9 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Tue, 4 Aug 2020 18:03:16 +0300 Subject: [PATCH 12/18] Include stellar-core binary to the docker image --- Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 14c215e..534b683 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,16 @@ RUN GO111MODULE=on go build # =============================================================================================== +FROM stellar/base AS stellar-core + +ENV STELLAR_CORE_VERSION 13.2.0-1260-e45018ea + +ADD install.sh . +RUN ["chmod", "+x", "./install.sh"] +RUN ./install.sh + +# =============================================================================================== + FROM alpine:latest ENV DATABASE_URL=postgres://localhost/core?sslmode=disable @@ -20,7 +30,9 @@ ENV INGEST_GAP=-50 WORKDIR /root COPY --from=build /go/src/github.com/astroband/astrologer/astrologer . -RUN chmod +x ./astrologer +RUN ["chmod", "+x", "./astrologer"] + +COPY --from=stellar-core /usr/local/bin/stellar-core /usr/local/bin/ COPY entry.sh /entry.sh From c5c4b39e8cbb292986ab5ea665beb234a3a42cba Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Thu, 6 Aug 2020 16:18:23 +0300 Subject: [PATCH 13/18] Fix docker image --- Dockerfile | 35 +++++++++++++++-------------------- dependencies.sh | 8 ++++++++ 2 files changed, 23 insertions(+), 20 deletions(-) create mode 100755 dependencies.sh diff --git a/Dockerfile b/Dockerfile index 534b683..69ac5e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,19 @@ -FROM golang:alpine AS build - -RUN apk add --no-cache git +FROM golang:stretch AS build RUN mkdir -p $GOPATH/src/github.com/astroband/astrologer WORKDIR $GOPATH/src/github.com/astroband/astrologer -ADD . . - -RUN GO111MODULE=on go build +COPY . . -# =============================================================================================== +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -tags netgo -ldflags '-w' -FROM stellar/base AS stellar-core +#=========================== -ENV STELLAR_CORE_VERSION 13.2.0-1260-e45018ea +FROM stellar/stellar-core:13.2.0-1297-b5dda51e AS stellar-core -ADD install.sh . -RUN ["chmod", "+x", "./install.sh"] -RUN ./install.sh +#=========================== -# =============================================================================================== - -FROM alpine:latest +FROM stellar/base ENV DATABASE_URL=postgres://localhost/core?sslmode=disable ENV ES_URL=http://localhost:9200 @@ -29,12 +21,15 @@ ENV INGEST_GAP=-50 WORKDIR /root -COPY --from=build /go/src/github.com/astroband/astrologer/astrologer . -RUN ["chmod", "+x", "./astrologer"] +COPY dependencies.sh entry.sh ./ + +RUN ["chmod", "+x", "./dependencies.sh"] +RUN ./dependencies.sh COPY --from=stellar-core /usr/local/bin/stellar-core /usr/local/bin/ -COPY entry.sh /entry.sh +COPY --from=build /go/src/github.com/astroband/astrologer/astrologer . +RUN ["chmod", "+x", "./astrologer"] -ENTRYPOINT ["/entry.sh"] -CMD /root/astrologer ingest -- $INGEST_GAP +ENTRYPOINT ["./entry.sh"] +CMD ./astrologer ingest -- $INGEST_GAP diff --git a/dependencies.sh b/dependencies.sh new file mode 100755 index 0000000..8df3775 --- /dev/null +++ b/dependencies.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +apt-get update +apt-get install -y curl libpq-dev +apt-get clean + +echo "\nDone installing stellar-core dependencies...\n" From 1688976ca5a3a505844814d0a4f4d2871e717104 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Thu, 6 Aug 2020 16:19:04 +0300 Subject: [PATCH 14/18] Remove dead code --- commands/fast_replay.go | 105 ---------------------------------------- main.go | 6 --- 2 files changed, 111 deletions(-) delete mode 100644 commands/fast_replay.go diff --git a/commands/fast_replay.go b/commands/fast_replay.go deleted file mode 100644 index 79b480d..0000000 --- a/commands/fast_replay.go +++ /dev/null @@ -1,105 +0,0 @@ -package commands - -import ( - "bufio" - "encoding/base64" - "encoding/binary" - "fmt" - "io" - "log" - "os" - "os/exec" - "syscall" - - "github.com/astroband/astrologer/es" -) - -type FastReplayCommandConfig struct { - UpTo int - Count int -} - -type FastReplayCommand struct { - ES es.Adapter - Config FastReplayCommandConfig -} - -// Execute starts the export process -func (cmd *FastReplayCommand) Execute() { - log.Printf("Fast replay %d ledgers up to %d\n", cmd.Config.Count, cmd.Config.UpTo) - - pipeFile := "astrologer_meta_stream" - - os.Remove(pipeFile) - err := syscall.Mkfifo(pipeFile, 0666) - - if err != nil { - log.Fatal("Make named pipe file error:", err) - } - - stellarCoreInstance := exec.Command( - "stellar-core", - "catchup", - fmt.Sprintf("%d/%d", cmd.Config.UpTo, cmd.Config.Count), - "--replay-in-memory", - "--conf", - "./stellar-core.cfg", - ) - - // stdout, err := stellarCoreInstance.StdoutPipe() - - // if err != nil { - // log.Fatal(err) - // } - - go func() { - if err := stellarCoreInstance.Start(); err != nil { - log.Fatal(err) - } - // scanner := bufio.NewScanner(stdout) - - // for scanner.Scan() { - // log.Println(scanner.Text()) - // } - - // if err := scanner.Err(); err != nil { - // log.Fatal(err) - // } - }() - - file, err := os.OpenFile(pipeFile, os.O_RDONLY, os.ModeNamedPipe) - - if err != nil { - log.Fatal("Open named pipe file error:", err) - } - - reader := bufio.NewReader(file) - sizeBytes := make([]byte, 4) - - for { - bytesRead, err := io.ReadFull(reader, sizeBytes) - - if err != nil { - log.Fatal(err) - } - - if bytesRead == 0 { - break - } - - sizeBytes[0] &= 0x7f - size := binary.BigEndian.Uint32(sizeBytes) - log.Printf("Gonna read %d bytes\n", size) - - data := make([]byte, size) - - _, err = io.ReadFull(reader, data) - - if err == nil { - sEnc := base64.StdEncoding.EncodeToString(data) - log.Printf("META: %s\n\n", sEnc) - } else { - log.Fatal("Error reading from pipe:", err) - } - } -} diff --git a/main.go b/main.go index 5dccef2..d8d6bf2 100644 --- a/main.go +++ b/main.go @@ -38,12 +38,6 @@ func main() { command = &cmd.IngestCommand{ES: esClient, DB: dbClient} case "es-stats": command = &cmd.EsStatsCommand{ES: esClient} - case "fast-replay": - config := cmd.FastReplayCommandConfig{ - UpTo: *cfg.FastReplayUpTo, - Count: *cfg.FastReplayCount, - } - command = &cmd.FastReplayCommand{ES: esClient, Config: config} } command.Execute() From ee0bce93161872ac84b50db7fa5a985946077b16 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Thu, 6 Aug 2020 16:19:15 +0300 Subject: [PATCH 15/18] Upgrade Go SDK --- commands/export.go | 14 ++++++++++---- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/commands/export.go b/commands/export.go index a411b8b..600e2bc 100644 --- a/commands/export.go +++ b/commands/export.go @@ -41,16 +41,22 @@ func (cmd *ExportCommand) Execute() { log.Infof("Exporting ledgers from %d to %d. Total: %d ledgers\n", cmd.firstLedger, cmd.lastLedger, total) log.Infof("Will insert %d batches %d ledgers each\n", cmd.blockCount(total), cmd.Config.BatchSize) - ledgerBackend := lb.NewCaptive( + ledgerBackend, err := lb.NewCaptive( "stellar-core", + "", cmd.Config.NetworkPassphrase, getHistoryURLs(cmd.Config.NetworkPassphrase), ) - err := ledgerBackend.PrepareRange(cmd.firstLedger, cmd.lastLedger) + if err != nil { + log.Fatal("error creating captive core backend", err) + } + + log.Info("Preparing range...") + err = ledgerBackend.PrepareRange(lb.BoundedRange(cmd.firstLedger, cmd.lastLedger)) if err != nil { - log.Fatal(err) + log.Fatal("Failed preparing range", err) } var batchBuffer bytes.Buffer @@ -68,7 +74,7 @@ func (cmd *ExportCommand) Execute() { if (ledgerSeq-cmd.firstLedger+1)%uint32(cmd.Config.BatchSize) == 0 || ledgerSeq == cmd.lastLedger { payload := batchBuffer.String() pool.Submit(func() { - log.Printf("Gonna bulk insert %d bytes\n", len(payload)) + log.Infof("Gonna bulk insert %d bytes\n", len(payload)) err := cmd.ES.BulkInsert(payload) if err != nil { diff --git a/go.mod b/go.mod index fe0215f..839034a 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/olekukonko/tablewriter v0.0.4 github.com/sirupsen/logrus v1.6.0 - github.com/stellar/go v0.0.0-20200719182404-6934026f350b + github.com/stellar/go v0.0.0-20200804130752-fdc4eca4f6ce github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.6.1 // indirect golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect diff --git a/go.sum b/go.sum index de08dbb..1de977f 100644 --- a/go.sum +++ b/go.sum @@ -207,6 +207,8 @@ github.com/spf13/pflag v0.0.0-20161005214240-4bd69631f475/go.mod h1:DYY7MBk1bdzu github.com/spf13/viper v0.0.0-20150621231900-db7ff930a189/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/stellar/go v0.0.0-20200719182404-6934026f350b h1:BIM4A/5MhgfBabSv+MIimHwdeQlamXFD0zlhkOXjvmg= github.com/stellar/go v0.0.0-20200719182404-6934026f350b/go.mod h1:LDU2cdYFQbYGt7UtgciOw8g5DXdc93r6N3e1mdHWems= +github.com/stellar/go v0.0.0-20200804130752-fdc4eca4f6ce h1:Lixum59yJNsfxgKO1e99ZwWKgfmQfFWRD5U4oyFxazc= +github.com/stellar/go v0.0.0-20200804130752-fdc4eca4f6ce/go.mod h1:LDU2cdYFQbYGt7UtgciOw8g5DXdc93r6N3e1mdHWems= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 h1:K9H+A+eWe8ZlnpNha+pXbEK+jtIluQp/2dKxkK8k7OE= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible/go.mod h1:7CJ23pXirXBJq45DqvO6clzTEGM/l1SfKrgrzLry8b4= From c8d83f228bc9e62e8e54e9c6ccdf56a7f631d63a Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Mon, 10 Aug 2020 13:22:18 +0300 Subject: [PATCH 16/18] Revert "Add ids to the document bodies" This reverts commit f8b9080d1603d045d8f814aec16d69544c45e197. --- es/balance.go | 1 - es/balance_extractor.go | 4 ---- es/indices.go | 5 ----- es/ledger_header.go | 6 +----- es/operation.go | 11 +---------- es/signer_history.go | 2 -- es/trade.go | 1 - es/trade_extractor.go | 4 ---- 8 files changed, 2 insertions(+), 32 deletions(-) diff --git a/es/balance.go b/es/balance.go index c2a6bcf..5c73d0f 100644 --- a/es/balance.go +++ b/es/balance.go @@ -20,7 +20,6 @@ const ( // Balance represents balance log entry type Balance struct { - ID string `json:"id"` PagingToken PagingToken `json:"paging_token"` AccountID string `json:"account_id"` Value string `json:"value"` diff --git a/es/balance_extractor.go b/es/balance_extractor.go index aea0a33..3cfb28a 100644 --- a/es/balance_extractor.go +++ b/es/balance_extractor.go @@ -54,10 +54,6 @@ func (e *BalanceExtractor) extract() []*Balance { } } - for i, balance := range e.balances { - e.balances[i].ID = balance.PagingToken.String() - } - return e.balances } diff --git a/es/indices.go b/es/indices.go index 70f7890..141fea5 100644 --- a/es/indices.go +++ b/es/indices.go @@ -30,7 +30,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "hash": { "type": "keyword", "index": true }, "prev_hash": { "type": "keyword", "index": false }, "bucket_list_hash": { "type": "keyword", "index": false }, @@ -99,7 +98,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "tx_id": { "type": "keyword", "index": true }, "tx_idx": { "type": "integer" }, "idx": { "type": "integer" }, @@ -235,7 +233,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "paging_token": { "type": "keyword", "index": true }, "account_id": { "type": "keyword", "index": true }, "value": { "type": "scaled_float", "scaling_factor": 10000000 }, @@ -266,7 +263,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "paging_token": { "type": "keyword", "index": true }, "sold": { "type": "scaled_float", "scaling_factor": 10000000 }, "bought": { "type": "scaled_float", "scaling_factor": 10000000 }, @@ -305,7 +301,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "paging_token": { "type": "keyword", "index": true }, "account_id": { "type": "keyword", "index": true }, "signer": { "type": "keyword", "index": true }, diff --git a/es/ledger_header.go b/es/ledger_header.go index 9292c2f..9546dd4 100644 --- a/es/ledger_header.go +++ b/es/ledger_header.go @@ -10,7 +10,6 @@ import ( // LedgerHeader represents json-serializable struct for LedgerHeader to index type LedgerHeader struct { - ID string `json:"id"` Hash string `json:"hash"` PrevHash string `json:"prev_hash"` BucketListHash string `json:"bucket_list_hash"` @@ -29,15 +28,12 @@ type LedgerHeader struct { // NewLedgerHeader creates LedgerHeader from LedgerHeaderRow func NewLedgerHeader(row *db.LedgerHeaderRow) *LedgerHeader { - pagingToken := PagingToken{LedgerSeq: row.LedgerSeq} - return &LedgerHeader{ - ID: pagingToken.String(), Hash: row.Hash, PrevHash: row.PrevHash, BucketListHash: row.BucketListHash, Seq: row.LedgerSeq, - PagingToken: pagingToken, + PagingToken: PagingToken{LedgerSeq: row.LedgerSeq}, CloseTime: time.Unix(row.CloseTime, 0), Version: int(row.Data.LedgerVersion), TotalCoins: int(row.Data.TotalCoins), diff --git a/es/operation.go b/es/operation.go index 234c4ac..11ee6fa 100644 --- a/es/operation.go +++ b/es/operation.go @@ -16,7 +16,6 @@ const ( // Operation represents ES-serializable transaction type Operation struct { - ID string `json:"id"` TxID string `json:"tx_id"` TxIndex int `json:"tx_idx"` Index int `json:"idx"` @@ -71,15 +70,7 @@ func NewOperation(t *Transaction, o *xdr.Operation, r *[]xdr.OperationResult, n result = &(*r)[n] } - operation, err := ProduceOperation(t, o, result, n) - - if err != nil { - return nil, err - } - - operation.ID = operation.PagingToken.String() - - return operation, nil + return ProduceOperation(t, o, result, n) } // DocID returns elastic document id diff --git a/es/signer_history.go b/es/signer_history.go index 2556e35..3402632 100644 --- a/es/signer_history.go +++ b/es/signer_history.go @@ -6,7 +6,6 @@ import ( // SignerHistory represents signer change entry, still the question if it is required type SignerHistory struct { - ID string `json:"id"` PagingToken PagingToken `json:"paging_token"` AccountID string `json:"account_id"` Signer string `json:"signer"` @@ -33,7 +32,6 @@ func ProduceSignerHistory(o *Operation) (h *SignerHistory) { } entry := &SignerHistory{ - ID: token.String(), PagingToken: token, AccountID: o.SourceAccountID, Signer: o.Signer.ID, diff --git a/es/trade.go b/es/trade.go index 479b22e..5de5a3b 100644 --- a/es/trade.go +++ b/es/trade.go @@ -6,7 +6,6 @@ import ( // Trade represents trade entry type Trade struct { - ID string `json:"id"` PagingToken PagingToken `json:"paging_token"` Sold string `json:"sold"` Bought string `json:"bought"` diff --git a/es/trade_extractor.go b/es/trade_extractor.go index d0759c9..eb7e0b4 100644 --- a/es/trade_extractor.go +++ b/es/trade_extractor.go @@ -55,10 +55,6 @@ func (e *TradeExtractor) extract() (trades []Trade) { } } - for i, trade := range trades { - trades[i].ID = trade.PagingToken.String() - } - return trades } From 78afe970b1cdc9356d45c80daf5561bf30ee73b3 Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Fri, 14 Aug 2020 10:50:05 +0300 Subject: [PATCH 17/18] Refactoring --- commands/export.go | 2 +- config/main.go | 12 +-- es/ledger_header.go | 26 ------ es/ledger_serializer.go | 59 ++++++++++++ es/ledger_serializer_xdr.go | 174 ------------------------------------ es/serialize.go | 19 +++- es/transaction.go | 116 ++---------------------- main.go | 12 ++- 8 files changed, 98 insertions(+), 322 deletions(-) delete mode 100644 es/ledger_serializer_xdr.go diff --git a/commands/export.go b/commands/export.go index 600e2bc..0f13022 100644 --- a/commands/export.go +++ b/commands/export.go @@ -69,7 +69,7 @@ func (cmd *ExportCommand) Execute() { log.Fatal(err) } - es.SerializeLedgerFromHistory(meta, &batchBuffer) + es.SerializeLedgerFromHistory(cmd.Config.NetworkPassphrase, meta, &batchBuffer) if (ledgerSeq-cmd.firstLedger+1)%uint32(cmd.Config.BatchSize) == 0 || ledgerSeq == cmd.lastLedger { payload := batchBuffer.String() diff --git a/config/main.go b/config/main.go index 9ed2468..5dd5621 100644 --- a/config/main.go +++ b/config/main.go @@ -53,7 +53,6 @@ var ( createIndexCommand = kingpin.Command("create-index", "Create ES indexes") exportCommand = kingpin.Command("export", "Run export") ingestCommand = kingpin.Command("ingest", "Start real time ingestion") - fastReplayCommand = kingpin.Command("fast-replay", "Experiment with using stellar-core fast in-memory replay catchup") _ = kingpin.Command("stats", "Print database ledger statistics") _ = kingpin.Command("es-stats", "Print ES ranges stats") @@ -95,12 +94,8 @@ var ( Start = exportCommand.Arg("start", "Ledger to start indexing, +100 means offset 100 from the first").Default("0").Int() // Count ledgers - Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() - NetworkPassphrase = exportCommand. - Flag("network-passphrase", "Network passphrase to use"). - Default("Test SDF Network ; September 2015"). - OverrideDefaultFromEnvar("NETWORK_PASSPHRASE"). - String() + Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() + Network = exportCommand.Flag("network", "Stellar network to use").Default("testnet").Enum("public", "test") // StartIngest ledger to start with ingesting StartIngest = ingestCommand.Arg("start", "Ledger to start ingesting").Int() @@ -113,7 +108,4 @@ var ( // ForceRecreateIndexes Allows indexes to be deleted before creation ForceRecreateIndexes = createIndexCommand.Flag("force", "Delete indexes before creation").Bool() - - FastReplayUpTo = fastReplayCommand.Arg("upto", "Ledger to start indexing").Int() - FastReplayCount = fastReplayCommand.Arg("count", "Ledgers count to catchup").Int() ) diff --git a/es/ledger_header.go b/es/ledger_header.go index 9546dd4..c3c5f16 100644 --- a/es/ledger_header.go +++ b/es/ledger_header.go @@ -1,11 +1,9 @@ package es import ( - "encoding/hex" "time" "github.com/astroband/astrologer/db" - "github.com/stellar/go/xdr" ) // LedgerHeader represents json-serializable struct for LedgerHeader to index @@ -46,30 +44,6 @@ func NewLedgerHeader(row *db.LedgerHeaderRow) *LedgerHeader { } } -func NewLedgerHeaderFromHistory(historyEntry xdr.LedgerHeaderHistoryEntry) *LedgerHeader { - header := historyEntry.Header - seq := int(header.LedgerSeq) - pagingToken := PagingToken{LedgerSeq: seq} - - return &LedgerHeader{ - ID: pagingToken.String(), - Hash: hex.EncodeToString(historyEntry.Hash[:]), - PrevHash: hex.EncodeToString(header.PreviousLedgerHash[:]), - BucketListHash: hex.EncodeToString(header.BucketListHash[:]), - Seq: seq, - PagingToken: pagingToken, - CloseTime: time.Unix(int64(header.ScpValue.CloseTime), 0), - Version: int(header.LedgerVersion), - TotalCoins: int(header.TotalCoins), - FeePool: int(header.FeePool), - InflationSeq: int(header.InflationSeq), - IDPool: int(header.IdPool), - BaseFee: int(header.BaseFee), - BaseReserve: int(header.BaseReserve), - MaxTxSetSize: int(header.MaxTxSetSize), - } -} - // DocID returns es id (seq number in this case) func (h *LedgerHeader) DocID() *string { s := h.PagingToken.String() diff --git a/es/ledger_serializer.go b/es/ledger_serializer.go index c9d810f..2fadb3b 100644 --- a/es/ledger_serializer.go +++ b/es/ledger_serializer.go @@ -2,9 +2,12 @@ package es import ( "bytes" + "encoding/hex" "fmt" + "log" "github.com/astroband/astrologer/db" + "github.com/astroband/astrologer/util" "github.com/stellar/go/xdr" ) @@ -32,6 +35,62 @@ func SerializeLedger(ledgerRow db.LedgerHeaderRow, transactionRows []db.TxHistor return serializer.serialize() } +// SerializeLedger serializes ledger data into ES bulk index data +func SerializeLedgerFromHistory(networkPassphrase string, meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { + ledgerHeader := meta.V0.LedgerHeader.Header + + ledgerRow := db.LedgerHeaderRow{ + Hash: hex.EncodeToString(meta.V0.LedgerHeader.Hash[:]), + PrevHash: hex.EncodeToString(ledgerHeader.PreviousLedgerHash[:]), + BucketListHash: hex.EncodeToString(ledgerHeader.BucketListHash[:]), + LedgerSeq: int(ledgerHeader.LedgerSeq), + CloseTime: int64(ledgerHeader.ScpValue.CloseTime), + Data: ledgerHeader, + } + + transactionRows := make([]db.TxHistoryRow, len(meta.V0.TxSet.Txs)) + feeRows := make([]db.TxFeeHistoryRow, len(meta.V0.TxSet.Txs)) + + for i, txe := range meta.V0.TxSet.Txs { + txHash, hashErr := util.HashTransactionInEnvelope(txe, networkPassphrase) + + if hashErr != nil { + log.Fatalf("Failed to hash transaction #%d in ledger %d\n", i, ledgerRow.LedgerSeq) + } + + transactionRows[i] = db.TxHistoryRow{ + ID: hex.EncodeToString(txHash[:]), + LedgerSeq: ledgerRow.LedgerSeq, + Index: i + 1, + Envelope: txe, + } + + feeRows[i] = db.TxFeeHistoryRow{ + TxID: transactionRows[i].ID, + LedgerSeq: ledgerRow.LedgerSeq, + Index: i + 1, + } + + for _, txp := range meta.V0.TxProcessing { + if transactionRows[i].ID == hex.EncodeToString(txp.Result.TransactionHash[:]) { + transactionRows[i].Result = txp.Result + transactionRows[i].Meta = txp.TxApplyProcessing + feeRows[i].Changes = txp.FeeProcessing + } + } + } + + serializer := &ledgerSerializer{ + ledgerRow: ledgerRow, + transactionRows: transactionRows, + feeRows: feeRows, + ledger: NewLedgerHeader(&ledgerRow), + buffer: buffer, + } + + serializer.serialize() +} + func (s *ledgerSerializer) serialize() error { SerializeForBulk(s.ledger, s.buffer) diff --git a/es/ledger_serializer_xdr.go b/es/ledger_serializer_xdr.go deleted file mode 100644 index 6422ee9..0000000 --- a/es/ledger_serializer_xdr.go +++ /dev/null @@ -1,174 +0,0 @@ -package es - -import ( - "bytes" - "log" - - "github.com/astroband/astrologer/support" - "github.com/stellar/go/xdr" -) - -type txVersion int - -const ( - v0 = iota - v1 = iota -) - -type txSetMember struct { - v txVersion - xdrV0 *xdr.TransactionV0 - xdrV1 *xdr.Transaction -} - -type ledgerSerializerXDR struct { - ledgerHeader *LedgerHeader - txSet []txSetMember - transactionResults []xdr.TransactionResultPair - transactionMetas []xdr.TransactionMeta - changes []xdr.LedgerEntryChanges - - buffer *bytes.Buffer -} - -// SerializeLedger serializes ledger data into ES bulk index data -func SerializeLedgerFromHistory(meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { - serializer := &ledgerSerializerXDR{ - txSet: make([]txSetMember, len(meta.V0.TxSet.Txs)), - ledgerHeader: NewLedgerHeaderFromHistory(meta.V0.LedgerHeader), - transactionResults: make([]xdr.TransactionResultPair, len(meta.V0.TxProcessing)), - changes: make([]xdr.LedgerEntryChanges, len(meta.V0.TxProcessing)), - transactionMetas: make([]xdr.TransactionMeta, len(meta.V0.TxProcessing)), - buffer: buffer, - } - - for i, txe := range meta.V0.TxSet.Txs { - switch txe.Type { - case xdr.EnvelopeTypeEnvelopeTypeTxV0: - serializer.txSet[i] = txSetMember{v: v0, xdrV0: &txe.V0.Tx} - case xdr.EnvelopeTypeEnvelopeTypeTx: - serializer.txSet[i] = txSetMember{v: v1, xdrV1: &txe.V1.Tx} - default: - log.Fatal("Unknown type") - } - } - - for i, txp := range meta.V0.TxProcessing { - serializer.transactionResults[i] = txp.Result - serializer.changes[i] = txp.FeeProcessing - serializer.transactionMetas[i] = txp.TxApplyProcessing - } - - serializer.serialize() -} - -func (s *ledgerSerializerXDR) serialize() { - SerializeForBulk(s.ledgerHeader, s.buffer) - - for i, tx := range s.txSet { - txData := transactionData{ - result: s.transactionResults[i], - index: i + 1, - ledgerSeq: s.ledgerHeader.Seq, - closeTime: s.ledgerHeader.CloseTime, - } - - switch tx.v { - case v0: - txData.v = v0 - txData.xdrV0 = tx.xdrV0 - case v1: - txData.v = v1 - txData.xdrV1 = tx.xdrV1 - } - - transaction, err := NewTransactionFromXDR(&txData) - - if err != nil { - log.Fatal(err) - } - - SerializeForBulk(transaction, s.buffer) - - if transaction.Successful { - changes := s.changes[i] - s.serializeBalances(changes, transaction, nil, BalanceSourceFee) - } - - // s.serializeOperations(transactionRow, transaction) - } -} - -func (s *ledgerSerializerXDR) serializeOperations(operations []xdr.Operation, operationResults *[]xdr.OperationResult, transaction *Transaction) error { - // effectsCount := 0 - - for index, operation := range operations { - result := (*operationResults)[index] - operation, err := ProduceOperation(transaction, &operation, &result, index+1) - - if err != nil { - return err - } - - SerializeForBulk(operation, s.buffer) - - if transaction.Successful { - metas := support.OperationMeta(s.transactionMetas[transaction.Index], index) - if metas != nil { - // effectsCount = s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) - s.serializeBalances(metas.Changes, transaction, operation, BalanceSourceMeta) - } - - // s.serializeTrades(result, transaction, operation, effectsCount) - - h := ProduceSignerHistory(operation) - if h != nil { - SerializeForBulk(h, s.buffer) - } - } - } - - return nil -} - -func (s *ledgerSerializerXDR) serializeBalances(changes xdr.LedgerEntryChanges, transaction *Transaction, operation *Operation, source BalanceSource) int { - if len(changes) == 0 { - return 0 - } - - pagingToken := PagingToken{ - LedgerSeq: s.ledgerHeader.Seq, - TransactionOrder: transaction.Index, - } - - if operation != nil { - pagingToken.OperationOrder = operation.Index - } - - balances := ProduceBalances(changes, s.ledgerHeader.CloseTime, source, pagingToken) - - if len(balances) > 0 { - for _, balance := range balances { - SerializeForBulk(balance, s.buffer) - } - } - - return len(balances) -} - -// func (s *ledgerSerializerXDR) serializeTrades(result *xdr.OperationResult, transaction *Transaction, operation *Operation, startIndex int) int { -// pagingToken := PagingToken{ -// LedgerSeq: s.ledger.Seq, -// TransactionOrder: transaction.Index, -// OperationOrder: operation.Index, -// } - -// trades := ProduceTrades(result, operation, s.ledger.CloseTime, pagingToken, startIndex) -// if len(trades) > 0 { -// for _, trade := range trades { -// SerializeForBulk(&trade, s.buffer) -// } -// } - -// return len(trades) -// } diff --git a/es/serialize.go b/es/serialize.go index a1b01bf..03386a6 100644 --- a/es/serialize.go +++ b/es/serialize.go @@ -9,9 +9,22 @@ import ( // SerializeForBulk returns object serialized for elastic bulk indexing func SerializeForBulk(obj Indexable, b *bytes.Buffer) { - meta := fmt.Sprintf( - `{ "index": { "_index": "%s", "_type": "_doc" } }%s`, obj.IndexName(), "\n", - ) + var meta string + + id := obj.DocID() + + if id != nil { + meta = fmt.Sprintf( + `{ "index": { "_index": "%s", "_id": "%s", "_type": "_doc" } }%s`, + obj.IndexName(), + *id, + "\n", + ) + } else { + meta = fmt.Sprintf( + `{ "index": { "_index": "%s", "_type": "_doc" } }%s`, obj.IndexName(), "\n", + ) + } data, err := json.Marshal(obj) if err != nil { diff --git a/es/transaction.go b/es/transaction.go index 6efe30d..f12bb3b 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -1,28 +1,28 @@ package es import ( - "encoding/hex" "time" "github.com/astroband/astrologer/db" "github.com/astroband/astrologer/support" - "github.com/astroband/astrologer/util" - "github.com/stellar/go/network" "github.com/stellar/go/xdr" ) // transactionData represents all necessary pieces we need // from stellar-core to ingest single transaction type transactionData struct { + id string ledgerSeq int // ledger close time closeTime time.Time // index of transaction in ledger - index int - v txVersion - xdrV0 *xdr.TransactionV0 - xdrV1 *xdr.Transaction - result xdr.TransactionResultPair + index int + xdrV0 *xdr.TransactionV0 + xdrV1 *xdr.Transaction + feeBump *xdr.FeeBumpTransaction + result xdr.TransactionResultPair + feeAccount string + maxFee int } // Transaction represents ES-serializable transaction @@ -44,7 +44,7 @@ type Transaction struct { *Memo `json:"memo,omitempty"` } -// NewTransaction creates LedgerHeader from LedgerHeaderRow +// NewTransaction creates Transaction from TxHistoryRow func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*Transaction, error) { var ( err error @@ -109,104 +109,6 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T return transaction, nil } -func NewTransactionFromXDR(data *transactionData) (*Transaction, error) { - resultCode := data.result.Result.Result.Code - - var ( - err error - txHash [32]byte - sourceAccountID string - maxFee int - operationsCount int - memo Memo - timeBounds TimeBounds - ) - - switch data.v { - case v0: - //FIXME remove hardcoded network passphrase - txHash, err = network.HashTransactionV0( - *data.xdrV0, - "Public Global Stellar Network ; September 2015", - ) - if err != nil { - return nil, err - } - - sourceAccountID, err = util.EncodeEd25519(data.xdrV0.SourceAccountEd25519) - maxFee = int(data.xdrV0.Fee) - operationsCount = len(data.xdrV0.Operations) - - if data.xdrV0.Memo.Type != xdr.MemoTypeMemoNone { - value := support.MemoValue(data.xdrV0.Memo) - - memo = Memo{ - Type: int(data.xdrV0.Memo.Type), - Value: value.String, - } - } - - if data.xdrV0.TimeBounds != nil { - timeBounds = TimeBounds{ - MinTime: int64(data.xdrV0.TimeBounds.MinTime), - MaxTime: int64(data.xdrV0.TimeBounds.MaxTime), - } - } - - case v1: - //FIXME remove hardcoded network passphrase - txHash, err = network.HashTransaction( - *data.xdrV1, - "Public Global Stellar Network ; September 2015", - ) - if err != nil { - return nil, err - } - - sourceAccountID, err = util.EncodeMuxedAccount(data.xdrV1.SourceAccount) - maxFee = int(data.xdrV1.Fee) - operationsCount = len(data.xdrV1.Operations) - - if data.xdrV1.Memo.Type != xdr.MemoTypeMemoNone { - value := support.MemoValue(data.xdrV1.Memo) - - memo = Memo{ - Type: int(data.xdrV1.Memo.Type), - Value: value.String, - } - } - - if data.xdrV1.TimeBounds != nil { - timeBounds = TimeBounds{ - MinTime: int64(data.xdrV1.TimeBounds.MinTime), - MaxTime: int64(data.xdrV1.TimeBounds.MaxTime), - } - } - } - - if err != nil { - return nil, err - } - - tx := &Transaction{ - ID: hex.EncodeToString(txHash[:]), - Index: data.index, - Seq: data.ledgerSeq, - PagingToken: PagingToken{LedgerSeq: data.ledgerSeq, TransactionOrder: data.index}, - MaxFee: maxFee, - FeeCharged: int(data.result.Result.FeeCharged), - OperationCount: operationsCount, - CloseTime: data.closeTime, - Successful: resultCode == xdr.TransactionResultCodeTxSuccess, - ResultCode: int(resultCode), - SourceAccountID: sourceAccountID, - Memo: &memo, - TimeBounds: &timeBounds, - } - - return tx, nil -} - // DocID return es transaction id (tx id in this case) func (tx *Transaction) DocID() *string { return &tx.ID diff --git a/main.go b/main.go index d8d6bf2..78cb38b 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( cfg "github.com/astroband/astrologer/config" "github.com/astroband/astrologer/db" "github.com/astroband/astrologer/es" + "github.com/stellar/go/network" "gopkg.in/alecthomas/kingpin.v2" ) @@ -24,13 +25,22 @@ func main() { config := cmd.CreateIndexCommandConfig{Force: *cfg.ForceRecreateIndexes} command = &cmd.CreateIndexCommand{ES: esClient, Config: config} case "export": + var networkPassphrase string + + switch *cfg.Network { + case "public": + networkPassphrase = network.PublicNetworkPassphrase + case "test": + networkPassphrase = network.TestNetworkPassphrase + } + config := cmd.ExportCommandConfig{ Start: *cfg.Start, Count: *cfg.Count, DryRun: *cfg.ExportDryRun, RetryCount: *cfg.Retries, BatchSize: *cfg.BatchSize, - NetworkPassphrase: *cfg.NetworkPassphrase, + NetworkPassphrase: networkPassphrase, } command = &cmd.ExportCommand{ES: esClient, Config: config} case "ingest": From ca0ac830d6490c881b4ef16356a147c463ed475f Mon Sep 17 00:00:00 2001 From: Timur Ramazanov Date: Fri, 14 Aug 2020 17:47:26 +0300 Subject: [PATCH 18/18] Fix tx indices assigning --- es/ledger_serializer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/es/ledger_serializer.go b/es/ledger_serializer.go index 2fadb3b..107d524 100644 --- a/es/ledger_serializer.go +++ b/es/ledger_serializer.go @@ -61,21 +61,21 @@ func SerializeLedgerFromHistory(networkPassphrase string, meta xdr.LedgerCloseMe transactionRows[i] = db.TxHistoryRow{ ID: hex.EncodeToString(txHash[:]), LedgerSeq: ledgerRow.LedgerSeq, - Index: i + 1, Envelope: txe, } feeRows[i] = db.TxFeeHistoryRow{ TxID: transactionRows[i].ID, LedgerSeq: ledgerRow.LedgerSeq, - Index: i + 1, } - for _, txp := range meta.V0.TxProcessing { + for j, txp := range meta.V0.TxProcessing { if transactionRows[i].ID == hex.EncodeToString(txp.Result.TransactionHash[:]) { transactionRows[i].Result = txp.Result transactionRows[i].Meta = txp.TxApplyProcessing + transactionRows[i].Index = j + 1 feeRows[i].Changes = txp.FeeProcessing + feeRows[i].Index = j + 1 } } }