diff --git a/.gitignore b/.gitignore index 9f3758e..6d85c82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /astrologer vendor dist + +stellar-core/buckets diff --git a/Dockerfile b/Dockerfile index 14c215e..69ac5e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,19 @@ -FROM golang:alpine AS build - -RUN apk add --no-cache git +FROM golang:stretch AS build RUN mkdir -p $GOPATH/src/github.com/astroband/astrologer WORKDIR $GOPATH/src/github.com/astroband/astrologer -ADD . . +COPY . . + +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -tags netgo -ldflags '-w' -RUN GO111MODULE=on go build +#=========================== -# =============================================================================================== +FROM stellar/stellar-core:13.2.0-1297-b5dda51e AS stellar-core -FROM alpine:latest +#=========================== + +FROM stellar/base ENV DATABASE_URL=postgres://localhost/core?sslmode=disable ENV ES_URL=http://localhost:9200 @@ -19,10 +21,15 @@ ENV INGEST_GAP=-50 WORKDIR /root -COPY --from=build /go/src/github.com/astroband/astrologer/astrologer . -RUN chmod +x ./astrologer +COPY dependencies.sh entry.sh ./ -COPY entry.sh /entry.sh +RUN ["chmod", "+x", "./dependencies.sh"] +RUN ./dependencies.sh + +COPY --from=stellar-core /usr/local/bin/stellar-core /usr/local/bin/ + +COPY --from=build /go/src/github.com/astroband/astrologer/astrologer . +RUN ["chmod", "+x", "./astrologer"] -ENTRYPOINT ["/entry.sh"] -CMD /root/astrologer ingest -- $INGEST_GAP +ENTRYPOINT ["./entry.sh"] +CMD ./astrologer ingest -- $INGEST_GAP diff --git a/commands/export.go b/commands/export.go index 571647f..0f13022 100644 --- a/commands/export.go +++ b/commands/export.go @@ -2,150 +2,134 @@ package commands import ( "bytes" - "log" - "math/rand" - "time" + log "github.com/sirupsen/logrus" - progressbar "github.com/schollz/progressbar/v2" - - "github.com/astroband/astrologer/config" - "github.com/astroband/astrologer/db" "github.com/astroband/astrologer/es" -) - -var ( - bar *progressbar.ProgressBar + lb "github.com/stellar/go/exp/ingest/ledgerbackend" ) // ExportCommandConfig represents configuration options for `export` CLI command type ExportCommandConfig struct { - Start config.NumberWithSign - Count int - RetryCount int - DryRun bool - BatchSize int + Start int + Count int + RetryCount int + DryRun bool + BatchSize int + NetworkPassphrase string } // ExportCommand represents the `export` CLI command type ExportCommand struct { ES es.Adapter - DB db.Adapter Config ExportCommandConfig - firstLedger int - lastLedger int + firstLedger uint32 + lastLedger uint32 } // Execute starts the export process func (cmd *ExportCommand) Execute() { - cmd.firstLedger, cmd.lastLedger = cmd.getRange() + total := cmd.Config.Count - total := cmd.DB.LedgerHeaderRowCount(cmd.firstLedger, cmd.lastLedger) + cmd.firstLedger = uint32(cmd.Config.Start) + cmd.lastLedger = cmd.firstLedger + uint32(cmd.Config.Count) - 1 if total == 0 { log.Fatal("Nothing to export within given range!", cmd.firstLedger, cmd.lastLedger) } - log.Println("Exporting ledgers from", cmd.firstLedger, "to", cmd.lastLedger, "total", total) + log.Infof("Exporting ledgers from %d to %d. Total: %d ledgers\n", cmd.firstLedger, cmd.lastLedger, total) + log.Infof("Will insert %d batches %d ledgers each\n", cmd.blockCount(total), cmd.Config.BatchSize) - createBar(total) + ledgerBackend, err := lb.NewCaptive( + "stellar-core", + "", + cmd.Config.NetworkPassphrase, + getHistoryURLs(cmd.Config.NetworkPassphrase), + ) - for i := 0; i < cmd.blockCount(total); i++ { - i := i - pool.Submit(func() { cmd.exportBlock(i) }) + if err != nil { + log.Fatal("error creating captive core backend", err) } - pool.StopWait() - finishBar() -} + log.Info("Preparing range...") + err = ledgerBackend.PrepareRange(lb.BoundedRange(cmd.firstLedger, cmd.lastLedger)) -func (cmd *ExportCommand) exportBlock(i int) { - var b bytes.Buffer - - rows := cmd.DB.LedgerHeaderRowFetchBatch(i, cmd.firstLedger, cmd.Config.BatchSize) + if err != nil { + log.Fatal("Failed preparing range", err) + } - for n := 0; n < len(rows); n++ { - txs := cmd.DB.TxHistoryRowForSeq(rows[n].LedgerSeq) - fees := cmd.DB.TxFeeHistoryRowsForRows(txs) + var batchBuffer bytes.Buffer - err := es.SerializeLedger(rows[n], txs, fees, &b) + for ledgerSeq := cmd.firstLedger; ledgerSeq <= cmd.lastLedger; ledgerSeq++ { + _, meta, err := ledgerBackend.GetLedger(ledgerSeq) if err != nil { - log.Fatalf("Failed to ingest ledger %d: %v\n", rows[n].LedgerSeq, err) + // FIXME skip instead of failing + log.Fatal(err) } - if !*config.Verbose { - bar.Add(1) - } - } + es.SerializeLedgerFromHistory(cmd.Config.NetworkPassphrase, meta, &batchBuffer) - if *config.Verbose { - log.Println(b.String()) - } + if (ledgerSeq-cmd.firstLedger+1)%uint32(cmd.Config.BatchSize) == 0 || ledgerSeq == cmd.lastLedger { + payload := batchBuffer.String() + pool.Submit(func() { + log.Infof("Gonna bulk insert %d bytes\n", len(payload)) + err := cmd.ES.BulkInsert(payload) + + if err != nil { + log.Fatal("Cannot bulk insert", err) + } else { + log.Printf("Batch successfully inserted\n") + } + }) - if !cmd.Config.DryRun { - cmd.ES.IndexWithRetries(&b, cmd.Config.RetryCount) + batchBuffer.Reset() + } } -} -func (cmd *ExportCommand) index(b *bytes.Buffer, retry int) { - indexed := cmd.ES.BulkInsert(b) + // for i := 0; i < cmd.blockCount(total); i++ { + // var b bytes.Buffer + // ledgerCounter := 0 + // batchNum := i + 1 - if !indexed { - if retry > cmd.Config.RetryCount { - log.Fatal("Retries for bulk failed, aborting") - } + // for meta := range channel { + // seq := int(meta.V0.LedgerHeader.Header.LedgerSeq) - delay := time.Duration((rand.Intn(10) + 5)) - time.Sleep(delay * time.Second) + // if seq < cmd.firstLedger || seq > cmd.lastLedger { + // continue + // } - cmd.index(b, retry+1) - } -} + // ledgerCounter += 1 -// Parses range of export command -func (cmd *ExportCommand) getRange() (first int, last int) { - firstLedger := cmd.DB.LedgerHeaderFirstRow() - lastLedger := cmd.DB.LedgerHeaderLastRow() + // log.Println(seq) - if cmd.Config.Start.Explicit { - if cmd.Config.Start.Value < 0 { - first = lastLedger.LedgerSeq + cmd.Config.Start.Value + 1 - } else if config.Start.Value > 0 { - first = firstLedger.LedgerSeq + cmd.Config.Start.Value - } - } else if cmd.Config.Start.Value != 0 { - first = cmd.Config.Start.Value - } else { - first = firstLedger.LedgerSeq - } + // es.SerializeLedgerFromHistory(meta, &b) - if cmd.Config.Count == 0 { - last = lastLedger.LedgerSeq - } else { - last = first + cmd.Config.Count - 1 - } + // log.Printf("Ledger %d of %d in batch %d\n", ledgerCounter, cmd.Config.BatchSize, batchNum) - return first, last -} + // if ledgerCounter == cmd.Config.BatchSize { + // break + // } + // } -func createBar(count int) { - bar = progressbar.NewOptions( - count, - progressbar.OptionEnableColorCodes(false), - progressbar.OptionShowCount(), - progressbar.OptionThrottle(500*time.Millisecond), - progressbar.OptionSetRenderBlankState(true), - progressbar.OptionSetWidth(100), - ) + // if cmd.Config.DryRun { + // continue + // } - bar.RenderBlank() -} + // pool.Submit(func() { + // log.Printf("Gonna bulk insert %d bytes\n", b.Len()) + // err := cmd.ES.BulkInsert(b) -func finishBar() { - if !*config.Verbose { - bar.Finish() - } + // if err != nil { + // log.Fatal("Cannot bulk insert", err) + // } else { + // log.Printf("Batch %d successfully inserted\n", batchNum) + // } + // }) + // } + + pool.StopWait() } func (cmd *ExportCommand) blockCount(count int) (blocks int) { @@ -157,3 +141,22 @@ func (cmd *ExportCommand) blockCount(count int) (blocks int) { return blocks } + +func getHistoryURLs(networkPassphrase string) []string { + switch networkPassphrase { + case "Public Global Stellar Network ; September 2015": + return []string{ + "https://history.stellar.org/prd/core-live/core_live_001", + "https://history.stellar.org/prd/core-live/core_live_002", + "https://history.stellar.org/prd/core-live/core_live_003", + } + case "Test SDF Network ; September 2015": + return []string{ + "http://history.stellar.org/prd/core-testnet/core_testnet_001", + "http://history.stellar.org/prd/core-testnet/core_testnet_002", + "http://history.stellar.org/prd/core-testnet/core_testnet_003", + } + default: + return []string{} + } +} diff --git a/config/main.go b/config/main.go index 860f1f4..5dd5621 100644 --- a/config/main.go +++ b/config/main.go @@ -91,11 +91,11 @@ var ( Default("25"). Int() - // Start ledger to start with - Start = NumberWithSignParse(exportCommand.Arg("start", "Ledger to start indexing, +100 means offset 100 from the first")) + Start = exportCommand.Arg("start", "Ledger to start indexing, +100 means offset 100 from the first").Default("0").Int() // Count ledgers - Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() + Count = exportCommand.Arg("count", "Count of ledgers to ingest, should be aliquout batch size").Default("0").Int() + Network = exportCommand.Flag("network", "Stellar network to use").Default("testnet").Enum("public", "test") // StartIngest ledger to start with ingesting StartIngest = ingestCommand.Arg("start", "Ledger to start ingesting").Int() diff --git a/db/main.go b/db/main.go index 813ca61..9a797e8 100644 --- a/db/main.go +++ b/db/main.go @@ -1,41 +1,13 @@ package db import ( - "bytes" "log" "net/url" - "unicode/utf8" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" // Postgres driver ) -// Copy paste from Horizon -func utf8Scrub(in string) string { - - // First check validity using the stdlib, returning if the string is already - // valid - if utf8.ValidString(in) { - return in - } - - left := []byte(in) - var result bytes.Buffer - - for len(left) > 0 { - r, n := utf8.DecodeRune(left) - - _, err := result.WriteRune(r) - if err != nil { - panic(err) - } - - left = left[n:] - } - - return result.String() -} - // Adapter defines the interface to work with ledger database type Adapter interface { LedgerHeaderRowCount(first int, last int) int diff --git a/db/tx_history_row.go b/db/tx_history_row.go index 1f31655..07c01cc 100644 --- a/db/tx_history_row.go +++ b/db/tx_history_row.go @@ -1,13 +1,9 @@ package db import ( - "encoding/base64" "fmt" - "log" - "strings" - - "github.com/guregu/null" "github.com/stellar/go/xdr" + "log" ) // TxHistoryRow represents row of txhistory table @@ -32,40 +28,6 @@ func (db *Client) TxHistoryRowForSeq(seq int) []TxHistoryRow { return txs } -// MemoValue Returns clean memo value, this is copy paste from horizon internal package -func (tx *TxHistoryRow) MemoValue() null.String { - var ( - value string - valid bool - memo = tx.Envelope.Memo() - ) - - switch memo.Type { - case xdr.MemoTypeMemoNone: - value, valid = "", false - case xdr.MemoTypeMemoText: - scrubbed := utf8Scrub(memo.MustText()) - notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") - value, valid = notnull, true - case xdr.MemoTypeMemoId: - value, valid = fmt.Sprintf("%d", memo.MustId()), true - case xdr.MemoTypeMemoHash: - hash := memo.MustHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - case xdr.MemoTypeMemoReturn: - hash := memo.MustRetHash() - value, valid = - base64.StdEncoding.EncodeToString(hash[:]), - true - default: - panic(fmt.Errorf("invalid memo type: %v", memo.Type)) - } - - return null.NewString(value, valid) -} - // Operations returns operations array func (tx *TxHistoryRow) Operations() (error, []xdr.Operation) { switch tx.Envelope.Type { diff --git a/dependencies.sh b/dependencies.sh new file mode 100755 index 0000000..8df3775 --- /dev/null +++ b/dependencies.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +apt-get update +apt-get install -y curl libpq-dev +apt-get clean + +echo "\nDone installing stellar-core dependencies...\n" diff --git a/es/adapter.go b/es/adapter.go index 1919aa8..e03b11f 100644 --- a/es/adapter.go +++ b/es/adapter.go @@ -3,6 +3,7 @@ package es import ( "bytes" "encoding/json" + "errors" "log" "math/rand" "net/http" @@ -106,14 +107,22 @@ func (es *Client) LedgerSeqRangeQuery(ranges []map[string]interface{}) map[strin } // BulkInsert sends the payload to ES using bulk operation -func (es *Client) BulkInsert(payload *bytes.Buffer) (success bool) { - res, err := es.rawClient.Bulk(bytes.NewReader(payload.Bytes())) +func (es *Client) BulkInsert(payload string) error { + res, err := es.rawClient.Bulk(strings.NewReader(payload)) if res != nil { defer res.Body.Close() } - return err == nil && (res == nil || !res.IsError()) + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil } // LedgerCountInRange counts number of ledgers from the given range persisted into ES @@ -183,9 +192,9 @@ func (es *Client) GetLedgerSeqsInRange(min, max int) (seqs []int) { // IndexWithRetries performs a bulk insert into ES cluster with retries on failures func (es *Client) IndexWithRetries(payload *bytes.Buffer, retryCount int) { - isIndexed := es.BulkInsert(payload) + err := es.BulkInsert(payload.String()) - if !isIndexed { + if err != nil { if retryCount-1 == 0 { log.Fatal("Retries for bulk failed, aborting") } diff --git a/es/balance.go b/es/balance.go index c2a6bcf..5c73d0f 100644 --- a/es/balance.go +++ b/es/balance.go @@ -20,7 +20,6 @@ const ( // Balance represents balance log entry type Balance struct { - ID string `json:"id"` PagingToken PagingToken `json:"paging_token"` AccountID string `json:"account_id"` Value string `json:"value"` diff --git a/es/balance_extractor.go b/es/balance_extractor.go index aea0a33..3cfb28a 100644 --- a/es/balance_extractor.go +++ b/es/balance_extractor.go @@ -54,10 +54,6 @@ func (e *BalanceExtractor) extract() []*Balance { } } - for i, balance := range e.balances { - e.balances[i].ID = balance.PagingToken.String() - } - return e.balances } diff --git a/es/indices.go b/es/indices.go index 70f7890..141fea5 100644 --- a/es/indices.go +++ b/es/indices.go @@ -30,7 +30,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "hash": { "type": "keyword", "index": true }, "prev_hash": { "type": "keyword", "index": false }, "bucket_list_hash": { "type": "keyword", "index": false }, @@ -99,7 +98,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "tx_id": { "type": "keyword", "index": true }, "tx_idx": { "type": "integer" }, "idx": { "type": "integer" }, @@ -235,7 +233,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "paging_token": { "type": "keyword", "index": true }, "account_id": { "type": "keyword", "index": true }, "value": { "type": "scaled_float", "scaling_factor": 10000000 }, @@ -266,7 +263,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "paging_token": { "type": "keyword", "index": true }, "sold": { "type": "scaled_float", "scaling_factor": 10000000 }, "bought": { "type": "scaled_float", "scaling_factor": 10000000 }, @@ -305,7 +301,6 @@ func GetIndexDefinitions() map[IndexName]IndexDefinition { }, "mappings": { "properties": { - "id": { "type": "keyword", "index": true }, "paging_token": { "type": "keyword", "index": true }, "account_id": { "type": "keyword", "index": true }, "signer": { "type": "keyword", "index": true }, diff --git a/es/ledger_header.go b/es/ledger_header.go index d8534b0..c3c5f16 100644 --- a/es/ledger_header.go +++ b/es/ledger_header.go @@ -8,7 +8,6 @@ import ( // LedgerHeader represents json-serializable struct for LedgerHeader to index type LedgerHeader struct { - ID string `json:"id"` Hash string `json:"hash"` PrevHash string `json:"prev_hash"` BucketListHash string `json:"bucket_list_hash"` @@ -27,15 +26,12 @@ type LedgerHeader struct { // NewLedgerHeader creates LedgerHeader from LedgerHeaderRow func NewLedgerHeader(row *db.LedgerHeaderRow) *LedgerHeader { - pagingToken := PagingToken{LedgerSeq: row.LedgerSeq} - return &LedgerHeader{ - ID: pagingToken.String(), Hash: row.Hash, PrevHash: row.PrevHash, BucketListHash: row.BucketListHash, Seq: row.LedgerSeq, - PagingToken: pagingToken, + PagingToken: PagingToken{LedgerSeq: row.LedgerSeq}, CloseTime: time.Unix(row.CloseTime, 0), Version: int(row.Data.LedgerVersion), TotalCoins: int(row.Data.TotalCoins), diff --git a/es/ledger_serializer.go b/es/ledger_serializer.go index c9d810f..107d524 100644 --- a/es/ledger_serializer.go +++ b/es/ledger_serializer.go @@ -2,9 +2,12 @@ package es import ( "bytes" + "encoding/hex" "fmt" + "log" "github.com/astroband/astrologer/db" + "github.com/astroband/astrologer/util" "github.com/stellar/go/xdr" ) @@ -32,6 +35,62 @@ func SerializeLedger(ledgerRow db.LedgerHeaderRow, transactionRows []db.TxHistor return serializer.serialize() } +// SerializeLedger serializes ledger data into ES bulk index data +func SerializeLedgerFromHistory(networkPassphrase string, meta xdr.LedgerCloseMeta, buffer *bytes.Buffer) { + ledgerHeader := meta.V0.LedgerHeader.Header + + ledgerRow := db.LedgerHeaderRow{ + Hash: hex.EncodeToString(meta.V0.LedgerHeader.Hash[:]), + PrevHash: hex.EncodeToString(ledgerHeader.PreviousLedgerHash[:]), + BucketListHash: hex.EncodeToString(ledgerHeader.BucketListHash[:]), + LedgerSeq: int(ledgerHeader.LedgerSeq), + CloseTime: int64(ledgerHeader.ScpValue.CloseTime), + Data: ledgerHeader, + } + + transactionRows := make([]db.TxHistoryRow, len(meta.V0.TxSet.Txs)) + feeRows := make([]db.TxFeeHistoryRow, len(meta.V0.TxSet.Txs)) + + for i, txe := range meta.V0.TxSet.Txs { + txHash, hashErr := util.HashTransactionInEnvelope(txe, networkPassphrase) + + if hashErr != nil { + log.Fatalf("Failed to hash transaction #%d in ledger %d\n", i, ledgerRow.LedgerSeq) + } + + transactionRows[i] = db.TxHistoryRow{ + ID: hex.EncodeToString(txHash[:]), + LedgerSeq: ledgerRow.LedgerSeq, + Envelope: txe, + } + + feeRows[i] = db.TxFeeHistoryRow{ + TxID: transactionRows[i].ID, + LedgerSeq: ledgerRow.LedgerSeq, + } + + for j, txp := range meta.V0.TxProcessing { + if transactionRows[i].ID == hex.EncodeToString(txp.Result.TransactionHash[:]) { + transactionRows[i].Result = txp.Result + transactionRows[i].Meta = txp.TxApplyProcessing + transactionRows[i].Index = j + 1 + feeRows[i].Changes = txp.FeeProcessing + feeRows[i].Index = j + 1 + } + } + } + + serializer := &ledgerSerializer{ + ledgerRow: ledgerRow, + transactionRows: transactionRows, + feeRows: feeRows, + ledger: NewLedgerHeader(&ledgerRow), + buffer: buffer, + } + + serializer.serialize() +} + func (s *ledgerSerializer) serialize() error { SerializeForBulk(s.ledger, s.buffer) diff --git a/es/main.go b/es/main.go index 7e7c43e..2e5ed14 100644 --- a/es/main.go +++ b/es/main.go @@ -22,7 +22,7 @@ type Adapter interface { IndexExists(name IndexName) bool CreateIndex(name IndexName, body IndexDefinition) DeleteIndex(name IndexName) - BulkInsert(payload *bytes.Buffer) (success bool) + BulkInsert(payload string) error IndexWithRetries(payload *bytes.Buffer, retriesCount int) } diff --git a/es/operation.go b/es/operation.go index 234c4ac..11ee6fa 100644 --- a/es/operation.go +++ b/es/operation.go @@ -16,7 +16,6 @@ const ( // Operation represents ES-serializable transaction type Operation struct { - ID string `json:"id"` TxID string `json:"tx_id"` TxIndex int `json:"tx_idx"` Index int `json:"idx"` @@ -71,15 +70,7 @@ func NewOperation(t *Transaction, o *xdr.Operation, r *[]xdr.OperationResult, n result = &(*r)[n] } - operation, err := ProduceOperation(t, o, result, n) - - if err != nil { - return nil, err - } - - operation.ID = operation.PagingToken.String() - - return operation, nil + return ProduceOperation(t, o, result, n) } // DocID returns elastic document id diff --git a/es/serialize.go b/es/serialize.go index a1b01bf..03386a6 100644 --- a/es/serialize.go +++ b/es/serialize.go @@ -9,9 +9,22 @@ import ( // SerializeForBulk returns object serialized for elastic bulk indexing func SerializeForBulk(obj Indexable, b *bytes.Buffer) { - meta := fmt.Sprintf( - `{ "index": { "_index": "%s", "_type": "_doc" } }%s`, obj.IndexName(), "\n", - ) + var meta string + + id := obj.DocID() + + if id != nil { + meta = fmt.Sprintf( + `{ "index": { "_index": "%s", "_id": "%s", "_type": "_doc" } }%s`, + obj.IndexName(), + *id, + "\n", + ) + } else { + meta = fmt.Sprintf( + `{ "index": { "_index": "%s", "_type": "_doc" } }%s`, obj.IndexName(), "\n", + ) + } data, err := json.Marshal(obj) if err != nil { diff --git a/es/signer_history.go b/es/signer_history.go index 2556e35..3402632 100644 --- a/es/signer_history.go +++ b/es/signer_history.go @@ -6,7 +6,6 @@ import ( // SignerHistory represents signer change entry, still the question if it is required type SignerHistory struct { - ID string `json:"id"` PagingToken PagingToken `json:"paging_token"` AccountID string `json:"account_id"` Signer string `json:"signer"` @@ -33,7 +32,6 @@ func ProduceSignerHistory(o *Operation) (h *SignerHistory) { } entry := &SignerHistory{ - ID: token.String(), PagingToken: token, AccountID: o.SourceAccountID, Signer: o.Signer.ID, diff --git a/es/trade.go b/es/trade.go index 479b22e..5de5a3b 100644 --- a/es/trade.go +++ b/es/trade.go @@ -6,7 +6,6 @@ import ( // Trade represents trade entry type Trade struct { - ID string `json:"id"` PagingToken PagingToken `json:"paging_token"` Sold string `json:"sold"` Bought string `json:"bought"` diff --git a/es/trade_extractor.go b/es/trade_extractor.go index d0759c9..eb7e0b4 100644 --- a/es/trade_extractor.go +++ b/es/trade_extractor.go @@ -55,10 +55,6 @@ func (e *TradeExtractor) extract() (trades []Trade) { } } - for i, trade := range trades { - trades[i].ID = trade.PagingToken.String() - } - return trades } diff --git a/es/transaction.go b/es/transaction.go index 88ee007..f12bb3b 100644 --- a/es/transaction.go +++ b/es/transaction.go @@ -4,9 +4,27 @@ import ( "time" "github.com/astroband/astrologer/db" + "github.com/astroband/astrologer/support" "github.com/stellar/go/xdr" ) +// transactionData represents all necessary pieces we need +// from stellar-core to ingest single transaction +type transactionData struct { + id string + ledgerSeq int + // ledger close time + closeTime time.Time + // index of transaction in ledger + index int + xdrV0 *xdr.TransactionV0 + xdrV1 *xdr.Transaction + feeBump *xdr.FeeBumpTransaction + result xdr.TransactionResultPair + feeAccount string + maxFee int +} + // Transaction represents ES-serializable transaction type Transaction struct { ID string `json:"id"` @@ -26,7 +44,7 @@ type Transaction struct { *Memo `json:"memo,omitempty"` } -// NewTransaction creates LedgerHeader from LedgerHeaderRow +// NewTransaction creates Transaction from TxHistoryRow func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*Transaction, error) { var ( err error @@ -58,8 +76,8 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T CloseTime: t, Successful: success, ResultCode: int(result.Code), - OperationCount: len(envelope.Operations()), SourceAccountID: sourceAccountAddress, + OperationCount: len(envelope.Operations()), } if envelope.IsFeeBump() { @@ -77,7 +95,7 @@ func (s *ledgerSerializer) NewTransaction(row *db.TxHistoryRow, t time.Time) (*T if envelope.Memo().Type != xdr.MemoTypeMemoNone { transaction.Memo = &Memo{ Type: int(envelope.Memo().Type), - Value: row.MemoValue().String, + Value: support.MemoValue(row.Envelope.Memo()).String, } } diff --git a/go.mod b/go.mod index 7f54791..839034a 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,26 @@ module github.com/astroband/astrologer go 1.12 require ( + github.com/Masterminds/squirrel v1.4.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/elastic/go-elasticsearch/v7 v7.7.0 + github.com/aws/aws-sdk-go v1.33.7 // indirect + github.com/elastic/go-elasticsearch/v7 v7.8.0 github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896 // indirect - github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6 + github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae + github.com/go-errors/errors v1.1.1 // indirect github.com/guregu/null v4.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 - github.com/lib/pq v1.5.2 + github.com/lib/pq v1.7.0 github.com/mattn/go-runewidth v0.0.9 // indirect github.com/olekukonko/tablewriter v0.0.4 - github.com/pkg/errors v0.9.1 // indirect - github.com/schollz/progressbar/v2 v2.15.0 - github.com/stellar/go v0.0.0-20200526231405-08ec13c54232 - github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 // indirect + github.com/sirupsen/logrus v1.6.0 + github.com/stellar/go v0.0.0-20200804130752-fdc4eca4f6ce + github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/testify v1.6.1 // indirect + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 + gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect ) diff --git a/go.sum b/go.sum index c549f44..1de977f 100644 --- a/go.sum +++ b/go.sum @@ -2,11 +2,16 @@ bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c/go.mod h1:1vhO7Mn/ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= firebase.google.com/go v3.12.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= -github.com/BurntSushi/toml v0.2.1-0.20160717150709-99064174e013/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 h1:PPfYWScYacO3Q6JMCLkyh6Ea2Q/REDTMgmiTAeiV8Jg= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= +github.com/Masterminds/squirrel v1.4.0 h1:he5i/EXixZxrBUWcxzDYMiju9WZ3ld/l7QBNuo/eN3w= +github.com/Masterminds/squirrel v1.4.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -18,8 +23,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/asaskevich/govalidator v0.0.0-20180319081651-7d2e70ef918f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.6.11-0.20170104181648-8649d278323e/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= +github.com/aws/aws-sdk-go v1.25.25 h1:j3HLOqcDWjNox1DyvJRs+kVQF42Ghtv6oL6cVBfXS3U= github.com/aws/aws-sdk-go v1.25.25/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.33.7 h1:vOozL5hmWHHriRviVTQnUwz8l05RS0rehmEFymI+/x8= +github.com/aws/aws-sdk-go v1.33.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,118 +36,126 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-elasticsearch/v7 v7.2.0 h1:9gdDHRMHjM+rPV5lLWiG8vkWn8OCgQHoGwr1pINn8aE= -github.com/elastic/go-elasticsearch/v7 v7.2.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= -github.com/elastic/go-elasticsearch/v7 v7.7.0 h1:oQBx/S3RiaH0/kiP0scYSay9xgSmVAYJpuqEf+e9GZg= -github.com/elastic/go-elasticsearch/v7 v7.7.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/elastic/go-elasticsearch/v7 v7.8.0 h1:M9D55OK13IEgg51Jb57mZgseag1AsncwAUn4C6j1vlc= +github.com/elastic/go-elasticsearch/v7 v7.8.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/inject v0.0.0-20161006174721-cc1aa653e50f/go.mod h1:oO8UHw+fDHjDsk4CTy/E96WDzFUYozAtBAaGNoVL0+c= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/structtag v0.0.0-20150214074306-217e25fb9691/go.mod h1:sKLL1iua/0etWfo/nPCmyz+v2XDMXy+Ho53W7RAuZNY= -github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gammazero/deque v0.0.0-20190515085511-50f5fc498a27 h1:eqgrF1fSzfhRQet8I3ws3MrGPLS7z2n1emuetE3yr2w= -github.com/gammazero/deque v0.0.0-20190515085511-50f5fc498a27/go.mod h1:GeIq9qoE43YdGnDXURnmKTnGg15pQz4mYkXSTChbneI= github.com/gammazero/deque v0.0.0-20200227231300-1e9af0e52b46/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w= github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896 h1:p5u6mxVWDmHzLSxFbtMmlSTL9svFPkBXMvTEIfvEXzY= github.com/gammazero/deque v0.0.0-20200310222745-50fa758af896/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w= -github.com/gammazero/workerpool v0.0.0-20190515092108-567ff334f003 h1:9FFqhJ//pGQuuogGF9kRZqtlmrc+BREVC0+dk42Q5VY= -github.com/gammazero/workerpool v0.0.0-20190515092108-567ff334f003/go.mod h1:lU9Kl5oGNQzGAc+k1OcuqtXCPBqnz8zrxoxEwqc9mRc= -github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6 h1:1Cy/haf7XO4OyrkGid0Wq5CMluIErbvDptVAt8UTy38= -github.com/gammazero/workerpool v0.0.0-20200311205957-7b00833861c6/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= +github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae h1:kNWKQA116SMe5ydvryvVlSV8qVfuv+NS8cO74LacufM= +github.com/gammazero/workerpool v0.0.0-20200719191849-1c428e5ee8ae/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs= +github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955 h1:gmtGRvSexPU4B1T/yYo0sLOKzER1YT+b4kPxPpm0Ty4= github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= github.com/getsentry/raven-go v0.0.0-20160805001729-c9d3cc542ad1/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/go-chi/chi v3.1.5+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.0.3+incompatible h1:gakN3pDJnzZN5jqFV2TEdF66rTfKeITyR8qu6ekICEY= github.com/go-chi/chi v4.0.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-errors/errors v0.0.0-20150906023321-a41850380601 h1:jxTbmDuqQUTI6MscgbqB39vtxGfr2fi61nYIcFQUnlE= github.com/go-errors/errors v0.0.0-20150906023321-a41850380601/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-ini/ini v1.23.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/packr v1.12.1/go.mod h1:H2dZhQFqHeZwr/5A/uGQkBp7xYuMGuzXFeKhYdcz5No= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 h1:oERTZ1buOUYlpmKaqlO5fYmz8cZ1rYu5DieJzF4ZVmU= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/schema v1.1.0 h1:CamqUDOFUBqzrvxuz2vEwo8+SUdwsluFh7IlzJh30LY= github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/graph-gophers/graphql-go v0.0.0-20190225005345-3e8838d4614c/go.mod h1:uJhtPXrcJLqyi0H5IuMFh+fgW+8cMMakK3Txrbk/WJE= github.com/guregu/null v2.1.3-0.20151024101046-79c5bd36b615+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= -github.com/guregu/null v3.4.0+incompatible h1:a4mw37gBO7ypcBlTJeZGuMpSxxFTV9qFfFKgWxQSGaM= -github.com/guregu/null v3.4.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31 h1:Aw95BEvxJ3K6o9GGv5ppCd1P8hkeIeEJ30FO+OhOJpM= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v0.0.0-20161106143436-e3b7981a12dd h1:vQ0EEfHpdFUtNRj1ri25MUq5jb3Vma+kKhLyjeUTVow= github.com/klauspost/compress v0.0.0-20161106143436-e3b7981a12dd/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc h1:WW8B7p7QBnFlqRVv/k6ro/S8Z7tCnYjJHcQNScx9YVs= github.com/klauspost/cpuid v0.0.0-20160302075316-09cded8978dc/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.0.0-20150520163514-e6ac2fc51e89 h1:Smt4CPhAnATQEGlX/nyqGETX4Tj8bg/7shBT5gH8d7s= github.com/kr/pretty v0.0.0-20150520163514-e6ac2fc51e89/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA= github.com/kr/text v0.0.0-20150520163712-e373e137fafd h1:ohpc+F5FseC/PPrR1wz2WcZxOc4kplnJ39pGbFlj/eY= github.com/kr/text v0.0.0-20150520163712-e373e137fafd/go.mod h1:sjUstKUATFIcff4qlB53Kml0wQPtJVc/3fWrmuUmcfA= +github.com/lann/builder v0.0.0-20140829050551-c603884a2c1f h1:GYBg1t6ujjhgyYsiO9i0qwbnUZzTiPVLCA/QUkD7ECQ= github.com/lann/builder v0.0.0-20140829050551-c603884a2c1f/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw= -github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= +github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.5.4/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 h1:ykXz+pRRTibcSjG1yRhpdSHInF8yZY/mfn+Rz2Nd1rE= github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739/go.mod h1:zUx1mhth20V3VKgL5jbd1BSQcW4Fy6Qs4PZvQwRFwzM= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mndrix/ps v0.0.0-20131111202200-33ddf69629c1 h1:kCroTjOY+wyp+iHA2lZOV5aJ6WfBVjGnW8bCYmXmLPo= github.com/mndrix/ps v0.0.0-20131111202200-33ddf69629c1/go.mod h1:dHgTaDInzkAqJv67VaX1IkK449M2UoBY68CZeI/bNCU= +github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db h1:eZgFHVkk9uOTaOQLC6tgjkzdp7Ays8eEVecBcfHZlJQ= github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077 h1:A804awGqaW7i61y8KnbtHmh3scqbNuTJqcycq3u5ZAU= -github.com/nullstyle/go-xdr v0.0.0-20180726165426-f4c839f75077/go.mod h1:sZZi9x5aHXGZ/RRp7Ne5rkvtDxZb7pd7vgVA+gmE35A= -github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -166,66 +181,78 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rcrowley/go-metrics v0.0.0-20160113235030-51425a2415d2/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rubenv/sql-migrate v0.0.0-20190717103323-87ce952f7079/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= -github.com/schollz/progressbar/v2 v2.13.3-0.20190604152910-b6d7f92524db h1:skK2aM0X0H72NfIqzqBGytur9ymS2pnF/qFrtleDuvU= -github.com/schollz/progressbar/v2 v2.13.3-0.20190604152910-b6d7f92524db/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= -github.com/schollz/progressbar/v2 v2.15.0 h1:dVzHQ8fHRmtPjD3K10jT3Qgn/+H+92jhPrhmxIJfDz8= -github.com/schollz/progressbar/v2 v2.15.0/go.mod h1:UdPq3prGkfQ7MOzZKlDRpYKcFqEMczbD7YmbPgpzKMI= github.com/sebest/xff v0.0.0-20150611211316-7a36e3a787b5/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0= +github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 h1:S4OC0+OBKz6mJnzuHioeEat74PuQ4Sgvbf8eus695sc= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2/go.mod h1:8zLRYR5npGjaOXgPSKat5+oOh+UHd8OdbS18iqX9F6Y= +github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca h1:oR/RycYTFTVXzND5r4FdsvbnBn0HJXSVeNAnwaTXRwk= github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/sirupsen/logrus v1.0.6-0.20180530095059-070c81def33f/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/cobra v0.0.0-20160830174925-9c28e4bbd74e/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20161005214240-4bd69631f475/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v0.0.0-20150621231900-db7ff930a189/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= -github.com/stellar/go v0.0.0-20191016082916-2f64a5e02f78 h1:fIdPEXodE+DAGOioOVd0lT2mqsxzWqrqVL4TqbgmfQg= -github.com/stellar/go v0.0.0-20191016082916-2f64a5e02f78/go.mod h1:U4zGmzxsNw5Kr0Z8f0clhHVFOIfi80iVh8MBp7y8pY0= -github.com/stellar/go v0.0.0-20200515153832-820becd8083b h1:bXG6jOM0MGSyR4euTMYZPazBbtUZbt/ZzphYN1vZ04I= -github.com/stellar/go v0.0.0-20200515153832-820becd8083b/go.mod h1:kfZjGLxoc7P6QAk7BOrXL1qVk7gJV3+OSk4FCf0dXsM= -github.com/stellar/go v0.0.0-20200526231405-08ec13c54232 h1:HweN0TbrsTmdp58L+I1ozu2JSzn+w6fhaGSiLohEtxI= -github.com/stellar/go v0.0.0-20200526231405-08ec13c54232/go.mod h1:Q4JDgZhGw7Ytu1924PzNkwr2GsLptrzAjwC/icuXnLU= -github.com/stellar/go v0.0.0-20200528062442-f08b35a3f034 h1:YbQySRDlm+qU2+khzooq+bCEteADDprTrqh2YMNehqI= -github.com/stellar/go v0.0.0-20200617134134-abd5c5f8ad8f h1:vARqQGSAvss4qUaK+/7vJOLoNAOrod9+LYiqVVUxtrI= -github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e h1:n/hfey8pO+RYMoGXyvyzuw5pdO8IFDoyAL/g5OiCesY= -github.com/stellar/go-xdr v0.0.0-20180917104419-0bc96f33a18e/go.mod h1:gpOLVzy6TVYTQ3LvHSN9RJC700FkhFCpSE82u37aNRM= +github.com/stellar/go v0.0.0-20200719182404-6934026f350b h1:BIM4A/5MhgfBabSv+MIimHwdeQlamXFD0zlhkOXjvmg= +github.com/stellar/go v0.0.0-20200719182404-6934026f350b/go.mod h1:LDU2cdYFQbYGt7UtgciOw8g5DXdc93r6N3e1mdHWems= +github.com/stellar/go v0.0.0-20200804130752-fdc4eca4f6ce h1:Lixum59yJNsfxgKO1e99ZwWKgfmQfFWRD5U4oyFxazc= +github.com/stellar/go v0.0.0-20200804130752-fdc4eca4f6ce/go.mod h1:LDU2cdYFQbYGt7UtgciOw8g5DXdc93r6N3e1mdHWems= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2 h1:K9H+A+eWe8ZlnpNha+pXbEK+jtIluQp/2dKxkK8k7OE= github.com/stellar/go-xdr v0.0.0-20200331223602-71a1e6d555f2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps= github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible/go.mod h1:7CJ23pXirXBJq45DqvO6clzTEGM/l1SfKrgrzLry8b4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tyler-smith/go-bip39 v0.0.0-20180618194314-52158e4697b8/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v0.0.0-20170109085056-0a7f0a797cd6 h1:s0IDmR1jFyWvOK7jVIuAsmHQaGkXUuTas8NXFUOwuAI= github.com/valyala/fasthttp v0.0.0-20170109085056-0a7f0a797cd6/go.mod h1:+g/po7GqyG5E+1CNgquiIxJnsXEi5vwFn5weFujbO78= +github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 h1:KM4T3G70MiR+JtqplcYkNVoNz7pDwYaBxWBXQK804So= github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c h1:XZWnr3bsDQWAZg4Ne+cPoXRPILrNlPNQfxBuwLl43is= github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce h1:cVSRGH8cOveJNwFEEZLXtB+XMnRqKLjUP6V/ZFYQCXI= github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb h1:06WAhQa+mYv7BiOk13B/ywyTlkoE/S7uu6TBKU6FHnE= github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M= github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce h1:888GrqRxabUce7lj4OaoShPxodm3kXOMpSa85wdYzfY= github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -243,6 +270,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -258,11 +287,17 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c h1:+EXw7AwNOKzPFXMZ1yNjO40aWCh3PIquJB2fYlv9wcs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -280,7 +315,6 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -288,8 +322,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 h1:r5ptJ1tBxVAeqw4CrYWhXIMr0SybY3CDHuIbCg5CFVw= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0/go.mod h1:WtiW9ZA1LdaWqtQRo1VbIL/v4XZ8NDta+O/kSpGgVek= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -298,5 +332,11 @@ gopkg.in/tylerb/graceful.v1 v1.2.13/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index cb0b966..78cb38b 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( cfg "github.com/astroband/astrologer/config" "github.com/astroband/astrologer/db" "github.com/astroband/astrologer/es" + "github.com/stellar/go/network" "gopkg.in/alecthomas/kingpin.v2" ) @@ -24,15 +25,24 @@ func main() { config := cmd.CreateIndexCommandConfig{Force: *cfg.ForceRecreateIndexes} command = &cmd.CreateIndexCommand{ES: esClient, Config: config} case "export": - dbClient := db.Connect(*cfg.DatabaseURL) + var networkPassphrase string + + switch *cfg.Network { + case "public": + networkPassphrase = network.PublicNetworkPassphrase + case "test": + networkPassphrase = network.TestNetworkPassphrase + } + config := cmd.ExportCommandConfig{ - Start: *cfg.Start, - Count: *cfg.Count, - DryRun: *cfg.ExportDryRun, - RetryCount: *cfg.Retries, - BatchSize: *cfg.BatchSize, + Start: *cfg.Start, + Count: *cfg.Count, + DryRun: *cfg.ExportDryRun, + RetryCount: *cfg.Retries, + BatchSize: *cfg.BatchSize, + NetworkPassphrase: networkPassphrase, } - command = &cmd.ExportCommand{ES: esClient, DB: dbClient, Config: config} + command = &cmd.ExportCommand{ES: esClient, Config: config} case "ingest": dbClient := db.Connect(*cfg.DatabaseURL) command = &cmd.IngestCommand{ES: esClient, DB: dbClient} diff --git a/stellar-core/pubnet.cfg b/stellar-core/pubnet.cfg new file mode 100644 index 0000000..f3f7675 --- /dev/null +++ b/stellar-core/pubnet.cfg @@ -0,0 +1,63 @@ +LOG_FILE_PATH="" +BUCKET_DIR_PATH="./stellar-core/buckets" +DATABASE="postgresql://dbname=stellar user=charlie host=localhost connect_timeout=5" +HTTP_PORT=11626 +PUBLIC_HTTP_PORT=true +HTTP_MAX_CLIENT=128 + +NODE_NAMES=[ +"GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH sdf_1", +"GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK sdf_2", +"GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ sdf_3", +"GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE satoshipay_de", +"GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT satoshipay_sg", +"GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY satoshipay_us" +] +NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" +PEER_PORT=11625 +KNOWN_PEERS=[ +"core-live-a.stellar.org", +"core-live-b.stellar.org", +"core-live-c.stellar.org", +"stellar-de-fra.satoshipay.io", +"stellar-sg-sin.satoshipay.io", +"stellar-us-iowa.satoshipay.io", +"stellar.256kw.com", +"stellar.smartlands.io", +"ohio-1.stellar.stellarport.io", +"ohio-2.stellar.stellarport.io", +"stellar1.tempo.eu.com" +] +NODE_SEED="SB6W7PNMIGC7AF47D3UV5XLNKE6SVIZ24UX2RTHEGXJD2I5FKHMIYN5H" +MAX_CONCURRENT_SUBPROCESSES=2 +AUTOMATIC_MAINTENANCE_PERIOD=0 +METADATA_OUTPUT_STREAM="astrologer_meta_stream" + +[HISTORY.satoshipay_de] +get="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}" + +[HISTORY.satoshipay_sg] +get="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}" + +[HISTORY.satoshipay_us] +get="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}" + +[HISTORY.sdf_1] +get="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" + +[HISTORY.sdf_2] +get="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" + +[HISTORY.sdf_3] +get="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" + +[QUORUM_SET] + +VALIDATORS=[ +"$sdf_1", +"$sdf_2", +"$sdf_3", +"$satoshipay_de", +"$satoshipay_sg", +"$satoshipay_us" +] diff --git a/support/main.go b/support/main.go new file mode 100644 index 0000000..bb7f9b2 --- /dev/null +++ b/support/main.go @@ -0,0 +1,83 @@ +package support + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "unicode/utf8" + + "github.com/guregu/null" + "github.com/stellar/go/xdr" +) + +func MemoValue(memo xdr.Memo) null.String { + var ( + value string + valid bool + ) + switch memo.Type { + case xdr.MemoTypeMemoNone: + value, valid = "", false + case xdr.MemoTypeMemoText: + scrubbed := Utf8Scrub(memo.MustText()) + notnull := strings.Join(strings.Split(scrubbed, "\x00"), "") + value, valid = notnull, true + case xdr.MemoTypeMemoId: + value, valid = fmt.Sprintf("%d", memo.MustId()), true + case xdr.MemoTypeMemoHash: + hash := memo.MustHash() + value, valid = + base64.StdEncoding.EncodeToString(hash[:]), + true + case xdr.MemoTypeMemoReturn: + hash := memo.MustRetHash() + value, valid = + base64.StdEncoding.EncodeToString(hash[:]), + true + default: + panic(fmt.Errorf("invalid memo type: %v", memo.Type)) + } + + return null.NewString(value, valid) +} + +func OperationMeta(txMeta xdr.TransactionMeta, opIndex int) *xdr.OperationMeta { + if v1, ok := txMeta.GetV1(); ok { + ops := v1.Operations + return &ops[opIndex] + } + + ops, ok := txMeta.GetOperations() + if !ok { + return nil + } + + return &ops[opIndex] +} + +// Copy paste from Horizon +func Utf8Scrub(in string) string { + + // First check validity using the stdlib, returning if the string is already + // valid + if utf8.ValidString(in) { + return in + } + + left := []byte(in) + var result bytes.Buffer + + for len(left) > 0 { + r, n := utf8.DecodeRune(left) + + _, err := result.WriteRune(r) + if err != nil { + panic(err) + } + + left = left[n:] + } + + return result.String() +}