diff --git a/README.md b/README.md index d7d6824..88129c6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ Universal crypto candlestick iterator library & CLI - [x] Binance - [x] Binance USDM Futures -- [x] FTX - [x] Coinbase - [x] Kucoin - [x] Bitstamp diff --git a/candles/candles.go b/candles/candles.go index e1e1ea2..c308cee 100644 --- a/candles/candles.go +++ b/candles/candles.go @@ -12,34 +12,38 @@ // package main // // import ( -// "fmt" -// "log" -// "time" -// "encoding/json" // -// "github.com/marianogappa/crypto-candles/candles" -// "github.com/marianogappa/crypto-candles/candles/common" +// "fmt" +// "log" +// "time" +// "encoding/json" +// +// "github.com/marianogappa/crypto-candles/candles" +// "github.com/marianogappa/crypto-candles/candles/common" +// // ) -// func main() { -// m := candles.NewMarket() -// iter, err := m.Iterator( -// common.MarketSource{Type: common.COIN, Provider: common.BINANCE, BaseAsset: "BTC", QuoteAsset: "USDT"}, -// time.Now().Add(-12*time.Hour), // Start time -// 1*time.Hour, // Candlestick interval -// ) -// if err != nil { -// log.Fatal(err) -// } // -// for i := 0; i < 10; i++ { -// candlestick, err := iter.Next() -// if err != nil { -// log.Fatal(err) -// } -// bs, _ := json.Marshal(candlestick) -// fmt.Printf("%+v\n", string(bs)) -// } -// } +// func main() { +// m := candles.NewMarket() +// iter, err := m.Iterator( +// common.MarketSource{Type: common.COIN, Provider: common.BINANCE, BaseAsset: "BTC", QuoteAsset: "USDT"}, +// time.Now().Add(-12*time.Hour), // Start time +// 1*time.Hour, // Candlestick interval +// ) +// if err != nil { +// log.Fatal(err) +// } +// +// for i := 0; i < 10; i++ { +// candlestick, err := iter.Next() +// if err != nil { +// log.Fatal(err) +// } +// bs, _ := json.Marshal(candlestick) +// fmt.Printf("%+v\n", string(bs)) +// } +// } +// // ``` package candles @@ -55,7 +59,6 @@ import ( "github.com/marianogappa/crypto-candles/candles/cache" "github.com/marianogappa/crypto-candles/candles/coinbase" "github.com/marianogappa/crypto-candles/candles/common" - "github.com/marianogappa/crypto-candles/candles/ftx" "github.com/marianogappa/crypto-candles/candles/iterator" "github.com/marianogappa/crypto-candles/candles/kucoin" ) @@ -123,7 +126,6 @@ func (m Market) CalculateCacheHitRatio() float64 { func buildExchanges() map[string]common.Exchange { return map[string]common.Exchange{ common.BINANCE: binance.NewBinance(), - common.FTX: ftx.NewFTX(), common.COINBASE: coinbase.NewCoinbase(), common.KUCOIN: kucoin.NewKucoin(), common.BINANCEUSDMFUTURES: binanceusdmfutures.NewBinanceUSDMFutures(), diff --git a/candles/common/types.go b/candles/common/types.go index 16872d3..6f774ac 100644 --- a/candles/common/types.go +++ b/candles/common/types.go @@ -11,8 +11,6 @@ import ( const ( // BINANCE is an enumesque string value representing the BINANCE exchange BINANCE = "BINANCE" - // FTX is an enumesque string value representing the FTX exchange - FTX = "FTX" // COINBASE is an enumesque string value representing the COINBASE exchange COINBASE = "COINBASE" // KUCOIN is an enumesque string value representing the KUCOIN exchange diff --git a/candles/ftx/api_klines.go b/candles/ftx/api_klines.go deleted file mode 100644 index 7238105..0000000 --- a/candles/ftx/api_klines.go +++ /dev/null @@ -1,152 +0,0 @@ -package ftx - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - "time" - - "github.com/marianogappa/crypto-candles/candles/common" - "github.com/rs/zerolog/log" -) - -//[ -// { -// "startTime":"2021-07-05T18:20:00+00:00", -// "time":1625509200000.0, -// "open":33831.0, -// "high":33837.0, -// "low":33810.0, -// "close":33837.0, -// "volume":11679.9302 -// } -//] -type responseCandlestick struct { - StartTime string `json:"startTime"` - Time float64 `json:"time"` - Open float64 `json:"open"` - High float64 `json:"high"` - Low float64 `json:"low"` - Close float64 `json:"close"` - Volume float64 `json:"volume"` -} - -type response struct { - Success bool `json:"success"` - Error string `json:"error"` - Result []responseCandlestick `json:"result"` -} - -func (r response) toCandlesticks() []common.Candlestick { - candlesticks := make([]common.Candlestick, len(r.Result)) - for i := 0; i < len(r.Result); i++ { - raw := r.Result[i] - candlestick := common.Candlestick{ - Timestamp: int(raw.Time) / 1000, - OpenPrice: common.JSONFloat64(raw.Open), - ClosePrice: common.JSONFloat64(raw.Close), - LowestPrice: common.JSONFloat64(raw.Low), - HighestPrice: common.JSONFloat64(raw.High), - } - candlesticks[i] = candlestick - } - - return candlesticks -} - -func (e *FTX) requestCandlesticks(baseAsset string, quoteAsset string, startTime time.Time, candlestickInterval time.Duration) ([]common.Candlestick, error) { - req, _ := http.NewRequest("GET", fmt.Sprintf("%vmarkets/%v/%v/candles", e.apiURL, strings.ToUpper(baseAsset), strings.ToUpper(quoteAsset)), nil) - q := req.URL.Query() - - resolution := int(candlestickInterval / time.Second) - - validResolutions := map[int]bool{ - 15: true, - 60: true, - 300: true, - 900: true, - 3600: true, - 14400: true, - 86400: true, - // All multiples of 86400 up to 30*86400 are actually valid - // https://docs.ftx.com/#get-historical-prices - 86400 * 7: true, - } - if isValid := validResolutions[resolution]; !isValid { - return nil, common.CandleReqError{IsNotRetryable: true, Err: common.ErrUnsupportedCandlestickInterval} - } - - q.Add("resolution", fmt.Sprintf("%v", resolution)) - q.Add("start_time", fmt.Sprintf("%v", startTime.Unix())) - - // N.B.: if you don't supply end_time, or if you supply a very large range, FTX silently ignores this and - // instead gives you recent data. - q.Add("end_time", fmt.Sprintf("%v", int(startTime.Unix())+1000*resolution)) - - req.URL.RawQuery = q.Encode() - - client := &http.Client{Timeout: 10 * time.Second} - - resp, err := client.Do(req) - if err != nil { - return nil, common.CandleReqError{IsNotRetryable: true, Err: fmt.Errorf("%w: %v", common.ErrExecutingRequest, err)} - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusNotFound { - return nil, common.CandleReqError{IsNotRetryable: true, Err: common.ErrInvalidMarketPair} - } - - byts, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, common.CandleReqError{IsNotRetryable: false, Err: common.ErrBrokenBodyResponse} - } - - maybeResponse := response{} - err = json.Unmarshal(byts, &maybeResponse) - if err != nil { - return nil, common.CandleReqError{IsNotRetryable: false, Err: common.ErrInvalidJSONResponse} - } - - if !maybeResponse.Success { - return nil, common.CandleReqError{ - IsNotRetryable: false, - Err: fmt.Errorf("FTX returned error: %v", maybeResponse.Error), - Code: resp.StatusCode, - } - } - - if e.debug { - log.Info().Str("exchange", "FTX").Str("market", fmt.Sprintf("%v/%v", baseAsset, quoteAsset)).Int("candlestick_count", len(maybeResponse.Result)).Msg("Candlestick request successful!") - } - - candlesticks := maybeResponse.toCandlesticks() - if len(candlesticks) == 0 { - return nil, common.CandleReqError{IsNotRetryable: false, Err: common.ErrOutOfCandlesticks} - } - - return candlesticks, nil -} - -// FTX uses the strategy of having candlesticks on multiples of an hour or a day, and truncating the requested -// millisecond timestamps to the closest mutiple in the future. To test this, use the following snippet: -// -// curl -s "https://ftx.com/api/markets/BTC/USDT/candles?resolution=60&start_time="$(date -j -f "%Y-%m-%d %H:%M:%S" "2020-04-07 00:00:00" "+%s")"&end_time="$(date -j -f "%Y-%m-%d %H:%M:%S" "2020-04-07 00:03:00" "+%s")"" | jq '.result | .[] | .startTime' -// -// Two important caveats for FTX: -// -// 1) if end_time is not specified, start_time is ignored silently and recent data is returned. -// 2) if the range between start_time & end_time is too broad, start_time will be pushed upwards until the range spans 1500 candlesticks. -// -// On the 15 resolution, candlesticks exist at: 00, 15, 30, 45 -// On the 60 resolution, candlesticks exist at every minute -// On the 300 resolution, candlesticks exist at: 00, 05, 10 ... -// On the 900 resolution, candlesticks exist at: 00, 15, 30 & 45 -// On the 3600 resolution, candlesticks exist at every hour at :00 -// On the 14400 resolution, candlesticks exist at: 00:00, 04:00, 08:00 ... -// -// From the 86400 resolution and onwards, FTX is a unique case: -// - it first truncates the date to the beginning of the supplied start_time's day -// - then it returns candlesticks at multiples of the truncated date, starting at that date rather than a prescribed one diff --git a/candles/ftx/api_klines_test.go b/candles/ftx/api_klines_test.go deleted file mode 100644 index 460b572..0000000 --- a/candles/ftx/api_klines_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package ftx - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/marianogappa/crypto-candles/candles/common" - "github.com/stretchr/testify/require" -) - -func TestHappyToCandlesticks(t *testing.T) { - testCandlestick := ` - { - "success": true, - "result": [ - { - "startTime": "2020-04-06T23:00:00+00:00", - "time": 1586214000000, - "open": 7274, - "high": 7281.5, - "low": 7272, - "close": 7281.5, - "volume": 0 - }, - { - "startTime": "2020-04-06T23:01:00+00:00", - "time": 1586214060000, - "open": 7281.5, - "high": 7281.5, - "low": 7277, - "close": 7280, - "volume": 0 - }, - { - "startTime": "2020-04-06T23:02:00+00:00", - "time": 1586214120000, - "open": 7280, - "high": 7280, - "low": 7271.5, - "close": 7274, - "volume": 0 - } - ] - } - ` - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, testCandlestick) - })) - defer ts.Close() - - b := NewFTX() - b.SetDebug(true) - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = ts.URL + "/" - - actual, err := b.RequestCandlesticks(msBTCUSDT, tp("2020-04-06T23:00:00+00:00"), time.Minute) - require.Nil(t, err) - require.Len(t, actual, 3) - expected := []common.Candlestick{ - { - Timestamp: 1586214000, - OpenPrice: f(7274), - HighestPrice: f(7281.5), - LowestPrice: f(7272), - ClosePrice: f(7281.5), - }, - { - Timestamp: 1586214060, - OpenPrice: f(7281.5), - HighestPrice: f(7281.5), - LowestPrice: f(7277), - ClosePrice: f(7280), - }, - { - Timestamp: 1586214120, - OpenPrice: f(7280), - HighestPrice: f(7280), - LowestPrice: f(7271.5), - ClosePrice: f(7274), - }, - } - require.Equal(t, expected, actual) -} - -func TestOutOfCandlesticks(t *testing.T) { - testCandlestick := ` - { - "success": true, - "result": [] - } - ` - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, testCandlestick) - })) - defer ts.Close() - - b := NewFTX() - b.SetDebug(true) - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = ts.URL + "/" - - _, err := b.RequestCandlesticks(msBTCUSDT, tp("2020-04-06T23:00:00+00:00"), time.Minute) - require.ErrorIs(t, err.(common.CandleReqError).Err, common.ErrOutOfCandlesticks) -} - -func TestKlinesInvalidUrl(t *testing.T) { - i := 0 - replies := []string{ - `{"success": true, "result": [{"startTime": "2020-04-06T23:00:00+00:00", "time": 1586214000000, "open": 7274, "high": 7281.5, "low": 7272, "close": 7281.5, "volume": 0 } ] }`, - } - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, replies[i%len(replies)]) - i++ - })) - defer ts.Close() - - b := NewFTX() - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = "invalid url" - - _, err := b.RequestCandlesticks(msBTCUSDT, tp("2021-07-04T14:14:18+00:00"), time.Minute) - if err == nil { - t.Fatalf("should have failed due to invalid url") - } -} - -func TestKlinesErrReadingResponseBody(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Length", "1") - })) - defer ts.Close() - - b := NewFTX() - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = ts.URL + "/" - - _, err := b.RequestCandlesticks(msBTCUSDT, tp("2021-07-04T14:14:18+00:00"), time.Minute) - if err == nil { - t.Fatalf("should have failed due to invalid response body") - } -} - -func TestKlinesErrorResponse(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, `{"success": false, "error": "Invalid parameter end_time"}`) - })) - defer ts.Close() - - b := NewFTX() - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = ts.URL + "/" - - _, err := b.RequestCandlesticks(msBTCUSDT, tp("2021-07-04T14:14:18+00:00"), time.Minute) - require.NotNil(t, err) -} - -func TestKlines404Response(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(404) - })) - defer ts.Close() - - b := NewFTX() - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = ts.URL + "/" - - _, err := b.RequestCandlesticks(msBTCUSDT, tp("2021-07-04T14:14:18+00:00"), time.Minute) - require.NotNil(t, err) -} - -func TestKlinesInvalidJSONResponse(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, `invalid json`) - })) - defer ts.Close() - - b := NewFTX() - b.SetDebug(false) - b.requester.Strategy = common.RetryStrategy{Attempts: 1} - b.apiURL = ts.URL + "/" - - _, err := b.RequestCandlesticks(msBTCUSDT, tp("2021-07-04T14:14:18+00:00"), time.Minute) - require.NotNil(t, err) -} - -func TestName(t *testing.T) { - require.Equal(t, "FTX", NewFTX().Name()) -} -func TestPatience(t *testing.T) { - require.Equal(t, 2*time.Minute, NewFTX().Patience()) -} -func TestInvalidCandlestickInterval(t *testing.T) { - _, err := NewFTX().RequestCandlesticks(msBTCUSDT, tp("2021-07-04T14:14:18+00:00"), 60*time.Hour) - require.ErrorIs(t, err.(common.CandleReqError).Err, common.ErrUnsupportedCandlestickInterval) -} - -func f(fl float64) common.JSONFloat64 { - return common.JSONFloat64(fl) -} - -func tp(s string) time.Time { - t, _ := time.Parse(time.RFC3339, s) - return t -} - -var ( - msBTCUSDT = common.MarketSource{ - Type: common.COIN, - Provider: "BINANCE", - BaseAsset: "BTC", - QuoteAsset: "USDT", - } -) diff --git a/candles/ftx/ftx.go b/candles/ftx/ftx.go deleted file mode 100644 index e40ca87..0000000 --- a/candles/ftx/ftx.go +++ /dev/null @@ -1,69 +0,0 @@ -package ftx - -import ( - "sync" - "time" - - "github.com/marianogappa/crypto-candles/candles/common" -) - -// FTX struct enables requesting candlesticks from FTX -type FTX struct { - apiURL string - debug bool - lock sync.Mutex - requester common.RequesterWithRetry -} - -// NewFTX is the constructor for FTX -func NewFTX() *FTX { - e := &FTX{ - apiURL: "https://ftx.com/api/", - } - - e.requester = common.NewRequesterWithRetry( - e.requestCandlesticks, - common.RetryStrategy{Attempts: 3, FirstSleepTime: 1 * time.Second, SleepTimeMultiplier: 2.0}, - &e.debug, - ) - - return e -} - -// RequestCandlesticks requests candlesticks for the given market source, of a given candlestick interval, -// starting at a given time.Time. -// -// The supplied candlestick interval may not be supported by this exchange. -// -// Candlesticks will start at the next multiple of startTime as defined by -// time.Truncate(candlestickInterval), except in some documented exceptions. -// -// Some exchanges return candlesticks with gaps, but this method will patch the gaps by cloning the candlestick -// received right before the gap as many times as gaps, or the first candlestick if the gaps is at the start. -// -// Most of the usage of this method is with 1 minute intervals, the interval used to follow predictions. -func (e *FTX) RequestCandlesticks(marketSource common.MarketSource, startTime time.Time, candlestickInterval time.Duration) ([]common.Candlestick, error) { - e.lock.Lock() - defer e.lock.Unlock() - - candlesticks, err := e.requestCandlesticks(marketSource.BaseAsset, marketSource.QuoteAsset, startTime, candlestickInterval) - if err != nil { - return nil, err - } - - return common.PatchCandlestickHoles(candlesticks, int(startTime.Unix()), int(candlestickInterval/time.Second)), nil -} - -// Patience returns the delay that this exchange usually takes in order for it to return candlesticks. -// -// Some exchanges may return results for unfinished candles (e.g. the current minute) and some may not, so callers -// should not request unfinished candles. This patience should be taken into account in addition to unfinished candles. -func (e *FTX) Patience() time.Duration { return 2 * time.Minute } - -// Name is the name of this candlestick provider. -func (e *FTX) Name() string { return common.FTX } - -// SetDebug sets exchange-wide debug logging. It's useful to know how many times requests are being sent to exchanges. -func (e *FTX) SetDebug(debug bool) { - e.debug = debug -} diff --git a/candles/ftx/ftx_integration_test.go b/candles/ftx/ftx_integration_test.go deleted file mode 100644 index eb63d24..0000000 --- a/candles/ftx/ftx_integration_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package ftx_test - -import ( - "testing" - "time" - - "github.com/marianogappa/crypto-candles/candles" - "github.com/marianogappa/crypto-candles/candles/common" - "github.com/stretchr/testify/require" -) - -func TestIntegration(t *testing.T) { - testCases := []struct { - name string - marketSource common.MarketSource - startTime time.Time - startFromNext bool - candlestickInterval time.Duration - expectedCandlesticks []common.Candlestick - expectedErrs []error - }{ - - { - name: "FTX", - marketSource: common.MarketSource{Type: common.COIN, Provider: common.FTX, BaseAsset: "BTC", QuoteAsset: "USDT"}, - startTime: tp("2022-07-09T15:00:00Z"), - startFromNext: false, - candlestickInterval: time.Hour, - expectedCandlesticks: []common.Candlestick{ - { - Timestamp: 1657378800, - OpenPrice: 21597, - ClosePrice: 21550, - LowestPrice: 21541, - HighestPrice: 21649, - }, - { - Timestamp: 1657382400, - OpenPrice: 21550, - ClosePrice: 21691, - LowestPrice: 21536, - HighestPrice: 21714, - }, - { - Timestamp: 1657386000, - OpenPrice: 21691, - ClosePrice: 21883, - LowestPrice: 21669, - HighestPrice: 21973, - }, - }, - expectedErrs: []error{nil, nil, nil}, - }, - } - mkt := candles.NewMarket(candles.WithCacheSizes(map[time.Duration]int{})) - for _, ts := range testCases { - t.Run(ts.name, func(t *testing.T) { - it, err := mkt.Iterator(ts.marketSource, ts.startTime, ts.candlestickInterval) - it.SetStartFromNext(ts.startFromNext) - require.Nil(t, err) - for i, expectedCandlestick := range ts.expectedCandlesticks { - candlestick, err := it.Next() - require.ErrorIs(t, err, ts.expectedErrs[i]) - require.Equal(t, expectedCandlestick, candlestick) - } - }) - } -} - -func tp(s string) time.Time { - tm, _ := time.Parse(time.RFC3339, s) - return tm.UTC() -} diff --git a/main.go b/main.go index 43bc31d..4863d73 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,7 @@ import ( func main() { var ( flagMarketType = flag.String("marketType", "COIN", "for now only 'COIN' is supported, representing market pairs e.g. BTC/USDT") - flagProvider = flag.String("provider", "BINANCE", "one of BINANCE|FTX|COINBASE|KUCOIN|BINANCEUSDMFUTURES|BITSTAMP|BITFINEX") + flagProvider = flag.String("provider", "BINANCE", "one of BINANCE|COINBASE|KUCOIN|BINANCEUSDMFUTURES|BITSTAMP|BITFINEX") flagBaseAsset = flag.String("baseAsset", "", "e.g. BTC in BTC/USDT") flagQuoteAsset = flag.String("quoteAsset", "", "e.g. USDT in BTC/USDT") flagStartTime = flag.String("startTime", "", "ISO8601/RFC3339 date to start retrieving candlesticks e.g. 2022-07-10T14:01:00Z")