From 38bd5479f249792151a13c07daecea731e5c953c Mon Sep 17 00:00:00 2001 From: edwin Date: Thu, 14 Mar 2024 17:17:09 +0800 Subject: [PATCH 1/2] pkg/exchange: gen 3 day and regen history transaction api --- pkg/exchange/okex/okexapi/client_test.go | 30 ++ ..._three_days_transaction_history_request.go | 39 +++ ..._transaction_history_request_requestgen.go | 322 ++++++++++++++++++ .../get_transaction_history_request.go | 4 +- ..._transaction_history_request_requestgen.go | 43 ++- ...hree_days_transaction_history_request.json | 32 ++ .../get_transaction_history_request.json | 32 ++ 7 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 pkg/exchange/okex/okexapi/get_three_days_transaction_history_request.go create mode 100644 pkg/exchange/okex/okexapi/get_three_days_transaction_history_request_requestgen.go create mode 100644 pkg/exchange/okex/okexapi/testdata/get_three_days_transaction_history_request.json create mode 100644 pkg/exchange/okex/okexapi/testdata/get_transaction_history_request.json diff --git a/pkg/exchange/okex/okexapi/client_test.go b/pkg/exchange/okex/okexapi/client_test.go index 3d12fcc8f5..f9282761ac 100644 --- a/pkg/exchange/okex/okexapi/client_test.go +++ b/pkg/exchange/okex/okexapi/client_test.go @@ -264,6 +264,36 @@ func TestClient_TransactionHistoryWithTime(t *testing.T) { } } +func TestClient_ThreeDaysTransactionHistoryWithTime(t *testing.T) { + client := getTestClientOrSkip(t) + ctx := context.Background() + + beforeId := int64(0) + startTime := time.Now().Add(-3 * 24 * time.Hour) + end := time.Now() + + for { + // [{"side":"sell","fillSz":"1","fillPx":"46446.4","fillPxVol":"","fillFwdPx":"","fee":"-46.4464","fillPnl":"0","ordId":"665951654130348158","feeRate":"-0.001","instType":"SPOT","fillPxUsd":"","instId":"BTC-USDT","clOrdId":"","posSide":"net","billId":"665951654138736652","fillMarkVol":"","tag":"","fillTime":"1705047247128","execType":"T","fillIdxPx":"","tradeId":"724072849","fillMarkPx":"","feeCcy":"USDT","ts":"1705047247130"}] + // [{"side":"sell","fillSz":"11.053006","fillPx":"54.17","fillPxVol":"","fillFwdPx":"","fee":"-0.59874133502","fillPnl":"0","ordId":"665951812901531754","feeRate":"-0.001","instType":"SPOT","fillPxUsd":"","instId":"OKB-USDT","clOrdId":"","posSide":"net","billId":"665951812905726068","fillMarkVol":"","tag":"","fillTime":"1705047284982","execType":"T","fillIdxPx":"","tradeId":"589438381","fillMarkPx":"","feeCcy":"USDT","ts":"1705047284983"}] + // [{"side":"sell","fillSz":"88.946994","filollPx":"54.16","fillPxVol":"","fillFwdPx":"","fee":"-4.81736919504","fillPnl":"0","ordId":"665951812901531754","feeRate":"-0.001","instType":"SPOT","fillPxUsd":"","instId":"OKB-USDT","clOrdId":"","posSide":"net","billId":"665951812905726084","fillMarkVol":"","tag":"","fillTime":"1705047284982","execType":"T","fillIdxPx":"","tradeId":"589438382","fillMarkPx":"","feeCcy":"USDT","ts":"1705047284983"}] + c := client.NewGetThreeDaysTransactionHistoryRequest(). + StartTime(types.NewMillisecondTimestampFromInt(startTime.UnixMilli()).Time()). + EndTime(types.NewMillisecondTimestampFromInt(end.UnixMilli()).Time()). + Limit(1) + if beforeId != 0 { + c.Before(strconv.FormatInt(beforeId, 10)) + } + res, err := c.Do(ctx) + assert.NoError(t, err) + + if len(res) != 1 { + break + } + t.Log(res[0].FillTime, res[0].Timestamp, res[0].BillId, res) + beforeId = int64(res[0].BillId) + } +} + func TestClient_BatchCancelOrderRequest(t *testing.T) { client := getTestClientOrSkip(t) ctx := context.Background() diff --git a/pkg/exchange/okex/okexapi/get_three_days_transaction_history_request.go b/pkg/exchange/okex/okexapi/get_three_days_transaction_history_request.go new file mode 100644 index 0000000000..3a5168ce25 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_three_days_transaction_history_request.go @@ -0,0 +1,39 @@ +package okexapi + +import ( + "time" + + "github.com/c9s/requestgen" +) + +//go:generate -command GetRequest requestgen -method GET -responseType .APIResponse -responseDataField Data +//go:generate -command PostRequest requestgen -method POST -responseType .APIResponse -responseDataField Data + +//go:generate GetRequest -url "/api/v5/trade/fills" -type GetThreeDaysTransactionHistoryRequest -responseDataType []Trade -rateLimiter 1+60/2s +type GetThreeDaysTransactionHistoryRequest struct { + client requestgen.AuthenticatedAPIClient + + instrumentType InstrumentType `param:"instType,query"` + instrumentID *string `param:"instId,query"` + orderID *string `param:"ordId,query"` + + underlying *string `param:"uly,query"` + instrumentFamily *string `param:"instFamily,query"` + + after *string `param:"after,query"` + before *string `param:"before,query"` + startTime *time.Time `param:"begin,query,milliseconds"` + + // endTime for each request, startTime and endTime can be any interval, but should be in last 3 months + endTime *time.Time `param:"end,query,milliseconds"` + + // limit for data size per page. Default: 100 + limit *uint64 `param:"limit,query"` +} + +func (c *RestClient) NewGetThreeDaysTransactionHistoryRequest() *GetThreeDaysTransactionHistoryRequest { + return &GetThreeDaysTransactionHistoryRequest{ + client: c, + instrumentType: InstrumentTypeSpot, + } +} diff --git a/pkg/exchange/okex/okexapi/get_three_days_transaction_history_request_requestgen.go b/pkg/exchange/okex/okexapi/get_three_days_transaction_history_request_requestgen.go new file mode 100644 index 0000000000..594b7ba030 --- /dev/null +++ b/pkg/exchange/okex/okexapi/get_three_days_transaction_history_request_requestgen.go @@ -0,0 +1,322 @@ +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/fills -type GetThreeDaysTransactionHistoryRequest -responseDataType []Trade -rateLimiter 1+60/2s"; DO NOT EDIT. + +package okexapi + +import ( + "context" + "encoding/json" + "fmt" + "golang.org/x/time/rate" + "net/url" + "reflect" + "regexp" + "strconv" + "time" +) + +var GetThreeDaysTransactionHistoryRequestLimiter = rate.NewLimiter(30.000000300000004, 1) + +func (g *GetThreeDaysTransactionHistoryRequest) InstrumentType(instrumentType InstrumentType) *GetThreeDaysTransactionHistoryRequest { + g.instrumentType = instrumentType + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) InstrumentID(instrumentID string) *GetThreeDaysTransactionHistoryRequest { + g.instrumentID = &instrumentID + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) OrderID(orderID string) *GetThreeDaysTransactionHistoryRequest { + g.orderID = &orderID + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) Underlying(underlying string) *GetThreeDaysTransactionHistoryRequest { + g.underlying = &underlying + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) InstrumentFamily(instrumentFamily string) *GetThreeDaysTransactionHistoryRequest { + g.instrumentFamily = &instrumentFamily + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) After(after string) *GetThreeDaysTransactionHistoryRequest { + g.after = &after + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) Before(before string) *GetThreeDaysTransactionHistoryRequest { + g.before = &before + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) StartTime(startTime time.Time) *GetThreeDaysTransactionHistoryRequest { + g.startTime = &startTime + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) EndTime(endTime time.Time) *GetThreeDaysTransactionHistoryRequest { + g.endTime = &endTime + return g +} + +func (g *GetThreeDaysTransactionHistoryRequest) Limit(limit uint64) *GetThreeDaysTransactionHistoryRequest { + g.limit = &limit + return g +} + +// GetQueryParameters builds and checks the query parameters and returns url.Values +func (g *GetThreeDaysTransactionHistoryRequest) GetQueryParameters() (url.Values, error) { + var params = map[string]interface{}{} + // check instrumentType field -> json key instType + instrumentType := g.instrumentType + + // TEMPLATE check-valid-values + switch instrumentType { + case InstrumentTypeSpot, InstrumentTypeSwap, InstrumentTypeFutures, InstrumentTypeOption, InstrumentTypeMARGIN: + params["instType"] = instrumentType + + default: + return nil, fmt.Errorf("instType value %v is invalid", instrumentType) + + } + // END TEMPLATE check-valid-values + + // assign parameter of instrumentType + params["instType"] = instrumentType + // check instrumentID field -> json key instId + if g.instrumentID != nil { + instrumentID := *g.instrumentID + + // assign parameter of instrumentID + params["instId"] = instrumentID + } else { + } + // check orderID field -> json key ordId + if g.orderID != nil { + orderID := *g.orderID + + // assign parameter of orderID + params["ordId"] = orderID + } else { + } + // check underlying field -> json key uly + if g.underlying != nil { + underlying := *g.underlying + + // assign parameter of underlying + params["uly"] = underlying + } else { + } + // check instrumentFamily field -> json key instFamily + if g.instrumentFamily != nil { + instrumentFamily := *g.instrumentFamily + + // assign parameter of instrumentFamily + params["instFamily"] = instrumentFamily + } else { + } + // check after field -> json key after + if g.after != nil { + after := *g.after + + // assign parameter of after + params["after"] = after + } else { + } + // check before field -> json key before + if g.before != nil { + before := *g.before + + // assign parameter of before + params["before"] = before + } else { + } + // check startTime field -> json key begin + if g.startTime != nil { + startTime := *g.startTime + + // assign parameter of startTime + // convert time.Time to milliseconds time stamp + params["begin"] = strconv.FormatInt(startTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check endTime field -> json key end + if g.endTime != nil { + endTime := *g.endTime + + // assign parameter of endTime + // convert time.Time to milliseconds time stamp + params["end"] = strconv.FormatInt(endTime.UnixNano()/int64(time.Millisecond), 10) + } else { + } + // check limit field -> json key limit + if g.limit != nil { + limit := *g.limit + + // assign parameter of limit + params["limit"] = limit + } else { + } + + query := url.Values{} + for _k, _v := range params { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + + return query, nil +} + +// GetParameters builds and checks the parameters and return the result in a map object +func (g *GetThreeDaysTransactionHistoryRequest) GetParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +// GetParametersQuery converts the parameters from GetParameters into the url.Values format +func (g *GetThreeDaysTransactionHistoryRequest) GetParametersQuery() (url.Values, error) { + query := url.Values{} + + params, err := g.GetParameters() + if err != nil { + return query, err + } + + for _k, _v := range params { + if g.isVarSlice(_v) { + g.iterateSlice(_v, func(it interface{}) { + query.Add(_k+"[]", fmt.Sprintf("%v", it)) + }) + } else { + query.Add(_k, fmt.Sprintf("%v", _v)) + } + } + + return query, nil +} + +// GetParametersJSON converts the parameters from GetParameters into the JSON format +func (g *GetThreeDaysTransactionHistoryRequest) GetParametersJSON() ([]byte, error) { + params, err := g.GetParameters() + if err != nil { + return nil, err + } + + return json.Marshal(params) +} + +// GetSlugParameters builds and checks the slug parameters and return the result in a map object +func (g *GetThreeDaysTransactionHistoryRequest) GetSlugParameters() (map[string]interface{}, error) { + var params = map[string]interface{}{} + + return params, nil +} + +func (g *GetThreeDaysTransactionHistoryRequest) applySlugsToUrl(url string, slugs map[string]string) string { + for _k, _v := range slugs { + needleRE := regexp.MustCompile(":" + _k + "\\b") + url = needleRE.ReplaceAllString(url, _v) + } + + return url +} + +func (g *GetThreeDaysTransactionHistoryRequest) iterateSlice(slice interface{}, _f func(it interface{})) { + sliceValue := reflect.ValueOf(slice) + for _i := 0; _i < sliceValue.Len(); _i++ { + it := sliceValue.Index(_i).Interface() + _f(it) + } +} + +func (g *GetThreeDaysTransactionHistoryRequest) isVarSlice(_v interface{}) bool { + rt := reflect.TypeOf(_v) + switch rt.Kind() { + case reflect.Slice: + return true + } + return false +} + +func (g *GetThreeDaysTransactionHistoryRequest) GetSlugsMap() (map[string]string, error) { + slugs := map[string]string{} + params, err := g.GetSlugParameters() + if err != nil { + return slugs, nil + } + + for _k, _v := range params { + slugs[_k] = fmt.Sprintf("%v", _v) + } + + return slugs, nil +} + +// GetPath returns the request path of the API +func (g *GetThreeDaysTransactionHistoryRequest) GetPath() string { + return "/api/v5/trade/fills" +} + +// Do generates the request object and send the request object to the API endpoint +func (g *GetThreeDaysTransactionHistoryRequest) Do(ctx context.Context) ([]Trade, error) { + if err := GetThreeDaysTransactionHistoryRequestLimiter.Wait(ctx); err != nil { + return nil, err + } + + // no body params + var params interface{} + query, err := g.GetQueryParameters() + if err != nil { + return nil, err + } + + var apiURL string + + apiURL = g.GetPath() + + req, err := g.client.NewAuthenticatedRequest(ctx, "GET", apiURL, query, params) + if err != nil { + return nil, err + } + + response, err := g.client.SendRequest(req) + if err != nil { + return nil, err + } + + var apiResponse APIResponse + + type responseUnmarshaler interface { + Unmarshal(data []byte) error + } + + if unmarshaler, ok := interface{}(&apiResponse).(responseUnmarshaler); ok { + if err := unmarshaler.Unmarshal(response.Body); err != nil { + return nil, err + } + } else { + // The line below checks the content type, however, some API server might not send the correct content type header, + // Hence, this is commented for backward compatibility + // response.IsJSON() + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } + } + + type responseValidator interface { + Validate() error + } + + if validator, ok := interface{}(&apiResponse).(responseValidator); ok { + if err := validator.Validate(); err != nil { + return nil, err + } + } + var data []Trade + if err := json.Unmarshal(apiResponse.Data, &data); err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/exchange/okex/okexapi/get_transaction_history_request.go b/pkg/exchange/okex/okexapi/get_transaction_history_request.go index 7022439369..57d626bcbf 100644 --- a/pkg/exchange/okex/okexapi/get_transaction_history_request.go +++ b/pkg/exchange/okex/okexapi/get_transaction_history_request.go @@ -53,13 +53,13 @@ type Trade struct { PosSide string `json:"posSide"` } -//go:generate GetRequest -url "/api/v5/trade/fills-history" -type GetTransactionHistoryRequest -responseDataType []Trade +//go:generate GetRequest -url "/api/v5/trade/fills-history" -type GetTransactionHistoryRequest -responseDataType []Trade -rateLimiter 1+10/2s type GetTransactionHistoryRequest struct { client requestgen.AuthenticatedAPIClient instrumentType InstrumentType `param:"instType,query"` instrumentID *string `param:"instId,query"` - orderID string `param:"ordId,query"` + orderID *string `param:"ordId,query"` // Underlying and InstrumentFamily Applicable to FUTURES/SWAP/OPTION underlying *string `param:"uly,query"` diff --git a/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go b/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go index 8451627122..ad45f52960 100644 --- a/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go +++ b/pkg/exchange/okex/okexapi/get_transaction_history_request_requestgen.go @@ -1,4 +1,4 @@ -// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/fills-history -type GetTransactionHistoryRequest -responseDataType []Trade"; DO NOT EDIT. +// Code generated by "requestgen -method GET -responseType .APIResponse -responseDataField Data -url /api/v5/trade/fills-history -type GetTransactionHistoryRequest -responseDataType []Trade -rateLimiter 1+10/2s"; DO NOT EDIT. package okexapi @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "golang.org/x/time/rate" "net/url" "reflect" "regexp" @@ -13,6 +14,8 @@ import ( "time" ) +var GetTransactionHistoryRequestLimiter = rate.NewLimiter(5, 1) + func (g *GetTransactionHistoryRequest) InstrumentType(instrumentType InstrumentType) *GetTransactionHistoryRequest { g.instrumentType = instrumentType return g @@ -24,7 +27,7 @@ func (g *GetTransactionHistoryRequest) InstrumentID(instrumentID string) *GetTra } func (g *GetTransactionHistoryRequest) OrderID(orderID string) *GetTransactionHistoryRequest { - g.orderID = orderID + g.orderID = &orderID return g } @@ -91,10 +94,13 @@ func (g *GetTransactionHistoryRequest) GetQueryParameters() (url.Values, error) } else { } // check orderID field -> json key ordId - orderID := g.orderID + if g.orderID != nil { + orderID := *g.orderID - // assign parameter of orderID - params["ordId"] = orderID + // assign parameter of orderID + params["ordId"] = orderID + } else { + } // check underlying field -> json key uly if g.underlying != nil { underlying := *g.underlying @@ -255,6 +261,9 @@ func (g *GetTransactionHistoryRequest) GetPath() string { // Do generates the request object and send the request object to the API endpoint func (g *GetTransactionHistoryRequest) Do(ctx context.Context) ([]Trade, error) { + if err := GetTransactionHistoryRequestLimiter.Wait(ctx); err != nil { + return nil, err + } // no body params var params interface{} @@ -272,27 +281,39 @@ func (g *GetTransactionHistoryRequest) Do(ctx context.Context) ([]Trade, error) return nil, err } - fmt.Println(">>", req.URL) response, err := g.client.SendRequest(req) if err != nil { return nil, err } var apiResponse APIResponse - if err := response.DecodeJSON(&apiResponse); err != nil { - return nil, err + + type responseUnmarshaler interface { + Unmarshal(data []byte) error + } + + if unmarshaler, ok := interface{}(&apiResponse).(responseUnmarshaler); ok { + if err := unmarshaler.Unmarshal(response.Body); err != nil { + return nil, err + } + } else { + // The line below checks the content type, however, some API server might not send the correct content type header, + // Hence, this is commented for backward compatibility + // response.IsJSON() + if err := response.DecodeJSON(&apiResponse); err != nil { + return nil, err + } } type responseValidator interface { Validate() error } - validator, ok := interface{}(apiResponse).(responseValidator) - if ok { + + if validator, ok := interface{}(&apiResponse).(responseValidator); ok { if err := validator.Validate(); err != nil { return nil, err } } - var data []Trade if err := json.Unmarshal(apiResponse.Data, &data); err != nil { return nil, err diff --git a/pkg/exchange/okex/okexapi/testdata/get_three_days_transaction_history_request.json b/pkg/exchange/okex/okexapi/testdata/get_three_days_transaction_history_request.json new file mode 100644 index 0000000000..d3cbf4b224 --- /dev/null +++ b/pkg/exchange/okex/okexapi/testdata/get_three_days_transaction_history_request.json @@ -0,0 +1,32 @@ +{ + "code":"0", + "data":[ + { + "side":"buy", + "fillSz":"0.001", + "fillPx":"73397.8", + "fillPxVol":"", + "fillFwdPx":"", + "fee":"-0.000001", + "fillPnl":"0", + "ordId":"688362711456706560", + "feeRate":"-0.001", + "instType":"SPOT", + "fillPxUsd":"", + "instId":"BTC-USDT", + "clOrdId":"1229606897", + "posSide":"net", + "billId":"688362711465447466", + "fillMarkVol":"", + "tag":"", + "fillTime":"1710390459571", + "execType":"T", + "fillIdxPx":"", + "tradeId":"749554213", + "fillMarkPx":"", + "feeCcy":"BTC", + "ts":"1710390459574" + } + ], + "msg":"" +} diff --git a/pkg/exchange/okex/okexapi/testdata/get_transaction_history_request.json b/pkg/exchange/okex/okexapi/testdata/get_transaction_history_request.json new file mode 100644 index 0000000000..d3cbf4b224 --- /dev/null +++ b/pkg/exchange/okex/okexapi/testdata/get_transaction_history_request.json @@ -0,0 +1,32 @@ +{ + "code":"0", + "data":[ + { + "side":"buy", + "fillSz":"0.001", + "fillPx":"73397.8", + "fillPxVol":"", + "fillFwdPx":"", + "fee":"-0.000001", + "fillPnl":"0", + "ordId":"688362711456706560", + "feeRate":"-0.001", + "instType":"SPOT", + "fillPxUsd":"", + "instId":"BTC-USDT", + "clOrdId":"1229606897", + "posSide":"net", + "billId":"688362711465447466", + "fillMarkVol":"", + "tag":"", + "fillTime":"1710390459571", + "execType":"T", + "fillIdxPx":"", + "tradeId":"749554213", + "fillMarkPx":"", + "feeCcy":"BTC", + "ts":"1710390459574" + } + ], + "msg":"" +} From 2ae1933d7b2179ba70a90be06688bd895061da48 Mon Sep 17 00:00:00 2001 From: edwin Date: Thu, 14 Mar 2024 17:17:41 +0800 Subject: [PATCH 2/2] pkg/exchange: use 3 days trade api if start time - now < 3 days --- pkg/exchange/okex/exchange.go | 61 +++-- pkg/exchange/okex/exchange_test.go | 355 +++++++++++++++++++++++++++++ 2 files changed, 396 insertions(+), 20 deletions(-) diff --git a/pkg/exchange/okex/exchange.go b/pkg/exchange/okex/exchange.go index 622cdb0bc4..d7757618f4 100644 --- a/pkg/exchange/okex/exchange.go +++ b/pkg/exchange/okex/exchange.go @@ -55,6 +55,7 @@ const ( defaultQueryLimit = 100 maxHistoricalDataQueryPeriod = 90 * 24 * time.Hour + threeDaysHistoricalPeriod = 3 * 24 * time.Hour ) var log = logrus.WithFields(logrus.Fields{ @@ -66,7 +67,8 @@ var ErrSymbolRequired = errors.New("symbol is a required parameter") type Exchange struct { key, secret, passphrase string - client *okexapi.RestClient + client *okexapi.RestClient + timeNowFunc func() time.Time } func New(key, secret, passphrase string) *Exchange { @@ -77,10 +79,11 @@ func New(key, secret, passphrase string) *Exchange { } return &Exchange{ - key: key, - secret: secret, - passphrase: passphrase, - client: client, + key: key, + secret: secret, + passphrase: passphrase, + client: client, + timeNowFunc: time.Now, } } @@ -564,25 +567,23 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type return nil, ErrSymbolRequired } - req := e.client.NewGetTransactionHistoryRequest().InstrumentID(toLocalSymbol(symbol)) - limit := options.Limit if limit > defaultQueryLimit || limit <= 0 { log.Infof("limit is exceeded default limit %d or zero, got: %d, use default limit", defaultQueryLimit, limit) limit = defaultQueryLimit } - req.Limit(uint64(limit)) - var newStartTime time.Time + timeNow := e.timeNowFunc() + newStartTime := timeNow.Add(-threeDaysHistoricalPeriod) if options.StartTime != nil { newStartTime = *options.StartTime - if time.Since(newStartTime) > maxHistoricalDataQueryPeriod { - newStartTime = time.Now().Add(-maxHistoricalDataQueryPeriod) + if timeNow.Sub(newStartTime) > maxHistoricalDataQueryPeriod { + newStartTime = timeNow.Add(-maxHistoricalDataQueryPeriod) log.Warnf("!!!OKX EXCHANGE API NOTICE!!! The trade API cannot query data beyond 90 days from the current date, update %s -> %s", *options.StartTime, newStartTime) } - req.StartTime(newStartTime.UTC()) } + endTime := timeNow if options.EndTime != nil { if options.EndTime.Before(newStartTime) { return nil, fmt.Errorf("end time %s before start %s", *options.EndTime, newStartTime) @@ -590,7 +591,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type if options.EndTime.Sub(newStartTime) > maxHistoricalDataQueryPeriod { return nil, fmt.Errorf("start time %s and end time %s cannot greater than 90 days", newStartTime, options.EndTime) } - req.EndTime(options.EndTime.UTC()) + endTime = *options.EndTime } if options.LastTradeID != 0 { @@ -599,12 +600,33 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type log.Infof("Last trade id not supported on QueryTrades") } - for { - if err := queryTradeLimiter.Wait(ctx); err != nil { - return nil, fmt.Errorf("query trades rate limiter wait error: %w", err) - } + if timeNow.Sub(newStartTime) <= threeDaysHistoricalPeriod { + c := e.client.NewGetThreeDaysTransactionHistoryRequest(). + InstrumentID(toLocalSymbol(symbol)). + StartTime(newStartTime). + EndTime(endTime). + Limit(uint64(limit)) + return getTrades(ctx, limit, func(ctx context.Context, billId string) ([]okexapi.Trade, error) { + c.Before(billId) + return c.Do(ctx) + }) + } - response, err := req.Do(ctx) + c := e.client.NewGetTransactionHistoryRequest(). + InstrumentID(toLocalSymbol(symbol)). + StartTime(newStartTime). + EndTime(endTime). + Limit(uint64(limit)) + return getTrades(ctx, limit, func(ctx context.Context, billId string) ([]okexapi.Trade, error) { + c.Before(billId) + return c.Do(ctx) + }) +} + +func getTrades(ctx context.Context, limit int64, doFunc func(ctx context.Context, billId string) ([]okexapi.Trade, error)) (trades []types.Trade, err error) { + billId := "0" + for { + response, err := doFunc(ctx, billId) if err != nil { return nil, fmt.Errorf("failed to query trades, err: %w", err) } @@ -623,9 +645,8 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type break } // use Before filter to get all data. - req.Before(response[tradeLen-1].BillId.String()) + billId = response[tradeLen-1].BillId.String() } - return trades, nil } diff --git a/pkg/exchange/okex/exchange_test.go b/pkg/exchange/okex/exchange_test.go index 0282a1c254..3f42198483 100644 --- a/pkg/exchange/okex/exchange_test.go +++ b/pkg/exchange/okex/exchange_test.go @@ -1,11 +1,23 @@ package okex import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strconv" "strings" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/okex/okexapi" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/testing/httptesting" + "github.com/c9s/bbgo/pkg/types" ) func Test_clientOrderIdRegex(t *testing.T) { @@ -29,3 +41,346 @@ func Test_clientOrderIdRegex(t *testing.T) { assert.False(t, clientOrderIdRegex.MatchString(uuid.NewString())) }) } + +func TestExchange_QueryTrades(t *testing.T) { + var ( + assert = assert.New(t) + ex = New("key", "secret", "passphrase") + expBtcSymbol = "BTCUSDT" + expLocalBtcSymbol = "BTC-USDT" + until = time.Now() + since = until.Add(-threeDaysHistoricalPeriod) + + options = &types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + Limit: defaultQueryLimit, + LastTradeID: 0, + } + threeDayUrl = "/api/v5/trade/fills" + historyUrl = "/api/v5/trade/fills-history" + expOrder = []types.Trade{ + { + ID: 749554213, + OrderID: 688362711456706560, + Exchange: types.ExchangeOKEx, + Price: fixedpoint.MustNewFromString("73397.8"), + Quantity: fixedpoint.MustNewFromString("0.001"), + QuoteQuantity: fixedpoint.MustNewFromString("73.3978"), + Symbol: expBtcSymbol, + Side: types.SideTypeBuy, + IsBuyer: true, + IsMaker: false, + Time: types.Time(types.NewMillisecondTimestampFromInt(1710390459574).Time()), + Fee: fixedpoint.MustNewFromString("0.000001"), + FeeCurrency: "BTC", + }, + } + ) + ex.timeNowFunc = func() time.Time { return until } + + t.Run("3 days", func(t *testing.T) { + t.Run("succeeds with one record", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + + // order history + historyOrderFile, err := os.ReadFile("okexapi/testdata/get_three_days_transaction_history_request.json") + assert.NoError(err) + + transport.GET(threeDayUrl, func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + assert.Len(query, 6) + assert.Contains(query, "begin") + assert.Contains(query, "end") + assert.Contains(query, "limit") + assert.Contains(query, "instId") + assert.Contains(query, "instType") + assert.Contains(query, "before") + assert.Equal(query["begin"], []string{strconv.FormatInt(since.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["end"], []string{strconv.FormatInt(until.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["limit"], []string{strconv.FormatInt(defaultQueryLimit, 10)}) + assert.Equal(query["instId"], []string{expLocalBtcSymbol}) + assert.Equal(query["instType"], []string{string(okexapi.InstrumentTypeSpot)}) + assert.Equal(query["before"], []string{"0"}) + return httptesting.BuildResponseString(http.StatusOK, string(historyOrderFile)), nil + }) + + orders, err := ex.QueryTrades(context.Background(), expBtcSymbol, options) + assert.NoError(err) + assert.Equal(expOrder, orders) + }) + + t.Run("succeeds with exceeded max records", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + + tradeId := 749554213 + billId := 688362711465447466 + dataTemplace := ` + { + "side":"buy", + "fillSz":"0.001", + "fillPx":"73397.8", + "fillPxVol":"", + "fillFwdPx":"", + "fee":"-0.000001", + "fillPnl":"0", + "ordId":"688362711456706560", + "feeRate":"-0.001", + "instType":"SPOT", + "fillPxUsd":"", + "instId":"BTC-USDT", + "clOrdId":"1229606897", + "posSide":"net", + "billId":"%d", + "fillMarkVol":"", + "tag":"", + "fillTime":"1710390459571", + "execType":"T", + "fillIdxPx":"", + "tradeId":"%d", + "fillMarkPx":"", + "feeCcy":"BTC", + "ts":"1710390459574" + }` + + tradesStr := make([]string, 0, defaultQueryLimit+1) + expTrades := make([]types.Trade, 0, defaultQueryLimit+1) + for i := 0; i < defaultQueryLimit+1; i++ { + dataStr := fmt.Sprintf(dataTemplace, billId+i, tradeId+i) + tradesStr = append(tradesStr, dataStr) + + trade := &okexapi.Trade{} + err := json.Unmarshal([]byte(dataStr), &trade) + assert.NoError(err) + expTrades = append(expTrades, tradeToGlobal(*trade)) + } + + transport.GET(threeDayUrl, func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + assert.Contains(query, "begin") + assert.Contains(query, "end") + assert.Contains(query, "limit") + assert.Contains(query, "instId") + assert.Contains(query, "instType") + assert.Equal(query["begin"], []string{strconv.FormatInt(since.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["end"], []string{strconv.FormatInt(until.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["limit"], []string{strconv.FormatInt(defaultQueryLimit, 10)}) + assert.Equal(query["instId"], []string{expLocalBtcSymbol}) + assert.Equal(query["instType"], []string{string(okexapi.InstrumentTypeSpot)}) + assert.Len(query, 6) + + if query["before"][0] == "0" { + resp := &okexapi.APIResponse{ + Code: "0", + Data: []byte("[" + strings.Join(tradesStr[0:defaultQueryLimit], ",") + "]"), + } + respRaw, err := json.Marshal(resp) + assert.NoError(err) + + return httptesting.BuildResponseString(http.StatusOK, string(respRaw)), nil + } + + // second time query + // last order id, so need to -1 + assert.Equal(query["before"], []string{strconv.FormatInt(int64(billId+defaultQueryLimit-1), 10)}) + + resp := okexapi.APIResponse{ + Code: "0", + Data: []byte("[" + strings.Join(tradesStr[defaultQueryLimit:defaultQueryLimit+1], ",") + "]"), + } + respRaw, err := json.Marshal(resp) + assert.NoError(err) + return httptesting.BuildResponseString(http.StatusOK, string(respRaw)), nil + }) + + trades, err := ex.QueryTrades(context.Background(), expBtcSymbol, options) + assert.NoError(err) + assert.Equal(expTrades, trades) + }) + }) + t.Run("3 days < x < Max days", func(t *testing.T) { + t.Run("succeeds with one record", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + newSince := until.Add(-maxHistoricalDataQueryPeriod) + options.StartTime = &newSince + + // order history + historyOrderFile, err := os.ReadFile("okexapi/testdata/get_transaction_history_request.json") + assert.NoError(err) + + transport.GET(historyUrl, func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + assert.Len(query, 6) + assert.Contains(query, "begin") + assert.Contains(query, "end") + assert.Contains(query, "limit") + assert.Contains(query, "instId") + assert.Contains(query, "instType") + assert.Contains(query, "before") + assert.Equal(query["begin"], []string{strconv.FormatInt(newSince.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["end"], []string{strconv.FormatInt(until.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["limit"], []string{strconv.FormatInt(defaultQueryLimit, 10)}) + assert.Equal(query["instId"], []string{expLocalBtcSymbol}) + assert.Equal(query["instType"], []string{string(okexapi.InstrumentTypeSpot)}) + assert.Equal(query["before"], []string{"0"}) + return httptesting.BuildResponseString(http.StatusOK, string(historyOrderFile)), nil + }) + + orders, err := ex.QueryTrades(context.Background(), expBtcSymbol, options) + assert.NoError(err) + assert.Equal(expOrder, orders) + }) + + t.Run("succeeds with exceeded max records", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + newSince := until.Add(-maxHistoricalDataQueryPeriod) + options.StartTime = &newSince + + tradeId := 749554213 + billId := 688362711465447466 + dataTemplace := ` + { + "side":"buy", + "fillSz":"0.001", + "fillPx":"73397.8", + "fillPxVol":"", + "fillFwdPx":"", + "fee":"-0.000001", + "fillPnl":"0", + "ordId":"688362711456706560", + "feeRate":"-0.001", + "instType":"SPOT", + "fillPxUsd":"", + "instId":"BTC-USDT", + "clOrdId":"1229606897", + "posSide":"net", + "billId":"%d", + "fillMarkVol":"", + "tag":"", + "fillTime":"1710390459571", + "execType":"T", + "fillIdxPx":"", + "tradeId":"%d", + "fillMarkPx":"", + "feeCcy":"BTC", + "ts":"1710390459574" + }` + + tradesStr := make([]string, 0, defaultQueryLimit+1) + expTrades := make([]types.Trade, 0, defaultQueryLimit+1) + for i := 0; i < defaultQueryLimit+1; i++ { + dataStr := fmt.Sprintf(dataTemplace, billId+i, tradeId+i) + tradesStr = append(tradesStr, dataStr) + + trade := &okexapi.Trade{} + err := json.Unmarshal([]byte(dataStr), &trade) + assert.NoError(err) + expTrades = append(expTrades, tradeToGlobal(*trade)) + } + + transport.GET(historyUrl, func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + assert.Contains(query, "begin") + assert.Contains(query, "end") + assert.Contains(query, "limit") + assert.Contains(query, "instId") + assert.Contains(query, "instType") + assert.Equal(query["begin"], []string{strconv.FormatInt(newSince.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["end"], []string{strconv.FormatInt(until.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["limit"], []string{strconv.FormatInt(defaultQueryLimit, 10)}) + assert.Equal(query["instId"], []string{expLocalBtcSymbol}) + assert.Equal(query["instType"], []string{string(okexapi.InstrumentTypeSpot)}) + assert.Len(query, 6) + + if query["before"][0] == "0" { + resp := &okexapi.APIResponse{ + Code: "0", + Data: []byte("[" + strings.Join(tradesStr[0:defaultQueryLimit], ",") + "]"), + } + respRaw, err := json.Marshal(resp) + assert.NoError(err) + + return httptesting.BuildResponseString(http.StatusOK, string(respRaw)), nil + } + + // second time query + // last order id, so need to -1 + assert.Equal(query["before"], []string{strconv.FormatInt(int64(billId+defaultQueryLimit-1), 10)}) + + resp := okexapi.APIResponse{ + Code: "0", + Data: []byte("[" + strings.Join(tradesStr[defaultQueryLimit:defaultQueryLimit+1], ",") + "]"), + } + respRaw, err := json.Marshal(resp) + assert.NoError(err) + return httptesting.BuildResponseString(http.StatusOK, string(respRaw)), nil + }) + + trades, err := ex.QueryTrades(context.Background(), expBtcSymbol, options) + assert.NoError(err) + assert.Equal(expTrades, trades) + }) + }) + + t.Run("start time exceeded 3 months", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + newSince := options.StartTime.Add(-365 * 24 * time.Hour) + newOpts := *options + newOpts.StartTime = &newSince + + expSinceTime := until.Add(-maxHistoricalDataQueryPeriod) + + // order history + historyOrderFile, err := os.ReadFile("okexapi/testdata/get_three_days_transaction_history_request.json") + assert.NoError(err) + + transport.GET(historyUrl, func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + assert.Len(query, 6) + assert.Contains(query, "begin") + assert.Contains(query, "end") + assert.Contains(query, "limit") + assert.Contains(query, "instId") + assert.Contains(query, "instType") + assert.Contains(query, "before") + assert.Equal(query["begin"], []string{strconv.FormatInt(expSinceTime.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["end"], []string{strconv.FormatInt(until.UnixNano()/int64(time.Millisecond), 10)}) + assert.Equal(query["limit"], []string{strconv.FormatInt(defaultQueryLimit, 10)}) + assert.Equal(query["instId"], []string{expLocalBtcSymbol}) + assert.Equal(query["instType"], []string{string(okexapi.InstrumentTypeSpot)}) + assert.Equal(query["before"], []string{"0"}) + return httptesting.BuildResponseString(http.StatusOK, string(historyOrderFile)), nil + }) + + orders, err := ex.QueryTrades(context.Background(), expBtcSymbol, &newOpts) + assert.NoError(err) + assert.Equal(expOrder, orders) + }) + + t.Run("start time after end day", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + newSince := options.StartTime.Add(365 * 24 * time.Hour) + newOpts := *options + newOpts.StartTime = &newSince + + _, err := ex.QueryTrades(context.Background(), expBtcSymbol, &newOpts) + assert.ErrorContains(err, "before start") + }) + + t.Run("empty symbol", func(t *testing.T) { + transport := &httptesting.MockTransport{} + ex.client.HttpClient.Transport = transport + newSince := options.StartTime.Add(365 * 24 * time.Hour) + newOpts := *options + newOpts.StartTime = &newSince + + _, err := ex.QueryTrades(context.Background(), "", &newOpts) + assert.ErrorContains(err, ErrSymbolRequired.Error()) + }) +}