carbonserver: include a Limited signal in /list_query api result

carbonserver/list.go

@@ -3,7 +3,6 @@ package carbonserver
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"net/http"
 	_ "net/http/pprof"
 	"strconv"
@@ -143,6 +142,8 @@ type ListQueryResult struct {
 	PhysicalSize int64
 	LogicalSize  int64
 
+	Limited bool
+
 	Metrics []ListMetricInfo
 }
 
@@ -156,14 +157,21 @@ func (listener *CarbonserverListener) queryMetricsList(query string, limit int,
 		return nil, errMetricsListEmpty
 	}
 
-	log.Println(strings.ReplaceAll(query, ".", "/"))
-	names, isFiles, nodes, err := fidx.trieIdx.query(strings.ReplaceAll(query, ".", "/"), limit, nil)
+	// TODO: support topk queries?
+
+	names, isFiles, nodes, limited, err := fidx.trieIdx.query(strings.ReplaceAll(query, ".", "/"), limit, nil)
 	if err != nil {
 		return nil, err
 	}
+	// WHY: it's impossible for /list_query api user to be aware that a
+	// query is limited in nodes lookup.
+	// Returning a Limited signal allows api user to decide that if it should
+	// make a separate a call or show a warning.
+	result.Limited = limited
 
 	for i, name := range names {
 		if len(result.Metrics) >= limit {
+			result.Limited = true
 			break
 		}
 

carbonserver/trie.go

@@ -695,7 +695,7 @@ func (ti *trieIndex) newDir() *trieNode {
 
 // TODO: add some defensive logics agains bad queries?
 // depth first search
-func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ([]string, error)) (files []string, isFiles []bool, nodes []*trieNode, err error) {
+func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ([]string, error)) (files []string, isFiles []bool, nodes []*trieNode, limited bool, err error) {
 	expr = strings.TrimSpace(expr)
 	if expr == "" {
 		expr = "*"
@@ -708,14 +708,14 @@ func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) (
 		}
 		gs, err := newGlobState(node, expand)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, false, err
 		}
 		exact = exact && gs.exact
 		matchers = append(matchers, gs)
 	}
 
 	if len(matchers) == 0 {
-		return nil, nil, nil, nil
+		return nil, nil, nil, false, nil
 	}
 
 	var cur = ti.root
@@ -829,7 +829,12 @@ func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) (
 			nodes = append(nodes, dirNode)
 		}
 
-		if len(files) >= limit || exact {
+		if len(files) >= limit {
+			limited = true
+			break
+		}
+
+		if exact {
 			break
 		}
 
@@ -861,7 +866,7 @@ func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) (
 		continue
 	}
 
-	return files, isFiles, nodes, nil
+	return files, isFiles, nodes, limited, nil
 }
 
 // note: tn might be a root or file node, which has a nil c
@@ -1475,7 +1480,7 @@ func (listener *CarbonserverListener) expandGlobsTrie(query string) ([]string, [
 	var leafs []bool
 
 	for _, g := range globs {
-		f, l, _, err := fidx.trieIdx.query(g, listener.maxMetricsGlobbed-len(files), listener.expandGlobBraces)
+		f, l, _, _, err := fidx.trieIdx.query(g, listener.maxMetricsGlobbed-len(files), listener.expandGlobBraces)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -1655,7 +1660,7 @@ func (ti *trieIndex) applyQuotas(resetFrequency time.Duration, quotas ...*Quota)
 			continue
 		}
 
-		paths, _, nodes, err := ti.query(strings.ReplaceAll(quota.Pattern, ".", "/"), 1<<31-1, nil)
+		paths, _, nodes, _, err := ti.query(strings.ReplaceAll(quota.Pattern, ".", "/"), 1<<31-1, nil)
 		if err != nil {
 			return nil, err
 		}

carbonserver/trie_fuzz_query.go

@@ -41,7 +41,7 @@ var trie = func() *trieIndex {
 }()
 
 func Fuzz(data []byte) int {
-	_, _, err := trie.query(string(data), 1000, func([]string) ([]string, error) { return nil, nil })
+	_, _,_, _, err := trie.query(string(data), 1000, func([]string) ([]string, error) { return nil, nil })
 	if err != nil {
 		// panic(err)
 		return 0

carbonserver/trie_test.go

@@ -745,7 +745,7 @@ func TestTrieIndex(t *testing.T) {
 func TestTrieEdgeCases(t *testing.T) {
 	var trie = newTrie(".wsp", nil)
 
-	_, _, _, err := trie.query("[\xff\xff-\xff", 1000, func([]string) ([]string, error) { return nil, nil })
+	_, _, _, _, err := trie.query("[\xff\xff-\xff", 1000, func([]string) ([]string, error) { return nil, nil })
 	if err == nil || err.Error() != "glob: range overflow" {
 		t.Errorf("trie should return an range overflow error")
 	}
@@ -775,7 +775,7 @@ func TestTrieQueryOpts(t *testing.T) {
 		"sys.app.host-02.cpu.system",
 	}
 
-	files, _, nodes, err := trieIndex.query("sys/app/host-0{1,2}", 1000, nil)
+	files, _, nodes, _, err := trieIndex.query("sys/app/host-0{1,2}", 1000, nil)
 	if err != nil {
 		t.Error(err)
 	}
@@ -837,7 +837,7 @@ func TestTrieConcurrentReadWrite(t *testing.T) {
 		// case <-filec:
 		default:
 			// skipcq: GSC-G404
-			files, _, _, err := trieIndex.query(fmt.Sprintf("level-0-%d/level-1-%d/level-2-%d*", rand.Intn(factor), rand.Intn(factor), rand.Intn(factor)), int(math.MaxInt64), nil)
+			files, _, _, _, err := trieIndex.query(fmt.Sprintf("level-0-%d/level-1-%d/level-2-%d*", rand.Intn(factor), rand.Intn(factor), rand.Intn(factor)), int(math.MaxInt64), nil)
 			if err != nil {
 				panic(err)
 			}