Added config option max_per_page (#1577)

* Added limit page size parameter (#4) * Added max_per_page in cofig * added default topster size config * removing const default_topster_size * typo fix * Update tests.yml * few test fixes * bug fix * test fix * added couple missing lines * last test fix * Added limit page size parameter (#5) * Added max_per_page in cofig * added default topster size config * removing const default_topster_size * typo fix * Update tests.yml * few test fixes * bug fix * test fix * added couple missing lines * last test fix * few fixes after updates from origin * restore default_topster_size parameter to const * removing unrelated change and changing int to unsigned int --------- Co-authored-by: Kishore Nallan <kishorenc@gmail.com>
typesense · Apr 24, 2024 · 2dbbb43 · 2dbbb43
1 parent 7c95ce4
commit 2dbbb43
Show file tree

Hide file tree

Showing 7 changed files with 37 additions and 4 deletions.
diff --git a/include/collection.h b/include/collection.h
@@ -349,8 +349,6 @@ class Collection {
 
     enum {MAX_ARRAY_MATCHES = 5};
 
-    const size_t PER_PAGE_MAX = 250;
-
     const size_t GROUP_LIMIT_MAX = 99;
 
     // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store

diff --git a/include/tsconfig.h b/include/tsconfig.h
@@ -80,6 +80,8 @@ class Config {
 
     bool enable_search_logging;
 
+    uint32_t max_per_page;
+
     uint16_t filter_by_max_ops;
 
 protected:
@@ -115,6 +117,8 @@ class Config {
         this->enable_lazy_filter = false;
 
         this->enable_search_logging = false;
+
+        this->max_per_page = 250;
 
         this->filter_by_max_ops = FILTER_BY_DEFAULT_OPERATIONS;
     }
@@ -209,6 +213,10 @@ class Config {
         this->reset_peers_on_error = reset_peers_on_error;
     }
 
+    void set_max_per_page(int max_per_page) {
+        this->max_per_page = max_per_page;
+    }
+
     // getters
 
     std::string get_data_dir() const {
@@ -384,6 +392,10 @@ class Config {
         return skip_writes;
     }
 
+    int get_max_per_page() const {
+        return this->max_per_page;
+    }
+
     uint16_t get_filter_by_max_ops() const {
         return filter_by_max_ops;
     }

diff --git a/src/collection.cpp b/src/collection.cpp
@@ -2132,8 +2132,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
         }
     }
 
-    if(per_page > PER_PAGE_MAX) {
-        std::string message = "Only upto " + std::to_string(PER_PAGE_MAX) + " hits can be fetched per page.";
+    int per_page_max = Config::get_instance().get_max_per_page();
+
+    if(per_page > per_page_max) {
+        std::string message = "Only upto " + std::to_string(per_page_max) + " hits can be fetched per page.";
         return Option<nlohmann::json>(422, message);
     }
 

diff --git a/src/tsconfig.cpp b/src/tsconfig.cpp
@@ -257,6 +257,10 @@ void Config::load_config_env() {
     this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));
     this->enable_lazy_filter = ("TRUE" == get_env("TYPESENSE_ENABLE_LAZY_FILTER"));
     this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR"));
+
+    if(!get_env("TYPESENSE_MAX_PER_PAGE").empty()) {
+        this->max_per_page = std::stoi(get_env("TYPESENSE_MAX_PER_PAGE"));
+    }
 }
 
 void Config::load_config_file(cmdline::parser& options) {
@@ -457,6 +461,10 @@ void Config::load_config_file(cmdline::parser& options) {
         this->reset_peers_on_error = (reset_peers_on_error_str == "true");
     }
 
+    if(reader.Exists("server", "max-per-page")) {
+        this->max_per_page = reader.GetInteger("server", "max-per-page", 250);
+    }
+
     if(reader.Exists("server", "filter-by-max-ops")) {
         this->filter_by_max_ops = (uint16_t) reader.GetInteger("server", "filter-by-max-ops", FILTER_BY_DEFAULT_OPERATIONS);
     }
@@ -633,8 +641,13 @@ void Config::load_config_cmd_args(cmdline::parser& options)  {
         this->enable_search_logging = options.get<bool>("enable-search-logging");
     }
 
+    if(options.exist("max-per-page")) {
+        this->max_per_page = options.get<int>("max-per-page");
+    }
+
     if(options.exist("filter-by-max-ops")) {
         this->filter_by_max_ops = options.get<uint16_t>("filter-by-max-ops");
+
     }
 }
 
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
@@ -116,6 +116,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
     options.add<uint32_t>("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800);
     options.add<uint16_t>("filter-by-max-ops", '\0', "Maximum number of operations permitted in filtery_by.", false, Config::FILTER_BY_DEFAULT_OPERATIONS);
 
+    options.add<int>("max-per-page", '\0', "Max number of hits per page", false, 250);
+
     // DEPRECATED
     options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
     options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);

diff --git a/test/tsconfig_test.cpp b/test/tsconfig_test.cpp
@@ -30,6 +30,7 @@ TEST(ConfigTest, LoadCmdLineArguments) {
         "--data-dir=/tmp/data",
         "--api-key=abcd",
         "--listen-port=8080",
+        "--max-per-page=250",
     };
 
     std::vector<char*> argv = get_argv(args);
@@ -144,6 +145,7 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) {
         "--api-key=abcd",
         "--listen-address=192.168.10.10",
         "--cors-domains=http://localhost:8108",
+        "--max-per-page=250",
         std::string("--config=") + std::string(ROOT_DIR)+"test/valid_sparse_config.ini"
     };
 
@@ -172,6 +174,7 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) {
     ASSERT_EQ("abcd", config.get_api_key());  // cli parameter overrides file config
     ASSERT_EQ(1, config.get_cors_domains().size());  // cli parameter overrides file config
     ASSERT_EQ("http://localhost:8108", *(config.get_cors_domains().begin()));
+    ASSERT_EQ(250, config.get_max_per_page());
 }
 
 TEST(ConfigTest, CorsDefaults) {
@@ -182,6 +185,7 @@ TEST(ConfigTest, CorsDefaults) {
         "--data-dir=/tmp/data",
         "--api-key=abcd",
         "--listen-address=192.168.10.10",
+        "--max-per-page=250",
         std::string("--config=") + std::string(ROOT_DIR)+"test/valid_sparse_config.ini"
     };
 

diff --git a/test/valid_config.ini b/test/valid_config.ini
@@ -11,3 +11,5 @@ listen-port = 9090
 enable-cors = True
 
 analytics-dir=/tmp/analytics
+
+max-per-page=250