From 2dbbb43e941909537c556962f56c9b80ffa96311 Mon Sep 17 00:00:00 2001 From: Lukas Laurinaitis Date: Wed, 24 Apr 2024 06:05:07 +0300 Subject: [PATCH] Added config option max_per_page (#1577) * Added limit page size parameter (#4) * Added max_per_page in cofig * added default topster size config * removing const default_topster_size * typo fix * Update tests.yml * few test fixes * bug fix * test fix * added couple missing lines * last test fix * Added limit page size parameter (#5) * Added max_per_page in cofig * added default topster size config * removing const default_topster_size * typo fix * Update tests.yml * few test fixes * bug fix * test fix * added couple missing lines * last test fix * few fixes after updates from origin * restore default_topster_size parameter to const * removing unrelated change and changing int to unsigned int --------- Co-authored-by: Kishore Nallan --- include/collection.h | 2 -- include/tsconfig.h | 12 ++++++++++++ src/collection.cpp | 6 ++++-- src/tsconfig.cpp | 13 +++++++++++++ src/typesense_server_utils.cpp | 2 ++ test/tsconfig_test.cpp | 4 ++++ test/valid_config.ini | 2 ++ 7 files changed, 37 insertions(+), 4 deletions(-) diff --git a/include/collection.h b/include/collection.h index 458104f16..067cf84ed 100644 --- a/include/collection.h +++ b/include/collection.h @@ -349,8 +349,6 @@ class Collection { enum {MAX_ARRAY_MATCHES = 5}; - const size_t PER_PAGE_MAX = 250; - const size_t GROUP_LIMIT_MAX = 99; // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store diff --git a/include/tsconfig.h b/include/tsconfig.h index 59d059e52..720232cb1 100644 --- a/include/tsconfig.h +++ b/include/tsconfig.h @@ -80,6 +80,8 @@ class Config { bool enable_search_logging; + uint32_t max_per_page; + uint16_t filter_by_max_ops; protected: @@ -115,6 +117,8 @@ class Config { this->enable_lazy_filter = false; this->enable_search_logging = false; + + this->max_per_page = 250; this->filter_by_max_ops = FILTER_BY_DEFAULT_OPERATIONS; } @@ -209,6 +213,10 @@ class Config { this->reset_peers_on_error = reset_peers_on_error; } + void set_max_per_page(int max_per_page) { + this->max_per_page = max_per_page; + } + // getters std::string get_data_dir() const { @@ -384,6 +392,10 @@ class Config { return skip_writes; } + int get_max_per_page() const { + return this->max_per_page; + } + uint16_t get_filter_by_max_ops() const { return filter_by_max_ops; } diff --git a/src/collection.cpp b/src/collection.cpp index 0bec33508..b9cded16e 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -2132,8 +2132,10 @@ Option Collection::search(std::string raw_query, } } - if(per_page > PER_PAGE_MAX) { - std::string message = "Only upto " + std::to_string(PER_PAGE_MAX) + " hits can be fetched per page."; + int per_page_max = Config::get_instance().get_max_per_page(); + + if(per_page > per_page_max) { + std::string message = "Only upto " + std::to_string(per_page_max) + " hits can be fetched per page."; return Option(422, message); } diff --git a/src/tsconfig.cpp b/src/tsconfig.cpp index 5f6ddc8ac..641e260bc 100644 --- a/src/tsconfig.cpp +++ b/src/tsconfig.cpp @@ -257,6 +257,10 @@ void Config::load_config_env() { this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES")); this->enable_lazy_filter = ("TRUE" == get_env("TYPESENSE_ENABLE_LAZY_FILTER")); this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR")); + + if(!get_env("TYPESENSE_MAX_PER_PAGE").empty()) { + this->max_per_page = std::stoi(get_env("TYPESENSE_MAX_PER_PAGE")); + } } void Config::load_config_file(cmdline::parser& options) { @@ -457,6 +461,10 @@ void Config::load_config_file(cmdline::parser& options) { this->reset_peers_on_error = (reset_peers_on_error_str == "true"); } + if(reader.Exists("server", "max-per-page")) { + this->max_per_page = reader.GetInteger("server", "max-per-page", 250); + } + if(reader.Exists("server", "filter-by-max-ops")) { this->filter_by_max_ops = (uint16_t) reader.GetInteger("server", "filter-by-max-ops", FILTER_BY_DEFAULT_OPERATIONS); } @@ -633,8 +641,13 @@ void Config::load_config_cmd_args(cmdline::parser& options) { this->enable_search_logging = options.get("enable-search-logging"); } + if(options.exist("max-per-page")) { + this->max_per_page = options.get("max-per-page"); + } + if(options.exist("filter-by-max-ops")) { this->filter_by_max_ops = options.get("filter-by-max-ops"); + } } diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp index 5395dedbc..bd0cfbb01 100644 --- a/src/typesense_server_utils.cpp +++ b/src/typesense_server_utils.cpp @@ -116,6 +116,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) { options.add("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800); options.add("filter-by-max-ops", '\0', "Maximum number of operations permitted in filtery_by.", false, Config::FILTER_BY_DEFAULT_OPERATIONS); + options.add("max-per-page", '\0', "Max number of hits per page", false, 250); + // DEPRECATED options.add("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0"); options.add("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108); diff --git a/test/tsconfig_test.cpp b/test/tsconfig_test.cpp index 4338a8471..d15f77494 100644 --- a/test/tsconfig_test.cpp +++ b/test/tsconfig_test.cpp @@ -30,6 +30,7 @@ TEST(ConfigTest, LoadCmdLineArguments) { "--data-dir=/tmp/data", "--api-key=abcd", "--listen-port=8080", + "--max-per-page=250", }; std::vector argv = get_argv(args); @@ -144,6 +145,7 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) { "--api-key=abcd", "--listen-address=192.168.10.10", "--cors-domains=http://localhost:8108", + "--max-per-page=250", std::string("--config=") + std::string(ROOT_DIR)+"test/valid_sparse_config.ini" }; @@ -172,6 +174,7 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) { ASSERT_EQ("abcd", config.get_api_key()); // cli parameter overrides file config ASSERT_EQ(1, config.get_cors_domains().size()); // cli parameter overrides file config ASSERT_EQ("http://localhost:8108", *(config.get_cors_domains().begin())); + ASSERT_EQ(250, config.get_max_per_page()); } TEST(ConfigTest, CorsDefaults) { @@ -182,6 +185,7 @@ TEST(ConfigTest, CorsDefaults) { "--data-dir=/tmp/data", "--api-key=abcd", "--listen-address=192.168.10.10", + "--max-per-page=250", std::string("--config=") + std::string(ROOT_DIR)+"test/valid_sparse_config.ini" }; diff --git a/test/valid_config.ini b/test/valid_config.ini index 037362d79..e49efd844 100644 --- a/test/valid_config.ini +++ b/test/valid_config.ini @@ -11,3 +11,5 @@ listen-port = 9090 enable-cors = True analytics-dir=/tmp/analytics + +max-per-page=250