Ignore DROP queries in stress test with 1/2 probability and TRUNCATE Memory/JOIN table instead of ignoring DROP in upgrade check

programs/client/Client.cpp

@@ -953,7 +953,9 @@ void Client::addOptions(OptionsDescription & options_description)
         ("opentelemetry-tracestate", po::value<std::string>(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation")
 
         ("no-warnings", "disable warnings when client connects to server")
+        /// TODO: Left for compatibility as it's used in upgrade check, remove after next release and use ignore-drop-queries-probability
         ("fake-drop", "Ignore all DROP queries, should be used only for testing")
+        ("ignore-drop-queries-probability", po::value<double>(), "With specified probability ignore all DROP queries (replace them to TRUNCATE for engines like Memory/JOIN), should be used only for testing")
         ("accept-invalid-certificate", "Ignore certificate verification errors, equal to config parameters openSSL.client.invalidCertificateHandler.name=AcceptCertificateHandler and openSSL.client.verificationMode=none")
     ;
 
@@ -1096,7 +1098,9 @@ void Client::processOptions(const OptionsDescription & options_description,
     if (options.count("no-warnings"))
         config().setBool("no-warnings", true);
     if (options.count("fake-drop"))
-        fake_drop = true;
+        ignore_drop_queries_probability = 1;
+    if (options.count("ignore-drop-queries-probability"))
+        ignore_drop_queries_probability = std::min(options["ignore-drop-queries-probability"].as<double>(), 1.);
     if (options.count("accept-invalid-certificate"))
     {
         config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler");

src/Client/ClientBase.cpp

@@ -68,6 +68,9 @@
 #include <Access/AccessControl.h>
 #include <Storages/ColumnsDescription.h>
 
+#include <DataTypes/DataTypeString.h>
+#include <IO/WriteBufferFromString.h>
+
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/replace.hpp>
 #include <iostream>
@@ -559,6 +562,11 @@ try
             pager_cmd = ShellCommand::execute(config);
             out_buf = &pager_cmd->in;
         }
+        /// We can use special buffer for query output for internal queries.
+        else if (output_format_buffer)
+        {
+            out_buf = output_format_buffer.get();
+        }
         else
         {
             out_buf = &std_out;
@@ -868,11 +876,68 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
         processError(full_query);
 }
 
+String ClientBase::getTableEngine(const String & database, const String & table)
+{
+    auto is_interactive_copy = is_interactive;
+    auto format_copy = format;
+
+    is_interactive = false;
+    format = "TSVRaw";
+    String result;
+    output_format_buffer = std::make_unique<WriteBufferFromString>(result);
+    String query;
+    if (database.empty())
+        query = fmt::format("SELECT engine FROM system.tables where name='{}' and database=currentDatabase()", table);
+    else
+        query = fmt::format("SELECT engine FROM system.tables where name='{}' and database='{}'", table, database);
+
+    try
+    {
+        processTextAsSingleQuery(query);
+    }
+    catch (...)
+    {
+        result = "";
+    }
+
+    output_format_buffer->finalize();
+    output_format_buffer.reset();
+    is_interactive = is_interactive_copy;
+    format = format_copy;
+    boost::trim(result);
+    return result;
+}
+
+void ClientBase::ignoreDropQueryOrTruncateTable(const DB::ASTDropQuery * drop_query)
+{
+    const auto & database = drop_query->getDatabase();
+    const auto & table = drop_query->getTable();
+    /// Use TRUNCATE for Memory/JOIN table engines to reduce memory usage in tests.
+    String table_engine = getTableEngine(database, table);
+    if (table_engine == "Memory" || table_engine == "JOIN")
+    {
+        String truncate_query;
+        if (database.empty())
+            truncate_query = fmt::format("TRUNCATE TABLE {}", table);
+        else
+            truncate_query = fmt::format("TRUNCATE TABLE {}.{}", database, table);
+
+        auto is_interactive_copy = is_interactive;
+        is_interactive = false;
+        processTextAsSingleQuery(truncate_query);
+        is_interactive = is_interactive_copy;
+    }
+}
 
 void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr parsed_query)
 {
-    if (fake_drop && parsed_query->as<ASTDropQuery>())
+    /// In tests we can ignore DROP queries with some probability.
+    const auto * drop_query = parsed_query->as<ASTDropQuery>();
+    if (ignore_drop_queries_probability != 0 && drop_query && drop_query->kind == ASTDropQuery::Kind::Drop && std::uniform_real_distribution<>(0.0, 1.0)(thread_local_rng) <= ignore_drop_queries_probability)
+    {
+        ignoreDropQueryOrTruncateTable(drop_query);
         return;
+    }
 
     auto query = query_to_execute;
 

src/Client/ClientBase.h

@@ -178,6 +178,12 @@ class ClientBase : public Poco::Util::Application, public IHints<2>
     void initQueryIdFormats();
     bool addMergeTreeSettings(ASTCreateQuery & ast_create);
 
+    void ignoreDropQueryOrTruncateTable(const ASTDropQuery * drop_query);
+    /// Request table engine from system.tables from server.
+    String getTableEngine(const String & database, const String & table);
+    /// Send TRUNCATE query for specific table.
+    void truncateTable(const String & database, const String & table);
+
 protected:
     static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context);
     bool processMultiQueryFromFile(const String & file_name);
@@ -248,6 +254,7 @@ class ClientBase : public Poco::Util::Application, public IHints<2>
     /// The user can specify to redirect query output to a file.
     std::unique_ptr<WriteBuffer> out_file_buf;
     std::shared_ptr<IOutputFormat> output_format;
+    std::unique_ptr<WriteBuffer> output_format_buffer;
 
     /// The user could specify special file for server logs (stderr by default)
     std::unique_ptr<WriteBuffer> out_logs_buf;
@@ -307,7 +314,7 @@ class ClientBase : public Poco::Util::Application, public IHints<2>
     QueryProcessingStage::Enum query_processing_stage;
     ClientInfo::QueryKind query_kind;
 
-    bool fake_drop = false;
+    double ignore_drop_queries_probability = 0;
 
     struct HostAndPort
     {

tests/ci/stress.py

@@ -66,6 +66,11 @@ def get_options(i: int, upgrade_check: bool) -> str:
     if random.random() < 0.3:
         client_options.append(f"http_make_head_request={random.randint(0, 1)}")
 
+    # TODO: After release 23.3 use ignore-drop-queries-probability for both
+    #       stress test and upgrade check
+    if not upgrade_check:
+        client_options.append("ignore-drop-queries-probability=0.5")
+
     if client_options:
         options.append(" --client-option " + " ".join(client_options))
 

tests/clickhouse-test

@@ -3170,6 +3170,8 @@ def parse_args():
         help="Do not run shard related tests",
     )
 
+    # TODO: Remove upgrade-check option after release 23.3 and use
+    #       ignore-drop-queries-probability option in stress.py as in stress tests
     group.add_argument(
         "--upgrade-check",
         action="store_true",