Merge pull request ClickHouse#89332 from ClickHouse/allow_statistics_optimize-by-default

Enable `allow_statistics_optimize` by default

Author: hanfei1991

src/Core/Settings.cpp

@@ -7188,9 +7188,9 @@ The maximum number of rows in the right table to determine whether to rerange th
 If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join.
 )", EXPERIMENTAL) \
     \
-    DECLARE_WITH_ALIAS(Bool, allow_statistics_optimize, false, R"(
+    DECLARE_WITH_ALIAS(Bool, allow_statistics_optimize, true, R"(
 Allows using statistics to optimize queries
-)", EXPERIMENTAL, allow_statistic_optimize) \
+)", BETA, allow_statistic_optimize) \
     DECLARE_WITH_ALIAS(Bool, allow_experimental_statistics, false, R"(
 Allows defining columns with [statistics](../../engines/table-engines/mergetree-family/mergetree.md/#table_engine-mergetree-creating-a-table) and [manipulate statistics](../../engines/table-engines/mergetree-family/mergetree.md/#column-statistics).
 )", EXPERIMENTAL, allow_experimental_statistic) \

src/Core/SettingsChangesHistory.cpp

@@ -46,6 +46,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
             {"query_plan_optimize_join_order_limit", 1, 10, "Allow JOIN reordering with more tables by default"},
             {"iceberg_insert_max_partitions", 100, 100, "New setting."},
             {"use_paimon_partition_pruning", false, false, "New setting."},
+            {"allow_statistics_optimize", false, true, "Enable this optimization by default."},
+            {"allow_statistic_optimize", false, true, "Enable this optimization by default."},
             {"query_plan_text_index_add_hint", true, true, "New setting"},
             {"text_index_hint_max_selectivity", 0.2, 0.2, "New setting"},
             {"allow_experimental_time_time64_type", false, true, "Enable Time and Time64 type by default"},

src/Storages/Statistics/ConditionSelectivityEstimator.cpp

@@ -127,7 +127,9 @@ RelationProfile ConditionSelectivityEstimator::estimateRelationProfileImpl(std::
     auto* final_element = rpn_stack.top();
     final_element->finalize(column_estimators);
     RelationProfile result;
-    result.rows = static_cast<UInt64>(final_element->selectivity * total_rows);
+    Float64 final_rows = final_element->selectivity * total_rows;
+    final_rows = std::max<Float64>(final_rows, 0);
+    result.rows = static_cast<UInt64>(final_rows);
     for (const auto & [column_name, estimator] : column_estimators)
     {
         UInt64 cardinality = std::min(result.rows, estimator.estimateCardinality());
@@ -259,7 +261,7 @@ bool ConditionSelectivityEstimator::extractAtomFromTree(const StorageMetadataPtr
                     column_type = removeLowCardinalityAndNullable(column_desc->type);
             }
             /// In some cases we need to cast the type of const
-            bool cast_not_needed = !column_type ||
+            bool cast_not_needed = !column_type || !const_type ||
                 ((isNativeInteger(column_type) || isDateTime(column_type))
                 && (isNativeInteger(const_type) || isDateTime(const_type)));
 
@@ -347,7 +349,9 @@ Float64 ConditionSelectivityEstimator::ColumnEstimator::estimateRanges(const Pla
     /// In case that there is an empty statistics.
     if (stats->rowCount() == 0)
         return 0;
-    return result / stats->rowCount();
+    Float64 selectivity = result / stats->rowCount();
+    selectivity = std::max<Float64>(selectivity, 0);
+    return selectivity;
 }
 
 UInt64 ConditionSelectivityEstimator::ColumnEstimator::estimateCardinality() const

src/Storages/Statistics/Statistics.cpp

@@ -4,6 +4,7 @@
 #include <Common/FieldVisitorConvertToNumber.h>
 #include <Common/logger_useful.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <Functions/FunctionFactory.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/convertFieldToType.h>
@@ -44,19 +45,14 @@ std::optional<Float64> StatisticsUtils::tryConvertToFloat64(const Field & value,
     if (!data_type->isValueRepresentedByNumber())
         return {};
 
-    Field value_converted;
-    if (isInteger(data_type) && !isBool(data_type))
-        /// For case val_int32 < 10.5 or val_int32 < '10.5' we should convert 10.5 to Float64.
-        value_converted = convertFieldToType(value, DataTypeFloat64());
-    else
-        /// We should convert value to the real column data type and then translate it to Float64.
-        /// For example for expression col_date > '2024-08-07', if we directly convert '2024-08-07' to Float64, we will get null.
-        value_converted = convertFieldToType(value, *data_type);
+    auto column = data_type->createColumn();
+    column->insert(value);
+    ColumnsWithTypeAndName arguments({ColumnWithTypeAndName(std::move(column), data_type, "stats_const")});
 
-    if (value_converted.isNull())
-        return {};
-
-    return applyVisitor(FieldVisitorConvertToNumber<Float64>(), value_converted);
+    auto cast_resolver = FunctionFactory::instance().get("toFloat64", nullptr);
+    auto cast_function = cast_resolver->build(arguments);
+    ColumnPtr result = cast_function->execute(arguments, std::make_shared<DataTypeFloat64>(), 1, false);
+    return result->getFloat64(0);
 }
 
 IStatistics::IStatistics(const SingleStatisticsDescription & stat_)

src/Storages/Statistics/StatisticsCountMinSketch.cpp

@@ -38,7 +38,7 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
     /// For example: if data_type is Int32:
     ///     1. For 1.0, 1, '1', return Field(1)
     ///     2. For 1.1, max_value_int64, return null
-    Field val_converted = convertFieldToType(val, *data_type);
+    Field val_converted = convertFieldToType(val, *data_type, data_type.get());
     if (val_converted.isNull())
         return 0;
 

tests/queries/0_stateless/01763_filter_push_down_bugs.sql

@@ -1,3 +1,4 @@
+SET allow_statistics_optimize = 0;
 SELECT * FROM (SELECT col1, col2 FROM (select '1' as col1, '2' as col2) GROUP by col1, col2) AS expr_qry WHERE col2 != '';
 SELECT * FROM (SELECT materialize('1') AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2';
 SELECT * FROM (SELECT materialize([1]) AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2';

tests/queries/0_stateless/02842_move_pk_to_end_of_prewhere.sql

@@ -1,6 +1,7 @@
 SET optimize_move_to_prewhere = 1;
 SET enable_multiple_prewhere_read_steps = 1;
 SET optimize_functions_to_subcolumns = 0;
+SET allow_statistics_optimize = 0;
 
 DROP TABLE IF EXISTS t_02848_mt1;
 DROP TABLE IF EXISTS t_02848_mt2;

tests/queries/0_stateless/03279_join_choose_build_table.sql

@@ -1,4 +1,4 @@
-
+SET allow_statistics_optimize = 0;
 SET use_skip_indexes_on_data_read = 0; -- for correct row count estimation in join order planning
 DROP TABLE IF EXISTS products;
 DROP TABLE IF EXISTS sales;

tests/queries/0_stateless/03357_join_pk_sharding.sql

@@ -1,5 +1,6 @@
 -- Tags: long
 
+SET allow_statistics_optimize = 0;
 drop table if exists tab_l;
 drop table if exists tab_m;
 drop table if exists tab_r;

tests/queries/0_stateless/03591_optimize_prewhere_row_policy.reference

@@ -2,6 +2,7 @@
 
 SET use_query_condition_cache = 0;
 SET enable_parallel_replicas = 0;
+SET allow_statistics_optimize = 0;
 DROP TABLE IF EXISTS 03591_test;
 DROP ROW POLICY IF EXISTS 03591_rp ON 03591_test;
 CREATE TABLE 03591_test (a Int32, b Int32) ENGINE=MergeTree ORDER BY tuple();