diff --git a/executor/builder.go b/executor/builder.go index 7f7921a23119..48dafed8d9e3 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -84,6 +84,8 @@ func (b *executorBuilder) build(p plannercore.Plan) Executor { return b.buildCheckIndexRange(v) case *plannercore.ChecksumTable: return b.buildChecksumTable(v) + case *plannercore.ReloadExprPushdownBlacklist: + return b.buildReloadExprPushdownBlacklist(v) case *plannercore.DDL: return b.buildDDL(v) case *plannercore.Deallocate: @@ -426,6 +428,10 @@ func (b *executorBuilder) buildChecksumTable(v *plannercore.ChecksumTable) Execu return e } +func (b *executorBuilder) buildReloadExprPushdownBlacklist(v *plannercore.ReloadExprPushdownBlacklist) Executor { + return &ReloadExprPushdownBlacklistExec{baseExecutor{ctx: b.ctx}} +} + func (b *executorBuilder) buildDeallocate(v *plannercore.Deallocate) Executor { base := newBaseExecutor(b.ctx, nil, v.ExplainID()) base.initCap = chunk.ZeroCapacity diff --git a/executor/reload_expr_pushdown_blacklist.go b/executor/reload_expr_pushdown_blacklist.go new file mode 100644 index 000000000000..e80a9ea9cf65 --- /dev/null +++ b/executor/reload_expr_pushdown_blacklist.go @@ -0,0 +1,50 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor + +import ( + "context" + "strings" + + "github.com/pingcap/tidb/expression" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tidb/util/sqlexec" +) + +// ReloadExprPushdownBlacklistExec indicates ReloadExprPushdownBlacklist executor. +type ReloadExprPushdownBlacklistExec struct { + baseExecutor +} + +// Next implements the Executor Next interface. +func (e *ReloadExprPushdownBlacklistExec) Next(ctx context.Context, _ *chunk.Chunk) error { + return LoadExprPushdownBlacklist(e.ctx) +} + +// LoadExprPushdownBlacklist loads the latest data from table mysql.expr_pushdown_blacklist. +func LoadExprPushdownBlacklist(ctx sessionctx.Context) (err error) { + sql := "select HIGH_PRIORITY name from mysql.expr_pushdown_blacklist" + rows, _, err := ctx.(sqlexec.RestrictedSQLExecutor).ExecRestrictedSQL(ctx, sql) + if err != nil { + return err + } + newBlacklist := make(map[string]struct{}) + for _, row := range rows { + name := row.GetString(0) + newBlacklist[strings.ToLower(name)] = struct{}{} + } + expression.DefaultExprPushdownBlacklist.Store(newBlacklist) + return nil +} diff --git a/executor/reload_expr_pushdown_blacklist_test.go b/executor/reload_expr_pushdown_blacklist_test.go new file mode 100644 index 000000000000..10092c37334b --- /dev/null +++ b/executor/reload_expr_pushdown_blacklist_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package executor_test + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/tidb/util/testkit" +) + +func (s *testSuite) TestReloadExprPushdownBlacklist(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("create database expr_pushdown_blacklist") + tk.MustExec("use expr_pushdown_blacklist") + tk.MustExec("create table t (a int)") + tk.MustQuery("desc select * from t where a < 1").Check(testkit.Rows( + "TableReader_7 3323.33 root data:Selection_6", + "└─Selection_6 3323.33 cop lt(expr_pushdown_blacklist.t.a, 1)", + " └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) + + tk.MustExec("insert into mysql.expr_pushdown_blacklist values('lt')") + tk.MustQuery("desc select * from t where a < 1").Check(testkit.Rows( + "TableReader_7 3323.33 root data:Selection_6", + "└─Selection_6 3323.33 cop lt(expr_pushdown_blacklist.t.a, 1)", + " └─TableScan_5 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) + + tk.MustExec("admin reload expr_pushdown_blacklist") + tk.MustQuery("desc select * from t where a < 1").Check(testkit.Rows( + "Selection_5 8000.00 root lt(expr_pushdown_blacklist.t.a, 1)", + "└─TableReader_7 10000.00 root data:TableScan_6", + " └─TableScan_6 10000.00 cop table:t, range:[-inf,+inf], keep order:false, stats:pseudo")) +} diff --git a/expression/expr_to_pb.go b/expression/expr_to_pb.go index 4416b464c1f4..a863f28ba47b 100644 --- a/expression/expr_to_pb.go +++ b/expression/expr_to_pb.go @@ -15,6 +15,7 @@ package expression import ( "context" + "sync/atomic" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/charset" @@ -321,8 +322,16 @@ func (pc PbConverter) canFuncBePushed(sf *ScalarFunction) bool { // date functions. ast.DateFormat: - - return true + _, disallowPushdown := DefaultExprPushdownBlacklist.Load().(map[string]struct{})[sf.FuncName.L] + return true && !disallowPushdown } return false } + +// DefaultExprPushdownBlacklist indicates the expressions which can not be pushed down to TiKV. +var DefaultExprPushdownBlacklist *atomic.Value + +func init() { + DefaultExprPushdownBlacklist = new(atomic.Value) + DefaultExprPushdownBlacklist.Store(make(map[string]struct{})) +} diff --git a/expression/integration_test.go b/expression/integration_test.go index 564bceb032bc..067a25c0b4bb 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -4155,3 +4155,8 @@ func (s *testIntegrationSuite) TestDaynameArithmetic(c *C) { tk.MustQuery(c.sql).Check(testkit.Rows(c.result)) } } + +func (s *testIntegrationSuite) TestExprPushdownBlacklist(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustQuery(`select * from mysql.expr_pushdown_blacklist`).Check(testkit.Rows()) +} diff --git a/go.mod b/go.mod index c737f3a0a993..00f2e82b8e51 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/pingcap/goleveldb v0.0.0-20171020084629-8d44bfdf1030 github.com/pingcap/kvproto v0.0.0-20190429124202-32a5ba2af0f7 github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 - github.com/pingcap/parser v0.0.0-20190701060323-a2aa507d6352 + github.com/pingcap/parser v0.0.0-20190703021824-628683061f71 github.com/pingcap/pd v2.1.12+incompatible github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 diff --git a/go.sum b/go.sum index 7ec2eed7b7f4..9f45eec77ba5 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/pingcap/kvproto v0.0.0-20190429124202-32a5ba2af0f7 h1:+wEqJTc74Jvoxen github.com/pingcap/kvproto v0.0.0-20190429124202-32a5ba2af0f7/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596 h1:t2OQTpPJnrPDGlvA+3FwJptMTt6MEPdzK1Wt99oaefQ= github.com/pingcap/log v0.0.0-20190307075452-bd41d9273596/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= -github.com/pingcap/parser v0.0.0-20190701060323-a2aa507d6352 h1:bqNncrTvLJyxLx8rHr5tDe4wYhfVFoegj+LrOoFwwuM= -github.com/pingcap/parser v0.0.0-20190701060323-a2aa507d6352/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20190703021824-628683061f71 h1:ErX0Ww42LBHTSlK2Q8f8/86CUEg+dUpeyYbMt0iNxkg= +github.com/pingcap/parser v0.0.0-20190703021824-628683061f71/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.12+incompatible h1:6N3LBxx2aSZqT+IWEG730EDNDttP7dXO8J6yvBh+HXw= github.com/pingcap/pd v2.1.12+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible h1:e9Gi/LP9181HT3gBfSOeSBA+5JfemuE4aEAhqNgoE4k= diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 19f8be2e454b..e79b0a8ec656 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -126,6 +126,11 @@ type CancelDDLJobs struct { JobIDs []int64 } +// ReloadExprPushdownBlacklist reloads the data from expr_pushdown_blacklist table. +type ReloadExprPushdownBlacklist struct { + baseSchemaProducer +} + // Prepare represents prepare plan. type Prepare struct { baseSchemaProducer diff --git a/planner/core/planbuilder.go b/planner/core/planbuilder.go index a39aae71b63a..4e0f9106a50f 100644 --- a/planner/core/planbuilder.go +++ b/planner/core/planbuilder.go @@ -581,6 +581,8 @@ func (b *planBuilder) buildAdmin(as *ast.AdminStmt) (Plan, error) { p := &ShowSlow{ShowSlow: as.ShowSlow} p.SetSchema(buildShowSlowSchema()) ret = p + case ast.AdminReloadExprPushdownBlacklist: + return &ReloadExprPushdownBlacklist{}, nil default: return nil, ErrUnsupportedType.GenWithStack("Unsupported ast.AdminStmt(%T) for buildAdmin", as) } diff --git a/session/bootstrap.go b/session/bootstrap.go index 333b8238d00d..fd093f90de1f 100644 --- a/session/bootstrap.go +++ b/session/bootstrap.go @@ -211,6 +211,11 @@ const ( feedback blob NOT NULL, index hist(table_id, is_index, hist_id) );` + + // CreateExprPushdownBlacklist stores the expressions which are not allowed to be pushed down. + CreateExprPushdownBlacklist = `CREATE TABLE IF NOT EXISTS mysql.expr_pushdown_blacklist ( + name char(100) NOT NULL + );` ) // bootstrap initiates system DB for a store. @@ -280,6 +285,7 @@ const ( version22 = 22 version23 = 23 version24 = 24 + version25 = 25 ) func checkBootstrapped(s Session) (bool, error) { @@ -435,6 +441,10 @@ func upgrade(s Session) { upgradeToVer24(s) } + if ver < version25 { + upgradeToVer25(s) + } + updateBootstrapVer(s) _, err = s.Execute(context.Background(), "COMMIT") @@ -694,6 +704,10 @@ func upgradeToVer24(s Session) { writeSystemTZ(s) } +func upgradeToVer25(s Session) { + doReentrantDDL(s, CreateExprPushdownBlacklist) +} + // updateBootstrapVer updates bootstrap version variable in mysql.TiDB table. func updateBootstrapVer(s Session) { // Update bootstrap version. @@ -744,6 +758,8 @@ func doDDLWorks(s Session) { mustExecute(s, CreateGCDeleteRangeDoneTable) // Create stats_feedback table. mustExecute(s, CreateStatsFeedbackTable) + // Create expr_pushdown_blacklist table. + mustExecute(s, CreateExprPushdownBlacklist) } // doDMLWorks executes DML statements in bootstrap stage. diff --git a/session/session.go b/session/session.go index d5e58aaee9bf..0d5822bb042c 100644 --- a/session/session.go +++ b/session/session.go @@ -1268,6 +1268,11 @@ func BootstrapSession(store kv.Storage) (*domain.Domain, error) { } } + err = executor.LoadExprPushdownBlacklist(se) + if err != nil { + return nil, err + } + se1, err := createSession(store) if err != nil { return nil, errors.Trace(err) @@ -1362,7 +1367,7 @@ func createSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, er const ( notBootstrapped = 0 - currentBootstrapVersion = 24 + currentBootstrapVersion = 25 ) func getStoreBootstrapVersion(store kv.Storage) int64 {