*: Keep the auto id allocator for single table renames | tidb-test=pr/2230 (#47892)

ddl/table.go

@@ -798,15 +798,21 @@ func onRenameTables(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error
 		return ver, errors.Trace(err)
 	}
 
+<<<<<<< HEAD:ddl/table.go
 	var tblInfos = make([]*model.TableInfo, 0, len(tableNames))
+=======
+	if job.SchemaState == model.StatePublic {
+		return finishJobRenameTables(d, t, job, tableNames, tableIDs, newSchemaIDs)
+	}
+
+>>>>>>> a3e2ddb5864 (*: Keep the auto id allocator for single table renames (#47892)):pkg/ddl/table.go
 	var err error
 	for i, oldSchemaID := range oldSchemaIDs {
 		job.TableID = tableIDs[i]
 		ver, tblInfo, err := checkAndRenameTables(t, job, oldSchemaID, newSchemaIDs[i], oldSchemaNames[i], tableNames[i])
 		if err != nil {
 			return ver, errors.Trace(err)
 		}
-		tblInfos = append(tblInfos, tblInfo)
 	}
 
 	ver, err = updateSchemaVersion(t, job)
@@ -845,13 +851,26 @@ func checkAndRenameTables(t *meta.Meta, job *model.Job, oldSchemaID, newSchemaID
 		return ver, tblInfo, errors.Wrapf(err, "failed to get old label rules from PD")
 	}
 
+	if tblInfo.AutoIDSchemaID == 0 && newSchemaID != oldSchemaID {
+		// The auto id is referenced by a schema id + table id
+		// Table ID is not changed between renames, but schema id can change.
+		// To allow concurrent use of the auto id during rename, keep the auto id
+		// by always reference it with the schema id it was originally created in.
+		tblInfo.AutoIDSchemaID = oldSchemaID
+	}
+	if newSchemaID == tblInfo.AutoIDSchemaID {
+		// Back to the original schema id, no longer needed.
+		tblInfo.AutoIDSchemaID = 0
+	}
+
 	tblInfo.Name = *tableName
 	err = t.CreateTableOrView(newSchemaID, tblInfo)
 	if err != nil {
 		job.State = model.JobStateCancelled
 		return ver, tblInfo, errors.Trace(err)
 	}
 
+<<<<<<< HEAD:ddl/table.go
 	if newSchemaID != oldSchemaID {
 		oldDBID := tblInfo.GetDBID(oldSchemaID)
 		err := meta.BackupAndRestoreAutoIDs(t, oldDBID, tblInfo.ID, newSchemaID, tblInfo.ID)
@@ -864,6 +883,8 @@ func checkAndRenameTables(t *meta.Meta, job *model.Job, oldSchemaID, newSchemaID
 		tblInfo.OldSchemaID = 0
 	}
 
+=======
+>>>>>>> a3e2ddb5864 (*: Keep the auto id allocator for single table renames (#47892)):pkg/ddl/table.go
 	err = updateLabelRules(job, tblInfo, oldRules, tableRuleID, partRuleIDs, oldRuleIDs, tblInfo.ID)
 	if err != nil {
 		job.State = model.JobStateCancelled

executor/seqtest/seq_executor_test.go

@@ -809,10 +809,19 @@ func HelperTestAdminShowNextID(t *testing.T, store kv.Storage, str string) {
 	tk.MustExec("rename table test.tt to test1.tt")
 	tk.MustExec("use test1")
 	r = tk.MustQuery(str + " tt next_row_id")
+<<<<<<< HEAD:executor/seqtest/seq_executor_test.go
 	r.Check(testkit.Rows("test1 tt id 31 AUTO_INCREMENT"))
 	tk.MustExec("insert test1.tt values ()")
 	r = tk.MustQuery(str + " tt next_row_id")
 	r.Check(testkit.Rows("test1 tt id 41 AUTO_INCREMENT"))
+=======
+	r.Check(testkit.Rows("test1 tt id 31 _TIDB_ROWID", "test1 tt id 1 AUTO_INCREMENT"))
+	tk.MustQuery(`select * from tt`).Sort().Check(testkit.Rows("20 1"))
+	tk.MustExec("insert test1.tt values ()")
+	r = tk.MustQuery(str + " tt next_row_id")
+	r.Check(testkit.Rows("test1 tt id 31 _TIDB_ROWID", "test1 tt id 1 AUTO_INCREMENT"))
+	tk.MustQuery(`select * from tt`).Sort().Check(testkit.Rows("20 1", "21 <nil>"))
+>>>>>>> a3e2ddb5864 (*: Keep the auto id allocator for single table renames (#47892)):pkg/executor/test/seqtest/seq_executor_test.go
 	tk.MustExec("drop table tt")
 
 	tk.MustExec("set @@allow_auto_random_explicit_insert = true")

infoschema/builder.go

@@ -213,15 +213,24 @@ func (b *Builder) applyTableUpdate(m *meta.Meta, diff *model.SchemaDiff) ([]int6
 	// We try to reuse the old allocator, so the cached auto ID can be reused.
 	var allocs autoid.Allocators
 	if tableIDIsValid(oldTableID) {
+<<<<<<< HEAD:infoschema/builder.go
 		if oldTableID == newTableID && (diff.Type != model.ActionRenameTable && diff.Type != model.ActionRenameTables) &&
 			diff.Type != model.ActionExchangeTablePartition &&
+=======
+		if oldTableID == newTableID &&
+			// For rename table, keep the old alloc.
+
+>>>>>>> a3e2ddb5864 (*: Keep the auto id allocator for single table renames (#47892)):pkg/infoschema/builder.go
 			// For repairing table in TiDB cluster, given 2 normal node and 1 repair node.
 			// For normal node's information schema, repaired table is existed.
 			// For repair node's information schema, repaired table is filtered (couldn't find it in `is`).
 			// So here skip to reserve the allocators when repairing table.
 			diff.Type != model.ActionRepairTable &&
 			// Alter sequence will change the sequence info in the allocator, so the old allocator is not valid any more.
 			diff.Type != model.ActionAlterSequence {
+			// TODO: Check how this would work with ADD/REMOVE Partitioning,
+			// which may have AutoID not connected to tableID
+			// TODO: can there be _tidb_rowid AutoID per partition?
 			oldAllocs, _ := b.is.AllocByID(oldTableID)
 			allocs = filterAllocators(diff, oldAllocs)
 		}

meta/autoid/autoid.go

@@ -198,7 +198,7 @@ type allocator struct {
 	base  int64
 	end   int64
 	store kv.Storage
-	// dbID is current database's ID.
+	// dbID is database ID where it was created.
 	dbID          int64
 	tbID          int64
 	tbVersion     uint16
@@ -525,7 +525,7 @@ func NewSequenceAllocator(store kv.Storage, dbID, tbID int64, info *model.Sequen
 // NewAllocatorsFromTblInfo creates an array of allocators of different types with the information of model.TableInfo.
 func NewAllocatorsFromTblInfo(store kv.Storage, schemaID int64, tblInfo *model.TableInfo) Allocators {
 	var allocs []Allocator
-	dbID := tblInfo.GetDBID(schemaID)
+	dbID := tblInfo.GetAutoIDSchemaID(schemaID)
 	idCacheOpt := CustomAutoIncCacheOption(tblInfo.AutoIdCache)
 	tblVer := AllocOptionTableInfoVersion(tblInfo.Version)
 

meta/meta_autoid.go

@@ -54,17 +54,17 @@ func (a *autoIDAccessor) Put(val int64) error {
 // Inc implements the interface AutoIDAccessor.
 func (a *autoIDAccessor) Inc(step int64) (int64, error) {
 	m := a.m
+	// Note that the databaseID may not match the current table,
+	// it may come from the original schema id the table was created
+	// in, but to allow concurrent use across renames etc. we keep
+	// the full ID (Schema ID + Table ID) as is.
+	// Meaning we cannot verify only the schema id.
+	// And a rename may have happened before the first id is set,
+	// as well as dropping the original schema.
+	// So no Schema ID or Table ID verifications can be done.
 	dbKey := m.dbKey(a.databaseID)
-	if err := m.checkDBExists(dbKey); err != nil {
-		return 0, errors.Trace(err)
-	}
-	// Check if table exists.
-	tableKey := m.tableKey(a.tableID)
-	if err := m.checkTableExists(dbKey, tableKey); err != nil {
-		return 0, errors.Trace(err)
-	}
-
-	return m.txn.HInc(dbKey, a.idEncodeFn(a.tableID), step)
+	tblKey := a.idEncodeFn(a.tableID)
+	return m.txn.HInc(dbKey, tblKey, step)
 }
 
 // Del implements the interface AutoIDAccessor.
@@ -192,25 +192,3 @@ type AutoIDGroup struct {
 	IncrementID int64
 	RandomID    int64
 }
-
-// BackupAndRestoreAutoIDs changes the meta key-values to fetch & delete
-// all the auto IDs from an old table, and set them to a new table.
-func BackupAndRestoreAutoIDs(m *Meta, databaseID, tableID int64, newDatabaseID, newTableID int64) (err error) {
-	acc := NewAutoIDAccessors(m, databaseID, tableID)
-	autoIDs, err := acc.Get()
-	if err != nil {
-		return errors.Trace(err)
-	}
-	overwriteIDs := databaseID == newDatabaseID && tableID == newTableID
-	if !overwriteIDs {
-		err = acc.Del()
-		if err != nil {
-			return errors.Trace(err)
-		}
-	}
-	err = NewAutoIDAccessors(m, newDatabaseID, newTableID).Put(autoIDs)
-	if err != nil {
-		return errors.Trace(err)
-	}
-	return nil
-}

meta/meta_test.go

@@ -102,16 +102,26 @@ func TestPlacementPolicy(t *testing.T) {
 	require.NoError(t, err)
 }
 
+<<<<<<< HEAD:meta/meta_test.go
 func TestBackupAndRestoreAutoIDs(t *testing.T) {
 	store, err := mockstore.NewMockStore()
 	require.NoError(t, err)
 	defer func() {
 		err := store.Close()
 		require.NoError(t, err)
+=======
+func TestResourceGroup(t *testing.T) {
+	store, err := mockstore.NewMockStore()
+	require.NoError(t, err)
+
+	defer func() {
+		require.NoError(t, store.Close())
+>>>>>>> a3e2ddb5864 (*: Keep the auto id allocator for single table renames (#47892)):pkg/meta/meta_test.go
 	}()
 
 	txn, err := store.Begin()
 	require.NoError(t, err)
+<<<<<<< HEAD:meta/meta_test.go
 	m := meta.NewMeta(txn)
 	acc := m.GetAutoIDAccessors(1, 1)
 	require.NoError(t, acc.RowID().Put(100))
@@ -147,6 +157,44 @@ func TestBackupAndRestoreAutoIDs(t *testing.T) {
 	acc2 = m.GetAutoIDAccessors(2, 2)
 	require.Equal(t, mustGet(acc2.RowID()), 100)
 	require.Equal(t, mustGet(acc2.RandomID()), 101)
+=======
+
+	// test the independent policy ID allocation.
+	m := meta.NewMeta(txn)
+	groups, err := m.ListResourceGroups()
+	require.NoError(t, err)
+	require.Equal(t, len(groups), 1)
+	require.Equal(t, groups[0], meta.DefaultGroupMeta4Test())
+
+	groupID := int64(2)
+	checkResourceGroup := func(ru uint64) {
+		rg, err := m.GetResourceGroup(groupID)
+		require.NoError(t, err)
+		require.Equal(t, rg.RURate, ru)
+	}
+
+	rg := &model.ResourceGroupInfo{
+		ID:   groupID,
+		Name: model.NewCIStr("aa"),
+		ResourceGroupSettings: &model.ResourceGroupSettings{
+			RURate: 100,
+		},
+	}
+	require.NoError(t, m.AddResourceGroup(rg))
+	checkResourceGroup(100)
+
+	groups, err = m.ListResourceGroups()
+	require.NoError(t, err)
+	require.Equal(t, len(groups), 2)
+
+	rg.RURate = 200
+	require.NoError(t, m.UpdateResourceGroup(rg))
+	checkResourceGroup(200)
+
+	m.DropResourceGroup(groupID)
+	_, err = m.GetResourceGroup(groupID)
+	require.Error(t, err)
+>>>>>>> a3e2ddb5864 (*: Keep the auto id allocator for single table renames (#47892)):pkg/meta/meta_test.go
 }
 
 func TestMeta(t *testing.T) {
@@ -305,18 +353,20 @@ func TestMeta(t *testing.T) {
 	n, err = m.GetAutoIDAccessors(currentDBID, tid).RowID().Inc(10)
 	require.NoError(t, err)
 	require.Equal(t, int64(10), n)
-	// Fail to update auto ID.
+	// Test to update non-existing auto ID.
 	// The table ID doesn't exist.
+	// We can no longer test for non-existing ids.
 	nonExistentID := int64(1234)
 	_, err = m.GetAutoIDAccessors(currentDBID, nonExistentID).RowID().Inc(10)
-	require.NotNil(t, err)
-	require.True(t, meta.ErrTableNotExists.Equal(err))
-	// Fail to update auto ID.
+	require.NoError(t, err)
+	//require.True(t, meta.ErrTableNotExists.Equal(err))
+	// Test to update non-existing auto ID.
 	// The current database ID doesn't exist.
+	// We can no longer test for non-existing ids.
 	currentDBID = nonExistentID
 	_, err = m.GetAutoIDAccessors(currentDBID, tid).RowID().Inc(10)
-	require.NotNil(t, err)
-	require.True(t, meta.ErrDBNotExists.Equal(err))
+	require.NoError(t, err)
+	//require.True(t, meta.ErrDBNotExists.Equal(err))
 	// Test case for CreateTableAndSetAutoID.
 	tbInfo3 := &model.TableInfo{
 		ID:   3,

parser/model/model.go

@@ -311,9 +311,9 @@ type TableInfo struct {
 	// Because auto increment ID has schemaID as prefix,
 	// We need to save original schemaID to keep autoID unchanged
 	// while renaming a table from one database to another.
-	// TODO: Remove it.
-	// Now it only uses for compatibility with the old version that already uses this field.
-	OldSchemaID int64 `json:"old_schema_id,omitempty"`
+	// Only set if table has been renamed across schemas
+	// Old name 'old_schema_id' is kept for backwards compatibility
+	AutoIDSchemaID int64 `json:"old_schema_id,omitempty"`
 
 	// ShardRowIDBits specify if the implicit row ID is sharded.
 	ShardRowIDBits uint64
@@ -516,11 +516,10 @@ func (t *TableInfo) GetUpdateTime() time.Time {
 	return TSConvert2Time(t.UpdateTS)
 }
 
-// GetDBID returns the schema ID that is used to create an allocator.
-// TODO: Remove it after removing OldSchemaID.
-func (t *TableInfo) GetDBID(dbID int64) int64 {
-	if t.OldSchemaID != 0 {
-		return t.OldSchemaID
+// GetAutoIDSchemaID returns the schema ID that was used to create an allocator.
+func (t *TableInfo) GetAutoIDSchemaID(dbID int64) int64 {
+	if t.AutoIDSchemaID != 0 {
+		return t.AutoIDSchemaID
 	}
 	return dbID
 }