[#19767] DocDB: Add Ysql syntax support for DB Cloning

Summary: The diff adds the YSQL syntax to support DB cloning: creating a database as a lightweight copy of another database. User can now create a database `db2` as a clone of database `db1` as of a point in time `t1`, by writing the following sql query: `CREATE DATABASE db2 TEMPLATE db1 AS OF t1;` where t1 is a linux timestamp in microseconds. Using the template syntax to support DB cloning is in line with postgres [[ https://www.postgresql.org/docs/current/manage-ag-templatedbs.html#MANAGE-AG-TEMPLATEDBS | intention ]] to support copying databases. Before this diff, it was not supported to use the TEMPLATE keyword with any database other than `template0` and `template1`. Now we removed this limitation. Yugabyte can also perform the cloning without the strict condition inherited from postgres: `no other sessions can be connected to the source database while it is being copied`. So removed this limitation in this diff and we can copy the database even if there are other connections to the source database. **Upgrade/Rollback safety:** The clone feature is guarded by the preview flag `enable_db_clone `. Jira: DB-8620 Test Plan: ybd --cxx-test integration-tests_minicluster-snapshot-test --gtest_filter PgCloneTest.CloneYsqlSyntax Removed `CREATE DATABASE test TEMPLATE = some_template;` from `CREATE DATABASE` unsupported options in regress tests as now we support copying databases from templates other than `template0` or `template1`. The same statement cannot be added to the regress test as supported because we require having a snapshot schedule on the template database as a requirement for copying the database. However, the unit test `PgCloneTest.CloneYsqlSyntax` is covering this case. Reviewers: asrivastava, yguan Reviewed By: asrivastava, yguan Subscribers: pjain, yql, ybase Differential Revision: https://phorge.dev.yugabyte.com/D33408
yugabyte · May 3, 2024 · cd5b86e · cd5b86e
1 parent 862a997
commit cd5b86e
Show file tree

Hide file tree

Showing 27 changed files with 332 additions and 130 deletions.
diff --git a/src/postgres/src/backend/bootstrap/bootstrap.c b/src/postgres/src/backend/bootstrap/bootstrap.c
@@ -525,9 +525,11 @@ BootstrapModeMain(void)
 		YBCCreateDatabase(TemplateDbOid,
 		                  "template1",
 		                  InvalidOid,
+		                  "template0",  
 		                  FirstBootstrapObjectId,
 		                  false /* colocated */,
-		                  NULL /* retry_on_oid_collision */);
+		                  NULL /* retry_on_oid_collision */,
+		                  0 /* clone_time */);
 	}
 
 	/*

diff --git a/src/postgres/src/backend/commands/dbcommands.c b/src/postgres/src/backend/commands/dbcommands.c
@@ -145,6 +145,7 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 	DefElem    *dencoding = NULL;
 	DefElem    *dcollate = NULL;
 	DefElem    *dcolocated = NULL;
+	DefElem	   *dclonetime = NULL;
 	DefElem    *dctype = NULL;
 	DefElem    *distemplate = NULL;
 	DefElem    *dallowconnections = NULL;
@@ -163,6 +164,7 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 	int			dbconnlimit = -1;
 	int			notherbackends;
 	int			npreparedxacts;
+	int64		dbclonetime = 0;
 	createdb_failure_params fparms;
 
 	/*
@@ -278,6 +280,15 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 						 parser_errposition(pstate, defel->location)));
 			dcolocated = defel;
 		}
+		else if (strcmp(defel->defname, "clone_time") == 0)
+		{
+			if (dclonetime)
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("conflicting or redundant options"),
+						 parser_errposition(pstate, defel->location)));
+			dclonetime = defel;
+		}
 		else
 			ereport(ERROR,
 					(errcode(ERRCODE_SYNTAX_ERROR),
@@ -337,7 +348,8 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 		dbcolocated = defGetBoolean(dcolocated);
 	else
 		dbcolocated = YBColocateDatabaseByDefault();
-
+	if (dclonetime && dclonetime->arg)
+		dbclonetime = defGetInt64(dclonetime);
 	/* obtain OID of proposed owner */
 	if (dbowner)
 		datdba = get_role_oid(dbowner, false);
@@ -387,16 +399,6 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 						 parser_errposition(pstate, option->location)));
 		}
 
-		if (strcmp(dbtemplate, "template0") != 0 &&
-			strcmp(dbtemplate, "template1") != 0)
-			ereport(YBUnsupportedFeatureSignalLevel(),
-					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-					 errmsg("Value other than default, template0 or template1 "
-							"for template option is not yet supported"),
-					 errhint("Please report the issue on "
-							 "https://github.com/YugaByte/yugabyte-db/issues"),
-					 parser_errposition(pstate, dtemplate->location)));
-
 		if (dbistemplate)
 			ereport(YBUnsupportedFeatureSignalLevel(),
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -590,21 +592,22 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 				(errcode(ERRCODE_DUPLICATE_DATABASE),
 				 errmsg("database \"%s\" already exists", dbname)));
 
-	/*
-	 * The source DB can't have any active backends, except this one
-	 * (exception is to allow CREATE DB while connected to template1).
-	 * Otherwise we might copy inconsistent data.
-	 *
-	 * This should be last among the basic error checks, because it involves
-	 * potential waiting; we may as well throw an error first if we're gonna
-	 * throw one.
-	 */
-	if (CountOtherDBBackends(src_dboid, &notherbackends, &npreparedxacts))
-		ereport(ERROR,
-				(errcode(ERRCODE_OBJECT_IN_USE),
-				 errmsg("source database \"%s\" is being accessed by other users",
-						dbtemplate),
-				 errdetail_busy_db(notherbackends, npreparedxacts)));
+	if (!IsYugaByteEnabled())
+		/*
+		* The source DB can't have any active backends, except this one
+		* (exception is to allow CREATE DB while connected to template1).
+		* Otherwise we might copy inconsistent data.
+		*
+		* This should be last among the basic error checks, because it involves
+		* potential waiting; we may as well throw an error first if we're gonna
+		* throw one.
+		*/
+		if (CountOtherDBBackends(src_dboid, &notherbackends, &npreparedxacts))
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_IN_USE),
+					errmsg("source database \"%s\" is being accessed by other users",
+							dbtemplate),
+					errdetail_busy_db(notherbackends, npreparedxacts)));
 
 	/*
 	 * Select an OID for the new database, checking that it doesn't have a
@@ -632,10 +635,26 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt)
 
 		retry_on_oid_collision = false;
 		if (IsYugaByteEnabled())
-			YBCCreateDatabase(dboid, dbname, src_dboid, InvalidOid, dbcolocated,
-							  &retry_on_oid_collision);
+			YBCCreateDatabase(dboid, dbname, src_dboid, dbtemplate, InvalidOid, dbcolocated,
+							  &retry_on_oid_collision, dbclonetime);
 	} while (retry_on_oid_collision);
 
+	/*
+	 * CREATE DATABASE using templates other than template0 and template1 will 
+	 * always go through the DB clone workflow.
+	 * A database created using the clone workflow already has an entry in
+	 * pg_database as it is created by executing ysql_dump script.
+	 * Thus, close pg_database relation and return the dboid in case of clone.
+	 */
+	if (strcmp(dbtemplate, "template0") != 0 &&
+		strcmp(dbtemplate, "template1") != 0)
+	{
+		heap_close(pg_database_rel, RowExclusiveLock);
+		// TODO(yamen): return the correct target dboid from the clone namespace.
+		// It is fine to return InvalidOid temporarely as it isn't used anywhere.
+		return InvalidOid;
+	}
+
 	/*
 	 * Insert a new tuple into pg_database.  This establishes our ownership of
 	 * the new database name (anyone else trying to insert the same name will

diff --git a/src/postgres/src/backend/commands/ybccmds.c b/src/postgres/src/backend/commands/ybccmds.c
@@ -102,8 +102,8 @@ ColumnSortingOptions(SortByDir dir, SortByNulls nulls, bool* is_desc, bool* is_n
 /*  Database Functions. */
 
 void
-YBCCreateDatabase(Oid dboid, const char *dbname, Oid src_dboid, Oid next_oid, bool colocated,
-				  bool *retry_on_oid_collision)
+YBCCreateDatabase(Oid dboid, const char *dbname, Oid src_dboid, const char *src_dbname, Oid next_oid, bool colocated,
+				  bool *retry_on_oid_collision, int64 clone_time)
 {
 	if (YBIsDBCatalogVersionMode())
 	{
@@ -125,8 +125,10 @@ YBCCreateDatabase(Oid dboid, const char *dbname, Oid src_dboid, Oid next_oid, bo
 	HandleYBStatus(YBCPgNewCreateDatabase(dbname,
 										  dboid,
 										  src_dboid,
+										  src_dbname,
 										  next_oid,
 										  colocated,
+										  clone_time,
 										  &handle));
 
 	YBCStatus createdb_status = YBCPgExecCreateDatabase(handle);

diff --git a/src/postgres/src/backend/parser/gram.y b/src/postgres/src/backend/parser/gram.y
@@ -11071,6 +11071,10 @@ createdb_opt_item:
 				{
 					$$ = makeDefElem($1, (Node *)makeString($3), @1);
 				}
+			| createdb_opt_name opt_equal FCONST
+				{
+					$$ = makeDefElem($1, (Node *)makeFloat($3), @1);
+				}
 			| createdb_opt_name opt_equal DEFAULT
 				{
 					$$ = makeDefElem($1, NULL, @1);
@@ -11106,6 +11110,7 @@ createdb_opt_name:
 					$$ = pstrdup($1);
 				}
 			| COLOCATION				    { $$ = pstrdup($1); }
+			| AS OF							{ $$ = pstrdup("clone_time"); }
 		;
 
 /*

diff --git a/src/postgres/src/include/commands/ybccmds.h b/src/postgres/src/include/commands/ybccmds.h
@@ -37,8 +37,8 @@
 /*  Database Functions -------------------------------------------------------------------------- */
 
 extern void YBCCreateDatabase(
-	Oid dboid, const char *dbname, Oid src_dboid, Oid next_oid, bool colocated,
-	bool *retry_on_oid_collision);
+	Oid dboid, const char *dbname, Oid src_dboid, const char *src_dbname, Oid next_oid,
+	bool colocated, bool *retry_on_oid_collision, int64 clone_time);
 
 extern void YBCDropDatabase(Oid dboid, const char *dbname);
 

diff --git a/src/postgres/src/test/regress/expected/yb_feature_db.out b/src/postgres/src/test/regress/expected/yb_feature_db.out
@@ -1,11 +1,6 @@
 --
 -- Test case for CREATE DATABASE unsupported options
 --
-CREATE DATABASE test TEMPLATE = some_template;
-ERROR:  Value other than default, template0 or template1 for template option is not yet supported
-LINE 1: CREATE DATABASE test TEMPLATE = some_template;
-                             ^
-HINT:  Please report the issue on https://github.com/YugaByte/yugabyte-db/issues
 CREATE DATABASE test IS_TEMPLATE = TRUE;
 ERROR:  Value other than default or false for is_template option is not yet supported
 LINE 1: CREATE DATABASE test IS_TEMPLATE = TRUE;

diff --git a/src/postgres/src/test/regress/sql/yb_feature_db.sql b/src/postgres/src/test/regress/sql/yb_feature_db.sql
@@ -1,7 +1,6 @@
 --
 -- Test case for CREATE DATABASE unsupported options
 --
-CREATE DATABASE test TEMPLATE = some_template;
 CREATE DATABASE test IS_TEMPLATE = TRUE;
 CREATE DATABASE test LC_COLLATE = "C";
 CREATE DATABASE test LC_CTYPE = "C";

diff --git a/src/yb/client/client-internal.cc b/src/yb/client/client-internal.cc
@@ -255,6 +255,7 @@ YB_CLIENT_SPECIALIZE_SIMPLE(IsAlterTableDone);
 YB_CLIENT_SPECIALIZE_SIMPLE(IsCreateNamespaceDone);
 YB_CLIENT_SPECIALIZE_SIMPLE(IsCreateTableDone);
 YB_CLIENT_SPECIALIZE_SIMPLE(IsDeleteNamespaceDone);
+YB_CLIENT_SPECIALIZE_SIMPLE(IsCloneDone);
 YB_CLIENT_SPECIALIZE_SIMPLE(IsDeleteTableDone);
 YB_CLIENT_SPECIALIZE_SIMPLE(IsFlushTablesDone);
 YB_CLIENT_SPECIALIZE_SIMPLE(GetCompactionStatus);
@@ -271,6 +272,7 @@ YB_CLIENT_SPECIALIZE_SIMPLE_EX(Admin, AddTransactionStatusTablet);
 YB_CLIENT_SPECIALIZE_SIMPLE_EX(Admin, AreNodesSafeToTakeDown);
 YB_CLIENT_SPECIALIZE_SIMPLE_EX(Admin, CreateTransactionStatusTable);
 YB_CLIENT_SPECIALIZE_SIMPLE_EX(Admin, WaitForYsqlBackendsCatalogVersion);
+YB_CLIENT_SPECIALIZE_SIMPLE_EX(Backup, CloneNamespace);
 YB_CLIENT_SPECIALIZE_SIMPLE_EX(Backup, CreateSnapshot);
 YB_CLIENT_SPECIALIZE_SIMPLE_EX(Backup, DeleteSnapshot);
 YB_CLIENT_SPECIALIZE_SIMPLE_EX(Backup, ListSnapshots);
@@ -1118,6 +1120,44 @@ Status YBClient::Data::WaitForDeleteNamespaceToFinish(YBClient* client,
           client, namespace_name, database_type, namespace_id, _1, _2));
 }
 
+Status YBClient::Data::IsCloneNamespaceInProgress(
+    YBClient* client, const std::string& source_namespace_id, int clone_seq_no,
+    CoarseTimePoint deadline, bool* create_in_progress) {
+  DCHECK_ONLY_NOTNULL(create_in_progress);
+  IsCloneDoneRequestPB req;
+  IsCloneDoneResponsePB resp;
+
+  req.set_source_namespace_id(source_namespace_id);
+  req.set_seq_no(clone_seq_no);
+  // RETURN_NOT_OK macro can't take templated function call as param,
+  // and SyncLeaderMasterRpc must be explicitly instantiated, else the
+  // compiler complains.
+  const Status s = SyncLeaderMasterRpc(
+      deadline, req, &resp, "IsCloneDone", &master::MasterBackupProxy::IsCloneDoneAsync);
+
+  RETURN_NOT_OK(s);
+  // IsCloneDone could return a terminal/done state as FAILED. This would result in an error'd
+  // Status.
+  if (resp.has_error()) {
+    return StatusFromPB(resp.error().status());
+  }
+
+  *create_in_progress = !resp.is_done();
+
+  return Status::OK();
+}
+
+Status YBClient::Data::WaitForCloneNamespaceToFinish(
+    YBClient* client, const std::string& source_namespace_id, int clone_seq_no,
+    CoarseTimePoint deadline) {
+  return RetryFunc(
+      deadline, "Waiting on Clone Namespace to be completed",
+      "Timed out waiting for Namespace Cloning",
+      std::bind(
+          &YBClient::Data::IsCloneNamespaceInProgress, this, client, source_namespace_id,
+          clone_seq_no, _1, _2));
+}
+
 Status YBClient::Data::AlterTable(YBClient* client,
                                   const AlterTableRequestPB& req,
                                   CoarseTimePoint deadline) {

diff --git a/src/yb/client/client-internal.h b/src/yb/client/client-internal.h
@@ -117,6 +117,13 @@ class YBClient::Data {
                                         const std::string& namespace_id,
                                         CoarseTimePoint deadline);
 
+  Status IsCloneNamespaceInProgress(
+      YBClient* client, const std::string& source_namespace_id, int clone_seq_no,
+      CoarseTimePoint deadline, bool* create_in_progress);
+  Status WaitForCloneNamespaceToFinish(
+      YBClient* client, const std::string& source_namespace_id, int clone_seq_no,
+      CoarseTimePoint deadline);
+
   Status CreateTable(YBClient* client,
                      const master::CreateTableRequestPB& req,
                      const YBSchema& schema,