feat(sqlite): add table and column information to sync schema (#582)

* feat(test): add test for sql execute and check database schema * feat: address comment * feat(test): add initial stub for fake gitlab * feat(test): add test for creating repository and fake createProjectHook * refactor: get rid of QueryParams and move some queries to database drivers * refactor: move insert history logic into its functor * feat(test): add test for simulating basic gitlab commit * feat: address comment * feat(test): implement test for vcs schema update, and fixed a few VCS flow bugs * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat: address comment * refactor: some refactoring of moving common code to helper functions * feat(test): add test for tenant schema update * Update tests/tenant_test.go Co-authored-by: tianzhou <t@bytebase.com> * Update tests/tenant_test.go Co-authored-by: tianzhou <t@bytebase.com> * feat: address comment * feat(test): test tenant schema update with VCS * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat(test): add tenant test for cases with database name template * feat(test): use temp for data directory otherwise files will be in tests code directory * fix(543): migrate clickhouse driver to v2.0.7 * refactor: enable parallel test so that testing can scale * feat: address comment * fix: wait for all runners to finish before we shutdown the server; otherwise, we will get database is closed error from runner when we shutdown the server * feat: address comment * fix(tenancy): disallow ENV_NAME token in file path template for projects in tenant mode * fix: address comment * refactor: fix lint warnings * refactor: re-enable errcheck in golinter * refactor: re-enable errcheck in golinter * refactor: re-enable errcheck in golinter * refactor: enable errcheck * refactor: enable errcheck * refactor: enable errcheck * refactor: enable errcheck lint * Update server/instance.go Co-authored-by: tianzhou <t@bytebase.com> * Update server/instance.go Co-authored-by: tianzhou <t@bytebase.com> * refactor: fix some lint warnings * feat(test): add test for migration history * Update tests/schema_update_test.go Co-authored-by: tianzhou <t@bytebase.com> * feat: address comment * feat: small change * feat: address comment * feat(sqlite): add table and column information to sync schema Co-authored-by: tianzhou <t@bytebase.com>
bytebase · Feb 11, 2022 · 14b05a8 · 14b05a8
1 parent 0b0d0dd
commit 14b05a8
Show file tree

Hide file tree

Showing 2 changed files with 94 additions and 12 deletions.
diff --git a/plugin/db/driver.go b/plugin/db/driver.go
@@ -69,34 +69,38 @@ type Column struct {
 	// Nullable isn't supported for ClickHouse.
 	Nullable bool
 	Type     string
-	// CharacterSet isn't supported for Postgres, ClickHouse..
+	// CharacterSet isn't supported for Postgres, ClickHouse, SQLite.
 	CharacterSet string
-	// Collation isn't supported for ClickHouse.
+	// Collation isn't supported for ClickHouse, SQLite.
 	Collation string
-	Comment   string
+	// Comment isn't supported for SQLite.
+	Comment string
 }
 
 // Table is the database table.
 type Table struct {
 	Name string
-	// CreatedTs isn't supported for ClickHouse.
+	// CreatedTs isn't supported for ClickHouse, SQLite.
 	CreatedTs int64
+	// UpdatedTs isn't supported for SQLite.
 	UpdatedTs int64
 	Type      string
-	// Engine isn't supported for Postgres, Snowflake.
+	// Engine isn't supported for Postgres, Snowflake, SQLite.
 	Engine string
-	// Collation isn't supported for Postgres, ClickHouse, Snowflake.
+	// Collation isn't supported for Postgres, ClickHouse, Snowflake, SQLite.
 	Collation string
 	RowCount  int64
-	DataSize  int64
-	// IndexSize isn't supported for ClickHouse, Snowflake.
+	// DataSize isn't supported for SQLite.
+	DataSize int64
+	// IndexSize isn't supported for ClickHouse, Snowflake, SQLite.
 	IndexSize int64
-	// DataFree isn't supported for Postgres, ClickHouse, Snowflake.
+	// DataFree isn't supported for Postgres, ClickHouse, Snowflake, SQLite.
 	DataFree int64
-	// CreateOptions isn't supported for Postgres, ClickHouse, Snowflake.
+	// CreateOptions isn't supported for Postgres, ClickHouse, Snowflake, SQLite.
 	CreateOptions string
-	Comment       string
-	ColumnList    []Column
+	// Comment isn't supported for SQLite.
+	Comment    string
+	ColumnList []Column
 	// IndexList isn't supported for ClickHouse, Snowflake.
 	IndexList []Index
 }

diff --git a/plugin/db/sqlite/sqlite.go b/plugin/db/sqlite/sqlite.go
@@ -123,12 +123,90 @@ func (driver *Driver) SyncSchema(ctx context.Context) ([]*db.User, []*db.Schema,
 
 		var schema db.Schema
 		schema.Name = dbName
+
+		sqldb, err := driver.GetDbConnection(ctx, dbName)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get database connection for %q: %s", dbName, err)
+		}
+		txn, err := sqldb.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
+		if err != nil {
+			return nil, nil, err
+		}
+		defer txn.Rollback()
+
 		// TODO(d-bytebase): retrieve database schema such as tables and indices.
+		tbls, err := getTables(txn)
+		if err != nil {
+			return nil, nil, err
+		}
+		schema.TableList = tbls
+
+		if err := txn.Commit(); err != nil {
+			return nil, nil, err
+		}
+
 		schemaList = append(schemaList, &schema)
 	}
 	return nil, schemaList, nil
 }
 
+// getTables gets all tables of a database.
+func getTables(txn *sql.Tx) ([]db.Table, error) {
+	var tables []db.Table
+	query := "SELECT name FROM sqlite_schema WHERE type ='table' AND name NOT LIKE 'sqlite_%';"
+	rows, err := txn.Query(query)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var tableNames []string
+	for rows.Next() {
+		var name string
+		if err := rows.Scan(&name); err != nil {
+			return nil, err
+		}
+		tableNames = append(tableNames, name)
+	}
+	for _, name := range tableNames {
+		var tbl db.Table
+		tbl.Name = name
+		tbl.Type = "BASE TABLE"
+
+		// Get columns: cid, name, type, notnull, dflt_value, pk.
+		query := fmt.Sprintf("pragma table_info(%s);", name)
+		rows, err := txn.Query(query)
+		if err != nil {
+			return nil, err
+		}
+		defer rows.Close()
+
+		for rows.Next() {
+			var col db.Column
+
+			var cid int
+			var notnull, pk bool
+			var name, ctype string
+			var dfltValue sql.NullString
+			if err := rows.Scan(&cid, &name, &ctype, &notnull, &dfltValue, &pk); err != nil {
+				return nil, err
+			}
+			col.Position = cid
+			col.Name = name
+			col.Nullable = !notnull
+			col.Type = ctype
+			if dfltValue.Valid {
+				col.Default = &dfltValue.String
+			}
+
+			tbl.ColumnList = append(tbl.ColumnList, col)
+		}
+
+		tables = append(tables, tbl)
+	}
+	return tables, nil
+}
+
 func (driver *Driver) getDatabases() ([]string, error) {
 	files, err := ioutil.ReadDir(driver.dir)
 	if err != nil {