From 14b05a81df4f0d3cf808736807f391966bbcd324 Mon Sep 17 00:00:00 2001 From: Danny Xu <98006139+d-bytebase@users.noreply.github.com> Date: Fri, 11 Feb 2022 17:09:25 +0800 Subject: [PATCH] feat(sqlite): add table and column information to sync schema (#582) * feat(test): add test for sql execute and check database schema * feat: address comment * feat(test): add initial stub for fake gitlab * feat(test): add test for creating repository and fake createProjectHook * refactor: get rid of QueryParams and move some queries to database drivers * refactor: move insert history logic into its functor * feat(test): add test for simulating basic gitlab commit * feat: address comment * feat(test): implement test for vcs schema update, and fixed a few VCS flow bugs * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat: address comment * refactor: some refactoring of moving common code to helper functions * feat(test): add test for tenant schema update * Update tests/tenant_test.go Co-authored-by: tianzhou * Update tests/tenant_test.go Co-authored-by: tianzhou * feat: address comment * feat(test): test tenant schema update with VCS * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat: address comment * feat(test): add tenant test for cases with database name template * feat(test): use temp for data directory otherwise files will be in tests code directory * fix(543): migrate clickhouse driver to v2.0.7 * refactor: enable parallel test so that testing can scale * feat: address comment * fix: wait for all runners to finish before we shutdown the server; otherwise, we will get database is closed error from runner when we shutdown the server * feat: address comment * fix(tenancy): disallow ENV_NAME token in file path template for projects in tenant mode * fix: address comment * refactor: fix lint warnings * refactor: re-enable errcheck in golinter * refactor: re-enable errcheck in golinter * refactor: re-enable errcheck in golinter * refactor: enable errcheck * refactor: enable errcheck * refactor: enable errcheck * refactor: enable errcheck lint * Update server/instance.go Co-authored-by: tianzhou * Update server/instance.go Co-authored-by: tianzhou * refactor: fix some lint warnings * feat(test): add test for migration history * Update tests/schema_update_test.go Co-authored-by: tianzhou * feat: address comment * feat: small change * feat: address comment * feat(sqlite): add table and column information to sync schema Co-authored-by: tianzhou --- plugin/db/driver.go | 28 ++++++++------ plugin/db/sqlite/sqlite.go | 78 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/plugin/db/driver.go b/plugin/db/driver.go index fae54183e12898..da90700a19f68e 100644 --- a/plugin/db/driver.go +++ b/plugin/db/driver.go @@ -69,34 +69,38 @@ type Column struct { // Nullable isn't supported for ClickHouse. Nullable bool Type string - // CharacterSet isn't supported for Postgres, ClickHouse.. + // CharacterSet isn't supported for Postgres, ClickHouse, SQLite. CharacterSet string - // Collation isn't supported for ClickHouse. + // Collation isn't supported for ClickHouse, SQLite. Collation string - Comment string + // Comment isn't supported for SQLite. + Comment string } // Table is the database table. type Table struct { Name string - // CreatedTs isn't supported for ClickHouse. + // CreatedTs isn't supported for ClickHouse, SQLite. CreatedTs int64 + // UpdatedTs isn't supported for SQLite. UpdatedTs int64 Type string - // Engine isn't supported for Postgres, Snowflake. + // Engine isn't supported for Postgres, Snowflake, SQLite. Engine string - // Collation isn't supported for Postgres, ClickHouse, Snowflake. + // Collation isn't supported for Postgres, ClickHouse, Snowflake, SQLite. Collation string RowCount int64 - DataSize int64 - // IndexSize isn't supported for ClickHouse, Snowflake. + // DataSize isn't supported for SQLite. + DataSize int64 + // IndexSize isn't supported for ClickHouse, Snowflake, SQLite. IndexSize int64 - // DataFree isn't supported for Postgres, ClickHouse, Snowflake. + // DataFree isn't supported for Postgres, ClickHouse, Snowflake, SQLite. DataFree int64 - // CreateOptions isn't supported for Postgres, ClickHouse, Snowflake. + // CreateOptions isn't supported for Postgres, ClickHouse, Snowflake, SQLite. CreateOptions string - Comment string - ColumnList []Column + // Comment isn't supported for SQLite. + Comment string + ColumnList []Column // IndexList isn't supported for ClickHouse, Snowflake. IndexList []Index } diff --git a/plugin/db/sqlite/sqlite.go b/plugin/db/sqlite/sqlite.go index 56ef9c2bbbab3b..e4ad2473a0520e 100644 --- a/plugin/db/sqlite/sqlite.go +++ b/plugin/db/sqlite/sqlite.go @@ -123,12 +123,90 @@ func (driver *Driver) SyncSchema(ctx context.Context) ([]*db.User, []*db.Schema, var schema db.Schema schema.Name = dbName + + sqldb, err := driver.GetDbConnection(ctx, dbName) + if err != nil { + return nil, nil, fmt.Errorf("failed to get database connection for %q: %s", dbName, err) + } + txn, err := sqldb.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return nil, nil, err + } + defer txn.Rollback() + // TODO(d-bytebase): retrieve database schema such as tables and indices. + tbls, err := getTables(txn) + if err != nil { + return nil, nil, err + } + schema.TableList = tbls + + if err := txn.Commit(); err != nil { + return nil, nil, err + } + schemaList = append(schemaList, &schema) } return nil, schemaList, nil } +// getTables gets all tables of a database. +func getTables(txn *sql.Tx) ([]db.Table, error) { + var tables []db.Table + query := "SELECT name FROM sqlite_schema WHERE type ='table' AND name NOT LIKE 'sqlite_%';" + rows, err := txn.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var tableNames []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + tableNames = append(tableNames, name) + } + for _, name := range tableNames { + var tbl db.Table + tbl.Name = name + tbl.Type = "BASE TABLE" + + // Get columns: cid, name, type, notnull, dflt_value, pk. + query := fmt.Sprintf("pragma table_info(%s);", name) + rows, err := txn.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var col db.Column + + var cid int + var notnull, pk bool + var name, ctype string + var dfltValue sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { + return nil, err + } + col.Position = cid + col.Name = name + col.Nullable = !notnull + col.Type = ctype + if dfltValue.Valid { + col.Default = &dfltValue.String + } + + tbl.ColumnList = append(tbl.ColumnList, col) + } + + tables = append(tables, tbl) + } + return tables, nil +} + func (driver *Driver) getDatabases() ([]string, error) { files, err := ioutil.ReadDir(driver.dir) if err != nil {