feat: new standalone lineage table + associated migration (#173)

* feat: add lineage table and update gorm model

* feat(lineage): custom migration for lineage db changes

* fix: code review tweaks | linters issues
* removed lineage field in State

Co-authored-by: Raphaël Pinson <github+aem1eeshi1@raphink.net>

db/db.go

@@ -2,6 +2,7 @@ package db
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
 	"strconv"
@@ -49,6 +50,7 @@ func Init(config config.DBConfig, debug bool) *Database {
 
 	log.Infof("Automigrate")
 	err = db.AutoMigrate(
+		&types.Lineage{},
 		&types.Version{},
 		&types.State{},
 		&types.Module{},
@@ -76,20 +78,64 @@ func Init(config config.DBConfig, debug bool) *Database {
 	if debug {
 		db.Config.Logger.LogMode(logger.Info)
 	}
-	return &Database{db}
+
+	d := &Database{db}
+	if err = d.MigrateLineage(); err != nil {
+		log.Fatalf("Lineage migration failed: %v\n", err)
+	}
+
+	return d
+}
+
+// MigrateLineage is a migration function to update db and its data to the
+// new lineage db scheme. It will update State table data, delete "lineage" column
+// and add corresponding Lineage entries
+func (db *Database) MigrateLineage() error {
+	if db.Migrator().HasColumn(&types.State{}, "lineage") {
+		states := db.ListStates()
+		for _, stPath := range states {
+			// Recover State from db for update
+			var st types.State
+			res := db.First(&st, types.State{Path: stPath})
+			if errors.Is(res.Error, gorm.ErrRecordNotFound) {
+				return fmt.Errorf("State not found in db")
+			}
+
+			if err := db.UpdateState(st); err != nil {
+				return fmt.Errorf("Failed to update %s state during lineage migration: %v", stPath, err)
+			}
+		}
+
+		// Custom migration rules
+		if err := db.Migrator().DropColumn(&types.State{}, "lineage"); err != nil {
+			return fmt.Errorf("Failed to drop lineage column during migration: %v", err)
+		}
+	}
+
+	return nil
 }
 
 type attributeValues map[string]interface{}
 
-func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID string) (st types.State) {
+func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID string) (st types.State, err error) {
 	var version types.Version
 	db.First(&version, types.Version{VersionID: versionID})
+
+	// Check if the associated lineage is already present in lineages table
+	// If so, it recovers its ID otherwise it inserts it at the same time as the state
+	var lineage types.Lineage
+	err = db.FirstOrCreate(&lineage, types.Lineage{Value: sf.Lineage}).Error
+	if err != nil || lineage.ID == 0 {
+		log.Error("Unknown error in stateS3toDB during lineage finding")
+		return types.State{}, err
+	}
+
 	st = types.State{
 		Path:      path,
 		Version:   version,
 		TFVersion: sf.TerraformVersion.String(),
 		Serial:    int64(sf.Serial),
-		Lineage:   sf.Lineage,
+		LineageID: lineage.ID,
 	}
 
 	for _, m := range sf.State.Modules {
@@ -164,11 +210,35 @@ func marshalAttributeValues(src *states.ResourceInstanceObjectSrc) (attrs []type
 
 // InsertState inserts a Terraform State in the Database
 func (db *Database) InsertState(path string, versionID string, sf *statefile.File) error {
-	st := db.stateS3toDB(sf, path, versionID)
-	db.Create(&st)
+	st, err := db.stateS3toDB(sf, path, versionID)
+	if err == nil {
+		db.Create(&st)
+	}
 	return nil
 }
 
+// UpdateState update a Terraform State in the Database with Lineage foreign constraint
+// It will also insert Lineage entry in the db if needed.
+// This method is only use during the Lineage migration since States are immutable
+func (db *Database) UpdateState(st types.State) error {
+	// Get lineage from old column
+	var lineage types.Lineage
+	if err := db.Raw("SELECT lineage FROM states WHERE path = ?", st.Path).Scan(&lineage.Value).Error; err != nil {
+		return fmt.Errorf("Error on %s lineage recovering during migration: %v", st.Path, err)
+	}
+
+	// Create Lineage entry if not exist (value column is unique)
+	tx := db.FirstOrCreate(&lineage)
+	if tx.Error != nil || lineage.ID == 0 {
+		return tx.Error
+	}
+
+	// Get Lineage ID for foreign constraint
+	st.LineageID = lineage.ID
+
+	return db.Save(&st).Error
+}
+
 // InsertVersion inserts an AWS S3 Version in the Database
 func (db *Database) InsertVersion(version *state.Version) error {
 	var v types.Version

types/db.go

@@ -29,10 +29,17 @@ type State struct {
 	VersionID  sql.NullInt64 `gorm:"index" json:"-"`
 	TFVersion  string        `gorm:"varchar(10)" json:"terraform_version"`
 	Serial     int64         `json:"serial"`
-	Lineage    string        `json:"lineage"`
+	LineageID  uint          `gorm:"index" json:"-"`
 	Modules    []Module      `json:"modules"`
 }
 
+type Lineage struct {
+	gorm.Model
+	Value  string  `gorm:"index;unique" json:"lineage"`
+	States []State `json:"states"`
+	Plans  []Plan  `json:"plans"`
+}
+
 // Module is a Terraform module in a State
 type Module struct {
 	ID           uint          `sql:"AUTO_INCREMENT" gorm:"primary_key" json:"-"`
@@ -72,7 +79,7 @@ type Attribute struct {
 // Plan is a Terraform plan
 type Plan struct {
 	gorm.Model
-	Lineage      string         `json:"lineage"`
+	LineageID    uint           `gorm:"index" json:"-"`
 	TFVersion    string         `gorm:"varchar(10)" json:"terraform_version"`
 	GitRemote    string         `json:"git_remote"`
 	GitCommit    string         `gorm:"varchar(50)" json:"git_commit"`