Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat!: use lineage instead of path to link states on overview (#179)
* feat: add ListLineageStats api endpoint and model

* feat: update Terraboard overview to list by lineage

* feat: add last modified path on lineage overview

* feat: adapt total item count for states by lineage view + add distinct on lineage_id

* feat: add lineage column on search view with filter

* feat(lineage): update search view and state explorer to use lineages

* fix(lineage): code review issues
* rename GetStateActivity functions to GetLineageActivity
* change /api/state/activity to /api/lineage/activity
* restore dynamic lock status on overview
* fix missing version_id attribute in states quick access links
* remove lineage display from overview

* refactor: remove obsolete lineage.html file

* feat(api): rewrite server using gorilla/mux for routing
* refactor(api): update some endpoints path to match new lineage system
* feat(server): add CORS middleware to abstract headers modifications
* refactor(api): update DefaultVersion function to use lineage
  • Loading branch information
hbollon committed Jul 29, 2021
1 parent e44ebce commit c576d95
Show file tree
Hide file tree
Showing 11 changed files with 208 additions and 265 deletions.
55 changes: 11 additions & 44 deletions api/api.go
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/camptocamp/terraboard/compare"
"github.com/camptocamp/terraboard/db"
"github.com/camptocamp/terraboard/state"
"github.com/camptocamp/terraboard/util"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
)

Expand All @@ -27,25 +27,9 @@ func JSONError(w http.ResponseWriter, message string, err error) {
}
}

// ListStates lists States
func ListStates(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
states := d.ListStates()

j, err := json.Marshal(states)
if err != nil {
JSONError(w, "Failed to marshal states", err)
return
}
if _, err := io.WriteString(w, string(j)); err != nil {
log.Error(err.Error())
}
}

// ListTerraformVersionsWithCount lists Terraform versions with their associated
// counts, sorted by the 'orderBy' parameter (version by default)
func ListTerraformVersionsWithCount(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
query := r.URL.Query()
versions, _ := d.ListTerraformVersionsWithCount(query)

Expand All @@ -61,7 +45,6 @@ func ListTerraformVersionsWithCount(w http.ResponseWriter, r *http.Request, d *d

// ListStateStats returns State information for a given path as parameter
func ListStateStats(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
query := r.URL.Query()
states, page, total := d.ListStateStats(query)

Expand All @@ -82,18 +65,17 @@ func ListStateStats(w http.ResponseWriter, r *http.Request, d *db.Database) {

// GetState provides information on a State
func GetState(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
st := util.TrimBasePath(r, "api/state/")
params := mux.Vars(r)
versionID := r.URL.Query().Get("versionid")
var err error
if versionID == "" {
versionID, err = d.DefaultVersion(st)
versionID, err = d.DefaultVersion(params["lineage"])
if err != nil {
JSONError(w, "Failed to retrieve default version", err)
return
}
}
state := d.GetState(st, versionID)
state := d.GetState(params["lineage"], versionID)

j, err := json.Marshal(state)
if err != nil {
Expand All @@ -105,11 +87,10 @@ func GetState(w http.ResponseWriter, r *http.Request, d *db.Database) {
}
}

// GetStateActivity returns the activity (version history) of a State
func GetStateActivity(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
st := util.TrimBasePath(r, "api/state/activity/")
activity := d.GetStateActivity(st)
// GetLineageActivity returns the activity (version history) of a Lineage
func GetLineageActivity(w http.ResponseWriter, r *http.Request, d *db.Database) {
params := mux.Vars(r)
activity := d.GetLineageActivity(params["lineage"])

j, err := json.Marshal(activity)
if err != nil {
Expand All @@ -123,14 +104,13 @@ func GetStateActivity(w http.ResponseWriter, r *http.Request, d *db.Database) {

// StateCompare compares two versions ('from' and 'to') of a State
func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
st := util.TrimBasePath(r, "api/state/compare/")
params := mux.Vars(r)
query := r.URL.Query()
fromVersion := query.Get("from")
toVersion := query.Get("to")

from := d.GetState(st, fromVersion)
to := d.GetState(st, toVersion)
from := d.GetState(params["lineage"], fromVersion)
to := d.GetState(params["lineage"], toVersion)
compare, err := compare.Compare(from, to)
if err != nil {
JSONError(w, "Failed to compare state versions", err)
Expand All @@ -149,7 +129,6 @@ func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {

// GetLocks returns information on locked States
func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) {
w.Header().Set("Access-Control-Allow-Origin", "*")
allLocks := make(map[string]state.LockInfo)
for _, sp := range sps {
locks, err := sp.GetLocks()
Expand All @@ -175,7 +154,6 @@ func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) {
// SearchAttribute performs a search on Resource Attributes
// by various parameters
func SearchAttribute(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
query := r.URL.Query()
result, page, total := d.SearchAttribute(query)

Expand All @@ -197,7 +175,6 @@ func SearchAttribute(w http.ResponseWriter, r *http.Request, d *db.Database) {

// ListResourceTypes lists all Resource types
func ListResourceTypes(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListResourceTypes()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -211,7 +188,6 @@ func ListResourceTypes(w http.ResponseWriter, _ *http.Request, d *db.Database) {

// ListResourceTypesWithCount lists all Resource types with their associated count
func ListResourceTypesWithCount(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListResourceTypesWithCount()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -225,7 +201,6 @@ func ListResourceTypesWithCount(w http.ResponseWriter, _ *http.Request, d *db.Da

// ListResourceNames lists all Resource names
func ListResourceNames(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListResourceNames()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -240,7 +215,6 @@ func ListResourceNames(w http.ResponseWriter, _ *http.Request, d *db.Database) {
// ListAttributeKeys lists all Resource Attribute Keys,
// optionally filtered by resource_type
func ListAttributeKeys(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
resourceType := r.URL.Query().Get("resource_type")
result, _ := d.ListAttributeKeys(resourceType)
j, err := json.Marshal(result)
Expand All @@ -255,7 +229,6 @@ func ListAttributeKeys(w http.ResponseWriter, r *http.Request, d *db.Database) {

// ListTfVersions lists all Terraform versions
func ListTfVersions(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListTfVersions()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -269,8 +242,6 @@ func ListTfVersions(w http.ResponseWriter, _ *http.Request, d *db.Database) {

// GetUser returns information about the logged user
func GetUser(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")

name := r.Header.Get("X-Forwarded-User")
email := r.Header.Get("X-Forwarded-Email")

Expand All @@ -289,8 +260,6 @@ func GetUser(w http.ResponseWriter, r *http.Request) {
// SubmitPlan inserts a new Terraform plan in the database.
// /api/plans POST endpoint callback
func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")

body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Errorf("Failed to read body: %v", err)
Expand All @@ -310,7 +279,6 @@ func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) {
// Sorted by most recent to oldest.
// /api/plans GET endpoint callback
func GetPlans(w http.ResponseWriter, r *http.Request, db *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
lineage := r.URL.Query().Get("lineage")
limit := r.URL.Query().Get("limit")
plans := db.GetPlans(lineage, limit)
Expand Down Expand Up @@ -342,7 +310,6 @@ func ManagePlans(w http.ResponseWriter, r *http.Request, db *db.Database) {
// Optional "&limit=X" parameter to limit requested quantity of them.
// Sorted by most recent to oldest.
func GetLineages(w http.ResponseWriter, r *http.Request, db *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
limit := r.URL.Query().Get("limit")
lineages := db.GetLineages(limit)

Expand Down
75 changes: 37 additions & 38 deletions db/db.go
Expand Up @@ -136,7 +136,7 @@ func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID strin
Version: version,
TFVersion: sf.TerraformVersion.String(),
Serial: int64(sf.Serial),
LineageID: sql.NullInt64{Int64: int64(lineage.ID)},
LineageID: sql.NullInt64{Int64: int64(lineage.ID), Valid: true},
}

for _, m := range sf.State.Modules {
Expand Down Expand Up @@ -279,25 +279,26 @@ func (db *Database) InsertVersion(version *state.Version) error {
}

// GetState retrieves a State from the database by its path and versionID
func (db *Database) GetState(path, versionID string) (state types.State) {
db.Joins("JOIN versions on states.version_id=versions.id").
func (db *Database) GetState(lineage, versionID string) (state types.State) {
db.Joins("JOIN lineages on states.lineage_id=lineages.id").
Joins("JOIN versions on states.version_id=versions.id").
Preload("Version").Preload("Modules").Preload("Modules.Resources").Preload("Modules.Resources.Attributes").
Preload("Modules.OutputValues").
Find(&state, "states.path = ? AND versions.version_id = ?", path, versionID)
Find(&state, "lineages.value = ? AND versions.version_id = ?", lineage, versionID)
return
}

// GetStateActivity returns a slice of StateStat from the Database
// for a given State path representing the State activity over time (Versions)
func (db *Database) GetStateActivity(path string) (states []types.StateStat) {
// GetLineageActivity returns a slice of StateStat from the Database
// for a given lineage representing the State activity over time (Versions)
func (db *Database) GetLineageActivity(lineage string) (states []types.StateStat) {
sql := "SELECT t.path, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" +
" FROM (SELECT states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id WHERE states.path = ? ORDER BY states.path, versions.last_modified ASC) t" +
" FROM (SELECT states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN lineages ON lineages.id = states.lineage_id JOIN versions ON versions.id = states.version_id WHERE lineages.value = ? ORDER BY states.path, versions.last_modified ASC) t" +
" JOIN modules ON modules.state_id = t.id" +
" JOIN resources ON resources.module_id = modules.id" +
" GROUP BY t.path, t.serial, t.tf_version, t.version_id, t.last_modified" +
" ORDER BY last_modified ASC"

db.Raw(sql, path).Find(&states)
db.Raw(sql, lineage).Find(&states)
return
}

Expand Down Expand Up @@ -336,7 +337,8 @@ func (db *Database) SearchAttribute(query url.Values) (results []types.SearchRes

sqlQuery += " JOIN modules ON states.id = modules.state_id" +
" JOIN resources ON modules.id = resources.module_id" +
" JOIN attributes ON resources.id = attributes.resource_id"
" JOIN attributes ON resources.id = attributes.resource_id" +
" JOIN lineages ON lineages.id = states.lineage_id"

var where []string
var params []interface{}
Expand Down Expand Up @@ -370,6 +372,10 @@ func (db *Database) SearchAttribute(query url.Values) (results []types.SearchRes
where = append(where, fmt.Sprintf("states.tf_version LIKE '%s'", fmt.Sprintf("%%%s%%", v)))
}

if v := query.Get("lineage_value"); string(v) != "" {
where = append(where, fmt.Sprintf("lineages.value LIKE '%s'", fmt.Sprintf("%%%s%%", v)))
}

if len(where) > 0 {
sqlQuery += " WHERE " + strings.Join(where, " AND ")
}
Expand All @@ -382,11 +388,12 @@ func (db *Database) SearchAttribute(query url.Values) (results []types.SearchRes

// Now get results
// gorm doesn't support subqueries...
sql := "SELECT states.path, states.version_id, states.tf_version, states.serial, modules.path as module_path, resources.type, resources.name, resources.index, attributes.key, attributes.value" +
sql := "SELECT states.path, states.version_id, states.tf_version, states.serial, lineages.value as lineage_value, modules.path as module_path, resources.type, resources.name, resources.index, attributes.key, attributes.value" +
sqlQuery +
" ORDER BY states.path, states.serial, modules.path, resources.type, resources.name, resources.index, attributes.key" +
" ORDER BY states.path, states.serial, lineage_value, modules.path, resources.type, resources.name, resources.index, attributes.key" +
" LIMIT ?"

log.Info(sql)
params = append(params, pageSize)

if v := string(query.Get("page")); v != "" {
Expand Down Expand Up @@ -422,20 +429,6 @@ func (db *Database) ListStatesVersions() (statesVersions map[string][]string) {
return
}

// ListStates returns a slice of all State paths from the Database
func (db *Database) ListStates() (states []string) {
rows, _ := db.Table("states").Select("DISTINCT path").Rows()
defer rows.Close()
for rows.Next() {
var state string
if err := rows.Scan(&state); err != nil {
log.Error(err.Error())
}
states = append(states, state)
}
return
}

// ListTerraformVersionsWithCount returns a slice of maps of Terraform versions
// mapped to the count of most recent State paths using them.
// ListTerraformVersionsWithCount also takes a query with possible parameter 'orderBy'
Expand Down Expand Up @@ -475,28 +468,33 @@ func (db *Database) ListTerraformVersionsWithCount(query url.Values) (results []

// ListStateStats returns a slice of StateStat, along with paging information
func (db *Database) ListStateStats(query url.Values) (states []types.StateStat, page int, total int) {
row := db.Table("states").Select("count(DISTINCT path)").Row()
row := db.Raw("SELECT count(*) FROM (SELECT DISTINCT lineage_id FROM states) AS t").Row()
if err := row.Scan(&total); err != nil {
log.Error(err.Error())
}

offset := 0
var paginationQuery string
var params []interface{}
page = 1
if v := string(query.Get("page")); v != "" {
page, _ = strconv.Atoi(v) // TODO: err
offset = (page - 1) * pageSize
offset := (page - 1) * pageSize
params = append(params, offset)
paginationQuery = " LIMIT 20 OFFSET ?"
} else {
page = -1
}

sql := "SELECT t.path, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" +
" FROM (SELECT DISTINCT ON(states.path) states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.path, versions.last_modified DESC) t" +
sql := "SELECT t.path, lineages.value as lineage_value, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" +
" FROM (SELECT DISTINCT ON(states.lineage_id) states.id, states.lineage_id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.lineage_id, versions.last_modified DESC) t" +
" JOIN modules ON modules.state_id = t.id" +
" JOIN resources ON resources.module_id = modules.id" +
" GROUP BY t.path, t.serial, t.tf_version, t.version_id, t.last_modified" +
" JOIN lineages ON lineages.id = t.lineage_id" +
" GROUP BY t.path, lineages.value, t.serial, t.tf_version, t.version_id, t.last_modified" +
" ORDER BY last_modified DESC" +
" LIMIT 20" +
" OFFSET ?"
paginationQuery

db.Raw(sql, offset).Find(&states)
db.Raw(sql, params...).Find(&states)
return
}

Expand Down Expand Up @@ -682,14 +680,15 @@ func (db *Database) GetLineages(limitStr string) (lineages []types.Lineage) {

// DefaultVersion returns the detault VersionID for a given State path
// Copied and adapted from github.com/hashicorp/terraform/command/jsonstate/state.go
func (db *Database) DefaultVersion(path string) (version string, err error) {
func (db *Database) DefaultVersion(lineage string) (version string, err error) {
sqlQuery := "SELECT versions.version_id FROM" +
" (SELECT states.path, max(states.serial) as mx FROM states GROUP BY states.path) t" +
" JOIN states ON t.path = states.path AND t.mx = states.serial" +
" JOIN versions on states.version_id=versions.id" +
" WHERE states.path = ?"
" JOIN lineages on lineages.id=states.lineage_id" +
" WHERE lineages.value = ?"

row := db.Raw(sqlQuery, path).Row()
row := db.Raw(sqlQuery, lineage).Row()
err = row.Scan(&version)
return
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Expand Up @@ -11,6 +11,7 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/go-test/deep v1.0.3
github.com/google/go-cmp v0.5.5
github.com/gorilla/mux v1.8.0
github.com/hashicorp/errwrap v1.1.0
github.com/hashicorp/go-cleanhttp v0.5.1
github.com/hashicorp/go-hclog v0.15.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Expand Up @@ -315,6 +315,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
Expand Down

0 comments on commit c576d95

Please sign in to comment.