Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat!: use lineage instead of path to link states on overview #179

Merged
merged 9 commits into from Jul 29, 2021
Merged

feat!: use lineage instead of path to link states on overview #179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c2272d4
feat: add ListLineageStats api endpoint and model
hbollon Jun 28, 2021
787469c
feat: update Terraboard overview to list by lineage
hbollon Jun 28, 2021
620fdb1
feat: add last modified path on lineage overview
hbollon Jun 29, 2021
9ada232
feat: adapt total item count for states by lineage view + add distinc…
hbollon Jun 30, 2021
95394ef
feat: add lineage column on search view with filter
hbollon Jun 30, 2021
25cb5fd
feat(lineage): update search view and state explorer to use lineages
hbollon Jul 8, 2021
5ca49f8
fix(lineage): code review issues
hbollon Jul 15, 2021
f41a383
refactor: remove obsolete lineage.html file
hbollon Jul 15, 2021
f79f764
feat(api): rewrite server using gorilla/mux for routing
hbollon Jul 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
55 changes: 11 additions & 44 deletions api/api.go
View file
Open in desktop
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/camptocamp/terraboard/compare"
"github.com/camptocamp/terraboard/db"
"github.com/camptocamp/terraboard/state"
"github.com/camptocamp/terraboard/util"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
)

Expand All @@ -27,25 +27,9 @@ func JSONError(w http.ResponseWriter, message string, err error) {
}
}

// ListStates lists States
func ListStates(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
states := d.ListStates()

j, err := json.Marshal(states)
if err != nil {
JSONError(w, "Failed to marshal states", err)
return
}
if _, err := io.WriteString(w, string(j)); err != nil {
log.Error(err.Error())
}
}

// ListTerraformVersionsWithCount lists Terraform versions with their associated
// counts, sorted by the 'orderBy' parameter (version by default)
func ListTerraformVersionsWithCount(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
query := r.URL.Query()
versions, _ := d.ListTerraformVersionsWithCount(query)

Expand All @@ -61,7 +45,6 @@ func ListTerraformVersionsWithCount(w http.ResponseWriter, r *http.Request, d *d

// ListStateStats returns State information for a given path as parameter
func ListStateStats(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
query := r.URL.Query()
states, page, total := d.ListStateStats(query)

Expand All @@ -82,18 +65,17 @@ func ListStateStats(w http.ResponseWriter, r *http.Request, d *db.Database) {

// GetState provides information on a State
func GetState(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
st := util.TrimBasePath(r, "api/state/")
params := mux.Vars(r)
versionID := r.URL.Query().Get("versionid")
var err error
if versionID == "" {
versionID, err = d.DefaultVersion(st)
versionID, err = d.DefaultVersion(params["lineage"])
if err != nil {
JSONError(w, "Failed to retrieve default version", err)
return
}
}
state := d.GetState(st, versionID)
state := d.GetState(params["lineage"], versionID)

j, err := json.Marshal(state)
if err != nil {
Expand All @@ -105,11 +87,10 @@ func GetState(w http.ResponseWriter, r *http.Request, d *db.Database) {
}
}

// GetStateActivity returns the activity (version history) of a State
func GetStateActivity(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
st := util.TrimBasePath(r, "api/state/activity/")
activity := d.GetStateActivity(st)
// GetLineageActivity returns the activity (version history) of a Lineage
func GetLineageActivity(w http.ResponseWriter, r *http.Request, d *db.Database) {
params := mux.Vars(r)
activity := d.GetLineageActivity(params["lineage"])

j, err := json.Marshal(activity)
if err != nil {
Expand All @@ -123,14 +104,13 @@ func GetStateActivity(w http.ResponseWriter, r *http.Request, d *db.Database) {

// StateCompare compares two versions ('from' and 'to') of a State
func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
st := util.TrimBasePath(r, "api/state/compare/")
params := mux.Vars(r)
query := r.URL.Query()
fromVersion := query.Get("from")
toVersion := query.Get("to")

from := d.GetState(st, fromVersion)
to := d.GetState(st, toVersion)
from := d.GetState(params["lineage"], fromVersion)
to := d.GetState(params["lineage"], toVersion)
compare, err := compare.Compare(from, to)
if err != nil {
JSONError(w, "Failed to compare state versions", err)
Expand All @@ -149,7 +129,6 @@ func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) {

// GetLocks returns information on locked States
func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) {
w.Header().Set("Access-Control-Allow-Origin", "*")
allLocks := make(map[string]state.LockInfo)
for _, sp := range sps {
locks, err := sp.GetLocks()
Expand All @@ -175,7 +154,6 @@ func GetLocks(w http.ResponseWriter, _ *http.Request, sps []state.Provider) {
// SearchAttribute performs a search on Resource Attributes
// by various parameters
func SearchAttribute(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
query := r.URL.Query()
result, page, total := d.SearchAttribute(query)

Expand All @@ -197,7 +175,6 @@ func SearchAttribute(w http.ResponseWriter, r *http.Request, d *db.Database) {

// ListResourceTypes lists all Resource types
func ListResourceTypes(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListResourceTypes()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -211,7 +188,6 @@ func ListResourceTypes(w http.ResponseWriter, _ *http.Request, d *db.Database) {

// ListResourceTypesWithCount lists all Resource types with their associated count
func ListResourceTypesWithCount(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListResourceTypesWithCount()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -225,7 +201,6 @@ func ListResourceTypesWithCount(w http.ResponseWriter, _ *http.Request, d *db.Da

// ListResourceNames lists all Resource names
func ListResourceNames(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListResourceNames()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -240,7 +215,6 @@ func ListResourceNames(w http.ResponseWriter, _ *http.Request, d *db.Database) {
// ListAttributeKeys lists all Resource Attribute Keys,
// optionally filtered by resource_type
func ListAttributeKeys(w http.ResponseWriter, r *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
resourceType := r.URL.Query().Get("resource_type")
result, _ := d.ListAttributeKeys(resourceType)
j, err := json.Marshal(result)
Expand All @@ -255,7 +229,6 @@ func ListAttributeKeys(w http.ResponseWriter, r *http.Request, d *db.Database) {

// ListTfVersions lists all Terraform versions
func ListTfVersions(w http.ResponseWriter, _ *http.Request, d *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
result, _ := d.ListTfVersions()
j, err := json.Marshal(result)
if err != nil {
Expand All @@ -269,8 +242,6 @@ func ListTfVersions(w http.ResponseWriter, _ *http.Request, d *db.Database) {

// GetUser returns information about the logged user
func GetUser(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")

name := r.Header.Get("X-Forwarded-User")
email := r.Header.Get("X-Forwarded-Email")

Expand All @@ -289,8 +260,6 @@ func GetUser(w http.ResponseWriter, r *http.Request) {
// SubmitPlan inserts a new Terraform plan in the database.
// /api/plans POST endpoint callback
func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")

body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Errorf("Failed to read body: %v", err)
Expand All @@ -310,7 +279,6 @@ func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) {
// Sorted by most recent to oldest.
// /api/plans GET endpoint callback
func GetPlans(w http.ResponseWriter, r *http.Request, db *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
lineage := r.URL.Query().Get("lineage")
limit := r.URL.Query().Get("limit")
plans := db.GetPlans(lineage, limit)
Expand Down Expand Up @@ -342,7 +310,6 @@ func ManagePlans(w http.ResponseWriter, r *http.Request, db *db.Database) {
// Optional "&limit=X" parameter to limit requested quantity of them.
// Sorted by most recent to oldest.
func GetLineages(w http.ResponseWriter, r *http.Request, db *db.Database) {
w.Header().Set("Access-Control-Allow-Origin", "*")
limit := r.URL.Query().Get("limit")
lineages := db.GetLineages(limit)

Expand Down
75 changes: 37 additions & 38 deletions db/db.go
View file
Open in desktop
Expand Up @@ -136,7 +136,7 @@ func (db *Database) stateS3toDB(sf *statefile.File, path string, versionID strin
Version: version,
TFVersion: sf.TerraformVersion.String(),
Serial: int64(sf.Serial),
LineageID: sql.NullInt64{Int64: int64(lineage.ID)},
LineageID: sql.NullInt64{Int64: int64(lineage.ID), Valid: true},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this Valid: true parameter do?

Copy link
Member Author

@hbollon hbollon Jul 15, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sql.NullInt64 or sql.NullString are types which handle null values for SQL operations.
Since we can't asign nil to int or string, the Valid field is here for that.
Valid: true means that it's not NULL, at false it's a NULL value

}

for _, m := range sf.State.Modules {
Expand Down Expand Up @@ -279,25 +279,26 @@ func (db *Database) InsertVersion(version *state.Version) error {
}

// GetState retrieves a State from the database by its path and versionID
func (db *Database) GetState(path, versionID string) (state types.State) {
db.Joins("JOIN versions on states.version_id=versions.id").
func (db *Database) GetState(lineage, versionID string) (state types.State) {
db.Joins("JOIN lineages on states.lineage_id=lineages.id").
Joins("JOIN versions on states.version_id=versions.id").
Preload("Version").Preload("Modules").Preload("Modules.Resources").Preload("Modules.Resources.Attributes").
Preload("Modules.OutputValues").
Find(&state, "states.path = ? AND versions.version_id = ?", path, versionID)
Find(&state, "lineages.value = ? AND versions.version_id = ?", lineage, versionID)
return
}

// GetStateActivity returns a slice of StateStat from the Database
// for a given State path representing the State activity over time (Versions)
func (db *Database) GetStateActivity(path string) (states []types.StateStat) {
// GetLineageActivity returns a slice of StateStat from the Database
// for a given lineage representing the State activity over time (Versions)
func (db *Database) GetLineageActivity(lineage string) (states []types.StateStat) {
sql := "SELECT t.path, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" +
" FROM (SELECT states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id WHERE states.path = ? ORDER BY states.path, versions.last_modified ASC) t" +
" FROM (SELECT states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN lineages ON lineages.id = states.lineage_id JOIN versions ON versions.id = states.version_id WHERE lineages.value = ? ORDER BY states.path, versions.last_modified ASC) t" +
" JOIN modules ON modules.state_id = t.id" +
" JOIN resources ON resources.module_id = modules.id" +
" GROUP BY t.path, t.serial, t.tf_version, t.version_id, t.last_modified" +
" ORDER BY last_modified ASC"

db.Raw(sql, path).Find(&states)
db.Raw(sql, lineage).Find(&states)
return
}

Expand Down Expand Up @@ -336,7 +337,8 @@ func (db *Database) SearchAttribute(query url.Values) (results []types.SearchRes

sqlQuery += " JOIN modules ON states.id = modules.state_id" +
" JOIN resources ON modules.id = resources.module_id" +
" JOIN attributes ON resources.id = attributes.resource_id"
" JOIN attributes ON resources.id = attributes.resource_id" +
" JOIN lineages ON lineages.id = states.lineage_id"

var where []string
var params []interface{}
Expand Down Expand Up @@ -370,6 +372,10 @@ func (db *Database) SearchAttribute(query url.Values) (results []types.SearchRes
where = append(where, fmt.Sprintf("states.tf_version LIKE '%s'", fmt.Sprintf("%%%s%%", v)))
}

if v := query.Get("lineage_value"); string(v) != "" {
hbollon marked this conversation as resolved.
Show resolved Hide resolved
where = append(where, fmt.Sprintf("lineages.value LIKE '%s'", fmt.Sprintf("%%%s%%", v)))
}

if len(where) > 0 {
sqlQuery += " WHERE " + strings.Join(where, " AND ")
}
Expand All @@ -382,11 +388,12 @@ func (db *Database) SearchAttribute(query url.Values) (results []types.SearchRes

// Now get results
// gorm doesn't support subqueries...
sql := "SELECT states.path, states.version_id, states.tf_version, states.serial, modules.path as module_path, resources.type, resources.name, resources.index, attributes.key, attributes.value" +
sql := "SELECT states.path, states.version_id, states.tf_version, states.serial, lineages.value as lineage_value, modules.path as module_path, resources.type, resources.name, resources.index, attributes.key, attributes.value" +
sqlQuery +
" ORDER BY states.path, states.serial, modules.path, resources.type, resources.name, resources.index, attributes.key" +
" ORDER BY states.path, states.serial, lineage_value, modules.path, resources.type, resources.name, resources.index, attributes.key" +
" LIMIT ?"

log.Info(sql)
params = append(params, pageSize)

if v := string(query.Get("page")); v != "" {
Expand Down Expand Up @@ -422,20 +429,6 @@ func (db *Database) ListStatesVersions() (statesVersions map[string][]string) {
return
}

// ListStates returns a slice of all State paths from the Database
func (db *Database) ListStates() (states []string) {
rows, _ := db.Table("states").Select("DISTINCT path").Rows()
defer rows.Close()
for rows.Next() {
var state string
if err := rows.Scan(&state); err != nil {
log.Error(err.Error())
}
states = append(states, state)
}
return
}

// ListTerraformVersionsWithCount returns a slice of maps of Terraform versions
// mapped to the count of most recent State paths using them.
// ListTerraformVersionsWithCount also takes a query with possible parameter 'orderBy'
Expand Down Expand Up @@ -475,28 +468,33 @@ func (db *Database) ListTerraformVersionsWithCount(query url.Values) (results []

// ListStateStats returns a slice of StateStat, along with paging information
func (db *Database) ListStateStats(query url.Values) (states []types.StateStat, page int, total int) {
row := db.Table("states").Select("count(DISTINCT path)").Row()
row := db.Raw("SELECT count(*) FROM (SELECT DISTINCT lineage_id FROM states) AS t").Row()
if err := row.Scan(&total); err != nil {
log.Error(err.Error())
}

offset := 0
var paginationQuery string
var params []interface{}
page = 1
if v := string(query.Get("page")); v != "" {
page, _ = strconv.Atoi(v) // TODO: err
offset = (page - 1) * pageSize
offset := (page - 1) * pageSize
params = append(params, offset)
paginationQuery = " LIMIT 20 OFFSET ?"
} else {
page = -1
}

sql := "SELECT t.path, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" +
" FROM (SELECT DISTINCT ON(states.path) states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.path, versions.last_modified DESC) t" +
sql := "SELECT t.path, lineages.value as lineage_value, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" +
" FROM (SELECT DISTINCT ON(states.lineage_id) states.id, states.lineage_id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.lineage_id, versions.last_modified DESC) t" +
" JOIN modules ON modules.state_id = t.id" +
" JOIN resources ON resources.module_id = modules.id" +
" GROUP BY t.path, t.serial, t.tf_version, t.version_id, t.last_modified" +
" JOIN lineages ON lineages.id = t.lineage_id" +
" GROUP BY t.path, lineages.value, t.serial, t.tf_version, t.version_id, t.last_modified" +
" ORDER BY last_modified DESC" +
" LIMIT 20" +
" OFFSET ?"
paginationQuery

db.Raw(sql, offset).Find(&states)
db.Raw(sql, params...).Find(&states)
return
}

Expand Down Expand Up @@ -682,14 +680,15 @@ func (db *Database) GetLineages(limitStr string) (lineages []types.Lineage) {

// DefaultVersion returns the detault VersionID for a given State path
// Copied and adapted from github.com/hashicorp/terraform/command/jsonstate/state.go
func (db *Database) DefaultVersion(path string) (version string, err error) {
func (db *Database) DefaultVersion(lineage string) (version string, err error) {
sqlQuery := "SELECT versions.version_id FROM" +
" (SELECT states.path, max(states.serial) as mx FROM states GROUP BY states.path) t" +
" JOIN states ON t.path = states.path AND t.mx = states.serial" +
" JOIN versions on states.version_id=versions.id" +
" WHERE states.path = ?"
" JOIN lineages on lineages.id=states.lineage_id" +
" WHERE lineages.value = ?"

row := db.Raw(sqlQuery, path).Row()
row := db.Raw(sqlQuery, lineage).Row()
err = row.Scan(&version)
return
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Expand Up @@ -11,6 +11,7 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/go-test/deep v1.0.3
github.com/google/go-cmp v0.5.5
github.com/gorilla/mux v1.8.0
github.com/hashicorp/errwrap v1.1.0
github.com/hashicorp/go-cleanhttp v0.5.1
github.com/hashicorp/go-hclog v0.15.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Expand Up @@ -315,6 +315,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
Expand Down