From de9188429aa3cd9f13d1ce81f936b592d63e9372 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Fri, 18 Jun 2021 13:32:56 +0200 Subject: [PATCH 1/3] feat(api): add plan submit endpoint --- api/api.go | 19 +++++++++++++++++++ db/db.go | 25 +++++++++++++++++++++++++ main.go | 1 + 3 files changed, 45 insertions(+) diff --git a/api/api.go b/api/api.go index 8af3e42e..0c992d61 100644 --- a/api/api.go +++ b/api/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "net/http" "github.com/camptocamp/terraboard/auth" @@ -278,3 +279,21 @@ func GetUser(w http.ResponseWriter, r *http.Request) { log.Error(err.Error()) } } + +// SubmitPlan insert a new Terraform plan in the database +func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) { + w.Header().Set("Access-Control-Allow-Origin", "*") + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errorf("Failed to read body: %v", err) + JSONError(w, "Failed to read body during plan submit", err) + return + } + + if err = db.InsertPlan(body); err != nil { + log.Errorf("Failed to insert plan to db: %v", err) + JSONError(w, "Failed to insert plan to db", err) + return + } +} diff --git a/db/db.go b/db/db.go index 0c2f86f3..260adf3a 100644 --- a/db/db.go +++ b/db/db.go @@ -566,6 +566,31 @@ func (db *Database) ListAttributeKeys(resourceType string) (results []string, er return } +// InsertPlan inserts a Terraform plan with associated information in the Database +func (db *Database) InsertPlan(plan []byte) error { + // Check for lineage existence + var lineage types.Lineage + if err := json.Unmarshal(plan, &lineage); err != nil { + return err + } + + res := db.First(&lineage, lineage) + if errors.Is(res.Error, gorm.ErrRecordNotFound) { + return fmt.Errorf("Plan's lineage not found in db") + } + + var p types.Plan + if err := json.Unmarshal(plan, &p); err != nil { + return err + } + if err := json.Unmarshal(p.PlanJSON, &p.ParsedPlan); err != nil { + return err + } + + p.LineageID = lineage.ID + return db.Create(&p).Error +} + // DefaultVersion returns the detault VersionID for a given State path // Copied and adapted from github.com/hashicorp/terraform/command/jsonstate/state.go func (db *Database) DefaultVersion(path string) (version string, err error) { diff --git a/main.go b/main.go index 6019093d..a1e217f0 100644 --- a/main.go +++ b/main.go @@ -196,6 +196,7 @@ func main() { http.HandleFunc(util.GetFullPath("api/resource/names"), handleWithDB(api.ListResourceNames, database)) http.HandleFunc(util.GetFullPath("api/attribute/keys"), handleWithDB(api.ListAttributeKeys, database)) http.HandleFunc(util.GetFullPath("api/tf_versions"), handleWithDB(api.ListTfVersions, database)) + http.HandleFunc(util.GetFullPath("api/plan/submit"), handleWithDB(api.SubmitPlan, database)) // Start server log.Debugf("Listening on port %d\n", c.Web.Port) From db01817a2ab152959fa9e9adb35d8f1450b874a9 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Mon, 21 Jun 2021 14:31:02 +0200 Subject: [PATCH 2/3] feat(api): add get plan endpoint + models tweaks * fix: missing omitempty tag on PlanStateResource.SchemaVersion * fix: replace PlanOutputChange by generic Change (adapt json unmarshalling) --- api/api.go | 48 +++++++++++++++++++++++++++++++++++++----------- db/db.go | 41 ++++++++++++++++++++++++++++++++++++++++- main.go | 10 +++++++++- types/db.go | 13 ++++--------- types/json.go | 9 ++++++--- 5 files changed, 96 insertions(+), 25 deletions(-) diff --git a/api/api.go b/api/api.go index 0c992d61..35d2619e 100644 --- a/api/api.go +++ b/api/api.go @@ -280,20 +280,46 @@ func GetUser(w http.ResponseWriter, r *http.Request) { } } -// SubmitPlan insert a new Terraform plan in the database +// SubmitPlan insert a new Terraform plan in the database. +// /api/plans POST endpoint callback func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) { - w.Header().Set("Access-Control-Allow-Origin", "*") + if r.Method == "POST" { + w.Header().Set("Access-Control-Allow-Origin", "*") - body, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Errorf("Failed to read body: %v", err) - JSONError(w, "Failed to read body during plan submit", err) - return + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errorf("Failed to read body: %v", err) + JSONError(w, "Failed to read body during plan submit", err) + return + } + + if err = db.InsertPlan(body); err != nil { + log.Errorf("Failed to insert plan to db: %v", err) + JSONError(w, "Failed to insert plan to db", err) + return + } } +} - if err = db.InsertPlan(body); err != nil { - log.Errorf("Failed to insert plan to db: %v", err) - JSONError(w, "Failed to insert plan to db", err) - return +// GetPlans provides all Plan by lineage. +// Optional "&limit=X" parameter to limit requested quantity of plans. +// Sorted by most recent to oldest. +// /api/plans GET endpoint callback +func GetPlans(w http.ResponseWriter, r *http.Request, db *db.Database) { + if r.Method == "GET" { + w.Header().Set("Access-Control-Allow-Origin", "*") + lineage := r.URL.Query().Get("lineage") + limit := r.URL.Query().Get("limit") + plans := db.GetPlans(lineage, limit) + + j, err := json.Marshal(plans) + if err != nil { + log.Errorf("Failed to marshal plans: %v", err) + JSONError(w, "Failed to marshal plans", err) + return + } + if _, err := io.WriteString(w, string(j)); err != nil { + log.Error(err.Error()) + } } } diff --git a/db/db.go b/db/db.go index 260adf3a..f9539bc9 100644 --- a/db/db.go +++ b/db/db.go @@ -61,7 +61,6 @@ func Init(config config.DBConfig, debug bool) *Database { &types.PlanModel{}, &types.PlanModelVariable{}, &types.PlanOutput{}, - &types.PlanOutputChange{}, &types.PlanResourceChange{}, &types.PlanState{}, &types.PlanStateModule{}, @@ -591,6 +590,46 @@ func (db *Database) InsertPlan(plan []byte) error { return db.Create(&p).Error } +// GetPlans retrieves all Plan of a lineage from the database +func (db *Database) GetPlans(lineage, limitStr string) (plans []types.Plan) { + var limit int + if limitStr == "" { + limit = -1 + } else { + var err error + limit, err = strconv.Atoi(limitStr) + if err != nil { + log.Warnf("GetPlans limit ignored: %v", err) + limit = -1 + } + } + + db.Joins("JOIN lineages on plans.lineage_id=lineages.id"). + Preload("ParsedPlan"). + Preload("ParsedPlan.PlanStateValue"). + Preload("ParsedPlan.PlanStateValue.PlanStateOutputs"). + Preload("ParsedPlan.PlanStateValue.PlanStateModule"). + Preload("ParsedPlan.PlanStateValue.PlanStateModule.PlanStateResources"). + Preload("ParsedPlan.PlanStateValue.PlanStateModule.PlanStateResources.PlanStateResourceAttributes"). + Preload("ParsedPlan.PlanStateValue.PlanStateModule.PlanStateModules"). + Preload("ParsedPlan.Variables"). + Preload("ParsedPlan.PlanResourceChanges"). + Preload("ParsedPlan.PlanResourceChanges.Change"). + Preload("ParsedPlan.PlanOutputs"). + Preload("ParsedPlan.PlanOutputs.Change"). + Preload("ParsedPlan.PlanState"). + Preload("ParsedPlan.PlanState.PlanStateValue"). + Preload("ParsedPlan.PlanState.PlanStateValue.PlanStateOutputs"). + Preload("ParsedPlan.PlanState.PlanStateValue.PlanStateModule"). + Preload("ParsedPlan.PlanState.PlanStateValue.PlanStateModule.PlanStateResources"). + Preload("ParsedPlan.PlanState.PlanStateValue.PlanStateModule.PlanStateResources.PlanStateResourceAttributes"). + Preload("ParsedPlan.PlanState.PlanStateValue.PlanStateModule.PlanStateModules"). + Order("created_at desc"). + Limit(limit). + Find(&plans, "lineages.value = ?", lineage) + return +} + // DefaultVersion returns the detault VersionID for a given State path // Copied and adapted from github.com/hashicorp/terraform/command/jsonstate/state.go func (db *Database) DefaultVersion(path string) (version string, err error) { diff --git a/main.go b/main.go index a1e217f0..25ccfe45 100644 --- a/main.go +++ b/main.go @@ -196,7 +196,15 @@ func main() { http.HandleFunc(util.GetFullPath("api/resource/names"), handleWithDB(api.ListResourceNames, database)) http.HandleFunc(util.GetFullPath("api/attribute/keys"), handleWithDB(api.ListAttributeKeys, database)) http.HandleFunc(util.GetFullPath("api/tf_versions"), handleWithDB(api.ListTfVersions, database)) - http.HandleFunc(util.GetFullPath("api/plan/submit"), handleWithDB(api.SubmitPlan, database)) + http.HandleFunc(util.GetFullPath("api/plans"), func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + api.GetPlans(w, r, database) + } else if r.Method == "POST" { + api.SubmitPlan(w, r, database) + } else { + http.Error(w, "Invalid request method.", 405) + } + }) // Start server log.Debugf("Listening on port %d\n", c.Web.Port) diff --git a/types/db.go b/types/db.go index 397c785e..3235f438 100644 --- a/types/db.go +++ b/types/db.go @@ -80,6 +80,7 @@ type Attribute struct { type Plan struct { gorm.Model LineageID uint `gorm:"index" json:"-"` + Lineage Lineage `json:"lineage_data"` TFVersion string `gorm:"varchar(10)" json:"terraform_version"` GitRemote string `json:"git_remote"` GitCommit string `gorm:"varchar(50)" json:"git_commit"` @@ -214,7 +215,7 @@ type PlanStateResource struct { // The version of the resource type schema the "values" property // conforms to. - SchemaVersion uint `json:"schema_version,"` + SchemaVersion uint `json:"schema_version,omitempty"` // The JSON representation of the attribute values of the resource, // whose structure depends on the resource type schema. Any unknown @@ -289,14 +290,8 @@ type PlanResourceChange struct { type PlanOutput struct { gorm.Model - Name string `gorm:"index" json:"key"` - PlanModelID sql.NullInt64 `gorm:"index" json:"-"` - PlanOutputChange PlanOutputChange `json:"value"` - PlanOutputChangeID sql.NullInt64 `gorm:"index" json:"-"` -} - -type PlanOutputChange struct { - gorm.Model + Name string `gorm:"index" json:"name"` + PlanModelID sql.NullInt64 `gorm:"index" json:"-"` // The data describing the change that will be made to this object. Change Change `json:"change,omitempty"` ChangeID sql.NullInt64 `gorm:"index" json:"-"` diff --git a/types/json.go b/types/json.go index 86b165d4..28f5f594 100644 --- a/types/json.go +++ b/types/json.go @@ -16,7 +16,7 @@ type planStateResourceAttributeList []PlanStateResourceAttribute type rawJSON string func (p *planOutputList) UnmarshalJSON(b []byte) error { - tmp := map[string]PlanOutput{} + tmp := map[string]Change{} err := json.Unmarshal(b, &tmp) if err != nil { return err @@ -24,8 +24,11 @@ func (p *planOutputList) UnmarshalJSON(b []byte) error { var list planOutputList for key, value := range tmp { - value.Name = key - list = append(list, value) + output := PlanOutput{ + Name: key, + Change: value, + } + list = append(list, output) } *p = list From 9814af5499291c34834bc4158ce44fffc22259c8 Mon Sep 17 00:00:00 2001 From: Hugo Bollon Date: Tue, 22 Jun 2021 13:52:16 +0200 Subject: [PATCH 3/3] feat(api): allow new lineage insertion if not exists on plan submit --- api/api.go | 64 ++++++++++++++++++++++++++++++------------------------ db/db.go | 8 +++---- main.go | 10 +-------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/api/api.go b/api/api.go index 35d2619e..650222ce 100644 --- a/api/api.go +++ b/api/api.go @@ -280,24 +280,22 @@ func GetUser(w http.ResponseWriter, r *http.Request) { } } -// SubmitPlan insert a new Terraform plan in the database. +// SubmitPlan inserts a new Terraform plan in the database. // /api/plans POST endpoint callback func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) { - if r.Method == "POST" { - w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Origin", "*") - body, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Errorf("Failed to read body: %v", err) - JSONError(w, "Failed to read body during plan submit", err) - return - } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errorf("Failed to read body: %v", err) + JSONError(w, "Failed to read body during plan submit", err) + return + } - if err = db.InsertPlan(body); err != nil { - log.Errorf("Failed to insert plan to db: %v", err) - JSONError(w, "Failed to insert plan to db", err) - return - } + if err = db.InsertPlan(body); err != nil { + log.Errorf("Failed to insert plan to db: %v", err) + JSONError(w, "Failed to insert plan to db", err) + return } } @@ -306,20 +304,30 @@ func SubmitPlan(w http.ResponseWriter, r *http.Request, db *db.Database) { // Sorted by most recent to oldest. // /api/plans GET endpoint callback func GetPlans(w http.ResponseWriter, r *http.Request, db *db.Database) { - if r.Method == "GET" { - w.Header().Set("Access-Control-Allow-Origin", "*") - lineage := r.URL.Query().Get("lineage") - limit := r.URL.Query().Get("limit") - plans := db.GetPlans(lineage, limit) + w.Header().Set("Access-Control-Allow-Origin", "*") + lineage := r.URL.Query().Get("lineage") + limit := r.URL.Query().Get("limit") + plans := db.GetPlans(lineage, limit) - j, err := json.Marshal(plans) - if err != nil { - log.Errorf("Failed to marshal plans: %v", err) - JSONError(w, "Failed to marshal plans", err) - return - } - if _, err := io.WriteString(w, string(j)); err != nil { - log.Error(err.Error()) - } + j, err := json.Marshal(plans) + if err != nil { + log.Errorf("Failed to marshal plans: %v", err) + JSONError(w, "Failed to marshal plans", err) + return + } + if _, err := io.WriteString(w, string(j)); err != nil { + log.Error(err.Error()) + } +} + +// ManagePlans is used to route the request to the appropriated handler function +// on /api/plans request +func ManagePlans(w http.ResponseWriter, r *http.Request, db *db.Database) { + if r.Method == "GET" { + GetPlans(w, r, db) + } else if r.Method == "POST" { + SubmitPlan(w, r, db) + } else { + http.Error(w, "Invalid request method.", 405) } } diff --git a/db/db.go b/db/db.go index f9539bc9..0e6a8e2a 100644 --- a/db/db.go +++ b/db/db.go @@ -567,15 +567,15 @@ func (db *Database) ListAttributeKeys(resourceType string) (results []string, er // InsertPlan inserts a Terraform plan with associated information in the Database func (db *Database) InsertPlan(plan []byte) error { - // Check for lineage existence var lineage types.Lineage if err := json.Unmarshal(plan, &lineage); err != nil { return err } - res := db.First(&lineage, lineage) - if errors.Is(res.Error, gorm.ErrRecordNotFound) { - return fmt.Errorf("Plan's lineage not found in db") + // Recover lineage from db if it's already exists or insert it + res := db.FirstOrCreate(&lineage, lineage) + if res.Error != nil { + return fmt.Errorf("Error on lineage retrival during plan insertion: %v", res.Error) } var p types.Plan diff --git a/main.go b/main.go index 25ccfe45..1d10406b 100644 --- a/main.go +++ b/main.go @@ -196,15 +196,7 @@ func main() { http.HandleFunc(util.GetFullPath("api/resource/names"), handleWithDB(api.ListResourceNames, database)) http.HandleFunc(util.GetFullPath("api/attribute/keys"), handleWithDB(api.ListAttributeKeys, database)) http.HandleFunc(util.GetFullPath("api/tf_versions"), handleWithDB(api.ListTfVersions, database)) - http.HandleFunc(util.GetFullPath("api/plans"), func(w http.ResponseWriter, r *http.Request) { - if r.Method == "GET" { - api.GetPlans(w, r, database) - } else if r.Method == "POST" { - api.SubmitPlan(w, r, database) - } else { - http.Error(w, "Invalid request method.", 405) - } - }) + http.HandleFunc(util.GetFullPath("api/plans"), handleWithDB(api.ManagePlans, database)) // Start server log.Debugf("Listening on port %d\n", c.Web.Port)