Merge pull request #1616 from xueqzhan/new-variant-api

TRT-1615: Create api to use the new variant registry

pkg/api/component_report.go

@@ -115,6 +115,15 @@ func GetComponentTestVariantsFromBigQuery(client *bqcachedclient.Client, gcsBuck
 	return getDataFromCacheOrGenerate[apitype.ComponentReportTestVariants](client.Cache, cache.RequestOptions{}, GetPrefixedCacheKey("TestVariants~", generator), generator.GenerateVariants, apitype.ComponentReportTestVariants{})
 }
 
+func GetJobVariantsFromBigQuery(client *bqcachedclient.Client, gcsBucket string) (apitype.JobVariants, []error) {
+	generator := componentReportGenerator{
+		client:    client,
+		gcsBucket: gcsBucket,
+	}
+
+	return getDataFromCacheOrGenerate[apitype.JobVariants](client.Cache, cache.RequestOptions{}, GetPrefixedCacheKey("TestAllVariants~", generator), generator.GenerateJobVariants, apitype.JobVariants{})
+}
+
 func GetComponentReportFromBigQuery(client *bqcachedclient.Client, prowURL, gcsBucket string,
 	baseRelease, sampleRelease apitype.ComponentReportRequestReleaseOptions,
 	testIDOption apitype.ComponentReportRequestTestIdentificationOptions,
@@ -215,6 +224,59 @@ func (c *componentReportGenerator) GenerateVariants() (apitype.ComponentReportTe
 	}, errs
 }
 
+func (c *componentReportGenerator) GenerateJobVariants() (apitype.JobVariants, []error) {
+	errs := []error{}
+	variants := apitype.JobVariants{Variants: map[string][]string{}}
+	queryString := fmt.Sprintf(`SELECT variant_name, ARRAY_AGG(DISTINCT variant_value ORDER BY variant_value) AS variant_values
+					FROM
+						%s.job_variants
+					WHERE
+						variant_value!=""
+					GROUP BY
+						variant_name`, c.client.Dataset)
+	query := c.client.BQ.Query(queryString)
+	it, err := query.Read(context.TODO())
+	if err != nil {
+		log.WithError(err).Errorf("error querying variants from bigquery for %s", queryString)
+		return variants, []error{err}
+	}
+
+	floatVariants := sets.NewString("FromRelease", "FromReleaseMajor", "FromReleaseMinor", "Release", "ReleaseMajor", "ReleaseMinor")
+	for {
+		row := apitype.JobVariant{}
+		err := it.Next(&row)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			wrappedErr := errors.Wrapf(err, "error fetching variant row")
+			log.WithError(err).Error("error fetching variants from bigquery")
+			errs = append(errs, wrappedErr)
+			return variants, errs
+		}
+
+		// Sort all releases in proper orders
+		if floatVariants.Has(row.VariantName) {
+			sort.Slice(row.VariantValues, func(i, j int) bool {
+				iStrings := strings.Split(row.VariantValues[i], ".")
+				jStrings := strings.Split(row.VariantValues[j], ".")
+				for idx, iString := range iStrings {
+					if iValue, err := strconv.ParseInt(iString, 10, 32); err == nil {
+						if jValue, err := strconv.ParseInt(jStrings[idx], 10, 32); err == nil {
+							if iValue != jValue {
+								return iValue < jValue
+							}
+						}
+					}
+				}
+				return false
+			})
+		}
+		variants.Variants[row.VariantName] = row.VariantValues
+	}
+	return variants, nil
+}
+
 func (c *componentReportGenerator) GenerateReport() (apitype.ComponentReport, []error) {
 	before := time.Now()
 	componentReportTestStatus, errs := c.GenerateComponentReportTestStatus()

pkg/apis/api/types.go

@@ -1026,6 +1026,17 @@ type ComponentReportTestVariants struct {
 	Variant  []string `json:"variant,omitempty"`
 }
 
+// JobVariant defines a variant and the possible values
+type JobVariant struct {
+	VariantName   string   `bigquery:"variant_name"`
+	VariantValues []string `bigquery:"variant_values"`
+}
+
+// JobVariants contains all variants supported in the system.
+type JobVariants struct {
+	Variants map[string][]string `json:"variants,omitempty"`
+}
+
 var FailureRiskLevelNone = RiskLevel{Name: "None", Level: 0}
 var FailureRiskLevelLow = RiskLevel{Name: "Low", Level: 1}
 var FailureRiskLevelUnknown = RiskLevel{Name: "Unknown", Level: 25}

pkg/sippyserver/server.go

@@ -627,6 +627,29 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re
 	api.RespondWithJSON(http.StatusOK, w, outputs)
 }
 
+func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Request) {
+	if s.bigQueryClient == nil {
+		api.RespondWithJSON(http.StatusBadRequest, w, map[string]interface{}{
+			"code":    http.StatusBadRequest,
+			"message": "job variants API is only available when google-service-account-credential-file is configured",
+		})
+		return
+	}
+	outputs, errs := api.GetJobVariantsFromBigQuery(s.bigQueryClient, s.gcsBucket)
+	if len(errs) > 0 {
+		log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs))
+		for _, err := range errs {
+			log.Error(err.Error())
+		}
+		api.RespondWithJSON(http.StatusInternalServerError, w, map[string]interface{}{
+			"code":    http.StatusInternalServerError,
+			"message": fmt.Sprintf("error querying job variants from big query: %v", errs),
+		})
+		return
+	}
+	api.RespondWithJSON(http.StatusOK, w, outputs)
+}
+
 func (s *Server) jsonComponentReportFromBigQuery(w http.ResponseWriter, req *http.Request) {
 	baseRelease, sampleRelease, testIDOption, variantOption, excludeOption, advancedOption, cacheOption, err := s.parseComponentReportRequest(req)
 	if err != nil {
@@ -1451,6 +1474,12 @@ func (s *Server) Serve() {
 			Capabilities: []string{LocalDBCapability},
 			HandlerFunc:  s.jsonJobBugsFromDB,
 		},
+		{
+			EndpointPath: "/api/job_variants",
+			Description:  "Reports all job variants defined in BigQuery",
+			Capabilities: []string{ComponentReadinessCapability},
+			HandlerFunc:  s.jsonJobVariantsFromBigQuery,
+		},
 		{
 			EndpointPath: "/api/pull_requests",
 			Description:  "Reports on pull requests",