feat: add support for normalising expected diffs (#1859)

* [Feature]: Create a normalisation command in keploy CLI (#1589) * Initialised Normalise Command Signed-off-by: Akash <akashsingh2210670@gmail.com> * Path init Signed-off-by: Akash <akashsingh2210670@gmail.com> * Successfully found the expected and actual responses for failing tests Signed-off-by: Akash <akashsingh2210670@gmail.com> * Files are being updated but structs are not proper: func WriteTestcase doesnt match yaml file Signed-off-by: Akash <akashsingh2210670@gmail.com> * Updating testcase files successfully Signed-off-by: Akash <akashsingh2210670@gmail.com> * Added flags to collect test-set and test-cases from user Signed-off-by: Akash <akashsingh2210670@gmail.com> * Fixed lint errors Signed-off-by: Akash <akashsingh2210670@gmail.com> * Fixed lint errors Signed-off-by: Akash <akashsingh2210670@gmail.com> * Refactored PR for Keploy v2 and fixed merge conflicts Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Normalise Test cases Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Addressed comments and refactored changes Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Fixed Lint Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Fixed Lint Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Refactoring and Adding Additional Flags Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Added Normalise to replay service Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> * Fixed lint Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> --------- Signed-off-by: Akash <akashsingh2210670@gmail.com> Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> Co-authored-by: Animesh Pathak <53110238+Sonichigo@users.noreply.github.com> * feat: add normalization Signed-off-by: charankamarapu <kamarapucharan@gmail.com> * fix: bugs Signed-off-by: charankamarapu <kamarapucharan@gmail.com> * fix: bugs Signed-off-by: charankamarapu <kamarapucharan@gmail.com> * fix: bugs Signed-off-by: charankamarapu <kamarapucharan@gmail.com> * fix: resolve comments Signed-off-by: charankamarapu <kamarapucharan@gmail.com> --------- Signed-off-by: Akash <akashsingh2210670@gmail.com> Signed-off-by: Akash Singh <akashsingh2210670@gmail.com> Signed-off-by: charankamarapu <kamarapucharan@gmail.com> Co-authored-by: Sky Singh <114267538+Akash-Singh04@users.noreply.github.com> Co-authored-by: Animesh Pathak <53110238+Sonichigo@users.noreply.github.com>
keploy · May 7, 2024 · 4410963 · 4410963
1 parent c0dbdbb
commit 4410963
Show file tree

Hide file tree

Showing 8 changed files with 231 additions and 15 deletions.
diff --git a/cli/normalise.go b/cli/normalise.go
@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+	"go.keploy.io/server/v2/config"
+	replaySvc "go.keploy.io/server/v2/pkg/service/replay"
+	"go.keploy.io/server/v2/utils"
+	"go.uber.org/zap"
+)
+
+func init() {
+	Register("normalize", Normalize)
+}
+
+// Normalize retrieves the command to normalize Keploy
+func Normalize(ctx context.Context, logger *zap.Logger, _ *config.Config, serviceFactory ServiceFactory, cmdConfigurator CmdConfigurator) *cobra.Command {
+	var normalizeCmd = &cobra.Command{
+		Use:     "normalize",
+		Short:   "Normalize Keploy",
+		Example: "keploy normalize  --test-run testrun --tests test-set-1:test-case-1 test-case-2,test-set-2:test-case-1 test-case-2 ",
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			return cmdConfigurator.ValidateFlags(ctx, cmd)
+		},
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			svc, err := serviceFactory.GetService(ctx, cmd.Name())
+			if err != nil {
+				utils.LogError(logger, err, "failed to get service")
+				return nil
+			}
+			var replay replaySvc.Service
+			var ok bool
+			if replay, ok = svc.(replaySvc.Service); !ok {
+				utils.LogError(logger, nil, "service doesn't satisfy replay service interface")
+				return nil
+			}
+			if err := replay.Normalize(ctx); err != nil {
+				utils.LogError(logger, err, "failed to normalize test cases")
+				return nil
+			}
+			return nil
+		},
+	}
+	if err := cmdConfigurator.AddFlags(normalizeCmd); err != nil {
+		utils.LogError(logger, err, "failed to add normalize cmd flags")
+		return nil
+	}
+	return normalizeCmd
+}
diff --git a/cli/provider/cmd.go b/cli/provider/cmd.go
@@ -169,6 +169,10 @@ func (c *CmdConfigurator) AddFlags(cmd *cobra.Command) error {
 	switch cmd.Name() {
 	case "update":
 		return nil
+	case "normalize":
+		cmd.Flags().StringP("path", "p", ".", "Path to local directory where generated testcases/mocks/reports are stored")
+		cmd.Flags().String("test-run", "", "Test Run to be normalized")
+		cmd.Flags().String("tests", "", "Test Sets to be normalized")
 	case "config":
 		cmd.Flags().StringP("path", "p", ".", "Path to local directory where generated config is stored")
 		cmd.Flags().Bool("generate", false, "Generate a new keploy configuration file")
@@ -399,9 +403,7 @@ func (c *CmdConfigurator) ValidateFlags(ctx context.Context, cmd *cobra.Command)
 					return errors.New("missing required --containerName flag or containerName in config file")
 				}
 			}
-
 		}
-
 		err = utils.StartInDocker(ctx, c.logger, c.cfg)
 		if err != nil {
 			return err
@@ -449,6 +451,36 @@ func (c *CmdConfigurator) ValidateFlags(ctx context.Context, cmd *cobra.Command)
 				}
 			}
 		}
+	case "normalize":
+		path := c.cfg.Path
+		//if user provides relative path
+		if len(path) > 0 && path[0] != '/' {
+			absPath, err := filepath.Abs(path)
+			if err != nil {
+				utils.LogError(c.logger, err, "failed to get the absolute path from relative path")
+			}
+			path = absPath
+		} else if len(path) == 0 { // if user doesn't provide any path
+			cdirPath, err := os.Getwd()
+			if err != nil {
+				utils.LogError(c.logger, err, "failed to get the path of current directory")
+			}
+			path = cdirPath
+		}
+		path += "/keploy"
+		c.cfg.Path = path
+		tests, err := cmd.Flags().GetString("tests")
+		if err != nil {
+			errMsg := "failed to read tests to be normalized"
+			utils.LogError(c.logger, err, errMsg)
+			return errors.New(errMsg)
+		}
+		err = config.SetSelectedTestsNormalize(c.cfg, tests)
+		if err != nil {
+			errMsg := "failed to normalize the selected tests"
+			utils.LogError(c.logger, err, errMsg)
+			return errors.New(errMsg)
+		}
 	}
 	return nil
 }
diff --git a/cli/provider/service.go b/cli/provider/service.go
@@ -83,12 +83,12 @@ func (n *ServiceProvider) GetService(ctx context.Context, cmd string) (interface
 	case "config", "update":
 		return tools.NewTools(n.logger, tel), nil
 	// TODO: add case for mock
-	case "record", "test", "mock":
+	case "record", "test", "mock", "normalize":
 		commonServices := n.GetCommonServices(*n.cfg)
 		if cmd == "record" {
 			return record.New(n.logger, commonServices.YamlTestDB, commonServices.YamlMockDb, tel, commonServices.Instrumentation, *n.cfg), nil
 		}
-		if cmd == "test" {
+		if cmd == "test" || cmd == "normalize" {
 			return replay.NewReplayer(n.logger, commonServices.YamlTestDB, commonServices.YamlMockDb, commonServices.YamlReportDb, tel, commonServices.Instrumentation, *n.cfg), nil
 		}
 		return nil, errors.New("invalid command")

diff --git a/config/config.go b/config/config.go
@@ -2,6 +2,8 @@
 package config
 
 import (
+	"fmt"
+	"strings"
 	"time"
 )
 
@@ -21,6 +23,7 @@ type Config struct {
 	BuildDelay            time.Duration `json:"buildDelay" yaml:"buildDelay" mapstructure:"buildDelay"`
 	Test                  Test          `json:"test" yaml:"test" mapstructure:"test"`
 	Record                Record        `json:"record" yaml:"record" mapstructure:"record"`
+	Normalize             Normalize     `json:"normalize" yaml:"normalize" mapstructure:"normalize"`
 	ConfigPath            string        `json:"configPath" yaml:"configPath" mapstructure:"configPath"`
 	BypassRules           []BypassRule  `json:"bypassRules" yaml:"bypassRules" mapstructure:"bypassRules"`
 	EnableTesting         bool          `json:"enableTesting" yaml:"enableTesting" mapstructure:"enableTesting"`
@@ -35,6 +38,11 @@ type Record struct {
 	RecordTimer time.Duration `json:"recordTimer" yaml:"recordTimer" mapstructure:"recordTimer"`
 }
 
+type Normalize struct {
+	SelectedTests []SelectedTests `json:"selectedTests" yaml:"selectedTests" mapstructure:"selectedTests"`
+	TestRun       string          `json:"testReport" yaml:"testReport" mapstructure:"testReport"`
+}
+
 type BypassRule struct {
 	Path string `json:"path" yaml:"path" mapstructure:"path"`
 	Host string `json:"host" yaml:"host" mapstructure:"host"`
@@ -67,6 +75,11 @@ type Globalnoise struct {
 	Testsets TestsetNoise `json:"test-sets" yaml:"test-sets" mapstructure:"test-sets"`
 }
 
+type SelectedTests struct {
+	TestSet string   `json:"testSet" yaml:"testSet" mapstructure:"testSet"`
+	Tests   []string `json:"tests" yaml:"tests" mapstructure:"tests"`
+}
+
 type (
 	Noise        map[string][]string
 	GlobalNoise  map[string]map[string][]string
@@ -100,3 +113,27 @@ func SetSelectedTests(conf *Config, testSets []string) {
 		conf.Test.SelectedTests[testSet] = []string{}
 	}
 }
+
+func SetSelectedTestsNormalize(conf *Config, value string) error {
+	testSets := strings.FieldsFunc(value, func(r rune) bool {
+		return r == ',' || r == ' '
+	})
+	var tests []SelectedTests
+	if len(testSets) == 0 {
+		conf.Normalize.SelectedTests = tests
+		return nil
+	}
+	for _, ts := range testSets {
+		parts := strings.Split(ts, ":")
+		if len(parts) != 2 {
+			return fmt.Errorf("invalid format: %s", ts)
+		}
+		testCases := strings.Split(parts[1], " ")
+		tests = append(tests, SelectedTests{
+			TestSet: parts[0],
+			Tests:   testCases,
+		})
+	}
+	conf.Normalize.SelectedTests = tests
+	return nil
+}
diff --git a/pkg/service/replay/replay.go b/pkg/service/replay/replay.go
@@ -468,16 +468,7 @@ func (r *Replayer) RunTestSet(ctx context.Context, testSetID string, testRunID s
 					Form:       testCase.HTTPReq.Form,
 					Timestamp:  testCase.HTTPReq.Timestamp,
 				},
-				Res: models.HTTPResp{
-					StatusCode:    testCase.HTTPResp.StatusCode,
-					Header:        testCase.HTTPResp.Header,
-					Body:          testCase.HTTPResp.Body,
-					StatusMessage: testCase.HTTPResp.StatusMessage,
-					ProtoMajor:    testCase.HTTPResp.ProtoMajor,
-					ProtoMinor:    testCase.HTTPResp.ProtoMinor,
-					Binary:        testCase.HTTPResp.Binary,
-					Timestamp:     testCase.HTTPResp.Timestamp,
-				},
+				Res:          *resp,
 				TestCasePath: filepath.Join(r.config.Path, testSetID),
 				MockPath:     filepath.Join(r.config.Path, testSetID, "mocks.yaml"),
 				Noise:        testCase.Noise,
@@ -727,3 +718,88 @@ func (r *Replayer) ProvideMocks(ctx context.Context) error {
 	<-ctx.Done()
 	return nil
 }
+
+func (r *Replayer) Normalize(ctx context.Context) error {
+
+	var testRun string
+	if r.config.Normalize.TestRun == "" {
+		testRunIDs, err := r.reportDB.GetAllTestRunIDs(ctx)
+		if err != nil {
+			if errors.Is(err, context.Canceled) {
+				return err
+			}
+			return fmt.Errorf("failed to get all test run ids: %w", err)
+		}
+		testRun = pkg.LastID(testRunIDs, models.TestRunTemplateName)
+	}
+
+	if len(r.config.Normalize.SelectedTests) == 0 {
+		testSetIDs, err := r.testDB.GetAllTestSetIDs(ctx)
+		if err != nil {
+			if errors.Is(err, context.Canceled) {
+				return err
+			}
+			return fmt.Errorf("failed to get all test set ids: %w", err)
+		}
+		for _, testSetID := range testSetIDs {
+			r.config.Normalize.SelectedTests = append(r.config.Normalize.SelectedTests, config.SelectedTests{TestSet: testSetID})
+		}
+	}
+
+	for _, testSet := range r.config.Normalize.SelectedTests {
+		testSetID := testSet.TestSet
+		testCases := testSet.Tests
+		err := r.normalizeTestCases(ctx, testRun, testSetID, testCases)
+		if err != nil {
+			return err
+		}
+	}
+	r.logger.Info("Normalized test cases successfully. Please run keploy tests to verify the changes.")
+	return nil
+}
+
+func (r *Replayer) normalizeTestCases(ctx context.Context, testRun string, testSetID string, selectedTestCaseIds []string) error {
+
+	testReport, err := r.reportDB.GetReport(ctx, testRun, testSetID)
+	if err != nil {
+		return fmt.Errorf("failed to get test report: %w", err)
+	}
+	testCaseResults := testReport.Tests
+	testCaseResultMap := make(map[string]models.TestResult)
+
+	testCases, err := r.testDB.GetTestCases(ctx, testSetID)
+	if err != nil {
+		return fmt.Errorf("failed to get test cases: %w", err)
+	}
+	selectedTestCases := make([]*models.TestCase, 0, len(selectedTestCaseIds))
+
+	if len(selectedTestCaseIds) == 0 {
+		selectedTestCases = testCases
+	} else {
+		for _, testCase := range testCases {
+			if _, ok := ArrayToMap(selectedTestCaseIds)[testCase.Name]; ok {
+				selectedTestCases = append(selectedTestCases, testCase)
+			}
+		}
+	}
+
+	for _, testCaseResult := range testCaseResults {
+		testCaseResultMap[testCaseResult.TestCaseID] = testCaseResult
+	}
+
+	for _, testCase := range selectedTestCases {
+		if _, ok := testCaseResultMap[testCase.Name]; !ok {
+			r.logger.Info("test case not found in the test report", zap.String("test-case-id", testCase.Name), zap.String("test-set-id", testSetID))
+			continue
+		}
+		if testCaseResultMap[testCase.Name].Status == models.TestStatusPassed {
+			continue
+		}
+		testCase.HTTPResp = testCaseResultMap[testCase.Name].Res
+		err = r.testDB.UpdateTestCase(ctx, testCase, testSetID)
+		if err != nil {
+			return fmt.Errorf("failed to update test case: %w", err)
+		}
+	}
+	return nil
+}
diff --git a/pkg/service/replay/service.go b/pkg/service/replay/service.go
@@ -31,11 +31,13 @@ type Service interface {
 	GetTestSetStatus(ctx context.Context, testRunID string, testSetID string) (models.TestSetStatus, error)
 	RunApplication(ctx context.Context, appID uint64, opts models.RunOptions) models.AppError
 	ProvideMocks(ctx context.Context) error
+	Normalize(ctx context.Context) error
 }
 
 type TestDB interface {
 	GetAllTestSetIDs(ctx context.Context) ([]string, error)
 	GetTestCases(ctx context.Context, testSetID string) ([]*models.TestCase, error)
+	UpdateTestCase(ctx context.Context, testCase *models.TestCase, testSetID string) error
 }
 
 type MockDB interface {

diff --git a/pkg/service/tools/service.go b/pkg/service/tools/service.go
@@ -1,7 +1,9 @@
 // Package tools provides utility functions for the service package.
 package tools
 
-import "context"
+import (
+	"context"
+)
 
 type Service interface {
 	Update(ctx context.Context) error

diff --git a/pkg/util.go b/pkg/util.go
@@ -231,6 +231,23 @@ func NewID(IDs []string, identifier string) string {
 	return fmt.Sprintf("%s%v", identifier, latestIndx)
 }
 
+func LastID(IDs []string, identifier string) string {
+	latestIndx := 0
+	for _, ID := range IDs {
+		namePackets := strings.Split(ID, "-")
+		if len(namePackets) == 3 {
+			Indx, err := strconv.Atoi(namePackets[2])
+			if err != nil {
+				continue
+			}
+			if latestIndx < Indx {
+				latestIndx = Indx
+			}
+		}
+	}
+	return fmt.Sprintf("%s%v", identifier, latestIndx)
+}
+
 var (
 	dateFormats = []string{
 		time.Layout,