Merge branch 'response-format-html-68928'

programmfabrik · Jul 18, 2023 · 8c109a9 · 8c109a9
2 parents d614cf3 + 0ec6a2e
commit 8c109a9
Show file tree

Hide file tree

Showing 66 changed files with 969 additions and 339 deletions.
diff --git a/README.md b/README.md
@@ -474,6 +474,14 @@ See also template [`file_xml2json`](#file_xml2json-path).
 
 On that json you can work as you are used to with the json syntax. For seeing how the converted json looks you can use the `--log-verbose` command line flag
 
+## HTML Data comparison
+
+If the response format is specified as `"type": "html"`, we internally marshal that HTML into json using [github.com/PuerkitoBio/goquery](https://github.com/PuerkitoBio/goquery).
+
+This marshalling is less strict than for [XHTML](#file_xhtml2json-path). For example it will not raise errors for unclosed tags like `<p>` or `<hr>`, as well as Javascript code inside the HTML code. But it is possible that unclosed tags are missing in the resulting JSON if the tokenizer can not find a matching closing tag.
+
+See also template [`file_html2json`](#file_html2json-path).
+
 ## XHTML Data comparison
 
 If the response format is specified as `"type": "xhtml"`, we internally marshal that XHTML into json using [github.com/clbanning/mxj](https://github.com/clbanning/mxj).
@@ -1855,6 +1863,102 @@ would result in
 }
 ```
 
+## `file_html2json [path]`
+
+Helper function to parse an HTML file and convert it into json
+- `@path`: string; a path to the HTML file that should be loaded. The path is either relative to the manifest or a weburl
+
+This marshalling is less strict than for [XHTML](#file_xhtml2json-path). For example it will not raise errors for unclosed tags like `<p>` or `<hr>`, as well as Javascript code inside the HTML code. But it is possible that unclosed tags are missing in the resulting JSON if the [goquery](https://github.com/PuerkitoBio/goquery) tokenizer can not find a matching closing tag.
+
+### Example
+
+Content of HTML file `some/path/example.html`:
+
+```html
+<!DOCTYPE html>
+<html lang="en">
+
+	<head>
+		<meta charset="utf-8" />
+		<title>fylr</title>
+		<meta name="description" content="fylr - manage your data" />
+		<script>
+			function onInputHandler(event) {
+				const form = event.currentTarget;
+				submitForm(form);
+			}
+		</script>
+	</head>
+
+	<body>
+		<div class="container">
+			<h1>Register</h1>
+
+			<p class="required-information"><sup>*</sup>Mandatory fields<br>
+			<p class="error-summary">Form has errors
+
+			<hr>
+		</div>
+	</body>
+
+</html>
+```
+
+The call
+
+```django
+{{ file_html2json "some/path/example.html" }}
+```
+
+would result in
+
+```json
+{
+    "html": {
+        "-lang": "en",
+        "head": {
+            "meta": [
+                {
+                    "-charset": "utf-8"
+                },
+                {
+                    "-content": "fylr - manage your data",
+                    "-name": "description"
+                }
+            ],
+            "title": {
+                "#text": "fylr"
+            },
+            "script": {
+                "#text": "function onInputHandler(event) {\n\t\t\t\tconst form = event.currentTarget;\n\t\t\t\tsubmitForm(form);\n\t\t\t}"
+            }
+        },
+        "body": {
+            "div": {
+                "-class": "container",
+                "h1": {
+                    "#text": "Register"
+                },
+                "p": [
+                    {
+                        "-class": "required-information",
+                        "sup": {
+                            "#text": "*"
+                        },
+                        "br": {}
+                    },
+                    {
+                        "#text": "Form has errors",
+                        "-class": "error-summary"
+                    }
+                ],
+                "hr": {}
+            }
+        }
+    }
+}
+```
+
 ## `file_xhtml2json [path]`
 
 Helper function to parse an XHTML file and convert it into json
@@ -1865,6 +1969,7 @@ Helper function to parse an XHTML file and convert it into json
 Content of XHTML file `some/path/example.xhtml`:
 
 ```html
+<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html>
 <html xmlns="http://www.w3.org/1999/xhtml">
     <head>

diff --git a/api_testcase.go b/api_testcase.go
@@ -23,20 +23,20 @@ import (
 // Case defines the structure of our single testcase
 // It gets read in by our config reader at the moment the mainfest.json gets parsed
 type Case struct {
-	Name              string                 `json:"name"`
-	Description       string                 `json:"description"`
-	RequestData       *interface{}           `json:"request"`
-	ResponseData      interface{}            `json:"response"`
-	ContinueOnFailure bool                   `json:"continue_on_failure"`
-	Store             map[string]interface{} `json:"store"`                // init datastore before testrun
-	StoreResponse     map[string]string      `json:"store_response_qjson"` // store qjson parsed response in datastore
-
-	Timeout         int           `json:"timeout_ms"`
-	WaitBefore      *int          `json:"wait_before_ms"`
-	WaitAfter       *int          `json:"wait_after_ms"`
-	Delay           *int          `json:"delay_ms"`
-	BreakResponse   []interface{} `json:"break_response"`
-	CollectResponse interface{}   `json:"collect_response"`
+	Name              string            `json:"name"`
+	Description       string            `json:"description"`
+	RequestData       *any              `json:"request"`
+	ResponseData      any               `json:"response"`
+	ContinueOnFailure bool              `json:"continue_on_failure"`
+	Store             map[string]any    `json:"store"`                // init datastore before testrun
+	StoreResponse     map[string]string `json:"store_response_qjson"` // store qjson parsed response in datastore
+
+	Timeout         int   `json:"timeout_ms"`
+	WaitBefore      *int  `json:"wait_before_ms"`
+	WaitAfter       *int  `json:"wait_after_ms"`
+	Delay           *int  `json:"delay_ms"`
+	BreakResponse   []any `json:"break_response"`
+	CollectResponse any   `json:"collect_response"`
 
 	LogNetwork *bool `json:"log_network"`
 	LogVerbose *bool `json:"log_verbose"`
@@ -543,7 +543,7 @@ func (testCase Case) loadRequestSerialization() (api.Request, error) {
 	return spec, nil
 }
 
-func (testCase Case) loadResponseSerialization(genJSON interface{}) (spec api.ResponseSerialization, err error) {
+func (testCase Case) loadResponseSerialization(genJSON any) (spec api.ResponseSerialization, err error) {
 	resLoader := testCase.loader
 	_, responseData, err := template.LoadManifestDataAsObject(genJSON, testCase.manifestDir, resLoader)
 	if err != nil {

diff --git a/api_testsuite.go b/api_testsuite.go
@@ -32,8 +32,8 @@ type Suite struct {
 		Testmode bool                  `json:"testmode"`
 		Proxy    httpproxy.ProxyConfig `json:"proxy"`
 	} `json:"http_server,omitempty"`
-	Tests []interface{}          `json:"tests"`
-	Store map[string]interface{} `json:"store"`
+	Tests []any          `json:"tests"`
+	Store map[string]any `json:"store"`
 
 	StandardHeader          map[string]*string `yaml:"header" json:"header"`
 	StandardHeaderFromStore map[string]string  `yaml:"header_from_store" json:"header_from_store"`
@@ -213,7 +213,7 @@ type TestContainer struct {
 	Path     string
 }
 
-func (ats *Suite) parseAndRunTest(v interface{}, manifestDir, testFilePath string, k, repeatNTimes int, runParallel bool, r *report.ReportElement, rootLoader template.Loader) bool {
+func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, k, repeatNTimes int, runParallel bool, r *report.ReportElement, rootLoader template.Loader) bool {
 	//Init variables
 	// logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters)
 	loader := template.NewLoader(ats.datastore)

diff --git a/config.go b/config.go
@@ -17,8 +17,8 @@ import (
 
 type ConfigStruct struct {
 	Apitest struct {
-		Server    string                 `mapstructure:"server"`
-		StoreInit map[string]interface{} `mapstructure:"store"`
+		Server    string         `mapstructure:"server"`
+		StoreInit map[string]any `mapstructure:"store"`
 		Limit     struct {
 			Request  int `mapstructure:"request"`
 			Response int `mapstructure:"response"`

diff --git a/go.mod b/go.mod
@@ -1,72 +1,68 @@
 module github.com/programmfabrik/apitest
 
-go 1.18
+go 1.20
 
 require (
-	github.com/Masterminds/sprig/v3 v3.2.2
+	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/clbanning/mxj v1.8.4
-	github.com/mattn/go-sqlite3 v1.14.12
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/moul/http2curl v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/programmfabrik/go-test-utils v0.0.0-20191114143449-b8e16b04adb1
-	github.com/programmfabrik/golib v0.0.0-20220506120914-94c5900ef991
-	github.com/sergi/go-diff v1.2.0
-	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/afero v1.8.2
-	github.com/spf13/cobra v1.4.0
-	github.com/spf13/viper v1.11.0
-	github.com/stretchr/testify v1.7.1
-	github.com/tidwall/gjson v1.14.1
+	github.com/programmfabrik/golib v0.0.0-20230614100546-9870ba66917d
+	github.com/sergi/go-diff v1.3.1
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/afero v1.9.5
+	github.com/spf13/cobra v1.7.0
+	github.com/spf13/viper v1.16.0
+	github.com/stretchr/testify v1.8.4
+	github.com/tidwall/gjson v1.14.4
 	github.com/tidwall/jsonc v0.3.2
-	golang.org/x/mod v0.5.1
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
+	golang.org/x/mod v0.12.0
+	golang.org/x/net v0.11.0
+	golang.org/x/oauth2 v0.9.0
 )
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver/v3 v3.1.1 // indirect
-	github.com/antchfx/xmlquery v1.3.10 // indirect
-	github.com/antchfx/xpath v1.2.0 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/antchfx/xmlquery v1.3.17 // indirect
+	github.com/antchfx/xpath v1.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.0 // indirect
-	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/smartystreets/goconvey v1.7.2 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
+	github.com/smartystreets/goconvey v1.8.1 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/yudai/pp v2.0.1+incompatible // indirect
-	github.com/yuin/goldmark v1.4.12 // indirect
-	golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect
-	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
-	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	github.com/yuin/goldmark v1.4.13 // indirect
+	golang.org/x/crypto v0.10.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
-	gopkg.in/ini.v1 v1.66.4 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	moul.io/http2curl v1.0.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )