[Ginkgo] Report on Fail

Signed-off-by: Eloy Coto <eloy.coto@gmail.com>

test/helpers/cilium.go

@@ -17,6 +17,7 @@ package helpers
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"strconv"
@@ -29,6 +30,8 @@ import (
 )
 
 const (
+	// MaxRetries is the number of times that a loop should iterate until a
+	// specified condition is not met
 	MaxRetries = 30
 )
 
@@ -264,6 +267,18 @@ func (c *Cilium) GetEndpointsIds() (map[string]string, error) {
 	return endpoints.KVOutput(), nil
 }
 
+// GetEndpointsIdentityIds returns a mapping of a Docker container name to it's
+// corresponding endpoint's security identity, it will return an error if the list
+// of endpoints cannot be retrieved via the Cilium CLI.
+func (c *Cilium) GetEndpointsIdentityIds() (map[string]string, error) {
+	filter := `{range [*]}{@.container-name}{"="}{@.identity.id}{"\n"}{end}`
+	endpoints := c.Exec(fmt.Sprintf("endpoint list -o jsonpath='%s'", filter))
+	if !endpoints.WasSuccessful() {
+		return nil, fmt.Errorf("cannot get endpoint list: %s", endpoints.CombineOutput())
+	}
+	return endpoints.KVOutput(), nil
+}
+
 // GetEndpointsNames returns the container-name field of each Cilium endpoint.
 func (c *Cilium) GetEndpointsNames() ([]string, error) {
 	data := c.GetEndpoints()
@@ -416,6 +431,95 @@ func (c *Cilium) ReportFailed(commands ...string) {
 		fmt.Fprint(wr, res.Output())
 	}
 	fmt.Fprint(wr, "StackTrace Ends\n")
+	c.ReportDump()
+	c.GatherLogs()
+}
+
+// ReportDump runs a variety of commands and writes the results to
+// testResultsPath
+func (c *Cilium) ReportDump() {
+
+	reportEndpointsCommands := map[string]string{
+		"cilium endpoint get %s":         "endpoint_%s.txt",
+		"sudo cilium bpf policy list %s": "bpf_policy_list_%s.txt",
+	}
+
+	testPath, err := ReportDirectory()
+	if err != nil {
+		c.logger.WithError(err).Errorf(
+			"cannot create test results path '%s'", testPath)
+		return
+	}
+	reportMap(testPath, ciliumCLICommands, c.Node)
+
+	// Endpoint specific information:
+	eps, err := c.GetEndpointsIds()
+	if err != nil {
+		c.logger.Errorf("cannot get endpoint ids")
+	}
+	for _, ep := range eps {
+		for cmd, logfile := range reportEndpointsCommands {
+			command := fmt.Sprintf(cmd, ep)
+			res := c.Node.Exec(command)
+
+			err = ioutil.WriteFile(
+				fmt.Sprintf("%s/%s", testPath, fmt.Sprintf(logfile, ep)),
+				res.CombineOutput().Bytes(),
+				os.ModePerm)
+
+			if err != nil {
+				c.logger.WithError(err).Errorf("cannot create test results for '%s'", command)
+			}
+		}
+	}
+
+	eps, err = c.GetEndpointsIdentityIds()
+	if err != nil {
+		c.logger.WithError(err).Error("cannot get endpoint identity ids")
+	}
+	for _, ep := range eps {
+		res := c.Node.Exec(fmt.Sprintf("cilium identity get %s", ep))
+		err = ioutil.WriteFile(
+			fmt.Sprintf("%s/%s", testPath, fmt.Sprintf("identity_%s.txt", ep)),
+			res.CombineOutput().Bytes(),
+			os.ModePerm)
+		if err != nil {
+			c.logger.WithError(err).Errorf("cannot create test results for identity get")
+		}
+	}
+}
+
+// GatherLogs dumps Cilium, Cilium Docker, key-value store logs, and gops output
+// to the directory testResultsPath
+func (c *Cilium) GatherLogs() {
+	ciliumLogCommands := map[string]string{
+		fmt.Sprintf("sudo journalctl -xe -u %s --no-pager", DaemonName):            "cilium.log",
+		fmt.Sprintf("sudo journalctl -xe -u %s--no-pager", CiliumDockerDaemonName): "cilium-docker.log",
+		"sudo docker logs cilium-consul":                                           "consul.log",
+		fmt.Sprintf(`sudo bash -c "gops memstats $(pgrep %s)"`, AgentDaemon):       "gops_memstats.txt",
+		fmt.Sprintf(`sudo bash -c "gops stack $(pgrep %s)"`, AgentDaemon):          "gops_stack.txt",
+		fmt.Sprintf(`sudo bash -c "gops stats $(pgrep %s)"`, AgentDaemon):          "gops_stats.txt",
+	}
+
+	testPath, err := ReportDirectory()
+	if err != nil {
+		c.logger.WithError(err).Errorf(
+			"cannot create test results path '%s'", testPath)
+		return
+	}
+	reportMap(testPath, ciliumLogCommands, c.Node)
+
+	ciliumBPFStateCommands := []string{
+		fmt.Sprintf("sudo cp -r %s %s/%s/lib", RunDir, BasePath, testPath),
+		fmt.Sprintf("sudo cp -r %s %s/%s/run", LibDir, BasePath, testPath),
+	}
+
+	for _, cmd := range ciliumBPFStateCommands {
+		res := c.Node.Exec(cmd)
+		if !res.WasSuccessful() {
+			c.logger.Errorf("cannot gather files for cmd '%s': %s", cmd, res.CombineOutput())
+		}
+	}
 }
 
 // ServiceAdd creates a new Cilium service with the provided ID, frontend,
@@ -532,7 +636,7 @@ PATH=/usr/lib/llvm-3.8/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sb
 CILIUM_OPTS=--kvstore consul --kvstore-opt consul.address=127.0.0.1:8500 --debug
 INITSYSTEM=SYSTEMD`
 
-	err := RenderTemplateToFile("cilium", template, 0777)
+	err := RenderTemplateToFile("cilium", template, os.ModePerm)
 	if err != nil {
 		return err
 	}

test/helpers/cons.go

@@ -119,8 +119,40 @@ const (
 
 	DefaultNamespace    = "default"
 	KubeSystemNamespace = "kube-system"
+
+	TestResultsPath = "test_results/"
+	RunDir          = "/var/run/cilium"
+	LibDir          = "/var/lib/cilium"
+
+	DaemonName             = "cilium"
+	CiliumDockerDaemonName = "cilium-docker"
+	AgentDaemon            = "cilium-agent"
 )
 
+var ciliumCLICommands = map[string]string{
+	"cilium endpoint list -o json":   "endpoint_list_txt",
+	"cilium service list -o json":    "service_list.txt",
+	"cilium config":                  "config.txt",
+	"sudo cilium bpf lb list":        "bpf_lb_list.txt",
+	"sudo cilium bpf ct list global": "bpf_ct_list.txt",
+	"sudo cilium bpf tunnel list":    "bpf_tunnel_list.txt",
+	"cilium policy get":              "policy_get.txt",
+	"cilium status":                  "status.txt",
+}
+
+// ciliumKubCLICommands these commands are the same as `ciliumCLICommands` but
+// it'll run inside a container and it does not have sudo support
+var ciliumKubCLICommands = map[string]string{
+	"cilium endpoint list -o json": "endpoint_list_txt",
+	"cilium service list -o json":  "service_list.txt",
+	"cilium config":                "config.txt",
+	"cilium bpf lb list":           "bpf_lb_list.txt",
+	"cilium bpf ct list global":    "bpf_ct_list.txt",
+	"cilium bpf tunnel list":       "bpf_tunnel_list.txt",
+	"cilium policy get":            "policy_get.txt",
+	"cilium status":                "status.txt",
+}
+
 //GetFilePath returns the absolute path of the provided filename
 func GetFilePath(filename string) string {
 	return fmt.Sprintf("%s%s", BasePath, filename)

test/helpers/kubectl.go

@@ -17,6 +17,7 @@ package helpers
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 	"time"
@@ -225,6 +226,22 @@ func (kub *Kubectl) CiliumEndpointsList(pod string) *CmdRes {
 	return kub.CiliumExec(pod, "cilium endpoint list -o json")
 }
 
+// CiliumEndpointsIDs returns a mapping  of a pod name to it is corresponding
+// endpoint's security identity
+func (kub *Kubectl) CiliumEndpointsIDs(pod string) map[string]string {
+	filter := `{range [*]}{@.container-name}{"="}{@.id}{"\n"}{end}`
+	return kub.CiliumExec(pod, fmt.Sprintf(
+		"cilium endpoint list -o jsonpath='%s'", filter)).KVOutput()
+}
+
+// CiliumEndpointsIdentityIDs returns a mapping with of a pod name to it is
+// corresponding endpoint's security identity
+func (kub *Kubectl) CiliumEndpointsIdentityIDs(pod string) map[string]string {
+	filter := `{range [*]}{@.container-name}{"="}{@.identity.id}{"\n"}{end}`
+	return kub.CiliumExec(pod, fmt.Sprintf(
+		"cilium endpoint list -o jsonpath='%s'", filter)).KVOutput()
+}
+
 // CiliumEndpointsListByLabel returns all endpoints that are labeled with label
 // in the form of an EndpointMap, which maps an endpoint's container name to its
 // corresponding Cilium API endpoint model. It returns an error if the extraction
@@ -442,9 +459,80 @@ func (kub *Kubectl) CiliumReport(namespace string, pod string, commands []string
 		fmt.Fprintln(wr, out.CombineOutput())
 	}
 	fmt.Fprint(wr, "StackTrace Ends\n")
+	kub.CiliumReportDump(namespace, pod)
+	kub.GatherLogs()
 	return nil
 }
 
+// CiliumReportDump runs a variety of commands (CiliumKubCLICommands) and writes the results to
+// TestResultsPath
+func (kub *Kubectl) CiliumReportDump(namespace string, pod string) {
+	reportEndpointCommands := map[string]string{
+		"cilium endpoint get %s":    "endpoint_get_%s.txt",
+		"cilium bpf policy list %s": "bpf_policy_list_%s.txt",
+	}
+
+	testPath, err := ReportDirectory()
+	if err != nil {
+		kub.logCxt.WithError(err).Errorf("cannot create test result path '%s'", testPath)
+		return
+	}
+
+	reportCmds := map[string]string{}
+	for cmd, logfile := range ciliumKubCLICommands {
+		command := fmt.Sprintf("kubectl exec -n %s %s -- %s", namespace, pod, cmd)
+		reportCmds[command] = logfile
+	}
+	reportMap(testPath, reportCmds, kub.Node)
+
+	for _, ep := range kub.CiliumEndpointsIDs(pod) {
+		for cmd, logfile := range reportEndpointCommands {
+			command := fmt.Sprintf(cmd, ep)
+			res := kub.Node.Exec(fmt.Sprintf(
+				"kubectl exec -n %s %s -- %s", namespace, pod, command))
+			err = ioutil.WriteFile(
+				fmt.Sprintf("%s/%s", testPath, fmt.Sprintf(logfile, ep)),
+				res.CombineOutput().Bytes(),
+				os.ModePerm)
+			if err != nil {
+				kub.logCxt.WithError(err).Errorf(
+					"cannot create test results for command '%s'", command)
+			}
+		}
+	}
+
+	for _, id := range kub.CiliumEndpointsIdentityIDs(pod) {
+		cmd := fmt.Sprintf("kubectl exec -n %s %s -- cilium identity get %s", namespace, pod, id)
+
+		res := kub.Node.Exec(cmd)
+		err = ioutil.WriteFile(
+			fmt.Sprintf("%s/%s", testPath, fmt.Sprintf("identity_%s.txt", id)),
+			res.CombineOutput().Bytes(),
+			os.ModePerm)
+		if err != nil {
+			kub.logCxt.WithError(err).Errorf("cannot create test results for command '%s'", cmd)
+		}
+	}
+}
+
+// GatherLogs dumps kubernetes pods, services, DaemonSet to the testResultsPath
+// directory
+func (kub *Kubectl) GatherLogs() {
+	reportCmds := map[string]string{
+		"kubectl get pods --all-namespaces":     "pods.txt",
+		"kubectl get services --all-namespaces": "svc.txt",
+		"kubectl get ds --all-namespaces":       "ds.txt",
+	}
+
+	testPath, err := ReportDirectory()
+	if err != nil {
+		kub.logCxt.WithError(err).Errorf(
+			"cannot create test results path '%s'", testPath)
+		return
+	}
+	reportMap(testPath, reportCmds, kub.Node)
+}
+
 // GetCiliumPodOnNode returns the name of the Cilium pod that is running on / in
 //the specified node / namespace.
 func (kub *Kubectl) GetCiliumPodOnNode(namespace string, node string) (string, error) {

test/helpers/utils.go

@@ -20,6 +20,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"os"
+	"strings"
 	"syscall"
 	"time"
 
@@ -138,3 +139,39 @@ func Fail(description string, callerSkip ...int) {
 	}
 	ginkgo.Fail(description, callerSkip...)
 }
+
+// ReportDirectory creates and returns the directory path to export all report
+// commands that need to be run in the case that a test has failed.
+// If the directory cannot be created it'll return an error
+func ReportDirectory() (string, error) {
+	testDesc := ginkgo.CurrentGinkgoTestDescription()
+	testPath := fmt.Sprintf("%s%s/",
+		TestResultsPath,
+		strings.Replace(testDesc.FullTestText, " ", "", -1))
+	if _, err := os.Stat(testPath); err == nil {
+		return testPath, nil
+	}
+	err := os.MkdirAll(testPath, os.ModePerm)
+	return testPath, err
+}
+
+// reportMap saves the output of the given commands to the specified filename.
+// Function needs a directory path where the files are going to be written and
+// a *SSHMeta instance to execute the commands
+func reportMap(path string, reportCmds map[string]string, node *SSHMeta) {
+	if node == nil {
+		log.Errorf("cannot execute reportMap due invalid node instance")
+		return
+	}
+
+	for cmd, logfile := range reportCmds {
+		res := node.Exec(cmd)
+		err := ioutil.WriteFile(
+			fmt.Sprintf("%s/%s", path, logfile),
+			res.CombineOutput().Bytes(),
+			os.ModePerm)
+		if err != nil {
+			log.WithError(err).Errorf("cannot create test results for command '%s'", cmd)
+		}
+	}
+}

test/runtime/monitor.go

@@ -68,6 +68,11 @@ var _ = Describe("RuntimeMonitorTest", func() {
 	})
 
 	AfterEach(func() {
+
+		if CurrentGinkgoTestDescription().Failed {
+			cilium.ReportFailed()
+		}
+
 		docker.SampleContainersActions(helpers.Delete, helpers.CiliumDockerNetwork)
 	})