Separate smart agent and integration tests (#4531)

* Separate smart agent and integration tests * fix tests * fix mysqldb name * make sure solr tests pass even with a different node resource attribute value * Do not check service_version when checking internal metrics * try to use redis-cli to make client writes work * make haproxy testing easier by grouping metrics into separate resources so comparison works best * do not start an extra redis server * remove one of the two redis checks since they both use the same config * fix more smartagent tests * remove clash of artifact names * skip one more test
signalfx · Mar 26, 2024 · 1280e73 · 1280e73
1 parent e5e5c3e
commit 1280e73
Show file tree

Hide file tree

Showing 32 changed files with 1,492 additions and 2,309 deletions.
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -101,6 +101,109 @@ jobs:
     runs-on: ubuntu-20.04
     needs: [docker-otelcol]
     services:
+      # Cassandra image for collectd-cassandra test:
+      cassandra:
+        image: quay.io/splunko11ytest/cassandra:latest
+        ports:
+          - "7199:7199"
+      # httpd_basicauth image for lightprometheus test
+      httpd_basicauth:
+        image: quay.io/splunko11ytest/httpd_basicauth:latest
+        ports:
+          - "8001:80"
+      kong:
+        image: quay.io/splunko11ytest/kong:latest
+        ports:
+          - "18001:8001"
+      oracle:
+        image: quay.io/splunko11ytest/oracle:latest
+        ports:
+          - "1521:1521"
+      postgres:
+        image: quay.io/splunko11ytest/postgres:latest
+        ports:
+          - "15432:5432"
+      redis_server:
+        image: quay.io/splunko11ytest/redis_server:latest
+        ports:
+          - "6379:6379"
+      mysql:
+        image: quay.io/splunko11ytest/mysql:latest
+        ports:
+          - "3306:3306"
+    strategy:
+      matrix:
+        ARCH: [ "amd64", "arm64" ]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: '**/go.sum'
+      - uses: actions/download-artifact@v4
+        with:
+          name: otelcol-${{ matrix.ARCH }}
+          path: ./bin
+      - uses: docker/setup-qemu-action@v3
+        if: ${{ matrix.ARCH != 'amd64' }}
+        with:
+          platforms: ${{ matrix.ARCH }}
+          image: tonistiigi/binfmt:qemu-v7.0.0
+      - run: docker load -i ./bin/image.tar
+      - run: chmod a+x ./bin/*
+      - uses: shogo82148/actions-setup-redis@v1
+        with:
+          auto-start: false
+          redis-port: "6379"
+      - run: redis-cli ping
+      - run: redis-cli set tempkey tempvalue
+      - name: Run Integration Test
+        run: |
+          set -o pipefail
+          make integration-test 2>&1 | tee ${{ github.job }}-${{ github.run_id }}.out
+          exit_status=${PIPESTATUS[0]}
+          echo "Exit status: $exit_status"
+          exit $exit_status
+        env:
+          SPLUNK_OTEL_COLLECTOR_IMAGE: 'otelcol:latest'
+      # The Integration Test output is extremely large so we upload it as an artifact
+      - name: Upload Integration Test Output as Artifact
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ github.job }}-${{ github.run_id }}-${{ matrix.ARCH }}.out
+          path: ${{ github.job }}-${{ github.run_id }}-${{ matrix.ARCH }}.out
+          retention-days: 5
+
+  smartagent-integration-test:
+    name: smartagent-integration-test
+    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
+    runs-on: ubuntu-20.04
+    needs: [docker-otelcol]
+    services:
+      # ActiveMQ image for collectd-activemq test:
+      activemq:
+        image: quay.io/splunko11ytest/activemq:latest
+        ports:
+          - "1099:1099"
+      # Apache image for collectd-apache test
+      apache:
+        image: quay.io/splunko11ytest/apache
+        ports:
+          - "18080:80"
+      # Cassandra image for collectd-cassandra test:
+      cassandra:
+        image: quay.io/splunko11ytest/cassandra:latest
+        ports:
+          - "7199:7199"
+      # Couchbase image for collectd-couchbase test:
+      couchbase:
+        image: quay.io/splunko11ytest/couchbase:latest
+        ports:
+          - "8091:8091"
       # Tomcat image for collectd-tomcat test:
       tomcat:
         image: tomcat
@@ -167,36 +270,16 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      # Cassandra image for collectd-cassandra test:
-      cassandra:
-        image: quay.io/splunko11ytest/cassandra:latest
-        ports:
-          - "7199:7199"
       # Nginx image for collectd-nginx test:
       nginx:
         image: quay.io/splunko11ytest/nginx:latest
         ports:
           - "8123:80"
-      # Couchbase image for collectd-couchbase test:
-      couchbase:
-        image: quay.io/splunko11ytest/couchbase:latest
-        ports:
-          - "8091:8091"
-      # ActiveMQ image for collectd-activemq test:
-      activemq:
-        image: quay.io/splunko11ytest/activemq:latest
-        ports:
-          - "1099:1099"
       # Haproxy image for haproxy test:
       haproxy:
         image: quay.io/splunko11ytest/haproxy:latest
         ports:
           - "8082:8080"
-      # Apache image for collectd-apache test
-      apache:
-        image: quay.io/splunko11ytest/apache
-        ports:
-          - "18080:80"
       # httpd image for prometheus-exporter test
       httpd:
         image: quay.io/splunko11ytest/httpd:latest
@@ -207,24 +290,6 @@ jobs:
         image: quay.io/splunko11ytest/httpd_basicauth:latest
         ports:
           - "8001:80"
-      kong:
-        image: quay.io/splunko11ytest/kong:latest
-        ports:
-          - "18001:8001"
-      oracle:
-        image: quay.io/splunko11ytest/oracle:latest
-        ports:
-          - "1521:1521"
-      postgres:
-        image: quay.io/splunko11ytest/postgres:latest
-        ports:
-          - "15432:5432"
-      redis_client:
-        image: quay.io/splunko11ytest/redis_client:latest
-      redis_server:
-        image: quay.io/splunko11ytest/redis_server:latest
-        ports:
-          - "6379:6379"
       solr:
         image: quay.io/splunko11ytest/solr:latest
         ports:
@@ -259,7 +324,7 @@ jobs:
       - name: Run Integration Test
         run: |
           set -o pipefail
-          make integration-test 2>&1 | tee ${{ github.job }}-${{ github.run_id }}.out
+          make smartagent-integration-test 2>&1 | tee ${{ github.job }}-${{ github.run_id }}.out
           exit_status=${PIPESTATUS[0]}
           echo "Exit status: $exit_status"
           exit $exit_status
@@ -270,6 +335,6 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: ${{ github.job }}-${{ github.run_id }}.out
-          path: ${{ github.job }}-${{ github.run_id }}.out
-          retention-days: 5
+          name: smartagent-${{ github.job }}-${{ github.run_id }}-${{ matrix.ARCH }}.out
+          path: smartagent-${{ github.job }}-${{ github.run_id }}-${{ matrix.ARCH }}.out
+          retention-days: 5
diff --git a/Makefile b/Makefile
@@ -72,9 +72,13 @@ integration-vet:
 	@set -e; cd tests && go vet -tags integration,testutilsintegration,zeroconfig,testutils ./... && $(GOTEST_SERIAL) $(BUILD_INFO_TESTS) -tags testutils,testutilsintegration -v -timeout 5m -count 1 ./...
 
 .PHONY: integration-test
-integration-test: integration-vet
+integration-test:
 	@set -e; cd tests && $(GOTEST_SERIAL) $(BUILD_INFO_TESTS) --tags=integration -v -timeout 5m -count 1 ./...
 
+.PHONY: smartagent-integration-test
+smartagent-integration-test:
+	@set -e; cd tests && $(GOTEST_SERIAL) $(BUILD_INFO_TESTS) --tags=smartagent_integration -v -timeout 5m -count 1 ./...
+
 .PHONY: test-with-cover
 test-with-cover:
 	@echo Verifying that all packages have test files to count in coverage

diff --git a/tests/general/container_test.go b/tests/general/container_test.go
@@ -12,13 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build integration
+//go:build smartagent_integration
 
 package tests
 
 import (
 	"fmt"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -177,13 +176,8 @@ func TestNonDefaultGIDCanAccessJavaInAgentBundle(t *testing.T) {
 func TestNonDefaultGIDCanAccessPythonInAgentBundle(t *testing.T) {
 	testutils.SkipIfNotContainerTest(t)
 	testutils.AssertAllMetricsReceived(t, "solr.yaml", "solr_config.yaml",
-		[]testutils.Container{
-			testutils.NewContainer().WithContext(
-				filepath.Join("..", "receivers", "smartagent", "collectd-solr", "testdata", "server"),
-			).WithExposedPorts("8983:8983").WithName(
-				"solr",
-			).WillWaitForPorts("8983").WillWaitForLogs("Time spent:"),
-		}, []testutils.CollectorBuilder{
+		nil,
+		[]testutils.CollectorBuilder{
 			func(collector testutils.Collector) testutils.Collector {
 				cc := collector.(*testutils.CollectorContainer)
 				cc.Container = cc.Container.WithUser("splunk-otel-collector:234567890")

diff --git a/tests/receivers/mysql/testdata/all_metrics_config.yaml b/tests/receivers/mysql/testdata/all_metrics_config.yaml
@@ -3,7 +3,7 @@ receivers:
     endpoint: "${MYSQLDB_ENDPOINT}"
     username: "${MYSQLDB_USERNAME}"
     password: "${MYSQLDB_PASSWORD}"
-    database: otel
+    database: testdb
     tls:
       insecure: true
     statement_events:                                                                                                                                                                                                           

diff --git a/tests/receivers/redis/bundled_test.go b/tests/receivers/redis/bundled_test.go
@@ -18,25 +18,21 @@ package tests
 
 import (
 	"fmt"
-	"path"
 	"runtime"
 	"testing"
 
 	"github.com/signalfx/splunk-otel-collector/tests/testutils"
 )
 
 func TestRedisDockerObserver(t *testing.T) {
+	t.Skip("Discovery mode picks up other Docker containers running, making the test fail.")
 	testutils.SkipIfNotContainerTest(t)
 	if runtime.GOOS == "darwin" {
 		t.Skip("unable to share sockets between mac and d4m vm: https://github.com/docker/for-mac/issues/483#issuecomment-758836836")
 	}
 
-	server := testutils.NewContainer().WithContext(path.Join(".", "testdata", "server")).WithExposedPorts("6379:6379").WithNetworks("redis_network").WithName("redis-server").WillWaitForLogs("Ready to accept connections")
-	client := testutils.NewContainer().WithContext(path.Join(".", "testdata", "client")).WithName("redis-client").WithNetworks("redis_network").WillWaitForLogs("redis client started")
-	containers := []testutils.Container{server, client}
-
 	testutils.AssertAllMetricsReceived(t, "all_server_bundled.yaml", "otlp_exporter.yaml",
-		containers, []testutils.CollectorBuilder{
+		nil, []testutils.CollectorBuilder{
 			func(c testutils.Collector) testutils.Collector {
 				cc := c.(*testutils.CollectorContainer)
 				cc.Container = cc.Container.WithBinds("/var/run/docker.sock:/var/run/docker.sock:ro")

diff --git a/tests/receivers/redis/redisreceiver_test.go b/tests/receivers/redis/redisreceiver_test.go
@@ -22,10 +22,6 @@ import (
 	"github.com/signalfx/splunk-otel-collector/tests/testutils"
 )
 
-func TestRedisReceiverProvidesAllMetrics(t *testing.T) {
-	testutils.AssertAllMetricsReceived(t, "all.yaml", "all_metrics_config.yaml", nil, nil)
-}
-
 func TestRedisReceiverProvidesAllMetricsWithServer(t *testing.T) {
 	testutils.AssertAllMetricsReceived(t, "all_server.yaml", "all_metrics_config.yaml", nil, nil)
 }
diff --git a/tests/receivers/redis/testdata/resource_metrics/all.yaml b/tests/receivers/redis/testdata/resource_metrics/all.yaml
diff --git a/tests/receivers/smartagent/collectd-activemq/collectd_activemq_test.go b/tests/receivers/smartagent/collectd-activemq/collectd_activemq_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build integration
+//go:build smartagent_integration
 
 package tests
 

diff --git a/tests/receivers/smartagent/collectd-apache/collectd_apache_test.go b/tests/receivers/smartagent/collectd-apache/collectd_apache_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build integration
+//go:build smartagent_integration
 
 package tests
 

diff --git a/tests/receivers/smartagent/collectd-cassandra/collectd_cassandra_test.go b/tests/receivers/smartagent/collectd-cassandra/collectd_cassandra_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build integration
+//go:build smartagent_integration
 
 package tests
 

diff --git a/tests/receivers/smartagent/collectd-couchbase/collectd_couchbase_test.go b/tests/receivers/smartagent/collectd-couchbase/collectd_couchbase_test.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build integration
+//go:build smartagent_integration
 
 package tests
 

diff --git a/tests/receivers/smartagent/collectd-custom/custom_upstat_test.go b/tests/receivers/smartagent/collectd-custom/custom_upstat_test.go
@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:build integration
+//go:build smartagent_integration
 
 package tests