Merge remote-tracking branch 'origin/opensearch'

NREL · Apr 17, 2024 · 211eb0d · 211eb0d
2 parents 28d2196 + 8617477
commit 211eb0d
Show file tree

Hide file tree

Showing 106 changed files with 3,132 additions and 3,806 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -10,7 +10,6 @@ env:
   IMAGE_NAME: nrel/api-umbrella
   DOCKER_BUILDKIT: 1
   TESTS_GLOB: "test/**/test_*.rb"
-  ELASTICSEARCH_TESTS_GLOB: "test/admin_ui/**/test_stats_*.rb test/admin_ui/**/test_stats_*.rb test/apis/admin/stats/**/test_*.rb test/apis/v0/test_analytics.rb test/apis/v1/analytics/**/test_*.rb test/proxy/logging/**/test_*.rb"
 
 jobs:
   build:
@@ -142,31 +141,6 @@ jobs:
   test:
     needs: [build]
     runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_PASSWORD: dev_password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      elasticsearch:
-        image: opensearchproject/opensearch:1.3.8
-        env:
-          OPENSEARCH_JAVA_OPTS: "-Xms32m -Xmx512m"
-          DISABLE_INSTALL_DEMO_CONFIG: "true"
-          DISABLE_SECURITY_PLUGIN: "true"
-          DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
-          DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI: "true"
-          discovery.type: single-node
-          cluster.routing.allocation.disk.threshold_enabled: "false"
-        options: >-
-          --health-cmd "curl -fsS http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 90s
-          --health-retries 10
     strategy:
       fail-fast: false
       matrix:
@@ -194,16 +168,16 @@ jobs:
           chmod +x /usr/local/bin/split_tests
 
           tests=$(split_tests --glob "${{ env.TESTS_GLOB }}" --split-total=${CI_NODE_TOTAL} --split-index=${CI_NODE_INDEX})
-          docker run --rm \
-            --network "${{ job.services.postgres.network }}" \
-            -v "$(pwd)/test/tmp/artifacts:/app/test/tmp/artifacts" \
-            -e CI=true \
+          docker compose run \
+            --rm \
             -e TESTS="${tests}" \
             -e MAXMIND_LICENSE_KEY="${MAXMIND_LICENSE_KEY}" \
-            "$DOCKER_IMAGE_CACHE_FROM" \
+            app \
             make test
         env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+          COMPOSE_FILE: docker-compose.ci.yml
+          TEST_IMAGE_REPO: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          TEST_IMAGE_TAG: ${{ steps.meta.outputs.version }}
           SPLIT_TEST_VERSION: v0.3.0
           CI_NODE_TOTAL: ${{ matrix.ci_node_total }}
           CI_NODE_INDEX: ${{ matrix.ci_node_index }}
@@ -230,171 +204,10 @@ jobs:
           name: test-artifacts-matrix-${{ matrix.ci_node_index }}
           path: test/tmp/artifacts/
 
-  test_elasticsearch_v6:
-    needs: [build]
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_PASSWORD: dev_password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.18
-        env:
-          ES_JAVA_OPTS: "-Xms32m -Xmx256m"
-          discovery.type: single-node
-          cluster.routing.allocation.disk.threshold_enabled: "false"
-        options: >-
-          --health-cmd "curl -fsS http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 90s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: dev-env container metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          flavor: |
-            suffix=-dev-env
-
-      - name: Pull
-        run: docker pull "$DOCKER_IMAGE_CACHE_FROM"
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-
-      - name: Test
-        run: |
-          shopt -s globstar
-          tests=(${{ env.ELASTICSEARCH_TESTS_GLOB }})
-          docker run --rm \
-            --network "${{ job.services.postgres.network }}" \
-            -v "$(pwd)/test/tmp/artifacts:/app/test/tmp/artifacts" \
-            -e ELASTICSEARCH_TEST_API_VERSION=6 \
-            -e ELASTICSEARCH_TEST_TEMPLATE_VERSION=2 \
-            -e CI=true \
-            -e TESTS="${tests[*]}" \
-            -e MAXMIND_LICENSE_KEY="${MAXMIND_LICENSE_KEY}" \
-            "$DOCKER_IMAGE_CACHE_FROM" \
-            make test
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-          MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
-
-      - name: Artifact permissions
-        if: ${{ always() }}
-        run: sudo chmod -R a+rwX test test/tmp/artifacts
-
-      - name: Dump docker logs
-        if: ${{ always() }}
-        uses: jwalton/gh-docker-logs@v2
-        with:
-          dest: test/tmp/artifacts/docker-logs
-
-      - name: Compress artifacts
-        if: ${{ always() }}
-        run: gzip -r ./test/tmp/artifacts/log ./test/tmp/artifacts/docker-logs
-
-      - name: Archive artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-artifacts-elasticsearch-v6
-          path: test/tmp/artifacts/
-
-  test_elasticsearch_v7_daily:
-    needs: [build]
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_PASSWORD: dev_password
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      elasticsearch:
-        image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
-        env:
-          ES_JAVA_OPTS: "-Xms32m -Xmx256m"
-          discovery.type: single-node
-          cluster.routing.allocation.disk.threshold_enabled: "false"
-        options: >-
-          --health-cmd "curl -fsS http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 90s
-          --health-retries 5
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: dev-env container metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          flavor: |
-            suffix=-dev-env
-
-      - name: Pull
-        run: docker pull "$DOCKER_IMAGE_CACHE_FROM"
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-
-      - name: Test
-        run: |
-          shopt -s globstar
-          tests=(${{ env.ELASTICSEARCH_TESTS_GLOB }})
-          docker run --rm \
-            --network "${{ job.services.postgres.network }}" \
-            -v "$(pwd)/test/tmp/artifacts:/app/test/tmp/artifacts" \
-            -e ELASTICSEARCH_TEST_API_VERSION=7 \
-            -e ELASTICSEARCH_TEST_TEMPLATE_VERSION=2 \
-            -e ELASTICSEARCH_TEST_INDEX_PARTITION=daily \
-            -e CI=true \
-            -e TESTS="${tests[*]}" \
-            -e MAXMIND_LICENSE_KEY="${MAXMIND_LICENSE_KEY}" \
-            "$DOCKER_IMAGE_CACHE_FROM" \
-            make test
-        env:
-          DOCKER_IMAGE_CACHE_FROM: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-          MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
-
-      - name: Artifact permissions
-        if: ${{ always() }}
-        run: sudo chmod -R a+rwX test/tmp/artifacts
-
-      - name: Dump docker logs
-        if: ${{ always() }}
-        uses: jwalton/gh-docker-logs@v2
-        with:
-          dest: test/tmp/artifacts/docker-logs
-
-      - name: Compress artifacts
-        if: ${{ always() }}
-        run: gzip -r ./test/tmp/artifacts/log ./test/tmp/artifacts/docker-logs
-
-      - name: Archive artifacts
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-artifacts-elasticsearch-v7-daily
-          path: test/tmp/artifacts/
-
   publish-test-results:
     runs-on: ubuntu-latest
     needs:
       - test
-      - test_elasticsearch_v6
-      - test_elasticsearch_v7_daily
     permissions:
       checks: write
     if: always()

diff --git a/Dockerfile b/Dockerfile
@@ -31,8 +31,8 @@ RUN make deps:openresty && make clean:dev
 COPY tasks/deps/trafficserver /app/tasks/deps/
 RUN make deps:trafficserver && make clean:dev
 
-COPY tasks/deps/rsyslog /app/tasks/deps/
-RUN make deps:rsyslog && make clean:dev
+COPY tasks/deps/fluent-bit /app/tasks/deps/
+RUN make deps:fluent-bit && make clean:dev
 
 COPY src/api-umbrella-git-1.rockspec src/luarocks.lock /app/src/
 COPY tasks/deps/luarocks /app/tasks/deps/

diff --git a/Dockerfile-opensearch b/Dockerfile-opensearch
@@ -0,0 +1,2 @@
+FROM public.ecr.aws/opensearchproject/opensearch:2.11.1
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch mapper-murmur3
diff --git a/Gemfile b/Gemfile
@@ -6,7 +6,7 @@ ruby "~> 3.1"
 gem "thor", "~> 1.3.0"
 
 # Tests
-gem "minitest", "~> 5.21.2"
+gem "minitest", "~> 5.22.2"
 
 # CLI helper for running tests
 gem "minitest-sprint", "~> 1.2.0"
@@ -37,7 +37,8 @@ gem "oj", "~> 3.10"
 # Database libraries
 gem "activerecord", "~> 7.1.1", :require => "active_record"
 gem "pg", "~> 1.5.3"
-gem "elasticsearch", "~> 7.13.3"
+gem "opensearch-ruby", "~> 3.1.0"
+gem "faraday-typhoeus", "~> 1.1.0"
 gem "active_attr", "~> 0.16.0"
 
 # Factories for test database data