CI update

autogluon · Nov 18, 2022 · 305ba92 · 305ba92
1 parent 6d7e7f0
commit 305ba92
Show file tree

Hide file tree

Showing 23 changed files with 137 additions and 44 deletions.
diff --git a/.github/workflow_scripts/build_all_docs.sh b/.github/workflow_scripts/build_all_docs.sh
@@ -67,7 +67,7 @@ sed -i -e "s@###_OTHER_VERSIONS_DOCUMENTATION_BRANCH_###@$other_doc_version_bran
 
 shopt -s extglob
 rm -rf ./docs/tutorials/!(index.rst)
-cd docs && d2lbook build rst && d2lbook build html && cp static/images/* _build/html/_static
+cd docs && d2lbook build rst && d2lbook build html
 
 COMMAND_EXIT_CODE=$?
 if [ $COMMAND_EXIT_CODE -ne 0 ]; then

diff --git a/.github/workflow_scripts/build_doc.sh b/.github/workflow_scripts/build_doc.sh
@@ -6,9 +6,11 @@ function build_doc {
     PR_NUMBER="$5"  # For push events, PR_NUMBER will be empty
 
     source $(dirname "$0")/write_to_s3.sh
+    source $(dirname "$0")/setup_mmcv.sh
 
     setup_build_contrib_env
     bash docs/build_pip_install.sh
+    setup_mmcv
     # only build for docs/$DOC
     rm -rf ./docs/tutorials/!($DOC)
     cd docs && rm -rf _build && d2lbook build rst

diff --git a/.github/workflow_scripts/build_multimodal_tutorial.sh b/.github/workflow_scripts/build_multimodal_tutorial.sh
@@ -11,4 +11,4 @@ PR_NUMBER=$4  # For push events, PR_NUMBER will be empty
 source $(dirname "$0")/env_setup.sh
 source $(dirname "$0")/build_doc.sh
 
-build_doc multimodal $BRANCH $GIT_REPO $COMMIT_SHA $PR_NUMBER
+build_doc multimodal $BRANCH $GIT_REPO $COMMIT_SHA $PR_NUMBER
diff --git a/.github/workflow_scripts/build_text_prediction_tutorial.sh b/.github/workflow_scripts/build_text_prediction_tutorial.sh
@@ -12,7 +12,6 @@ source $(dirname "$0")/env_setup.sh
 source $(dirname "$0")/build_doc.sh
 
 
-setup_mxnet_gpu
 export CUDA_VISIBLE_DEVICES=0
 
 build_doc text_prediction $BRANCH $GIT_REPO $COMMIT_SHA $PR_NUMBER
diff --git a/.github/workflow_scripts/env_setup.sh b/.github/workflow_scripts/env_setup.sh
@@ -1,5 +1,6 @@
 function setup_build_env {
     python3 -m pip install --upgrade pip
+    python3 -m pip install tox
     python3 -m pip install flake8
     python3 -m pip install black>=22.3
     python3 -m pip install isort>=5.10
@@ -47,7 +48,7 @@ function install_features {
 }
 
 function install_eda {
-    python3 -m pip install --upgrade -e eda/[tests]
+    echo "python3 -m pip install --upgrade -e eda/[tests]"
 }
 
 function install_tabular {
@@ -101,3 +102,12 @@ function install_all {
     install_eda
     install_autogluon
 }
+
+function build_all {
+    for module in common core features tabular multimodal text vision timeseries autogluon
+    do
+        cd "$module"/
+        python setup.py sdist bdist_wheel
+        cd ..
+    done
+}
diff --git a/.github/workflow_scripts/lint_check.sh b/.github/workflow_scripts/lint_check.sh
@@ -5,8 +5,4 @@ set -ex
 source $(dirname "$0")/env_setup.sh
 
 setup_build_env
-bandit -r multimodal/src -ll
-black --check --diff multimodal/
-isort --check --diff multimodal/
-black --check --diff timeseries/
-isort --check --diff timeseries/
+black --check --diff multimodal/src/autogluon/multimodal
diff --git a/.github/workflow_scripts/setup_mmcv.sh b/.github/workflow_scripts/setup_mmcv.sh
@@ -0,0 +1,4 @@
+function setup_mmcv {
+  mim install mmcv-full
+  python3 -m pip install mmdet
+}
diff --git a/.github/workflow_scripts/test_common.sh b/.github/workflow_scripts/test_common.sh
@@ -10,4 +10,9 @@ setup_build_env
 install_common
 
 cd common/
-python3 -m pytest --mypy --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflow_scripts/test_core.sh b/.github/workflow_scripts/test_core.sh
@@ -11,4 +11,9 @@ install_core_all_tests
 python3 -m pip install ray_lightning==0.2.0  # TODO Change this line once we support ray_lightning 0.3.0
 
 cd core/
-python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflow_scripts/test_features.sh b/.github/workflow_scripts/test_features.sh
@@ -11,4 +11,9 @@ install_common
 install_features
 
 cd features/
-python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflow_scripts/test_install.sh b/.github/workflow_scripts/test_install.sh
@@ -7,3 +7,4 @@ source $(dirname "$0")/env_setup.sh
 setup_build_env
 python3 -m pip install 'mxnet==1.9.*'
 install_all
+build_all
diff --git a/.github/workflow_scripts/test_multimodal.sh b/.github/workflow_scripts/test_multimodal.sh
diff --git a/.github/workflow_scripts/test_multimodal_others.sh b/.github/workflow_scripts/test_multimodal_others.sh
@@ -0,0 +1,21 @@
+##!/bin/bash
+#
+#set -ex
+#
+#ADDITIONAL_TEST_ARGS=$1
+#
+#source $(dirname "$0")/env_setup.sh
+#
+#setup_build_env
+#export CUDA_VISIBLE_DEVICES=0
+#install_core_all_tests
+#install_features
+#install_multimodal
+#
+#cd multimodal/
+#if [ -n "$ADDITIONAL_TEST_ARGS" ]
+#then
+#    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests/unittests/others/
+#else
+#    python3 -m pytest --junitxml=results.xml --runslow tests/unittests/others/
+#fi
diff --git a/.github/workflow_scripts/test_multimodal_predictor.sh b/.github/workflow_scripts/test_multimodal_predictor.sh
@@ -0,0 +1,16 @@
+##!/bin/bash
+#
+#set -ex
+#
+#ADDITIONAL_TEST_ARGS=$1
+#
+#source $(dirname "$0")/env_setup.sh
+#
+#setup_build_env
+#export CUDA_VISIBLE_DEVICES=0
+#install_core_all_tests
+#install_features
+#install_multimodal
+#
+#cd multimodal/
+#python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
diff --git a/.github/workflow_scripts/test_tabular.sh b/.github/workflow_scripts/test_tabular.sh
@@ -16,4 +16,9 @@ install_text
 install_vision
 
 cd tabular/
-python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflow_scripts/test_text.sh b/.github/workflow_scripts/test_text.sh
@@ -14,4 +14,9 @@ install_multimodal
 install_text
 
 cd text/
-python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflow_scripts/test_timeseries.sh b/.github/workflow_scripts/test_timeseries.sh
@@ -15,4 +15,9 @@ install_tabular_all
 install_timeseries
 
 cd timeseries/
-python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflow_scripts/test_vision.sh b/.github/workflow_scripts/test_vision.sh
@@ -11,7 +11,14 @@ setup_mxnet_gpu
 setup_torch_gpu
 export CUDA_VISIBLE_DEVICES=0
 install_core_all_tests
+install_features
+install_multimodal
 install_vision
 
 cd vision/
-python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+if [ -n "$ADDITIONAL_TEST_ARGS" ]
+then
+    python3 -m pytest --junitxml=results.xml --runslow "$ADDITIONAL_TEST_ARGS" tests
+else
+    python3 -m pytest --junitxml=results.xml --runslow tests
+fi
diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml
@@ -160,7 +160,7 @@ jobs:
           job-type: CI-GPU
           job-name: AutoGluon-Text
           command: chmod +x ./.github/workflow_scripts/test_text.sh && ./.github/workflow_scripts/test_text.sh
-  test_multimodal:
+  test_multimodal_predictor:
     needs: lint_check
     runs-on: ubuntu-latest
     steps:
@@ -173,13 +173,33 @@ jobs:
           filters: |
             other_than_docs:
               - '!(docs/**)**'
-      - name: Test Multimodal on AWS Batch
+      - name: Test Multimodal Predictor on AWS Batch
         if: steps.changes.outputs.other_than_docs == 'true'
         uses: ./.github/actions/submit-job
         with:
           job-type: CI-GPU
-          job-name: AutoGluon-Multimodal
-          command: chmod +x ./.github/workflow_scripts/test_multimodal.sh && ./.github/workflow_scripts/test_multimodal.sh
+          job-name: AutoGluon-Multimodal-Predictor
+          command: chmod +x ./.github/workflow_scripts/test_multimodal_predictor.sh && ./.github/workflow_scripts/test_multimodal_predictor.sh
+  test_multimodal_others:
+    needs: lint_check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Check if changes beside docs
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            other_than_docs:
+              - '!(docs/**)**'
+      - name: Test Multimodal Others on AWS Batch
+        if: steps.changes.outputs.other_than_docs == 'true'
+        uses: ./.github/actions/submit-job
+        with:
+          job-type: CI-GPU
+          job-name: AutoGluon-Multimodal-Others
+          command: chmod +x ./.github/workflow_scripts/test_multimodal_others.sh && ./.github/workflow_scripts/test_multimodal_others.sh
   test_vision:
     needs: lint_check
     runs-on: ubuntu-latest
@@ -241,7 +261,7 @@ jobs:
           job-name: AutoGluon-Install
           command: chmod +x ./.github/workflow_scripts/test_install.sh && ./.github/workflow_scripts/test_install.sh
   build_image_prediction_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -255,7 +275,7 @@ jobs:
           job-name: AutoGluon-BuildImagePrediction
           command: chmod +x ./.github/workflow_scripts/build_image_prediction_tutorial.sh && ./.github/workflow_scripts/build_image_prediction_tutorial.sh '${{ env.BRANCH }}' '${{ env.GIT_REPO }}' '${{ env.SHORT_SHA }}' '${{ env.PR_NUMBER }}'
   build_object_detection_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -269,7 +289,7 @@ jobs:
           job-name: AutoGluon-BuildObjectDetection
           command: chmod +x ./.github/workflow_scripts/build_object_detection_tutorial.sh && ./.github/workflow_scripts/build_object_detection_tutorial.sh '${{ env.BRANCH }}' '${{ env.GIT_REPO }}' '${{ env.SHORT_SHA }}' '${{ env.PR_NUMBER }}'
   build_tabular_prediction_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -283,7 +303,7 @@ jobs:
           job-name: AutoGluon-BuildTabularPrediction
           command: chmod +x ./.github/workflow_scripts/build_tabular_prediction_tutorial.sh && ./.github/workflow_scripts/build_tabular_prediction_tutorial.sh '${{ env.BRANCH }}' '${{ env.GIT_REPO }}' '${{ env.SHORT_SHA }}' '${{ env.PR_NUMBER }}'
   build_multimodal_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -297,7 +317,7 @@ jobs:
           job-name: AutoGluon-BuildMultimodal
           command: chmod +x ./.github/workflow_scripts/build_multimodal_tutorial.sh && ./.github/workflow_scripts/build_multimodal_tutorial.sh '${{ env.BRANCH }}' '${{ env.GIT_REPO }}' '${{ env.SHORT_SHA }}' '${{ env.PR_NUMBER }}'
   build_text_prediction_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -311,7 +331,7 @@ jobs:
           job-name: AutoGluon-BuildTextPrediction
           command: chmod +x ./.github/workflow_scripts/build_text_prediction_tutorial.sh && ./.github/workflow_scripts/build_text_prediction_tutorial.sh '${{ env.BRANCH }}' '${{ env.GIT_REPO }}' '${{ env.SHORT_SHA }}' '${{ env.PR_NUMBER }}'
   build_cloud_fit_deploy_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -325,7 +345,7 @@ jobs:
           job-name: AutoGluon-BuildCloudFitDeploy
           command: chmod +x ./.github/workflow_scripts/build_cloud_fit_deploy_tutorial.sh && ./.github/workflow_scripts/build_cloud_fit_deploy_tutorial.sh '${{ env.BRANCH }}' '${{ env.GIT_REPO }}' '${{ env.SHORT_SHA }}' '${{ env.PR_NUMBER }}'
   build_timeseries_tutorial:
-    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal, test_vision, test_timeseries, test_install]
+    needs: [test_common, test_core, test_features, test_tabular, test_eda, test_text, test_multimodal_predictor, test_multimodal_others, test_vision, test_timeseries, test_install]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository

diff --git a/docs/build.yml b/docs/build.yml
@@ -5,3 +5,4 @@ dependencies:
 - pip:
   - flake8
   - black>=22.3
+  - isort>=5.10
diff --git a/docs/requirements_doc.txt b/docs/requirements_doc.txt
@@ -15,3 +15,4 @@ sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.4
 sphinxcontrib-svg2pdfconverter==1.1.0
 jinja2==3.0.3
+numpydoc==1.4.0  # 1.5.0 requires sphinx>4.0. Our dependency doesn't ping numpydoc, so we have to ping it.
diff --git a/multimodal/setup.py b/multimodal/setup.py
@@ -45,6 +45,7 @@
     'pytorch-metric-learning>=1.3.0,<1.4.0',
     'nlpaug>=1.1.10,<=1.1.10',
     'nltk>=3.4.5,<4.0.0',
+    'openmim>0.1.5,<=0.2.1',
 ]
 
 install_requires = ag.get_dependency_version_ranges(install_requires)

diff --git a/multimodal/src/autogluon/multimodal/utils.py b/multimodal/src/autogluon/multimodal/utils.py
@@ -1803,7 +1803,7 @@ def sha1sum(filename: str):
             The sha1sum
         """
         with open(filename, mode="rb") as f:
-            d = hashlib.sha1()
+            d = hashlib.sha1()  # nosec B303 - not a cryptographic use
             for buf in iter(functools.partial(f.read, 1024 * 100), b""):
                 d.update(buf)
         return d.hexdigest()