Group colours by regex of the experiment name. (#6846) #11538
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Actions CI definition for TensorBoard. | |
| # | |
| # YAML schema for GitHub Actions: | |
| # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions | |
| # | |
| # Helpful YAML parser to clarify YAML syntax: | |
| # https://yaml-online-parser.appspot.com/ | |
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - '[0-9]+.*' | |
| - 'ci-*' | |
| tags: | |
| - '*' | |
| pull_request: {} | |
| workflow_call: {} | |
| permissions: | |
| contents: read | |
| env: | |
| # Keep this Bazel version in sync with the `versions.check` directive | |
| # in our WORKSPACE file. | |
| BAZEL_VERSION: '6.5.0' | |
| BAZEL_SHA256SUM: 'a40ac69263440761199fcb8da47ad4e3f328cbe79ffbf4ecc14e5ba252857307' | |
| BUILDTOOLS_VERSION: '3.0.0' | |
| BUILDIFIER_SHA256SUM: 'e92a6793c7134c5431c58fbc34700664f101e5c9b1c1fcd93b97978e8b7f88db' | |
| BUILDOZER_SHA256SUM: '3d58a0b6972e4535718cdd6c12778170ea7382de7c75bc3728f5719437ffb84d' | |
| TENSORFLOW_VERSION: 'tf-nightly' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-22.04 | |
| needs: lint-python-flake8 # fail fast in case of "undefined variable" errors | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tf_version_id: ['tf', 'notf'] | |
| python_version: ['3.9'] | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 | |
| with: | |
| python-version: ${{ matrix.python_version }} | |
| architecture: 'x64' | |
| - name: 'Set up Bazel' | |
| run: | | |
| ci/download_bazel.sh "${BAZEL_VERSION}" "${BAZEL_SHA256SUM}" ~/bazel | |
| sudo mv ~/bazel /usr/local/bin/bazel | |
| sudo chmod +x /usr/local/bin/bazel | |
| cp ./ci/bazelrc ~/.bazelrc | |
| - name: 'Configure build cache write credentials' | |
| env: | |
| CREDS: ${{ secrets.BAZEL_CACHE_SERVICE_ACCOUNT_CREDS }} | |
| EVENT_TYPE: ${{ github.event_name }} | |
| run: | | |
| if [ -z "${CREDS}" ]; then | |
| printf 'Using read-only cache (no credentials)\n' | |
| exit | |
| fi | |
| if [ "${EVENT_TYPE}" = pull_request ]; then | |
| printf 'Using read-only cache (PR build)\n' | |
| exit | |
| fi | |
| printf 'Using writable cache\n' | |
| creds_file=/tmp/service_account_creds.json | |
| printf '%s\n' "${CREDS}" >"${creds_file}" | |
| printf '%s\n' >>~/.bazelrc \ | |
| "common --google_credentials=${creds_file}" \ | |
| "common --remote_upload_local_results=true" \ | |
| ; | |
| - name: 'Install TensorFlow' | |
| run: | | |
| python -m pip install -U pip | |
| pip install "${TENSORFLOW_VERSION}" | |
| if: matrix.tf_version_id != 'notf' | |
| - name: 'Install Python dependencies' | |
| run: | | |
| python -m pip install -U pip | |
| pip install \ | |
| -r ./tensorboard/pip_package/requirements.txt \ | |
| -r ./tensorboard/pip_package/requirements_dev.txt \ | |
| ; | |
| - name: 'Check Pip state' | |
| run: pip freeze --all | |
| - name: 'Bazel: fetch' | |
| run: bazel fetch //tensorboard/... | |
| - name: 'Bazel: build' | |
| # Note we suppress a flood of warnings from the proto compiler. | |
| # Googlers see b/222706811 & b/182876485 discussion and preconditions | |
| # for removing this kludge. | |
| run: | | |
| ( set -o pipefail; | |
| bazel build //tensorboard/... |&\ | |
| grep -v 'external/com_google_protobuf/python: warning: directory' |&\ | |
| grep -v 'INFO: From ProtoCompile ' ) | |
| - name: 'Bazel: test (with TensorFlow support)' | |
| run: bazel test //tensorboard/... | |
| if: matrix.tf_version_id != 'notf' | |
| - name: 'Bazel: test (non-TensorFlow only)' | |
| run: bazel test //tensorboard/... --test_tag_filters="support_notf" | |
| if: matrix.tf_version_id == 'notf' | |
| - name: 'Bazel: run Pip package test (with TensorFlow support)' | |
| run: | | |
| bazel run //tensorboard/pip_package:test_pip_package -- \ | |
| --tf-version "${TENSORFLOW_VERSION}" | |
| if: matrix.tf_version_id != 'notf' | |
| - name: 'Bazel: run Pip package test (non-TensorFlow only)' | |
| run: | | |
| bazel run //tensorboard/pip_package:test_pip_package -- \ | |
| --tf-version notf | |
| if: matrix.tf_version_id == 'notf' | |
| - name: 'Bazel: run manual tests' | |
| run: | | |
| bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test && | |
| bazel test //tensorboard/summary/writer:event_file_writer_s3_test && | |
| bazel test //tensorboard/compat/tensorflow_stub:gfile_fsspec_test && | |
| bazel test //tensorboard/summary/writer:event_file_writer_fsspec_test | |
| - name: 'Bazel: build Pip package (master branch TF build only)' | |
| # This pip package is only needed if we will upload it, so run this step | |
| # only under the same conditions as the next step (see comment below). | |
| if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master' | |
| run: | | |
| ./tensorboard/tools/set_nightly_version.sh | |
| rm -rf /tmp/tb_nightly_pip_package && mkdir /tmp/tb_nightly_pip_package | |
| bazel run //tensorboard/pip_package:build_pip_package -- /tmp/tb_nightly_pip_package | |
| - name: 'Upload Pip package as an artifact (master branch TF build only)' | |
| # Prevent uploads when running on forks or non-master branch. | |
| if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master' | |
| uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 | |
| with: | |
| name: tb-nightly | |
| path: /tmp/tb_nightly_pip_package/* | |
| build-data-server-pip: | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| mode: ['native'] | |
| platform: ['ubuntu-20.04', 'macos-12'] | |
| rust_version: ['1.65.0'] | |
| include: | |
| - mode: 'universal' | |
| platform: 'ubuntu-20.04' | |
| rust_version: '1.65.0' | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 | |
| with: | |
| python-version: '3.9' | |
| architecture: 'x64' | |
| - name: 'Cache Cargo artifacts' | |
| if: matrix.mode == 'native' | |
| uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3.2.2 | |
| with: | |
| path: | | |
| tensorboard/data/server/target/ | |
| # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| # Needed if we `cargo install` binaries (at time of writing, this | |
| # job doesn't but the files are tiny, and this will mitigate | |
| # confusion if we need to `cargo install` later) | |
| ~/.cargo/.crates.toml | |
| ~/.cargo/.crates2.json | |
| key: build-data-server-pip-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }} | |
| - name: 'Install Rust toolchain' | |
| if: matrix.mode == 'native' | |
| uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023 | |
| with: | |
| toolchain: ${{ matrix.rust_version }} | |
| components: rustfmt | |
| - name: 'Install Python packaging deps' | |
| run: | | |
| python -m pip install -U pip | |
| pip install \ | |
| -c ./tensorboard/pip_package/requirements.txt \ | |
| -c ./tensorboard/pip_package/requirements_dev.txt \ | |
| setuptools wheel | |
| - name: 'Build' | |
| if: matrix.mode == 'native' | |
| run: cd tensorboard/data/server/ && cargo build --release | |
| - name: 'Test' | |
| if: matrix.mode == 'native' | |
| run: cd tensorboard/data/server/ && cargo test --release | |
| - name: 'Package (native)' | |
| if: matrix.mode == 'native' | |
| run: | | |
| rm -rf /tmp/pip_package && mkdir /tmp/pip_package | |
| python tensorboard/data/server/pip_package/build.py \ | |
| --server-binary tensorboard/data/server/target/release/rustboard \ | |
| --out-dir /tmp/pip_package \ | |
| ; | |
| - name: 'Package (universal)' | |
| if: matrix.mode == 'universal' | |
| run: | | |
| rm -rf /tmp/pip_package && mkdir /tmp/pip_package | |
| python tensorboard/data/server/pip_package/build.py \ | |
| --universal \ | |
| --out-dir /tmp/pip_package \ | |
| ; | |
| - name: 'Upload' | |
| uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 | |
| with: | |
| name: tensorboard-data-server | |
| path: /tmp/pip_package/* | |
| lint-python-flake8: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # flake8 should run on each Python version that we target, | |
| # because the errors and warnings can differ due to language | |
| # changes, and we want to catch them all. | |
| python_version: ['3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 | |
| with: | |
| python-version: ${{ matrix.python_version }} | |
| architecture: 'x64' | |
| - name: 'Install flake8' | |
| run: | | |
| python -m pip install -U pip | |
| pip install flake8 -c ./tensorboard/pip_package/requirements_dev.txt | |
| - run: pip freeze --all | |
| - name: 'Lint Python code for errors with flake8' | |
| # See: http://flake8.pycqa.org/en/3.7.8/user/error-codes.html | |
| # Use the comment '# noqa: <error code>' to suppress. | |
| run: flake8 . --count --select=E9,F63,F7,F82,F401 --show-source --statistics | |
| lint-python-yaml-docs: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 | |
| with: | |
| python-version: '3.10' | |
| architecture: 'x64' | |
| - name: 'Install black, yamllint, and the TensorFlow docs notebook tools' | |
| run: | | |
| python -m pip install -U pip | |
| nbfmt_version="84bd7bea0c468667dd0ec273d426ed50f23e1a15" | |
| pip install black yamllint -c ./tensorboard/pip_package/requirements_dev.txt | |
| pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version} | |
| # Workaround tensorflow incompatibility with protobuf>4. | |
| # See: https://github.com/tensorflow/tensorboard/issues/5708. | |
| # See: https://github.com/tensorflow/tensorflow/issues/53234. | |
| - name: 'Downgrade to compatible protobuf version' | |
| run: pip install -U "protobuf<3.20" | |
| - run: pip freeze --all | |
| - name: 'Lint Python code for style with Black' | |
| # You can run `black .` to fix all Black complaints. | |
| run: black --check --diff . | |
| - name: 'Lint YAML for gotchas with yamllint' | |
| # Use '# yamllint disable-line rule:foo' to suppress. | |
| run: yamllint -c docs/.yamllint docs docs/.yamllint | |
| - name: 'Lint Colab notebooks for formatting with nbfmt' | |
| run: git ls-files -z '*.ipynb' | xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test | |
| lint-rust: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| rust_version: ['1.65.0'] | |
| cargo_raze_version: ['0.16.1'] | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - name: 'Cache Cargo artifacts' | |
| uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3.2.2 | |
| with: | |
| path: | | |
| tensorboard/data/server/target/ | |
| # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| # Needed for installing binaries (`cargo-raze`) with cache | |
| ~/.cargo/.crates.toml | |
| ~/.cargo/.crates2.json | |
| key: lint-rust-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }} | |
| - name: 'Install Rust toolchain' | |
| uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023 | |
| with: | |
| toolchain: ${{ matrix.rust_version }} | |
| components: rustfmt, clippy | |
| - name: 'Install cargo-raze' | |
| run: cargo install cargo-raze --version ${{ matrix.cargo_raze_version }} --locked | |
| - name: 'Run Rustfmt' | |
| run: (cd tensorboard/data/server/ && cargo fmt -- --check) | |
| - name: 'Run Clippy' | |
| # You can run `cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml --fix` to fix all Clippy complaints. | |
| # This will only apply `MachineApplicable` fixes (https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/enum.Applicability.html), so some modifications may need to be done manually. | |
| run: cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml -- -D warnings | |
| - name: 'Check cargo-raze freshness' | |
| run: | | |
| rm -rf third_party/rust/ | |
| (cd tensorboard/data/server/ && cargo fetch && cargo raze) | |
| git add . | |
| git diff --staged --exit-code | |
| lint-frontend: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 | |
| with: | |
| node-version: 16 | |
| - run: yarn install --ignore-engines | |
| # You can run `yarn fix-lint` to fix all Prettier complaints. | |
| - run: yarn lint | |
| # Make sure no tests are skipped with "focused" tests. | |
| - run: | | |
| ! git grep -E 'f(it|describe)\(' 'tensorboard/*_test.ts' | |
| # Make sure no one depends on Angular material and CDK directly. Please | |
| # import the indirection in //tensorboard/webapp/angular. | |
| - run: | | |
| ! git grep -E '"@npm//@angular/material"|"@npm//@angular/cdk"' 'tensorboard/*/BUILD' ':!tensorboard/webapp/BUILD' ':!tensorboard/webapp/angular/BUILD' | |
| # Cannot directly depend on d3 in webapp. Must depend on | |
| # `//tensorboard/webapp/third_party:d3` instead. | |
| - run: | | |
| ! git grep -E '"@npm//d3"|"@npm//@types/d3"' 'tensorboard/webapp/**/*BUILD' ':!tensorboard/webapp/third_party/**' | |
| - run: | | |
| ! git grep -E '"@npm_angular_bazel//:index.bzl"' 'tensorboard/**/BUILD' | |
| - run: | | |
| ! git grep -E 'mat-color|$mat-' 'tensorboard/**/*.scss' | |
| # Make sure no one depends on the numeric library directly. Please | |
| # import the indirection in //tensorboard/webapp/third_party. | |
| - run: | | |
| ! git grep -E '@npm//numeric' 'tensorboard/*/BUILD' ':!tensorboard/webapp/third_party/**' | |
| lint-misc: # build, protos, etc. | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | |
| - name: 'Set up Buildifier' | |
| run: | | |
| ci/download_buildifier.sh "${BUILDTOOLS_VERSION}" "${BUILDIFIER_SHA256SUM}" ~/buildifier | |
| sudo mv ~/buildifier /usr/local/bin/buildifier | |
| - name: 'Set up Buildozer' | |
| run: | | |
| ci/download_buildozer.sh "${BUILDTOOLS_VERSION}" "${BUILDOZER_SHA256SUM}" ~/buildozer | |
| sudo mv ~/buildozer /usr/local/bin/buildozer | |
| - name: 'Lint BUILD files' | |
| # TODO(tensorboard-team): address all lint warnings and remove the exemption. | |
| run: | |
| git ls-files -z '*BUILD' third_party/js.bzl third_party/workspace.bzl WORKSPACE | xargs -0 buildifier --mode=check --lint=warn | |
| --warnings=-native-py,-native-java | |
| - run: ./tensorboard/tools/mirror_urls_test.sh | |
| - name: 'Lint for no py2 BUILD targets' | |
| # Use | to start a literal so YAML doesn't complain about the '!' character. | |
| run: | | |
| ! git grep 'python_version = "PY2"' '*BUILD' | |
| - name: 'No comments on licenses rule' | |
| # Assert buildozer error code for 'success, when no changes were made'. | |
| # https://github.com/bazelbuild/buildtools/blob/master/buildozer/README.md#error-code | |
| run: | | |
| buildozer '//tensorboard/...:%licenses' remove_comment && false || test $? = 3 | |
| - name: clang-format lint | |
| uses: DoozyX/clang-format-lint-action@0138140a2adaafd3032a3ff37f66366fd7dc88e0 | |
| with: | |
| source: ./tensorboard | |
| # Exclude tensorboard/compat because the source of truth is TensorFlow. | |
| exclude: ./tensorboard/compat/proto | |
| extensions: 'proto' | |
| clangFormatVersion: 9 | |
| - run: ./tensorboard/tools/do_not_submit_test.sh | |
| - run: ./tensorboard/tools/license_test.sh | |
| - run: ./tensorboard/tools/whitespace_hygiene_test.py |