Fix mixed language tokenization in JiebaTokenizer #11618
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish Documentation | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "documentation" | |
| tags: | |
| - "**" | |
| pull_request: | |
| concurrency: | |
| # group workflow runs based on the branch or the tag ref | |
| group: documentation-${{ github.ref }} | |
| cancel-in-progress: true | |
| # SECRETS | |
| # - GH_DOCS_WRITE_KEY: generated locally, added to github repo (public key) | |
| # `ssh-keygen -t rsa -b 4096 -C "Github CI Docs Key" -N "" -f key` | |
| # - GITHUB_TOKEN: (default, from github actions) | |
| # - NETLIFY_AUTH_TOKEN: an access token to use when authenticating commands on Netlify | |
| # - NETLIFY_SITE_ID: the API ID of the Netlify site for the docs | |
| env: | |
| DOCS_FOLDER: docs | |
| DOCS_BRANCH: documentation | |
| IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }} | |
| IS_MAIN_BRANCH: ${{ github.ref == 'refs/heads/main' }} | |
| jobs: | |
| changes: | |
| name: Check for file changes | |
| runs-on: ubuntu-22.04 | |
| # don't run this for pull requests of forks | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' | |
| outputs: | |
| # Both of the outputs below are strings but only one exists at any given time | |
| backend: ${{ steps.changed-files.outputs.backend || steps.run-all.outputs.backend }} | |
| docker: ${{ steps.changed-files.outputs.docker || steps.run-all.outputs.docker }} | |
| docs: ${{ steps.changed-files.outputs.docs || steps.run-all.outputs.docs }} | |
| steps: | |
| - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 | |
| # Run the normal filters if the all-tests-required label is not set | |
| id: changed-files | |
| if: contains(github.event.pull_request.labels.*.name, 'status:all-tests-required') == false | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| filters: .github/change_filters.yml | |
| - name: Set all filters to true if all tests are required | |
| # Set all filters to true if the all-tests-required label is set | |
| # Bypasses all the change filters in change_filters.yml and forces all outputs to true | |
| id: run-all | |
| if: contains(github.event.pull_request.labels.*.name, 'status:all-tests-required') | |
| run: | | |
| echo "backend=true" >> $GITHUB_OUTPUT | |
| echo "docker=true" >> $GITHUB_OUTPUT | |
| echo "docs=true" >> $GITHUB_OUTPUT | |
| evaluate_release_tag: | |
| name: Evaluate release tag | |
| runs-on: ubuntu-22.04 | |
| # don't run this for main branches of forks and on documentation branch | |
| if: github.repository == 'RasaHQ/rasa' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request' | |
| outputs: | |
| build_docs: ${{ steps.check_tag.outputs.build_docs }} | |
| steps: | |
| - name: Checkout git repository 🕝 | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Install version library | |
| run: | | |
| python3 -m pip install pep440_version_utils | |
| - name: Check if tag version is equal or higher than the latest tagged Rasa version | |
| id: check_tag | |
| if: env.IS_TAG_BUILD == 'true' || env.IS_MAIN_BRANCH == 'true' | |
| run: | | |
| if [[ "${IS_MAIN_BRANCH}" == "true" ]]; then | |
| echo "Main branch: setting build_docs to true." | |
| echo "build_docs=true" >> $GITHUB_OUTPUT | |
| else | |
| # Get latest tagged Rasa version | |
| git fetch --depth=1 origin "+refs/tags/*:refs/tags/*" | |
| # Fetch branch history | |
| TAG_NAME=${GITHUB_REF#refs/tags/} | |
| git fetch --prune --unshallow | |
| python scripts/evaluate_release_tag.py $TAG_NAME | |
| exit_status=$? | |
| if [[ ${exit_status} -eq 0 ]]; then | |
| echo "Setting build_docs to true." | |
| echo "build_docs=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Setting build_docs to false." | |
| echo "build_docs=false" >> $GITHUB_OUTPUT | |
| fi | |
| fi | |
| prebuild_docs: | |
| name: Prebuild Docs | |
| runs-on: ubuntu-22.04 | |
| needs: [evaluate_release_tag] | |
| # don't run this for main branches of forks, would fail anyways | |
| if: github.repository == 'RasaHQ/rasa' && needs.evaluate_release_tag.outputs.build_docs == 'true' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request' | |
| steps: | |
| - name: Checkout git repository 🕝 | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Set up Python 3.10 🐍 | |
| uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b | |
| with: | |
| python-version: '3.10' | |
| - name: Set up Node 12.x 🦙 | |
| uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c | |
| with: | |
| node-version: "12.x" | |
| - name: Read Poetry Version 🔢 | |
| run: | | |
| echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Install poetry 🦄 | |
| uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Load Poetry Cached Libraries ⬇ | |
| id: cache-poetry | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} | |
| restore-keys: ${{ runner.os }}-poetry-3.9-non-full | |
| - name: Clear Poetry cache | |
| if: steps.cache-poetry.outputs.cache-hit == 'true' | |
| run: rm -r .venv | |
| - name: Create virtual environment | |
| if: steps.cache-poetry.outputs.cache-hit != 'true' | |
| run: python -m venv create .venv | |
| - name: Set up virtual environment | |
| run: poetry config virtualenvs.in-project true | |
| - name: Load Yarn Cached Packages ⬇ | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 | |
| with: | |
| path: docs/node_modules | |
| key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} | |
| restore-keys: ${{ runner.os }}-yarn-12.x | |
| - name: Install Dependencies 📦 | |
| run: make install install-docs | |
| - name: Pre-build Docs 🧶 | |
| run: make prepare-docs | |
| - name: Push docs to documentation branch 🏃♀️ | |
| env: | |
| GH_DOCS_WRITE_KEY: ${{ secrets.GH_DOCS_WRITE_KEY }} | |
| TMP_DOCS_FOLDER: /tmp/documentation-${{ github.run_id }} | |
| TMP_SSH_KEY_PATH: /tmp/docs_key | |
| run: | | |
| eval "$(ssh-agent -s)"; touch $TMP_SSH_KEY_PATH; chmod 0600 $TMP_SSH_KEY_PATH | |
| echo "$GH_DOCS_WRITE_KEY" > $TMP_SSH_KEY_PATH | |
| ssh-add $TMP_SSH_KEY_PATH | |
| git config --global user.email "builds@github-ci.com" | |
| git config --global user.name "GitHub CI" | |
| git remote set-url --push origin "git@github.com:${{github.repository}}" | |
| ./scripts/push_docs_to_branch.sh | |
| - name: Notify slack on failure | |
| if: failure() | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e # v1.6.0 | |
| with: | |
| channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }} | |
| status: FAILED | |
| color: warning | |
| preview_docs: | |
| name: Preview Docs | |
| runs-on: ubuntu-22.04 | |
| needs: [changes] | |
| # don't run this for pull requests from forks | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' | |
| steps: | |
| - name: Checkout git repository 🕝 | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Set up Python 3.10 🐍 | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b | |
| with: | |
| python-version: '3.10' | |
| - name: Set up Node 12.x 🦙 | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c | |
| with: | |
| node-version: "12.x" | |
| - name: Read Poetry Version 🔢 | |
| if: needs.changes.outputs.docs == 'true' | |
| run: | | |
| echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Install poetry 🦄 | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Load Poetry Cached Libraries ⬇ | |
| id: cache-poetry | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 | |
| with: | |
| path: .venv | |
| key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} | |
| restore-keys: ${{ runner.os }}-poetry-3.9-non-full | |
| - name: Clear Poetry cache | |
| if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.docs == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-preview-docs') | |
| run: rm -r .venv | |
| - name: Create virtual environment | |
| if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-preview-docs')) && needs.changes.outputs.docs == 'true' | |
| run: python -m venv create .venv | |
| - name: Set up virtual environment | |
| if: needs.changes.outputs.docs == 'true' | |
| run: poetry config virtualenvs.in-project true | |
| - name: Load Yarn Cached Packages ⬇ | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 | |
| with: | |
| path: docs/node_modules | |
| key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} | |
| restore-keys: ${{ runner.os }}-yarn-12.x | |
| - name: Install Dependencies 📦 | |
| if: needs.changes.outputs.docs == 'true' | |
| run: make install install-docs | |
| - name: Pre-build Docs 🧶 | |
| if: needs.changes.outputs.docs == 'true' | |
| run: make prepare-docs | |
| - name: Preview draft build 🔬 | |
| if: needs.changes.outputs.docs == 'true' | |
| id: preview_draft_build | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} | |
| DOCS_SITE_BASE_URL: /docs/rasa | |
| PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| make preview-docs | |
| DEPLOY_URL="https://$PULL_REQUEST_NUMBER--rasahq-docs-rasa-v2.netlify.app${DOCS_SITE_BASE_URL}" | |
| echo "preview_url=$DEPLOY_URL" >> $GITHUB_OUTPUT | |
| - name: Create a comment with help description | |
| if: needs.changes.outputs.docs == 'true' | |
| uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486 | |
| with: | |
| mode: "delete-previous" | |
| id: comment_docs_previews | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| 🚀 A preview of the docs have been deployed at the following URL: ${{ steps.preview_draft_build.outputs.preview_url }} | |
| publish_docs: | |
| name: Publish Docs | |
| runs-on: ubuntu-22.04 | |
| # don't run this for main branches of forks; only run on documentation branch | |
| if: github.repository == 'RasaHQ/rasa' && github.ref == 'refs/heads/documentation' | |
| steps: | |
| - name: Checkout git repository 🕝 | |
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c | |
| - name: Set up Node 12.x 🦙 | |
| uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c | |
| with: | |
| node-version: "12.x" | |
| - name: Load Yarn Cached Packages ⬇ | |
| uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 | |
| with: | |
| path: docs/node_modules | |
| key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} | |
| restore-keys: ${{ runner.os }}-yarn-12.x | |
| - name: Install Dependencies 📦 | |
| run: make install-docs | |
| - name: Publish production build ✅ | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} | |
| run: make publish-docs | |
| - name: Notify slack on failure | |
| if: failure() | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e # v1.6.0 | |
| with: | |
| channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }} | |
| status: FAILED | |
| color: warning |