Merge 3.5.x into main

.github/workflows/continous-integration.yml

@@ -148,7 +148,7 @@ jobs:
         if: needs.changes.outputs.backend == 'true'
         run: |
           sudo apt-get -y install libpq-dev
-          make install-full
+          make install-full || make install-full || make install-full 
 
       - name: Checkout target branch to be able to diff
         if: needs.changes.outputs.backend == 'true' && github.event_name == 'pull_request'
@@ -215,6 +215,7 @@ jobs:
     name: Run Tests
     if: github.ref_type != 'tag'
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
     needs: [changes]
     strategy:
       fail-fast: false
@@ -293,7 +294,7 @@ jobs:
         if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04'
         run: |
           sudo apt-get -y install libpq-dev
-          make install-full | tee .output
+          make install-full | tee .output || make install-full | tee .output || make install-full | tee .output
           if grep 'The lock file is not up to date' .output; then exit 1; fi
           make prepare-tests-ubuntu
 
@@ -310,7 +311,7 @@ jobs:
             Remove-Item -Force -Recurse $spacy_data_dir
             New-Item -Path $spacy_data_dir -Type Directory
           }
-          make install-full
+          make install-full || make install-full || make install-full
           make prepare-tests-windows-gha
 
       - name: Add github workflow problem matchers
@@ -363,6 +364,7 @@ jobs:
     name: Run Flaky Tests
     if: github.ref_type != 'tag'
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
     needs: [changes]
     strategy:
       fail-fast: false
@@ -432,7 +434,7 @@ jobs:
         if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04'
         run: |
           sudo apt-get -y install libpq-dev
-          make install-full | tee .output
+          make install-full | tee .output || make install-full | tee .output || make install-full | tee .output
           if grep 'The lock file is not up to date' .output; then exit 1; fi
           make prepare-tests-ubuntu
 
@@ -449,7 +451,7 @@ jobs:
             Remove-Item -Force -Recurse $spacy_data_dir
             New-Item -Path $spacy_data_dir -Type Directory
           }
-          make install-full
+          make install-full || make install-full || make install-full
           make prepare-tests-windows-gha
 
       - name: Add github workflow problem matchers
@@ -554,6 +556,7 @@ jobs:
     name: Run Non-Sequential Integration Tests
     if: github.ref_type != 'tag'
     runs-on: ubuntu-22.04
+    timeout-minutes: 60
     needs: [changes]
     env:
       REDIS_HOST: localhost
@@ -668,7 +671,7 @@ jobs:
         if: needs.changes.outputs.backend == 'true'
         run: |
           sudo apt-get -y install libpq-dev
-          make install-full | tee .output
+          make install-full | tee .output || make install-full | tee .output || make install-full | tee .output
           if grep 'The lock file is not up to date' .output; then exit 1; fi
           make prepare-tests-ubuntu
 
@@ -690,6 +693,7 @@ jobs:
     name: Run Sequential Integration Tests
     if: github.ref_type != 'tag'
     runs-on: ubuntu-20.04
+    timeout-minutes: 60
     needs: [changes]
     env:
       POSTGRES_HOST: localhost
@@ -769,7 +773,7 @@ jobs:
         if: needs.changes.outputs.backend == 'true'
         run: |
           sudo apt-get -y install libpq-dev
-          make install-full | tee .output
+          make install-full | tee .output || make install-full | tee .output || make install-full | tee .output
           if grep 'The lock file is not up to date' .output; then exit 1; fi
           make prepare-tests-ubuntu
 

CHANGELOG.mdx

@@ -16,6 +16,45 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . -->
 
 <!-- TOWNCRIER -->
 
+## [3.5.17] - 2023-12-05
+
+Rasa 3.5.17 (2023-12-05)                         
+### Improvements
+- [#12851](https://github.com/rasahq/rasa/issues/12851): Added `username` to the connection parameters for `RedisLockStore` and `RedisTrackerStore`
+- [#1493](https://github.com/rasahq/rasa/issues/1493): Telemetry data is only send for licensed users.
+
+
+## [3.5.16] - 2023-08-30
+
+Rasa 3.5.16 (2023-08-30)                          
+
+No significant changes.
+
+
+## [3.5.15] - 2023-07-21
+
+Rasa 3.5.15 (2023-07-21)                          
+
+No significant changes.
+
+
+## [3.5.14] - 2023-07-12
+
+Rasa 3.5.14 (2023-07-12)                         
+### Bugfixes
+- [#12639](https://github.com/rasahq/rasa/issues/12639): Fix the issue with the most recent model not being selected if the owner or permissions where modified on the model file.
+
+### Miscellaneous internal changes
+- [#12649](https://github.com/rasahq/rasa/issues/12649)
+
+
+## [3.5.13] - 2023-07-05
+
+Rasa 3.5.13 (2023-07-05)                         
+### Bugfixes
+- [#12549](https://github.com/rasahq/rasa/issues/12549): Introduce a validation step in `rasa data validate` command to identify non-existent paths and empty domains.
+
+
 ## [3.5.12] - 2023-06-23
 
 Rasa 3.5.12 (2023-06-23)                         

Makefile

@@ -120,12 +120,8 @@ endif
 	rm data/MITIE*.bz2
 
 prepare-transformers:
-	if [ $(OS) = "Windows_NT" ]; then HOME_DIR="$(HOMEDRIVE)$(HOMEPATH)"; else HOME_DIR=$(HOME); fi;\
-	CACHE_DIR=$$HOME_DIR/.cache/torch/transformers;\
-	mkdir -p "$$CACHE_DIR";\
-	i=0;\
-	while read -r URL; do read -r CACHE_FILE; if { [ $(CI) ]  &&  [ $$i -gt 4 ]; } || ! [ $(CI) ]; then wget -nv $$URL -O $$CACHE_DIR/$$CACHE_FILE; fi; i=$$((i + 1)); done < "data/test/hf_transformers_models.txt"
-
+	while read -r MODEL; do poetry run python scripts/download_transformer_model.py $$MODEL ; done < data/test/hf_transformers_models.txt
+	if ! [ $(CI) ]; then poetry run python scripts/download_transformer_model.py rasa/LaBSE; fi
 prepare-tests-macos:
 	brew install wget graphviz || true
 
@@ -139,7 +135,7 @@ prepare-tests-windows:
 # It will retry the installation 5 times if it fails
 # See: https://github.com/actions/virtual-environments/blob/main/images/win/scripts/ImageHelpers/ChocoHelpers.ps1
 prepare-tests-windows-gha:
-	powershell -command "Choco-Install wget graphviz"
+	powershell -command "Install-ChocoPackage wget graphviz"
 
 test: clean
 	# OMP_NUM_THREADS can improve overall performance using one thread by process (on tensorflow), avoiding overload
@@ -177,7 +173,7 @@ test-nlu-predictors: prepare-spacy prepare-mitie test-marker
 
 test-full-model-training: PYTEST_MARKER=category_full_model_training and (not flaky)
 test-full-model-training: DD_ARGS := $(or $(DD_ARGS),)
-test-full-model-training: prepare-spacy prepare-mitie test-marker
+test-full-model-training: prepare-spacy prepare-mitie prepare-transformers test-marker
 
 test-other-unit-tests: PYTEST_MARKER=category_other_unit_tests and (not flaky)
 test-other-unit-tests: DD_ARGS := $(or $(DD_ARGS),)
@@ -197,7 +193,7 @@ test-gh-actions:
 test-marker: clean
     # OMP_NUM_THREADS can improve overall performance using one thread by process (on tensorflow), avoiding overload
 	# TF_CPP_MIN_LOG_LEVEL=2 sets C code log level for tensorflow to error suppressing lower log events
-	OMP_NUM_THREADS=1 TF_CPP_MIN_LOG_LEVEL=2 poetry run pytest tests -n $(JOBS) --dist loadscope -m "$(PYTEST_MARKER)" --cov rasa --ignore $(INTEGRATION_TEST_FOLDER) $(DD_ARGS)
+	TRANSFORMERS_OFFLINE=1 OMP_NUM_THREADS=1 TF_CPP_MIN_LOG_LEVEL=2 poetry run pytest tests -n $(JOBS) --dist loadscope -m "$(PYTEST_MARKER)" --cov rasa --ignore $(INTEGRATION_TEST_FOLDER) $(DD_ARGS)
 
 generate-pending-changelog:
 	poetry run python -c "from scripts import release; release.generate_changelog('major.minor.patch')"

data/test/hf_transformers_models.txt

@@ -1,54 +1,9 @@
-https://s3.amazonaws.com/models.huggingface.co/bert/rasa/LaBSE/vocab.txt
-21aa38329c730774d9f45df9ec5443a9bd4abd2191e1d510c27647c151c5437f.f2539f82b1008971c6ea6574f078d95c6eead57223fc74fdc420013fa9de391a
-https://s3.amazonaws.com/models.huggingface.co/bert/rasa/LaBSE/special_tokens_map.json
-99497d78492c90ab7d824d695b9a8d043369fbc2bf1112dcc7cdef9a6c4fa691.275045728fbf41c11d3dae08b8742c054377e18d92cc7b72b6351152a99b64e4
-https://s3.amazonaws.com/models.huggingface.co/bert/rasa/LaBSE/tokenizer_config.json
-527f618330e845c9d31826e7d9ce983aa816fafcf4f29f8c52f8ae1fdd097219.1c61d5d3dc67d88e0c74c64cda9b17bc30bdbd1c373cceeb740b9953729709aa
-https://s3.amazonaws.com/models.huggingface.co/bert/rasa/LaBSE/config.json
-90984a8da5021905af8679644b61bc5428ef16e9a307469152c163ec873db240.f1ba7080a92fc164a144311742f36dfb6a724bc9da532264b30d87040e15cc9d
-https://cdn.huggingface.co/rasa/LaBSE/tf_model.h5
-fd2ff7409cd4abbce31d54b8acebc305939787751dd697b6f38a3bf1f197a614.2589e15ea34b96d9bdcc478748ae77b629487da363566089fe6a8cdb1e6ea284.h5
-https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt
-8a0c070123c1f794c42a29c6904beb7c1b8715741e235bee04aca2c7636fc83f.9b42061518a39ca00b8b52059fd2bede8daa613f8a8671500e518a8c29de8c00
-https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-config.json
-8a3b1cfe5da58286e12a0f5d7d182b8d6eca88c08e26c332ee3817548cf7e60a.f12a4f986e43d8b328f5b067a641064d67b91597567a06c7b122d1ca7dfd9741
-https://cdn.huggingface.co/bert-base-chinese-tf_model.h5
-86a460b592673bcac3fe5d858ecf519e4890b4f6eddd1a46a077bd672dee6fe5.e6b974f59b54219496a89fd32be7afb020374df0976a796e5ccd3a1733d31537.h5
-https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json
-4ab93d0cd78ae80e746c27c9cd34e90b470abdabe0590c9ec742df61625ba310.b9628f6fe5519626534b82ce7ec72b22ce0ae79550325f45c604a25c0ad87fd6
-https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt
-0f8de0dbd6a2bb6bde7d758f4c120dd6dd20b46f2bf0a47bc899c89f46532fde.20808570f9a3169212a577f819c845330da870aeb14c40f7319819fce10c3b76
-https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-config.json
-a27bb7c70e9002d7558d2682d5a95f3c0a8b31034616309459e0b51ef07ade09.bd0797be126548711309ad2174d2afb16e3c37e891707667603d85e35a4ad001
-https://cdn.huggingface.co/openai-gpt-tf_model.h5
-642cba239b8eca22b702e71e92d507b8af47ddd2df74dc7751e2a4f65d8d434c.f26918df904593cca2dbd78a3bb760376f4f8ce1ce8d3b13bb6ab28228ee65fc.h5
-https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json
-f2808208f9bec2320371a9f5f891c184ae0b674ef866b79c58177067d15732dd.1512018be4ba4e8726e41b9145129dc30651ea4fec86aa61f4b9f40bf94eac71
-https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt
-d629f792e430b3c76a1291bb2766b0a047e36fae0588f9dbc1ae51decdff691b.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda
-https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json
-4be02c5697d91738003fb1685c9872f284166aa32e061576bbe6aaeb95649fcf.db13c9bc9c7bdd738ec89e069621d88e05dc670366092d809a9cbcac6798e24e
-https://cdn.huggingface.co/gpt2-tf_model.h5
-132dec44f9ced4b20f1b1c88a426b1d3dab5ba9e5f24a82541833dae44d5b8db.afd2261c07481427cd087f622388c2c086be9c62875f5945922c7adb2239b63a.h5
-https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-spiece.model
-dad589d582573df0293448af5109cb6981ca77239ed314e15ca63b7b8a318ddd.8b10bd978b5d01c21303cc761fc9ecd464419b3bf921864a355ba807cfbfafa8
-https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-config.json
-c9cc6e53904f7f3679a31ec4af244f4419e25ebc8e71ebf8c558a31cbcf07fc8.69e5e35e0b798cab5e473f253752f8bf4d280ee37682281a23eed80f6e2d09c6
-https://cdn.huggingface.co/xlnet-base-cased-tf_model.h5
-44ada4a49497a676c2d1fa2dbb7059df50f9cedb14f332862d2fca2c35d04a7d.42bc04b3944abffc38e9b60aadffb89f81aafa9c86473d157ed4b28953471ceb.h5
-https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt
-26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084
-https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json
-4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.7156163d5fdc189c3016baca0775ffce230789d7fa2a42ef516483e4ca884517
-https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json
-a41e817d5c0743e29e86ff85edc8c257e61bc8d88e4271bb1b243b6e7614c633.8949e27aafafa845a18d98a0e3a88bc2d248bbc32a1b75947366664658f23b1c
-https://cdn.huggingface.co/distilbert-base-uncased-tf_model.h5
-cce28882467f298a29fc905b9dd1683695d96198a83432fe707089dccd71c019.e02bd57e9d8507853eccc7c04ac2e938a6cdaff4b9bf941c10e781b61ddb9bbd.h5
-https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json
-d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b
-https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt
-b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda
-https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json
-e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.117c81977c5979de8c088352e74ec6e70f5c66096c28b61d3c50101609b39690
-https://cdn.huggingface.co/roberta-base-tf_model.h5
-2e18f106492efe1a8e6766e4d4e0bf4a82cee267c0cc52af431cf97005d4e3db.34733ed140f011f207fb07b32b443050356e99a9638db284a22d77bd3d5f54b3.h5
+bert-base-chinese
+openai-gpt
+gpt2
+xlnet-base-cased
+bert-base-uncased
+roberta-base
+distilbert-base-uncased
+camembert-base
+sentence-transformers/all-MiniLM-L6-v2

data/test_endpoints/example_endpoints.yml

@@ -13,13 +13,27 @@ tracker_store:
   url: localhost
   port: 6379
   db: 0
+  username: username
   password: password
   key_prefix: conversation
   record_exp: 30000
   use_ssl: True
   ssl_keyfile: "keyfile.key"
   ssl_certfile: "certfile.crt"
   ssl_ca_certs: "my-bundle.ca-bundle"
+# example of redis external lock store config
+lock_store:
+  type: redis
+  url: localhost
+  port: 6379
+  db: 0
+  username: username
+  password: password
+  key_prefix: lock
+  use_ssl: True
+  ssl_keyfile: "keyfile.key"
+  ssl_certfile: "certfile.crt"
+  ssl_ca_certs: "my-bundle.ca-bundle"
 # example of mongoDB external tracker store config
 #tracker_store:
   #type: mongod

docs/docs/lock-stores.mdx

@@ -156,6 +156,8 @@ The `ConcurrentRedisLockStore` recreates the `TicketLock` from the persisted `Ti
   - `key_prefix` (default: `None`): The prefix to prepend to lock store keys. Must
     be alphanumeric
 
+  - `username` (default: `None`): Username used for authentication
+
   - `password` (default: `None`): Password used for authentication
     (`None` equals no authentication)
 

docs/docs/tracker-stores.mdx

@@ -219,6 +219,8 @@ To set up Rasa with Redis the following steps are required:
 * `key_prefix` (default: `None`): The prefix to prepend to tracker store keys. Must
     be alphanumeric
 
+* `username` (default: `None`): Username used for authentication
+
 * `password` (default: `None`): Password used for authentication
 (`None` equals no authentication)