Error when running multiple_models template

[FioByr]

Hi,

I am trying to run a simulation using the multiple_models template on two datasets from the synergy dataset. However, when the template runs a simulation using the sbert model there is an error

Traceback (most recent call last):
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\review\base.py", line 103, in __init__
    self.X = self.project.get_feature_matrix(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\project.py", line 512, in get_feature_matrix
    return load_npz(Path(self.project_path, PATH_FEATURE_MATRICES,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\scipy\sparse\_matrix_io.py", line 125, in load_npz
    with np.load(file, **PICKLE_KWARGS) as loaded:
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\numpy\lib\npyio.py", line 405, in load
    fid = stack.enter_context(open(os_fspath(file), "rb"))
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: 'output\\simulation\\Nelson_2002\\state_files\\sim_Nelson_2002_logistic_sbert_0.asreview.tmp\\feature_matrices\\sbert_feature_matrix.npz'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\huggingface_hub\utils\_errors.py", line 259, in hf_raise_for_status
    response.raise_for_status()
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\requests\models.py", line 1021, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 504 Server Error: Gateway Time-out for url: https://huggingface.co/api/models/sentence-transformers/all-mpnet-base-v2

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Scripts\asreview.exe\__main__.py", line 7, in <module>
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\__main__.py", line 48, in main
    entry.load()().execute(sys.argv[2:])
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\entry_points\simulate.py", line 247, in execute
    raise err
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\entry_points\simulate.py", line 212, in execute
    reviewer = ReviewSimulate(
               ^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\review\simulate.py", line 156, in __init__
    super(ReviewSimulate, self).__init__(as_data,
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\review\base.py", line 106, in __init__
    self.X = self.feature_extraction.fit_transform(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\models\feature_extraction\base.py", line 57, in fit_transform
    X = self.transform(texts)
        ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\models\feature_extraction\sbert.py", line 77, in transform
    model = SentenceTransformer(self.transformer_model)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\sentence_transformers\SentenceTransformer.py", line 87, in __init__
    snapshot_download(model_name_or_path,
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\sentence_transformers\util.py", line 442, in snapshot_download
    model_info = _api.model_info(repo_id=repo_id, revision=revision, token=token)
                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\huggingface_hub\utils\_validators.py", line 120, in _inner_fn
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\huggingface_hub\hf_api.py", line 1604, in model_info
    hf_raise_for_status(r)
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\huggingface_hub\utils\_errors.py", line 301, in hf_raise_for_status
    raise HfHubHTTPError(str(e), response=response) from e
huggingface_hub.utils._errors.HfHubHTTPError: 504 Server Error: Gateway Time-out for url: https://huggingface.co/api/models/sentence-transformers/all-mpnet-base-v2
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Scripts\asreview.exe\__main__.py", line 7, in <module>
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\__main__.py", line 48, in main
    entry.load()().execute(sys.argv[2:])
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreviewcontrib\insights\entrypoint.py", line 162, in execute
    with open_state(args.asreview_files[0]) as s:
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\contextlib.py", line 137, in __enter__
    return next(self.gen)
           ^^^^^^^^^^^^^^
  File "C:\Users\fionn\AppData\Local\Programs\Python\Python311\Lib\site-packages\asreview\project.py", line 191, in open_state
    raise StateNotFoundError("State file does not exist, and in "
asreview.state.errors.StateNotFoundError: State file does not exist, and in read only mode."

I was wondering if anyone could help me with this?

All the best,
Fionn

[J535D165]

Thanks for reporting.
It looks like an issue with a model that uses this URL: requests.exceptions.HTTPError: 504 Server Error: Gateway Time-out for url: https://huggingface.co/api/models/sentence-transformers/all-mpnet-base-v2.

What extension is the origin of this model?

[FioByr]

Thank you. I'm not exactly sure what you mean by the extension of the origin of the model?

[J535D165]

I can't find this URL in our organisation: https://github.com/search?q=org%3Aasreview+https%3A%2F%2Fhuggingface.co%2Fapi%2Fmodels%2Fsentence-transformers%2Fall-mpnet-base-v2&type=code

[J535D165]

Hmmm, found it. Looks like this one

asreview/asreview/models/feature_extraction/sbert.py

Line 69 in 92e3ef6

def __init__(self, *args, transformer_model="all-mpnet-base-v2", **kwargs):

[J535D165]

Looks like their API is down as well as https://huggingface.co/sentence-transformers. @jteijema

[jteijema]

API is back up again. @FioByr will you have another go at running the simulation now?

[J535D165]

I moved this issue. Let's keep it open to work on a better error message cc @PeterLombaers.

[FioByr]

API is back up again. @FioByr will you have another go at running the simulation now?

I've run the simulation now, the error has now resolved itself. It wasn't an issue with ASReview, but an issue with the huggingface API.

[J535D165]

With a 401 error, I can't reproduce it.

% asreview simulate benchmark:van_de_Schoot_2017 -s $TMPDIR/$(uuidgen).asreview -e sbert

            _____ _____            _
     /\    / ____|  __ \          (_)
    /  \  | (___ | |__) |_____   ___  _____      __
   / /\ \  \___ \|  _  // _ \ \ / / |/ _ \ \ /\ / /
  / ____ \ ____) | | \ \  __/\ V /| |  __/\ V  V /
 /_/    \_\_____/|_|  \_\___| \_/ |_|\___| \_/\_/

---------------------------------------------------------------------------------
|                                                                                |
|  Welcome to ASReview LAB - AI-assisted systematic reviews software.            |
|  In simulation mode the computer will simulate how well ASReview LAB           |
|  could have accelerate the systematic review of your dataset.                  |
|  You can sit back and relax while the computer runs this simulation.           |
|                                                                                |
|  GitHub page:        https://github.com/asreview/asreview                      |
|  Questions/remarks:  asreview@uu.nl                                            |
|                                                                                |
---------------------------------------------------------------------------------

Traceback (most recent call last):
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/review/base.py", line 103, in __init__
    self.X = self.project.get_feature_matrix(self.feature_extraction.name)
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/project.py", line 506, in get_feature_matrix
    return load_npz(Path(self.project_path, PATH_FEATURE_MATRICES, matrix_filename))
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/scipy/sparse/_matrix_io.py", line 125, in load_npz
    with np.load(file, **PICKLE_KWARGS) as loaded:
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/numpy/lib/npyio.py", line 390, in load
    fid = stack.enter_context(open(os_fspath(file), "rb"))
FileNotFoundError: [Errno 2] No such file or directory: '/var/folders/jp/nykhdfms23v5pznbdssbq0980000gp/T/A403FAE6-A107-4909-835D-B3654F7CA1ED.asreview.tmp/feature_matrices/sbert_feature_matrix.npz'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/huggingface_hub/utils/_errors.py", line 259, in hf_raise_for_status
    response.raise_for_status()
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/requests/models.py", line 1021, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/api/models/sentence-transformers/all-mpnet-base-v2-broken

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/bin/asreview", line 33, in <module>
    sys.exit(load_entry_point('asreview', 'console_scripts', 'asreview')())
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/__main__.py", line 47, in main
    entry.load()().execute(sys.argv[2:])
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/entry_points/simulate.py", line 256, in execute
    raise err
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/entry_points/simulate.py", line 222, in execute
    reviewer = ReviewSimulate(
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/review/simulate.py", line 156, in __init__
    super(ReviewSimulate, self).__init__(
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/review/base.py", line 105, in __init__
    self.X = self.feature_extraction.fit_transform(
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/models/feature_extraction/base.py", line 59, in fit_transform
    X = self.transform(texts)
  File "/Users/Bruin056/Documents/GitHub/asreview/asreview/models/feature_extraction/sbert.py", line 76, in transform
    model = SentenceTransformer(self.transformer_model)
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/sentence_transformers/SentenceTransformer.py", line 87, in __init__
    snapshot_download(model_name_or_path,
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/sentence_transformers/util.py", line 442, in snapshot_download
    model_info = _api.model_info(repo_id=repo_id, revision=revision, token=token)
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/huggingface_hub/utils/_validators.py", line 120, in _inner_fn
    return fn(*args, **kwargs)
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/huggingface_hub/hf_api.py", line 1604, in model_info
    hf_raise_for_status(r)
  File "/Users/Bruin056/.pyenv/versions/asreview-dev/lib/python3.9/site-packages/huggingface_hub/utils/_errors.py", line 291, in hf_raise_for_status
    raise RepositoryNotFoundError(message, response) from e
huggingface_hub.utils._errors.RepositoryNotFoundError: 401 Client Error. (Request ID: Root=1-64635138-06f4253b67ccc28969e29958)

Repository Not Found for url: https://huggingface.co/api/models/sentence-transformers/all-mpnet-base-v2-broken.
Please make sure you specified the correct `repo_id` and `repo_type`.
If you are trying to access a private or gated repo, make sure you are authenticated.
Invalid username or password.