From 2de14e3247604cab976ec7332a5126cd66ed38f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 2 May 2022 13:15:10 +0200 Subject: [PATCH 1/3] feat: enable restricted access support in zenodo remote provider --- docs/snakefiles/remote_files.rst | 46 +++++++++++++++++++++++++++----- snakemake/remote/zenodo.py | 31 +++++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/snakefiles/remote_files.rst b/docs/snakefiles/remote_files.rst index 56be5305c..621d9c0f7 100644 --- a/docs/snakefiles/remote_files.rst +++ b/docs/snakefiles/remote_files.rst @@ -824,10 +824,9 @@ Avoid creating uploads with too many files, and instead group and zip them to ma # let Snakemake assert the presence of the required environment variable envvars: - "MYZENODO_PAT" + "ZENODO_ACCESS_TOKEN" - access_token = os.environ["MYZENODO_PAT"] - zenodo = RemoteProvider(deposition="your deposition id", access_token=access_token) + zenodo = RemoteProvider(deposition="your deposition id", access_token=os.environ["ZENODO_ACCESS_TOKEN"]) rule upload: input: @@ -841,11 +840,46 @@ Avoid creating uploads with too many files, and instead group and zip them to ma It is possible to use `Zenodo sandbox environment `_ for testing by setting ``sandbox=True`` argument. Using sandbox environment requires setting up sandbox account with its personal access token. -Auto -==== +Restricted access +----------------- +If you need to access a deposition with restricted access, you have to additionally provide a ``restricted_access_token``. +This can be obtained from the restricted access URL that Zenodo usually sends you via email once restricted access to a deposition (requested via the web interface) has been granted by the owner. +Let `` +https://zenodo.org/record/000000000?token=dlksajdlkjaslnflkndlfnjnn`` be the URL provided by Zenodo. +Then, the ``restricted_access_token`` is ``dlksajdlkjaslnflkndlfnjnn``, and it can be used as follows: + +.. code-block:: python + + from snakemake.remote.zenodo import RemoteProvider + import os + + # let Snakemake assert the presence of the required environment variable + envvars: + "ZENODO_ACCESS_TOKEN", + "ZENODO_RESTRICTED_ACCESS_TOKEN" + + zenodo = RemoteProvider( + deposition="your deposition id", + access_token=os.environ["ZENODO_ACCESS_TOKEN"], + restricted_access_token=os.environ["ZENODO_RESTRICTED_ACCESS_TOKEN"] + ) + + rule upload: + input: + "output/results.csv" + output: + zenodo.remote("results.csv") + shell: + "cp {input} {output}" + + +Auto remote provider +==================== A wrapper which automatically selects an appropriate remote provider based on the url's scheme. -It removes some of the boilerplate code required to download remote files from various providers: +It removes some of the boilerplate code required to download remote files from various providers. +The auto remote provider only works for those which do not require the passing of keyword arguments to the +``RemoteProvider`` object. .. code-block:: python diff --git a/snakemake/remote/zenodo.py b/snakemake/remote/zenodo.py index fb71ed127..c611b6447 100644 --- a/snakemake/remote/zenodo.py +++ b/snakemake/remote/zenodo.py @@ -48,7 +48,7 @@ def __init__( keep_local=keep_local, stay_on_remote=stay_on_remote, provider=provider, - **kwargs + **kwargs, ) if provider: self._zen = provider.remote_interface() @@ -138,6 +138,12 @@ def __init__(self, *args, **kwargs): # Creating a new deposition, as deposition id was not supplied. self.deposition, self.bucket = self.create_deposition().values() + self.restricted_access_token = None + self._restricted_access_cookies = None + + if "restricted_access_token" in kwargs: + self.restricted_access_token = kwargs["restricted_access_token"] + def _api_request( self, url, method="GET", data=None, headers={}, files=None, json=False ): @@ -148,9 +154,13 @@ def _api_request( session.headers["Authorization"] = "Bearer {}".format(self._access_token) session.headers.update(headers) + cookies = self.restricted_access_cookies + # Run query. try: - r = session.request(method=method, url=url, data=data, files=files) + r = session.request( + method=method, url=url, data=data, files=files, cookies=cookies + ) if json: msg = r.json() return msg @@ -189,3 +199,20 @@ def get_files(self): ) for f in files } + + @property + def restricted_access_cookies(self): + if self.restricted_access_token and self._restricted_access_cookies is None: + url = ( + self._baseurl + + f"/record/{self.deposition}?token={self.restricted_access_token}" + ) + resp = self._api_request(url) + if "session" in resp["cookies"]: + self._restricted_access_cookies = resp["cookies"] + else: + raise WorkflowError( + "Failure to retrieve session cookie with given restricted access token. " + f"Is the token valid? Please check by opening {url} manually in your browser." + ) + return self._restricted_access_cookies From bceb5022022e57dec561dc89c1d9a249b870c9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Mon, 2 May 2022 13:16:06 +0200 Subject: [PATCH 2/3] refer to source --- snakemake/remote/zenodo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/snakemake/remote/zenodo.py b/snakemake/remote/zenodo.py index c611b6447..5b0af570d 100644 --- a/snakemake/remote/zenodo.py +++ b/snakemake/remote/zenodo.py @@ -202,6 +202,10 @@ def get_files(self): @property def restricted_access_cookies(self): + """Retrieve cookies necessary for restricted access. + + Inspired by https://gist.github.com/slint/d47fe5628916d14b8d0b987ac45aeb66 + """ if self.restricted_access_token and self._restricted_access_cookies is None: url = ( self._baseurl From 347902fc5205b0abb4ca8b3216ccf2679e0da389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Tue, 3 May 2022 10:35:00 +0200 Subject: [PATCH 3/3] move declaration --- snakemake/remote/zenodo.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/snakemake/remote/zenodo.py b/snakemake/remote/zenodo.py index 5b0af570d..bfec83b07 100644 --- a/snakemake/remote/zenodo.py +++ b/snakemake/remote/zenodo.py @@ -121,6 +121,12 @@ def __init__(self, *args, **kwargs): "environment at https://sandbox.zenodo.org." ) + self.restricted_access_token = None + self._restricted_access_cookies = None + + if "restricted_access_token" in kwargs: + self.restricted_access_token = kwargs["restricted_access_token"] + if "sandbox" in kwargs: self._sandbox = kwargs.pop("sandbox") else: @@ -138,12 +144,6 @@ def __init__(self, *args, **kwargs): # Creating a new deposition, as deposition id was not supplied. self.deposition, self.bucket = self.create_deposition().values() - self.restricted_access_token = None - self._restricted_access_cookies = None - - if "restricted_access_token" in kwargs: - self.restricted_access_token = kwargs["restricted_access_token"] - def _api_request( self, url, method="GET", data=None, headers={}, files=None, json=False ):