Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support XRootD as a default remote provider #1017

Merged
merged 9 commits into from Aug 17, 2021
Merged

feat: support XRootD as a default remote provider #1017

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3fe88b1
Add the remote provider XRootD to the list of allowed default remote …
ryuwd May 26, 2021
e4fc85f
see what happens if we set supports_default = True for XRootD
ryuwd May 26, 2021
2166afc
make sure non-relative path is supplied to xrootd
ryuwd May 26, 2021
07993f5
ensure paths are always non-relative
ryuwd May 26, 2021
f541a76
missing slash
ryuwd May 26, 2021
dc5bebd
snakemake expects copying to overwrite the destination file if it alr…
ryuwd May 26, 2021
68425ee
Formatting.
ryuwd Jun 4, 2021
42cbe75
Remove extraneous forward-slashes
ryuwd Jun 4, 2021
4de4f43
Merge branch 'main' into patch-2
johanneskoester Aug 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions snakemake/__init__.py
View file
Open in desktop
Expand Up @@ -1858,6 +1858,7 @@ def get_argument_parser(profile=None):
"gridftp",
"iRODS",
"AzBlob",
"XRootD",
],
help="Specify default remote provider to be used for "
"all input and output files that don't yet specify "
Expand Down
20 changes: 17 additions & 3 deletions snakemake/remote/XRootD.py
View file
Open in desktop
Expand Up @@ -21,6 +21,8 @@


class RemoteProvider(AbstractRemoteProvider):
supports_default = True

def __init__(
self, *args, keep_local=False, stay_on_remote=False, is_default=False, **kwargs
):
Expand All @@ -29,7 +31,7 @@ def __init__(
keep_local=keep_local,
stay_on_remote=stay_on_remote,
is_default=is_default,
**kwargs
**kwargs,
)

self._xrd = XRootDHelper()
Expand Down Expand Up @@ -59,7 +61,7 @@ def __init__(
keep_local=keep_local,
stay_on_remote=stay_on_remote,
provider=provider,
**kwargs
**kwargs,
)

if provider:
Expand Down Expand Up @@ -131,6 +133,12 @@ def _parse_url(self, url):
dirname, filename = os.path.split(match.group("path"))
# We need a trailing / to keep XRootD happy
dirname += "/"

# and also make sure we supply a non-relative path
# (snakemake removes double-slash // characters)
if not dirname.startswith("/"):
dirname = "/" + dirname

return domain, dirname, filename

def exists(self, url):
Expand Down Expand Up @@ -188,18 +196,24 @@ def copy(self, source, destination):
# Prepare the source path for XRootD
if not self._parse_url(source):
source = abspath(source)
else:
domain, dirname, filename = self._parse_url(source)
source = f"{domain}/{dirname}/{filename}"

# Prepare the destination path for XRootD
assert os.path.basename(source) == os.path.basename(destination)
if self._parse_url(destination):
domain, dirname, filename = self._parse_url(destination)
destination = f"{domain}/{dirname}/{filename}"
self.makedirs(domain, dirname)
else:
destination = abspath(destination)
if not os.path.isdir(os.path.dirname(destination)):
os.makedirs(os.path.dirname(destination))

# Perform the copy operation
process = client.CopyProcess()
process.add_job(source, destination)
process.add_job(source, destination, force=True)
process.prepare()
status, returns = process.run()
if not status.ok or not returns[0]["status"].ok:
Expand Down