chore: change to pipenv (DEV-764) (#177)

.gitignore

@@ -45,8 +45,6 @@ venv/
 ENV/
 env.bak/
 venv.bak/
-Pipfile
-Pipfile.lock
 
 # Spyder project settings
 .spyderproject
@@ -70,6 +68,7 @@ id2iri_*
 **/~$*.*
 testdata/tmp/_*.json
 testdata/tmp/_*.xml
+testdata/test-list.json
 
 # for testing in development
 tmp/

Makefile

@@ -29,7 +29,7 @@ upload: ## upload distribution package to PyPI
 
 .PHONY: upgrade-dist-tools
 upgrade-dist-tools: ## upgrade packages necessary for testing, building, packaging and uploading to PyPI
-	python3 -m pip install --upgrade pip setuptools wheel tqdm twine pytest mkdocs
+	python3 -m pip install --upgrade pip setuptools wheel twine pytest mkdocs
 
 .PHONY: docs-build
 docs-build: ## build docs into the local 'site' folder
@@ -48,6 +48,7 @@ install-requirements: ## install requirements
 	python3 -m pip install --upgrade pip
 	pip3 install -r requirements.txt
 	pip3 install -r docs/requirements.txt
+	pip3 install -r dev-requirements.txt
 
 .PHONY: install
 install: ## install from source (runs setup.py)
@@ -86,4 +87,10 @@ run: ## create dist, install and run
 	$(MAKE) install
 	dsp-tools
 
+.PHONY: freeze-requirements
+freeze-requirements: ## update (dev-)requirements.txt and setup.py based on pipenv's Pipfile.lock
+	pipenv lock -r > requirements.txt
+	pipenv lock -r --dev-only > dev-requirements.txt
+	pipenv run pipenv-setup sync
+
 .DEFAULT_GOAL := help

Pipfile

@@ -0,0 +1,32 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+jsonpath-ng = "*"
+argparse = "*"
+rdflib = "*"
+lxml = "*"
+validators = "*"
+requests = "*"
+jsonschema = "*"
+click = "*"
+rfc3987 = "*"
+pystrict = "*"
+openpyxl = "*"
+
+[dev-packages]
+mkdocs = "*"
+mkdocs-material = "*"
+mkdocs-include-markdown-plugin = "*"
+mypy = "*"
+autopep8 = "*"
+setuptools = "*"
+wheel = "*"
+pipenv-setup = "*"
+pytest = "*"
+pandas = "*"
+
+[requires]
+python_version = "3.9"

README.md

@@ -1,74 +1,96 @@
 [![PyPI version](https://badge.fury.io/py/dsp-tools.svg)](https://badge.fury.io/py/dsp-tools)
 
 # DSP-TOOLS - DaSCH Service Platform Tools
-
-dsp-tools is a command line tool that helps you interact with the DaSCH Service Platform API.
-
+dsp-tools is a command line tool that helps you interacting with the DaSCH Service Platform API.  
 Go to [Full Documentation](https://dasch-swiss.github.io/dsp-tools/)
 
 ## Information for developers
+There is a `Makefile` for all the following tasks (and more). Type `make` to print the available targets. 
+
+For a quick start, use: 
+```
+pip install pipenv
+pipenv install --dev
+pipenv run make run
+```
 
-There is a `Makefile` for all the following tasks (and more). Type `make` to print the available targets. For a quick
-start use:
+This creates a pipenv-environment, installs all dependencies, and installs `dsp-tools` from source.
+
+If you prefer getting around pipenv, use instead:
 ```bash
 make upgrade-dist-tools
 make install-requirements
 make run
 ```
 
-## Install from source
+## Pipenv
+We use pipenv for our dependency management. There are two ways to get started:
+ - `pipenv install --dev` installs all dependencies, while giving them the opportunity to update themselves
+ - `pipenv install --ignore-pipfile` is used to get a deterministic build in production
 
-To install from source run:
+This works because there are two files defining the dependencies:
+ - `Pipfile` replaces `requirements.txt`, but lists only the core dependencies, ordered in two sections:
+   - `[packages]` lists the dependencies used to run the software.
+   - `[dev-packages]` lists additional dependencies used for tests and deployment.
+ - `Pipfile.lock` enables deterministic builds, by exactly pinning the version of all (sub-) dependencies. 
+   This is done automatically, you must not edit `Pipfile.lock`.
 
-```bash
-python3 setup.py install
-```
+The diverse `requirements.txt` files in this repo are only present for backwards compatibility
+and for GitHub CI.
 
-## Install the requirements
+If you want to install a new package, install it with `pipenv install package`. This 
+ - installs the package (incl. sub-dependencies) in your virtual environment
+ - adds the package to the section `[packages]` of `Pipfile`. By default, no versions are pinned
+ - adds the pinned versions of package and all sub-dependencies to `Pipfile.lock`
 
-To install the requirements run:
-
-```bash
-pip3 install -r requirements.txt
-```
+If a package is only needed for development, please install it with `pipenv install package --dev`,
+so it gets added to the `[dev-packages]` section of `Pipfile`.
 
-To generate a requirements file (usually requirements.txt), that you commit with your project, run:
+For security reasons, the maintainer regularly executes
+ - `pipenv check` to get informed about vulnerabilities
+ - `pipenv update` to update `Pipfile.lock` with the latest version of every package
+ - `make freeze-requirements` to update the requirement files and `setup.py`
 
-```bash
-pip3 freeze > requirements.txt
-```
+`make freeze-requirements` must also be executed after adding a new dependency. If you prefer working 
+without pipenv, you can freeze your requirements with `pip3 freeze > requirements.txt` and update `setup.py`
+manually.
 
-## Testing
+### Pipenv setup in PyCharm
+ - Go to Add Interpreter > Pipenv Environment
+ - Base Interpreter: PyCarm auto-detects one of your system-wide installed Pythons as base interpreter. 
+ - Pipenv executable: auto-detected
+ - After hitting OK, PyCharm creates a new pipenv environment and installs the dependencies from `Pipfile`
 
-Please note that testing requires launching the complete DSP API stack which is based on docker images. Therefore, we
-recommend installing the [docker desktop client](https://www.docker.com/products).
+If you already initialized a pipenv-environment via command line, you can add its interpreter in PyCharm,
+but this will create the pipenv-environment again.
 
+## Testing
+Please note that testing requires launching the complete DSP API stack which is based on docker images. 
+Therefore, we recommend installing the [docker desktop client](https://www.docker.com/products).  
 To run the complete test suite:
-
 ```bash
 make test
 ```
 
 ## Code style
-
 When contributing to the project please make sure you use the same code style rules as we do. We use
-[autopep8](https://pypi.org/project/autopep8/) and [mypy](https://pypi.org/project/mypy/). The configuration is defined
-in `pyproject.toml` in the root directory of the project.
+[autopep8](https://pypi.org/project/autopep8/) and [mypy](https://pypi.org/project/mypy/). The 
+configuration is defined in `pyproject.toml` in the root directory of the project.
 
 You can use the configuration with `autopep8 --global-config pyproject.toml [file path]` and 
 `mypy --config-file pyproject.toml [file path]`.
 
-If you are using PyCharm we recommend installing autopep8 as external tool. You can then use it with right-click on the
-file > `External Tools` > `autopep8` to reformat files in-place. Due to compatibility issues with VSCode, the argument 
-`--in-place=true` can not be declared in the `pyproject.toml` and needs to be passed to the external tool in the PyCharm settings.  
+If you are using PyCharm we recommend installing autopep8 as external tool. You can then use it with 
+right-click on the file > `External Tools` > `autopep8` to reformat files in-place. Due to compatibility 
+issues with VSCode, the argument  `--in-place=true` can not be declared in the `pyproject.toml` and 
+needs to be passed to the external tool in the PyCharm settings.  
 mypy is available as [plugin](https://plugins.jetbrains.com/plugin/11086-mypy).
 
 In VSCode, both mypy and autopep8 can be set up as default linter and formatter through the python extension.
 
 For formatting Markdown files (*.md) we use the default styling configuration provided by PyCharm.
 
 ## Publishing
-
 Publishing is automated with GitHub Actions and should _not_ be done manually. Please follow the
 [Pull Request Guidelines](https://docs.dasch.swiss/developers/dsp/contribution/#pull-request-guidelines). If done
 correctly, when merging a pull request into `main`, the `release-please` action will create or update a pull request for
@@ -79,7 +101,6 @@ create a release on GitHub, on PyPI and the docs.
 Please ensure you have only one pull request per feature.
 
 ## Publishing manually
-
 Publishing is automated with GitHub Actions and should _not_ be done manually. If you still need to do it, follow the
 steps below.
 
@@ -110,7 +131,6 @@ python3 setup.py develop
 ```
 
 ## Contributing to the documentation
-
 The documentation is a collection of [markdown](https://en.wikipedia.org/wiki/Markdown) files in the `docs` folder.  
 After updates of the files, build and check the result with the following commands:
 

dev-requirements.txt

@@ -0,0 +1,67 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements --dev-only
+#
+
+-i https://pypi.org/simple
+attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+autopep8==1.6.0
+cached-property==1.5.2
+cerberus==1.3.4
+certifi==2021.10.8
+chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+charset-normalizer==2.0.12; python_version >= '3'
+click==8.1.1
+colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+distlib==0.3.4
+ghp-import==2.0.2
+idna==3.3; python_version >= '3'
+importlib-metadata==4.11.3; python_version >= '3.7'
+iniconfig==1.1.1
+jinja2==3.1.1; python_version >= '3.7'
+markdown==3.3.6; python_version >= '3.6'
+markupsafe==2.1.1; python_version >= '3.7'
+mergedeep==1.3.4; python_version >= '3.6'
+mkdocs-include-markdown-plugin==3.3.0
+mkdocs-material-extensions==1.0.3; python_version >= '3.6'
+mkdocs-material==8.2.8
+mkdocs==1.3.0
+mypy-extensions==0.4.3
+mypy==0.942
+numpy==1.22.3; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'
+orderedmultidict==1.0.1
+packaging==20.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pandas==1.4.1
+pep517==0.12.0
+pip-shims==0.7.0; python_version >= '3.6'
+pip==22.0.4; python_version >= '3.7'
+pipenv-setup==3.2.0
+pipfile==0.0.2
+platformdirs==2.5.1; python_version >= '3.7'
+plette[validation]==0.2.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pluggy==1.0.0; python_version >= '3.6'
+py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pycodestyle==2.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pygments==2.11.2; python_version >= '3.5'
+pymdown-extensions==9.3; python_version >= '3.7'
+pyparsing==3.0.7; python_version >= '3.6'
+pytest==7.1.1
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pytz==2022.1
+pyyaml-env-tag==0.1; python_version >= '3.6'
+pyyaml==6.0; python_version >= '3.6'
+requests==2.27.1
+requirementslib==1.6.1; python_version >= '3.6'
+setuptools==61.2.0; python_version >= '3.7'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+tomli==2.0.1; python_version >= '3.7'
+tomlkit==0.10.1; python_version >= '3.6' and python_version < '4'
+typing-extensions==4.1.1; python_version >= '3.6'
+urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+vistir==0.5.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+watchdog==2.1.7; python_version >= '3.6'
+wheel==0.37.1
+zipp==3.7.0; python_version >= '3.7'

requirements.txt

@@ -1,71 +1,32 @@
-attrs==21.2.0
-bleach==4.1.0
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+-i https://pypi.org/simple
+argparse==1.4.0
+attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 certifi==2021.10.8
-charset-normalizer==2.0.7
-click==8.0.3
-colorama==0.4.4
-decorator==5.1.0
-docutils==0.18
-et-xmlfile==1.1.0
-future==0.18.2
-ghp-import==2.0.2
-idna==3.3
-importlib-metadata==4.8.1
-importlib-resources==5.4.0
-iniconfig==1.1.1
-isodate==0.6.0
-Jinja2==3.0.2
-joblib==1.1.0
+charset-normalizer==2.0.12; python_version >= '3'
+click==8.1.1
+decorator==5.1.1; python_version >= '3.5'
+et-xmlfile==1.1.0; python_version >= '3.6'
+idna==3.3; python_version >= '3'
+isodate==0.6.1
 jsonpath-ng==1.5.3
-jsonschema==4.2.1
-keyring==23.2.1
-livereload==2.6.3
-lunr==0.5.8
-lxml==4.6.5
-Markdown==3.3.4
-MarkupSafe==2.0.1
-mergedeep==1.3.4
-mkdocs==1.2.3
-mkdocs-autorefs==0.3.0
-mkdocs-include-markdown-plugin==3.2.3
-mkdocs-material==7.2.3
-mkdocs-material-extensions==1.0.3
-mkdocstrings==0.16.2
-nltk==3.6.6
-numpy==1.22.3
+jsonschema==4.4.0
+lxml==4.8.0
 openpyxl==3.0.9
-packaging==21.2
-pandas==1.4.1
-pkginfo==1.7.1
-pluggy==1.0.0
 ply==3.11
-py==1.11.0
-Pygments==2.10.0
-pymdown-extensions==9.0
-pyparsing==2.4.7
-pyrsistent==0.18.0
+pyparsing==3.0.7; python_version >= '3.6'
+pyrsistent==0.18.1; python_version >= '3.7'
 pystrict==1.1
-pytest==7.1.1
-python-dateutil==2.8.2
-pytkdocs==0.12.0
-pytz==2022.1
-PyYAML==6.0
-pyyaml_env_tag==0.1
-rdflib==6.0.2
-readme-renderer==30.0
-regex==2021.11.2
-requests==2.26.0
-requests-toolbelt==0.9.1
-rfc3986==1.5.0
+rdflib==6.1.1
+requests==2.27.1
 rfc3987==1.3.8
-six==1.16.0
-tomli==2.0.1
-tornado==6.1
-tqdm==4.62.3
-twine==3.5.0
-typing_extensions==4.0.1
-urllib3==1.26.7
+setuptools==61.2.0; python_version >= '3.7'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
 validators==0.18.2
-watchdog==2.1.6
-webencodings==0.5.1
-zipp==3.6.0