feat: change Makefile to generate dedicated python requirements.txt f…

…iles (#776) * change Makefile to generate dedicated python requirements.txt files Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * update Makefile removing unnecessary cp Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * ensure folders are generated for python versions Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * update Dockerfile Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * change Makefile to generate app_temp.txt Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * cleanup old requirements files Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * updated temp file creation Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * updated requirement text files Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * fix: missing future import for annotations Signed-off-by: F.N. Claessen <felix@seita.nl> * add missing import Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> * improve cashing restoration key and title in Github Action Signed-off-by: Nicolas Höning <nicolas@seita.nl> * add changelog entry Signed-off-by: Nicolas Höning <nicolas@seita.nl> * remove extra call to make ensure-deps-folder; rename make command to be more in line with other naming of dependency handling Signed-off-by: Nicolas Höning <nicolas@seita.nl> * update requirements to include highspy Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> --------- Signed-off-by: Nischay Ram Mamidi <NischayPro@gmail.com> Signed-off-by: F.N. Claessen <felix@seita.nl> Signed-off-by: Nicolas Höning <nicolas@seita.nl> Co-authored-by: F.N. Claessen <felix@seita.nl> Co-authored-by: Nicolas Höning <nicolas@seita.nl>
FlexMeasures · Aug 8, 2023 · b6687a4 · b6687a4
1 parent 9562473
commit b6687a4
Show file tree

Hide file tree

Showing 23 changed files with 2,007 additions and 28 deletions.
diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml
@@ -38,13 +38,13 @@ jobs:
         run: |
           git fetch --prune --unshallow
           git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - name: "Caching dependencies (txt)"
+      - name: "Caching for dependencies (.txt) - restore existing or ensure new cache will be made"
         uses: actions/cache@v2
         id: cache
         with:
           path: ${{ env.pythonLocation }}
           # manually disable a cache if needed by (re)setting CACHE_DATE
-          key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**.txt') }}
+          key: ${{ runner.os }}-pip-${{ env.pythonLocation }}-${{ SECRETS.CACHE_DATE }}-${{ hashFiles('**/requirements/**/*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - run: |

diff --git a/Dockerfile b/Dockerfile
@@ -12,7 +12,10 @@ WORKDIR /app
 COPY requirements /app/requirements
 
 # py dev tooling
-RUN python3 -m pip install --no-cache-dir  --upgrade pip && python3 --version && pip3 install --no-cache-dir --upgrade setuptools && pip3 install highspy && pip3 install --no-cache-dir -r requirements/app.txt -r requirements/dev.txt -r requirements/test.txt
+RUN python3 -m pip install --no-cache-dir --upgrade pip && python3 --version && \
+    pip3 install --no-cache-dir --upgrade setuptools && pip3 install highspy && \
+    PYV=$(python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)") && \
+    pip3 install --no-cache-dir -r requirements/$PYV/app.txt -r requirements/$PYV/dev.txt -r requirements/$PYV/test.txt
 
 # Copy code and meta/config data
 COPY setup.* .flaskenv wsgi.py /app/

diff --git a/Makefile b/Makefile
@@ -1,3 +1,7 @@
+# Check Python major and minor version
+# For more information, see https://stackoverflow.com/a/22105036
+PYV = $(shell python -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)")
+
 # Note: use tabs
 # actions which are virtual, i.e. not a script
 .PHONY: install install-for-dev install-for-test install-deps install-flexmeasures run-local test freeze-deps upgrade-deps update-docs update-docs-pdf show-file-space show-data-model clean-db
@@ -35,14 +39,15 @@ install: install-deps install-flexmeasures
 
 install-for-dev:
 	make freeze-deps
-	pip-sync requirements/app.txt requirements/dev.txt requirements/test.txt
+	make ensure-deps-folder
+	pip-sync requirements/${PYV}/app.txt requirements/${PYV}/dev.txt requirements/${PYV}/test.txt
 	make install-flexmeasures
 
 install-for-test:
 	make install-pip-tools
 # Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
 ifneq ($(pinned), no)
-	pip-sync requirements/app.txt requirements/test.txt
+	pip-sync requirements/${PYV}/app.txt requirements/${PYV}/test.txt
 else
 	# cutting off the -c inter-layer dependency (that's pip-tools specific)
 	tail -n +3 requirements/test.in >> temp-test.in
@@ -56,7 +61,7 @@ install-deps:
 	make freeze-deps
 # Pass pinned=no if you want to test against latest stable packages, default is our pinned dependency set
 ifneq ($(pinned), no)
-	pip-sync requirements/app.txt
+	pip-sync requirements/${PYV}/app.txt
 else
 	pip install --upgrade -r requirements/app.in
 endif
@@ -68,23 +73,35 @@ install-pip-tools:
 	pip3 install -q "pip-tools>=7.0"
 
 install-docs-dependencies:
-	pip install -r requirements/docs.txt
+	pip install -r requirements/${PYV}/docs.txt
 
 freeze-deps:
+	make ensure-deps-folder
 	make install-pip-tools
-	pip-compile -o requirements/app.txt requirements/app.in
-	pip-compile -o requirements/test.txt requirements/test.in
-	pip-compile -o requirements/dev.txt requirements/dev.in
-	pip-compile -o requirements/docs.txt requirements/docs.in
+	pip-compile -o requirements/${PYV}/app.txt requirements/app.in
+	# Create app.txt to create constraints for test.txt and dev.txt
+	cat requirements/${PYV}/app.txt > requirements/app.txt
+	pip-compile -o requirements/${PYV}/test.txt requirements/test.in
+	cat requirements/${PYV}/test.txt > requirements/test.txt
+	pip-compile -o requirements/${PYV}/dev.txt requirements/dev.in
+	pip-compile -o requirements/${PYV}/docs.txt requirements/docs.in
+	rm requirements/app.txt
+	rm requirements/test.txt
 
 upgrade-deps:
+	make ensure-deps-folder
 	make install-pip-tools
-	pip-compile --upgrade -o requirements/app.txt requirements/app.in
-	pip-compile --upgrade -o requirements/test.txt requirements/test.in
-	pip-compile --upgrade -o requirements/dev.txt requirements/dev.in
-	pip-compile --upgrade -o requirements/docs.txt requirements/docs.in
-	make test
+	pip-compile --upgrade -o requirements/${PYV}/app.txt requirements/app.in
+	# Create app.txt to create constraints for test.txt and dev.txt
+	cat requirements/${PYV}/app.txt > requirements/app.txt
+	pip-compile --upgrade -o requirements/${PYV}/test.txt requirements/test.in
+	cat requirements/${PYV}/test.txt > requirements/test.txt
+	pip-compile --upgrade -o requirements/${PYV}/dev.txt requirements/dev.in
+	pip-compile --upgrade -o requirements/${PYV}/docs.txt requirements/docs.in
+	rm requirements/app.txt
+	rm requirements/test.txt
 
+	make test
 
 # ---- Data ----
 
@@ -104,5 +121,8 @@ show-data-model:
 	# Use --help to learn more. 
 	./flexmeasures/data/scripts/visualize_data_model.py --uml
 
+ensure-deps-folder:
+	mkdir -p requirements/${PYV}
+
 clean-db:
-	./flexmeasures/data/scripts/clean_database.sh ${db_name} ${db_user}
+	./flexmeasures/data/scripts/clean_database.sh ${db_name} ${db_user}
diff --git a/documentation/changelog.rst b/documentation/changelog.rst
@@ -41,6 +41,7 @@ Infrastructure / Support
 * Document the `device_scheduler` linear program [see `PR #764 <https://www.github.com/FlexMeasures/flexmeasures/pull/764>`_].
 * Add support for `HiGHS <https://highs.dev/>`_ solver [see `PR #766 <https://www.github.com/FlexMeasures/flexmeasures/pull/766>`_].
 * Add support for installing FlexMeasures under Python 3.11 [see `PR #771 <https://www.github.com/FlexMeasures/flexmeasures/pull/771>`_].
+* Start keeping sets of pinned requirements per supported Python version. Also fixes recent Docker build problem. [see `PR #776 <https://www.github.com/FlexMeasures/flexmeasures/pull/776>`_]
 * Removed obsolete code dealing with deprecated data models (e.g. assets, markets and weather sensors), and sunset the fm0 scheme for entity addresses [see `PR #695 <https://www.github.com/FlexMeasures/flexmeasures/pull/695>`_ and `project 11 <https://www.github.com/FlexMeasures/flexmeasures/projects/11>`_]
 
 v0.14.2 | July 25, 2023

diff --git a/flexmeasures/api/v3_0/tests/conftest.py b/flexmeasures/api/v3_0/tests/conftest.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import timedelta
 
 import pandas as pd

diff --git a/requirements/app.txt → requirements/3.10/app.txt b/requirements/app.txt → requirements/3.10/app.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=requirements/app.txt requirements/app.in
+#    pip-compile --output-file=requirements/3.10/app.txt requirements/app.in
 #
 alembic==1.11.1
     # via flask-migrate
@@ -208,7 +208,7 @@ patsy==0.5.3
     # via statsmodels
 pillow==9.5.0
     # via matplotlib
-pint==0.22
+pint==0.21.1
     # via -r requirements/app.in
 ply==3.11
     # via pyomo
@@ -278,7 +278,7 @@ scikit-learn==1.2.2
     # via
     #   sktime
     #   timetomodel
-scipy==1.11.1
+scipy==1.10.1
     # via
     #   properscoring
     #   scikit-learn
@@ -323,7 +323,6 @@ typing-extensions==4.7.0
     # via
     #   alembic
     #   altair
-    #   pint
     #   py-moneyed
     #   pydantic
 tzdata==2023.3

diff --git a/requirements/dev.txt → requirements/3.10/dev.txt b/requirements/dev.txt → requirements/3.10/dev.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=requirements/dev.txt requirements/dev.in
+#    pip-compile --output-file=requirements/3.10/dev.txt requirements/dev.in
 #
 black==22.3.0
     # via -r requirements/dev.in
@@ -56,8 +56,10 @@ pyinstrument==4.5.0
     # via -r requirements/dev.in
 pytest-runner==6.0.0
     # via -r requirements/dev.in
-pyyaml==6.0
-    # via pre-commit
+pyyaml==6.0.1
+    # via
+    #   -c requirements/app.txt
+    #   pre-commit
 setuptools-scm==7.1.0
     # via -r requirements/dev.in
 tomli==2.0.1

diff --git a/requirements/docs.txt → requirements/3.10/docs.txt b/requirements/docs.txt → requirements/3.10/docs.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=requirements/docs.txt requirements/docs.in
+#    pip-compile --output-file=requirements/3.10/docs.txt requirements/docs.in
 #
 alabaster==0.7.13
     # via sphinx

diff --git a/requirements/test.txt → requirements/3.10/test.txt b/requirements/test.txt → requirements/3.10/test.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile --output-file=requirements/test.txt requirements/test.in
+#    pip-compile --output-file=requirements/3.10/test.txt requirements/test.in
 #
 async-timeout==4.0.2
     # via