Merge branch 'develop' into proforma

.bash_profile

@@ -29,25 +29,6 @@ export DYLD_LIBRARY_PATH  # for Mac
 #==============================================================================
 
 
-#------------------------------------------------------------------------------
-
-#  Uncomment the following if you are using the Xpress solver
-#------------------------------------------------------------------------------
-#XPRESSDIR="/usr/local/opt/xpress"
-#DYLD_LIBRARY_PATH="${XPRESSDIR}/lib:${SRC_DIR}:${DYLD_LIBRARY_PATH}"  # for Mac
-#LD_LIBRARY_PATH="${XPRESSDIR}/lib:${SRC_DIR}:${LD_LIBRARY_PATH}"  # for Linux
-#CLASSPATH="${XPRESSDIR}/lib/xprs.jar:${XPRESSDIR}/lib/xprb.jar:${XPRESSDIR}/lib/xprm.jar:${CLASSPATH}"
-#PATH="${XPRESSDIR}/bin:${PATH}"
-#
-#export PATH
-#export XPRESS="$XPRESSDIR/bin"
-#export DYLD_LIBRARY_PATH
-#export LD_LIBRARY_PATH
-#export CLASSPATH
-#source /opt/xpressmp/bin/xpvars.sh
-#==============================================================================
-
-
 #------------------------------------------------------------------------------
 #  You can change the following to add Julia to your system Path
 #------------------------------------------------------------------------------

.github/disabled_cbc_workflows/push_test_cbc.yml

@@ -26,4 +26,4 @@ jobs:
       - name: Check django logs
         run: docker logs django
       - name: test
-        run: docker-compose exec -T celery python manage.py test reo.tests.test_custom_rates reo.tests.test_demand_ratchet -v 2 --failfast --no-input
+        run: docker-compose exec -T celery python manage.py test reoptjl.test.test_job_endpoint -v 2 --failfast --no-input

.github/scripts/decrypt.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-transcrypt/transcrypt --flush-credentials -y || true
-transcrypt/transcrypt -c aes-256-cbc -p "$TRANSCRYPT_PASSWORD" -y
+# transcrypt/transcrypt --flush-credentials -y || true
+# transcrypt/transcrypt -c aes-256-cbc -p "$TRANSCRYPT_PASSWORD" -y
 
 

.github/workflows/pull_request_tests.yml

@@ -18,21 +18,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2      
-      - name: Decrypt
-        env:
-          TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
-        run: ./.github/scripts/decrypt.sh 
+      # - name: Decrypt
+      #   env:
+      #     TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
+      #   run: ./.github/scripts/decrypt.sh 
       - name: Make keys.py
         env:
           NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }}
         run: ./.github/scripts/make_keys.py.sh 
       - name: Build containers
-        run: docker-compose up -d
+        run: docker-compose -f docker-compose.highs.yml up -d
       - name: Check running containers
         run: docker ps -a
       - name: Wait for julia_api
         uses: jakejarvis/wait-action@master
         with:
           time: '150s'
-      - name: test
-        run: docker exec reopt_api_celery_1 python manage.py test -v 2 --failfast --no-input
+      - name: test v3 validator
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_validator -v 2 --failfast --no-input          
+      - name: test v3 http.jl endpoints
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_http_endpoints -v 2 --failfast --no-input

.github/workflows/push_tests.yml

@@ -14,21 +14,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Decrypt
-        env:
-          TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
-        run: ./.github/scripts/decrypt.sh 
+      # - name: Decrypt
+      #   env:
+      #     TRANSCRYPT_PASSWORD: ${{ secrets.TRANSCRYPT_PASSWORD }}
+      #   run: ./.github/scripts/decrypt.sh 
       - name: Make keys.py
         env:
           NREL_DEV_API_KEY: ${{ secrets.NREL_DEV_API_KEY }}
         run: ./.github/scripts/make_keys.py.sh 
       - name: Build containers
-        run: docker-compose up -d
+        run: docker-compose -f docker-compose.highs.yml up -d
       - name: Check running containers
         run: docker ps -a
       - name: Wait for julia_api
         uses: jakejarvis/wait-action@master
         with:
           time: '150s'
-      - name: test
-        run: docker exec reopt_api_celery_1 python manage.py test -v 2 --failfast --no-input
+      - name: test v3 validator
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_validator -v 2 --failfast --no-input          
+      - name: test v3 http.jl endpoints
+        run: docker exec reopt_api_celery_1 python manage.py test reoptjl.test.test_http_endpoints -v 2 --failfast --no-input

.gitignore

@@ -114,8 +114,10 @@ all_data*
 docs/formulation/*aux
 docs/formulation/*log
 
-# Customized docker files
-docker-compose.xpressOS.yml
-Dockerfile.xpressOS.pyjul
-
 /.werf_secret_key
+
+# Customized solver setup and docker files
+julia_src/xpress/licenseserver
+julia_src/Dockerfile.xpress
+docker-compose.xpress.yml
+julia_src/xpress/solver_setup.sh

.helm/templates/julia-deployment.yaml

@@ -34,7 +34,7 @@ spec:
       containers:
         - name: {{ .Chart.Name }}-julia
           image: {{ index .Values.werf.image "julia-api" }}
-          args: ["julia", "--project=/opt/julia_src", "http.jl"]
+          args: ["julia", "--project=/opt/julia_src/xpress", "xpress/http.jl"]
           ports:
             - containerPort: 8081
           envFrom:

CHANGELOG.md

@@ -27,6 +27,20 @@ Classify the change according to the following categories:
     ### Patches
 
 
+## v3.4.0
+### Minor Updates
+#### Added 
+- Added the following BAU outputs:  lifecycle_chp_standby_cost_after_tax, lifecycle_elecbill_after_tax, lifecycle_production_incentive_after_tax, lifecycle_outage_cost, lifecycle_MG_upgrade_and_fuel_cost
+### Changed
+- Updated REopt.jl version to 0.39.1 along with updates to other dependencies
+#### Fixed
+- Fixed setting of default Generator `installed_cost_per_kw` so that user inputs are not overridden 
+- Avoid /summary endpoint error with off-grid runs where there is no ElectricTariff
+
+## v3.3.0
+### Changed
+- Updates to REopt.jl for passing API key
+
 ## v3.2.3
 ### Minor Updates
 ##### Changed

Jenkinsfile

@@ -65,12 +65,22 @@ pipeline {
         WERF_LOG_VERBOSE = "true"
         WERF_SYNCHRONIZATION = ":local"
         XPRESS_LICENSE_HOST = credentials("reopt-api-xpress-license-host")
+        LICENSESERVER_URL = credentials("reopt-api-xpress-licenseserver-url")
         NREL_ROOT_CERT_URL_ROOT = credentials("reopt-api-nrel-root-cert-url-root")
       }
 
       stages {
         stage("deploy") {
           stages {
+            stage("solver setup") {
+              steps {
+                dir("julia_src/xpress/licenseserver") {
+                  git url: env.LICENSESERVER_URL
+                }
+                sh "cp julia_src/xpress/licenseserver/Dockerfile.xpress julia_src/"
+              }
+            }
+
             stage("lint") {
               steps {
                 withCredentials([string(credentialsId: "reopt-api-werf-secret-key", variable: "WERF_SECRET_KEY")]) {

docker-compose.highs.yml

@@ -0,0 +1,73 @@
+version: "2.1"
+
+services:
+
+  redis:
+    image: redis
+    command: redis-server
+    expose:
+      - 6379
+
+  db:
+    image: postgres
+    restart: always
+    environment:
+      - POSTGRES_USER=reopt
+      - POSTGRES_PASSWORD=reopt
+      - POSTGRES_DB=reopt
+    expose:
+      - 5432
+
+  celery:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: base-api-image
+    command:  >
+      "celery -A reopt_api worker -l info"
+    environment:
+      - APP_ENV=local
+      - SQL_HOST=db
+      - SQL_PORT=5432
+      - REDIS_HOST=redis
+    volumes:
+      - .:/opt/reopt
+    depends_on:
+      - db
+      - redis
+      - julia
+
+  django:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: base-api-image
+    command:  >
+      "python manage.py migrate
+      && /opt/reopt/bin/wait-for-it.bash -t 0 julia:8081 -- python manage.py runserver 0.0.0.0:8000"
+    environment:
+      - APP_ENV=local
+      - SQL_HOST=db
+      - SQL_PORT=5432
+      - REDIS_HOST=redis
+    depends_on:
+      - db
+      - redis
+      - julia
+    ports:
+      - 8000:8000
+    volumes:
+      - .:/opt/reopt
+
+  julia:
+    container_name: julia_api
+    build:
+      context: julia_src/
+      dockerfile: Dockerfile.highs
+    environment:
+      - SOLVER=HiGHS      
+    command: julia --project=/opt/julia_src/highs highs/http.jl
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./julia_src:/opt/julia_src

docker-compose.nginx.yml

@@ -74,25 +74,16 @@ services:
       - django-nginx
     restart: "on-failure"
 
-  licenseserver:
-    build:
-      context: julia_src/xpress/licenseserver
-    mac_address: c4:b3:01:d3:d5:eb
-    command: xpserver
-    expose:
-      - 27100
-
   julia-nginx:
     container_name: julia-nginx
     build:
       context: julia_src/
+      dockerfile: Dockerfile.highs
     command: bash ./run_julia_servers.sh 8
     environment:
       - JULIA_NUM_THREADS=2
     expose:
       - 8081
-    depends_on:
-      - licenseserver
     volumes:
       - ./julia_src:/opt/julia_src
 

docker-compose.nojulia.yml

@@ -65,14 +65,5 @@ services:
     volumes:
       - .:/opt/reopt
 
-  licenseserver:
-    container_name: licenseserver-nojul
-    build:
-      context: julia_src/xpress/licenseserver
-    mac_address: c4:b3:01:d3:d5:eb
-    command: xpserver
-    expose:
-      - 27100
-
 volumes:
   pgdata:

docker-compose.yml

@@ -58,26 +58,13 @@ services:
       - 8000:8000
     volumes:
       - .:/opt/reopt
-
-  licenseserver:
-    container_name: xpress_license_server
-    build:
-      context: julia_src/xpress/licenseserver
-    mac_address: c4:b3:01:d3:d5:eb
-    command: xpserver
-    expose:
-      - 27100
 
   julia:
     container_name: julia_api
     build:
       context: julia_src/
-      args:
-        - XPRESS_LICENSE_HOST=xpress_license_server
     command: julia --project=/opt/julia_src http.jl
     ports:
       - "8081:8081"
-    depends_on: 
-      - licenseserver
     volumes:
       - ./julia_src:/opt/julia_src

julia_src/Dockerfile

@@ -1,33 +1,10 @@
-FROM reopt/xpressbase AS xpress
-
 FROM julia:1.8.5
 
 # Install NREL root certs for machines running on NREL's network.
 ARG NREL_ROOT_CERT_URL_ROOT=""
 RUN set -x && if [ -n "$NREL_ROOT_CERT_URL_ROOT" ]; then curl -fsSLk -o /usr/local/share/ca-certificates/nrel_root.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_root.pem" && curl -fsSLk -o /usr/local/share/ca-certificates/nrel_xca1.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_xca1.pem" &&  update-ca-certificates; fi
 ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-# Install Xpress solver for client
-ENV XPRESSDIR=/opt/xpressmp
-ENV XPRESS=/opt/xpressmp/bin
-ENV LD_LIBRARY_PATH=${XPRESSDIR}/lib:${LD_LIBRARY_PATH}
-ENV LIBPATH=${XPRESSDIR}/lib:${LIBPATH}
-ARG CLASSPATH=${XPRESSDIR}/lib/xprs.jar:${CLASSPATH}
-ARG CLASSPATH=${XPRESSDIR}/lib/xprb.jar:${CLASSPATH}
-ENV CLASSPATH=${XPRESSDIR}/lib/xprm.jar:${CLASSPATH}
-ENV PATH=${XPRESSDIR}/bin:${PATH}
-
-WORKDIR /opt/reopt/solver
-COPY --from=xpress  /opt/reopt/solver .
-RUN sed -i -e 's/\r$//' install.sh
-ARG XPRESS_LICENSE_HOST=licenseserver
-RUN set -x && printf "f\nc\n\nn\n\n${XPRESS_LICENSE_HOST}\ny\n" | ./install.sh >> license_info.txt;
-RUN rm xp8.12.3_linux_x86_64.tar.gz
-
-# Install gfortran for /ghpghx
-RUN apt-get update && apt-get install -y \
-    gfortran
-
 # Install Julia packages
 ENV JULIA_NUM_THREADS=2
 WORKDIR /opt/julia_src/

julia_src/Dockerfile.highs

@@ -0,0 +1,17 @@
+FROM julia:1.8.5
+
+# Install NREL root certs for machines running on NREL's network.
+ARG NREL_ROOT_CERT_URL_ROOT=""
+RUN set -x && if [ -n "$NREL_ROOT_CERT_URL_ROOT" ]; then curl -fsSLk -o /usr/local/share/ca-certificates/nrel_root.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_root.pem" && curl -fsSLk -o /usr/local/share/ca-certificates/nrel_xca1.crt "${NREL_ROOT_CERT_URL_ROOT}/nrel_xca1.pem" &&  update-ca-certificates; fi
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+
+# Install Julia packages
+ENV JULIA_NUM_THREADS=2
+
+WORKDIR /opt/julia_src
+COPY . .
+RUN julia --project=/opt/julia_src/highs -e 'import Pkg; Pkg.instantiate();'
+RUN julia --project=/opt/julia_src/highs highs/precompile.jl
+EXPOSE 8081
+
+CMD ["bash"]

julia_src/TODO.txt

@@ -0,0 +1,3 @@
+Make separate functions which all/some http.jl uses for any solver
+Added HiGHS and Cbc to the same Julia env setup with http.jl, in julia_src/highs env
+Add input to Settings input for choice of solver between HiGHS and Cbc

julia_src/docker-compose.yml

@@ -1,21 +1,11 @@
 version: "2.1"
 
 services: 
-  licenseserver:
-    container_name: xpress_license_server_dev
-    build:
-      context: ./xpress/licenseserver
-    mac_address: c4:b3:01:d3:d5:eb
-    command: xpserver
-    expose:
-      - 27100
 
   julia:
     container_name: julia_api_dev
     build:
       context: .
-      args:
-        - XPRESS_LICENSE_HOST=xpress_license_server_dev
     command: julia --project="/opt/julia_src" http.jl
     ports:
       - "8081:8081"