Add AT submission import

seed/api/v3/urls.py

@@ -14,6 +14,7 @@
 from seed.views.v3.analysis_messages import AnalysisMessageViewSet
 from seed.views.v3.analysis_views import AnalysisPropertyViewViewSet
 from seed.views.v3.audit_template import AuditTemplateViewSet
+from seed.views.v3.audit_template_configs import AuditTemplateConfigViewSet
 from seed.views.v3.building_files import BuildingFileViewSet
 from seed.views.v3.column_list_profiles import ColumnListProfileViewSet
 from seed.views.v3.column_mapping_profiles import ColumnMappingProfileViewSet
@@ -67,6 +68,7 @@
 api_v3_router = routers.DefaultRouter()
 api_v3_router.register(r"analyses", AnalysisViewSet, basename="analyses")
 api_v3_router.register(r"audit_template", AuditTemplateViewSet, basename="audit_template")
+api_v3_router.register(r"audit_template_configs", AuditTemplateConfigViewSet, basename="audit_template_configs")
 api_v3_router.register(r"building_files", BuildingFileViewSet, basename="building_files")
 api_v3_router.register(r"column_list_profiles", ColumnListProfileViewSet, basename="column_list_profiles")
 api_v3_router.register(r"column_mapping_profiles", ColumnMappingProfileViewSet, basename="column_mapping_profiles")

seed/audit_template/audit_template.py

@@ -7,12 +7,15 @@
 import json
 import logging
 from datetime import datetime
-from typing import Any, Tuple
+from functools import wraps
 
 import requests
 from celery import shared_task
+from dateutil import parser
 from django.conf import settings
 from django.db.models import Q
+from django.utils.timezone import get_current_timezone
+from django_celery_beat.models import CrontabSchedule, PeriodicTask
 from lxml import etree
 from lxml.builder import ElementMaker
 from quantityfield.units import ureg
@@ -22,24 +25,81 @@
 from seed.lib.progress_data.progress_data import ProgressData
 from seed.lib.superperms.orgs.models import Organization
 from seed.models import PropertyView
-from seed.utils.encrypt import decrypt
 from seed.views.v3.properties import PropertyViewSet
 
 _log = logging.getLogger(__name__)
 
+AUTO_SYNC_NAME = "audit_template_sync_org-"
+
+
+def require_token(fn):
+    """Decorator to get an AT api token"""
+
+    @wraps(fn)
+    def wrapper(self, *args, **kwargs):
+        if not self.token:
+            token, message = self.get_api_token()
+            if not token:
+                return None, message
+        return fn(self, *args, **kwargs)
+
+    return wrapper
+
+
+def schedule_sync(data, org_id):
+    timezone = data.get("timezone", get_current_timezone())
+
+    if "update_at_hour" in data and "update_at_minute" in data:
+        # create crontab schedule
+        schedule, _ = CrontabSchedule.objects.get_or_create(
+            minute=data["update_at_minute"],
+            hour=data["update_at_hour"],
+            day_of_week=data["update_at_day"],
+            day_of_month="*",
+            month_of_year="*",
+            timezone=timezone,
+        )
+
+        # then schedule task (create/update with new crontab)
+        tasks = PeriodicTask.objects.filter(name=AUTO_SYNC_NAME + str(org_id))
+        if not tasks:
+            PeriodicTask.objects.create(
+                crontab=schedule, name=AUTO_SYNC_NAME + str(org_id), task="seed.tasks.sync_audit_template", args=json.dumps([org_id])
+            )
+        else:
+            task = tasks.first()
+            # update crontab (if changed)
+            task.crontab = schedule
+            task.save()
+
+            # Cleanup orphaned/unused crontab schedules
+            CrontabSchedule.objects.exclude(id__in=PeriodicTask.objects.values_list("crontab_id", flat=True)).delete()
+
+
+def toggle_audit_template_sync(audit_template_sync_enabled, org_id):
+    """when audit_template_sync_enabled value is toggled, also toggle the auto sync
+    task status if it exists
+    """
+    tasks = PeriodicTask.objects.filter(name=AUTO_SYNC_NAME + str(org_id))
+    if tasks:
+        task = tasks.first()
+        task.enabled = bool(audit_template_sync_enabled)
+        task.save()
+
 
 class AuditTemplate:
     HOST = settings.AUDIT_TEMPLATE_HOST
     API_URL = f"{HOST}/api/v2"
+    token = None
 
     def __init__(self, org_id):
         self.org_id = org_id
+        self.org = Organization.objects.get(id=self.org_id)
 
+    @require_token
     def get_building(self, audit_template_building_id):
-        token, message = self.get_api_token()
-        if not token:
-            return None, message
-        return self.get_building_xml(audit_template_building_id, token)
+        """Entry point for AuditTemplateViewSet"""
+        return self.get_building_xml(audit_template_building_id, self.token)
 
     def get_building_xml(self, audit_template_building_id, token):
         url = f"{self.API_URL}/building_sync/download/rp/buildings/{audit_template_building_id}.xml?token={token}"
@@ -57,7 +117,23 @@ def get_building_xml(self, audit_template_building_id, token):
 
         return response, ""
 
-    def get_submission(self, audit_template_submission_id: int, report_format: str = "pdf") -> Tuple[Any, str]:
+    def batch_get_city_submission_xml(self):
+        """
+        1. get_city_submissions
+        2. find views using custom_id_1 and cycle start/end bounds
+        3. get xmls corresponding to submissions matching a view
+        4. group data by cycles
+        5. update views in cycle batches
+        """
+        progress_data = ProgressData(func_name="batch_get_city_submission_xml", unique_id=self.org_id)
+
+        _batch_get_city_submission_xml.delay(self.org_id, self.org.audit_template_city_id, progress_data.key)
+
+        return progress_data.result(), ""
+
+    @require_token
+    def get_submission(self, audit_template_submission_id: int, report_format: str = "pdf"):
+        # def get_submission(self, audit_template_submission_id: int, report_format: str = 'pdf') -> Tuple[Any, str]:
         """Download an Audit Template submission report.
 
         Args:
@@ -68,10 +144,6 @@ def get_submission(self, audit_template_submission_id: int, report_format: str =
             requests.response: Result from Audit Template website
         """
         # supporting 'JSON', PDF', and 'XML' formats only for now
-        token, message = self.get_api_token()
-        if not token:
-            return None, message
-
         # validate format
         if report_format.lower() not in {"json", "xml", "pdf"}:
             report_format = "pdf"
@@ -80,7 +152,7 @@ def get_submission(self, audit_template_submission_id: int, report_format: str =
         accept_type = "application/" + report_format.lower()
         headers = {"accept": accept_type}
 
-        url = f"{self.API_URL}/rp/submissions/{audit_template_submission_id}.{report_format}?token={token}"
+        url = f"{self.API_URL}/rp/submissions/{audit_template_submission_id}.{report_format}?token={self.token}"
         try:
             response = requests.request("GET", url, headers=headers)
 
@@ -94,11 +166,25 @@ def get_submission(self, audit_template_submission_id: int, report_format: str =
 
         return response, ""
 
+    @require_token
+    def get_city_submissions(self, city_id):
+        """Return all submissions for a city"""
+
+        headers = {"accept": "application/xml"}
+        url = f"{self.API_URL}/rp/cities/{city_id}?token={self.token}"
+        try:
+            response = requests.request("GET", url, headers=headers)
+            if response.status_code != 200:
+                return None, f"Expected 200 response from Audit Template cities but got {response.stutus_code}: {response.content}"
+        except Exception as e:
+            return None, f"Unexpected error from Audit Template: {e}"
+
+        return response, ""
+
+    @require_token
     def get_buildings(self, cycle_id):
-        token, message = self.get_api_token()
-        if not token:
-            return None, message
-        url = f"{self.API_URL}/rp/buildings?token={token}"
+        """Entry point for AuditTemplateViewSet"""
+        url = f"{self.API_URL}/rp/buildings?token={self.token}"
         headers = {"accept": "application/xml"}
 
         return _get_buildings.delay(cycle_id, url, headers)
@@ -116,15 +202,15 @@ def batch_get_building_xml(self, cycle_id, properties):
         return progress_data.result()
 
     def get_api_token(self):
-        org = Organization.objects.get(pk=self.org_id)
-        if not org.at_organization_token or not org.audit_template_user or not org.audit_template_password:
+        if not self.org.at_organization_token or not self.org.audit_template_user or not self.org.audit_template_password:
             return None, "An Audit Template organization token, user email and password are required!"
+
         url = f"{self.API_URL}/users/authenticate"
         # Send data as form-data to handle special characters like '%'
         form_data = {
-            "organization_token": (None, org.at_organization_token),
-            "email": (None, org.audit_template_user),
-            "password": (None, decrypt(org.audit_template_password)[0]),
+            "organization_token": (None, self.org.at_organization_token),
+            "email": (None, self.org.audit_template_user),
+            "password": (None, self.org.audit_template_password),
         }
         headers = {"Accept": "application/xml"}
 
@@ -140,30 +226,29 @@ def get_api_token(self):
         except ValueError:
             raise validation_client.ValidationClientError(f"Expected JSON response from Audit Template: {response.text}")
 
-        return response_body["token"], ""
+        # instead of pinging AT for tokens every time, use existing token.
+        self.token = response_body.get("token")
+        return self.token, ""
 
+    @require_token
     def batch_export_to_audit_template(self, view_ids):
-        token, message = self.get_api_token()
-        if not token:
-            return None, message
         progress_data = ProgressData(func_name="batch_export_to_audit_template", unique_id=view_ids[0])
         progress_data.total = len(view_ids)
         progress_data.save()
 
-        _batch_export_to_audit_template.delay(self.org_id, view_ids, token, progress_data.key)
+        _batch_export_to_audit_template.delay(self.org_id, view_ids, self.token, progress_data.key)
 
         return progress_data.result(), []
 
     def export_to_audit_template(self, state, token):
-        org = Organization.objects.get(pk=self.org_id)
         url = f"{self.API_URL}/building_sync/upload"
-        display_field = getattr(state, org.property_display_field)
+        display_field = getattr(state, self.org.property_display_field)
 
         if state.audit_template_building_id:
             return None, ["info", f"{display_field}: Existing Audit Template Property"]
 
         try:
-            xml_string, messages = self.build_xml(state, org.audit_template_report_type, display_field)
+            xml_string, messages = self.build_xml(state, self.org.audit_template_report_type, display_field)
             if not xml_string:
                 return None, messages
         except Exception as e:
@@ -330,21 +415,91 @@ def _batch_get_building_xml(org_id, cycle_id, token, properties, progress_key):
     for property in properties:
         audit_template_building_id = property["audit_template_building_id"]
         xml, _ = AuditTemplate(org_id).get_building_xml(property["audit_template_building_id"], token)
-        result.append(
-            {
-                "property_view": property["property_view"],
-                "audit_template_building_id": audit_template_building_id,
-                "xml": xml.text,
-                "updated_at": property["updated_at"],
-            }
-        )
+        if hasattr(xml, "text"):
+            result.append(
+                {
+                    "property_view": property["property_view"],
+                    "matching_field": audit_template_building_id,
+                    "xml": xml.text,
+                    "updated_at": property["updated_at"],
+                }
+            )
         progress_data.step("Getting XML for buildings...")
 
     # Call the PropertyViewSet to update the property view with xml data
     property_view_set = PropertyViewSet()
     property_view_set.batch_update_with_building_sync(result, org_id, cycle_id, progress_data.key)
 
 
+@shared_task
+def _batch_get_city_submission_xml(org_id, city_id, progress_key):
+    """
+    1. find views using custom_id_1 and cycle start/end bounds
+    2. get xmls corresponding to submissions matching a view
+    3. group data by cycles
+    4. update views in cycle batches
+    """
+    audit_template = AuditTemplate(org_id)
+    progress_data = ProgressData.from_key(progress_key)
+
+    response, messages = audit_template.get_city_submissions(city_id)
+    if not response:
+        return None, messages
+    submissions = response.json()
+    # breakpoint()
+    # Progress data is difficult to calculate as not all submissions will need an xml
+    progress_data.total = len(submissions) * 2
+    progress_data.save()
+
+    # NEED TO SPECIFY CYCLE, based on updated_at?
+    # would be nice to have audit_date
+    # audit_date is not a relavent fields in the xml. what should we use?
+    # WHERE IS AUDIT_DATE
+    # Metering Year Start Dates?
+
+    # filering for cycles that contain {updated_at} makes the query more difficult
+    # without placing dates it could be a simple .filter(state__custom_id_1__in=custom_ids)
+    # however that could return multiple views across many cycles
+    # filtering by custom_id and {updated_at} will require looping through results to query views
+
+    xml_data_by_cycle = {}
+    for sub in submissions:
+        custom_id = sub["tax_id"]
+        # What if updated_at is None? default cycle?
+        updated_at = parser.parse(sub["updated_at"])
+
+        view = PropertyView.objects.filter(
+            state__custom_id_1=custom_id,
+            cycle__start__lte=updated_at,
+            cycle__end__gte=updated_at,
+            # updated_at__lte=updated_at  # only update old views?
+        ).first()
+
+        progress_data.step("Getting XML for submissions...")
+
+        if view:
+            xml, _ = audit_template.get_submission(sub["id"], "xml")
+
+            if hasattr(xml, "text"):
+                if not xml_data_by_cycle.get(view.cycle.id):
+                    xml_data_by_cycle[view.cycle.id] = []
+
+                xml_data_by_cycle[view.cycle.id].append(
+                    {"property_view": view.id, "matching_field": custom_id, "xml": xml.text, "updated_at": sub["updated_at"]}
+                )
+
+    property_view_set = PropertyViewSet()
+    # Update is cycle based, going to have update in cycle specific batches
+    combined_results = {"success": 0, "failure": 0}
+    for cycle, xmls in xml_data_by_cycle.items():
+        # does progress_data need to be recursively passed?
+        results = property_view_set.batch_update_with_building_sync(xmls, org_id, cycle, progress_data.key, finish=False)
+        combined_results["success"] += results["success"]
+        combined_results["failure"] += results["failure"]
+
+    progress_data.finish_with_success(combined_results)
+
+
 @shared_task
 def _batch_export_to_audit_template(org_id, view_ids, token, progress_key):
     audit_template = AuditTemplate(org_id)

seed/lib/superperms/orgs/migrations/0032_org_audit_template_fields.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.23 on 2024-03-13 15:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("orgs", "0031_encrypt_existing_audit_template_passwords"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="organization",
+            name="audit_template_city_id",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="organization",
+            name="audit_template_sync_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

seed/lib/superperms/orgs/models.py

@@ -267,6 +267,8 @@ class Meta:
     audit_template_user = models.EmailField(blank=True, max_length=128, default="")
     audit_template_password = models.CharField(blank=True, max_length=128, default="")
     audit_template_report_type = models.CharField(blank=True, max_length=128, default="Demo City Report")
+    audit_template_city_id = models.IntegerField(blank=True, null=True)
+    audit_template_sync_enabled = models.BooleanField(default=False)
 
     # Salesforce Functionality
     salesforce_enabled = models.BooleanField(default=False)