Localize notification_utils to the API

This changeset pulls in all of the notification_utils code directly into the API and removes it as an external dependency.  We are doing this to cut down on operational maintenance of the project and will begin removing parts of it no longer needed for the API.

Signed-off-by: Carlo Costino <carlo.costino@gsa.gov>

Makefile

@@ -81,7 +81,7 @@ test: ## Run tests and create coverage report
 	poetry run black .
 	poetry run flake8 .
 	poetry run isort --check-only ./app ./tests
-	poetry run coverage run --omit=*/notifications_utils/* -m pytest --maxfail=10
+	poetry run coverage run -m pytest --maxfail=10
 	poetry run coverage report -m --fail-under=95
 	poetry run coverage html -d .coverage_cache
 
@@ -90,13 +90,6 @@ py-lock: ## Syncs dependencies and updates lock file without performing recursiv
 	poetry lock --no-update
 	poetry install --sync
 
-.PHONY: update-utils
-update-utils: ## Forces Poetry to pull the latest changes from the notifications-utils repo; requires that you commit the changes to poetry.lock!
-	poetry update notifications-utils
-	@echo
-	@echo !!! PLEASE MAKE SURE TO COMMIT AND PUSH THE UPDATED poetry.lock FILE !!!
-	@echo
-
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
 	poetry export --without-hashes --format=requirements.txt > requirements.txt

README.md

@@ -431,23 +431,6 @@ In either situation, once you are finished and have verified the dependency
 changes are working, please be sure to commit both the `pyproject.toml` and
 `poetry.lock` files.
 
-### Keeping the notification-utils Dependency Up-to-Date
-
-The `notifications-utils` dependency references the other repository we have at
-https://github.com/GSA/notifications-utils - this dependency requires a bit of
-extra legwork to ensure it stays up-to-date.
-
-Whenever a PR is merged in the `notifications-utils` repository, we need to make
-sure the changes are pulled in here and committed to this repository as well.
-You can do this by going through these steps:
-
-- Make sure your local `main` branch is up-to-date
-- Create a new branch to work in
-- Run `make update-utils`
-- Commit the updated `poetry.lock` file and push the changes
-- Make a new PR with the change
-- Have the PR get reviewed and merged
-
 ## Known Installation Issues
 
 ### Python Installation Errors

app/__init__.py

@@ -12,10 +12,6 @@
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy as _SQLAlchemy
-from notifications_utils import logging, request_helper
-from notifications_utils.clients.encryption.encryption_client import Encryption
-from notifications_utils.clients.redis.redis_client import RedisClient
-from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 from sqlalchemy import event
 from werkzeug.exceptions import HTTPException as WerkzeugHTTPException
 from werkzeug.local import LocalProxy
@@ -26,6 +22,10 @@
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
 from app.clients.sms.aws_sns import AwsSnsClient
+from notifications_utils import logging, request_helper
+from notifications_utils.clients.encryption.encryption_client import Encryption
+from notifications_utils.clients.redis.redis_client import RedisClient
+from notifications_utils.clients.zendesk.zendesk_client import ZendeskClient
 
 
 class NotifyCelery(Celery):

app/authentication/auth.py

@@ -12,10 +12,10 @@
     TokenExpiredError,
     TokenIssuerError,
 )
-from notifications_utils import request_helper
 from sqlalchemy.orm.exc import NoResultFound
 
 from app.serialised_models import SerialisedService
+from notifications_utils import request_helper
 
 # stvnrlly - this is silly, but bandit has a multiline string bug (https://github.com/PyCQA/bandit/issues/658)
 # and flake8 wants a multiline quote here. TODO: check on bug status and restore sanity once possible

app/celery/scheduled_tasks.py

@@ -1,7 +1,6 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTicket
 from sqlalchemy import between
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -32,6 +31,7 @@
 from app.enums import JobStatus, NotificationType
 from app.models import Job
 from app.notifications.process_notifications import send_notification_to_queue
+from notifications_utils.clients.zendesk.zendesk_client import NotifySupportTicket
 
 MAX_NOTIFICATION_FAILS = 10000
 

app/celery/tasks.py

@@ -2,7 +2,6 @@
 from datetime import datetime
 
 from flask import current_app
-from notifications_utils.recipients import RecipientCSV
 from requests import HTTPError, RequestException, request
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
 
@@ -27,6 +26,7 @@
 from app.service.utils import service_allowed_to_send_to
 from app.utils import DATETIME_FORMAT
 from app.v2.errors import TotalRequestsError
+from notifications_utils.recipients import RecipientCSV
 
 
 @notify_celery.task(name="process-job")

app/commands.py

@@ -12,8 +12,6 @@
 from faker import Faker
 from flask import current_app, json
 from notifications_python_client.authentication import create_jwt_token
-from notifications_utils.recipients import RecipientCSV
-from notifications_utils.template import SMSMessageTemplate
 from sqlalchemy import and_, text
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
@@ -64,6 +62,8 @@
     User,
 )
 from app.utils import get_midnight_in_utc
+from notifications_utils.recipients import RecipientCSV
+from notifications_utils.template import SMSMessageTemplate
 from tests.app.db import (
     create_job,
     create_notification,

app/config.py

@@ -2,10 +2,10 @@
 from datetime import timedelta
 from os import getenv, path
 
-import notifications_utils
 from celery.schedules import crontab
 from kombu import Exchange, Queue
 
+import notifications_utils
 from app.cloudfoundry_config import cloud_config
 
 

app/dao/notifications_dao.py

@@ -1,12 +1,6 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
-from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
-from notifications_utils.recipients import (
-    InvalidEmailError,
-    try_validate_and_format_phone_number,
-    validate_and_format_email_address,
-)
 from sqlalchemy import asc, desc, or_, select, text, union
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.exc import NoResultFound
@@ -23,6 +17,12 @@
     get_midnight_in_utc,
     midnight_n_days_ago,
 )
+from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
+from notifications_utils.recipients import (
+    InvalidEmailError,
+    try_validate_and_format_phone_number,
+    validate_and_format_email_address,
+)
 
 
 def dao_get_last_date_template_was_used(template_id, service_id):

app/delivery/send_to_providers.py

@@ -4,11 +4,6 @@
 
 from cachetools import TTLCache, cached
 from flask import current_app
-from notifications_utils.template import (
-    HTMLEmailTemplate,
-    PlainTextEmailTemplate,
-    SMSMessageTemplate,
-)
 
 from app import create_uuid, db, notification_provider_clients, redis_store
 from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3
@@ -19,6 +14,11 @@
 from app.enums import BrandType, KeyType, NotificationStatus, NotificationType
 from app.exceptions import NotificationTechnicalFailureException
 from app.serialised_models import SerialisedService, SerialisedTemplate
+from notifications_utils.template import (
+    HTMLEmailTemplate,
+    PlainTextEmailTemplate,
+    SMSMessageTemplate,
+)
 
 
 def send_sms_to_provider(notification):

app/errors.py

@@ -1,12 +1,12 @@
 from flask import current_app, json, jsonify
 from jsonschema import ValidationError as JsonSchemaValidationError
 from marshmallow import ValidationError
-from notifications_utils.recipients import InvalidEmailError
 from sqlalchemy.exc import DataError
 from sqlalchemy.orm.exc import NoResultFound
 
 from app.authentication.auth import AuthError
 from app.exceptions import ArchiveValidationError
+from notifications_utils.recipients import InvalidEmailError
 
 
 class VirusScanError(Exception):

app/models.py

@@ -3,15 +3,6 @@
 import uuid
 
 from flask import current_app, url_for
-from notifications_utils.clients.encryption.encryption_client import EncryptionError
-from notifications_utils.recipients import (
-    InvalidEmailError,
-    InvalidPhoneError,
-    try_validate_and_format_phone_number,
-    validate_email_address,
-    validate_phone_number,
-)
-from notifications_utils.template import PlainTextEmailTemplate, SMSMessageTemplate
 from sqlalchemy import CheckConstraint, Index, UniqueConstraint
 from sqlalchemy.dialects.postgresql import JSON, JSONB, UUID
 from sqlalchemy.ext.associationproxy import association_proxy
@@ -46,6 +37,15 @@
     DATETIME_FORMAT_NO_TIMEZONE,
     get_dt_string_or_none,
 )
+from notifications_utils.clients.encryption.encryption_client import EncryptionError
+from notifications_utils.recipients import (
+    InvalidEmailError,
+    InvalidPhoneError,
+    try_validate_and_format_phone_number,
+    validate_email_address,
+    validate_phone_number,
+)
+from notifications_utils.template import PlainTextEmailTemplate, SMSMessageTemplate
 
 
 def filter_null_value_fields(obj):

app/notifications/process_notifications.py

@@ -2,12 +2,6 @@
 from datetime import datetime
 
 from flask import current_app
-from notifications_utils.recipients import (
-    format_email_address,
-    get_international_phone_info,
-    validate_and_format_phone_number,
-)
-from notifications_utils.template import PlainTextEmailTemplate, SMSMessageTemplate
 
 from app import redis_store
 from app.celery import provider_tasks
@@ -19,6 +13,12 @@
 from app.enums import KeyType, NotificationStatus, NotificationType
 from app.models import Notification
 from app.v2.errors import BadRequestError
+from notifications_utils.recipients import (
+    format_email_address,
+    get_international_phone_info,
+    validate_and_format_phone_number,
+)
+from notifications_utils.template import PlainTextEmailTemplate, SMSMessageTemplate
 
 
 def create_content_for_notification(template, personalisation):

app/notifications/receive_notifications.py

@@ -1,5 +1,4 @@
 from flask import Blueprint, current_app, json, jsonify, request
-from notifications_utils.recipients import try_validate_and_format_phone_number
 
 from app.celery import tasks
 from app.config import QueueNames
@@ -9,6 +8,7 @@
 from app.errors import InvalidRequest, register_errors
 from app.models import InboundSms
 from app.notifications.sns_handlers import sns_notification_handler
+from notifications_utils.recipients import try_validate_and_format_phone_number
 
 receive_notifications_blueprint = Blueprint("receive_notifications", __name__)
 register_errors(receive_notifications_blueprint)

app/notifications/rest.py

@@ -1,5 +1,4 @@
 from flask import Blueprint, current_app, jsonify, request
-from notifications_utils import SMS_CHAR_COUNT_LIMIT
 
 from app import api_user, authenticated_service
 from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3
@@ -26,6 +25,7 @@
 )
 from app.service.utils import service_allowed_to_send_to
 from app.utils import get_public_notify_type_text, pagination_links
+from notifications_utils import SMS_CHAR_COUNT_LIMIT
 
 notifications = Blueprint("notifications", __name__)
 

app/notifications/validators.py

@@ -1,14 +1,4 @@
 from flask import current_app
-from notifications_utils import SMS_CHAR_COUNT_LIMIT
-from notifications_utils.clients.redis import (
-    rate_limit_cache_key,
-    total_limit_cache_key,
-)
-from notifications_utils.recipients import (
-    get_international_phone_info,
-    validate_and_format_email_address,
-    validate_and_format_phone_number,
-)
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import redis_store
@@ -22,6 +12,16 @@
 from app.service.utils import service_allowed_to_send_to
 from app.utils import get_public_notify_type_text
 from app.v2.errors import BadRequestError, RateLimitError, TotalRequestsError
+from notifications_utils import SMS_CHAR_COUNT_LIMIT
+from notifications_utils.clients.redis import (
+    rate_limit_cache_key,
+    total_limit_cache_key,
+)
+from notifications_utils.recipients import (
+    get_international_phone_info,
+    validate_and_format_email_address,
+    validate_and_format_phone_number,
+)
 
 
 def check_service_over_api_rate_limit(service, api_key):

app/organization/invite_rest.py

@@ -3,7 +3,6 @@
 
 from flask import Blueprint, current_app, jsonify, request
 from itsdangerous import BadData, SignatureExpired
-from notifications_utils.url_safe_token import check_token, generate_token
 
 from app import redis_store
 from app.config import QueueNames
@@ -28,6 +27,7 @@
     post_update_invited_org_user_status_schema,
 )
 from app.schema_validation import validate
+from notifications_utils.url_safe_token import check_token, generate_token
 
 organization_invite_blueprint = Blueprint("organization_invite", __name__)
 

app/schema_validation/__init__.py

@@ -4,6 +4,7 @@
 
 from iso8601 import ParseError, iso8601
 from jsonschema import Draft7Validator, FormatChecker, ValidationError
+
 from notifications_utils.recipients import (
     InvalidEmailError,
     InvalidPhoneError,

app/schemas.py

@@ -14,6 +14,12 @@
     validates_schema,
 )
 from marshmallow_sqlalchemy import auto_field, field_for
+
+from app import ma, models
+from app.dao.permissions_dao import permission_dao
+from app.enums import ServicePermissionType, TemplateType
+from app.models import ServicePermission
+from app.utils import DATETIME_FORMAT_NO_TIMEZONE, get_template_instance
 from notifications_utils.recipients import (
     InvalidEmailError,
     InvalidPhoneError,
@@ -22,12 +28,6 @@
     validate_phone_number,
 )
 
-from app import ma, models
-from app.dao.permissions_dao import permission_dao
-from app.enums import ServicePermissionType, TemplateType
-from app.models import ServicePermission
-from app.utils import DATETIME_FORMAT_NO_TIMEZONE, get_template_instance
-
 
 def _validate_positive_number(value, msg="Not a positive integer"):
     try:

app/serialised_models.py

@@ -4,16 +4,16 @@
 
 import cachetools
 from flask import current_app
-from notifications_utils.clients.redis import RequestCache
-from notifications_utils.serialised_model import (
-    SerialisedModel,
-    SerialisedModelCollection,
-)
 from werkzeug.utils import cached_property
 
 from app import db, redis_store
 from app.dao.api_key_dao import get_model_api_keys
 from app.dao.services_dao import dao_fetch_service_by_id
+from notifications_utils.clients.redis import RequestCache
+from notifications_utils.serialised_model import (
+    SerialisedModel,
+    SerialisedModelCollection,
+)
 
 caches = defaultdict(partial(cachetools.TTLCache, maxsize=1024, ttl=2))
 locks = defaultdict(RLock)