Task/removeoldkeys (#373)

* remove notification counts v1

* update utils

* bump version

---------

Co-authored-by: Stephen McMurtry <smcmurtry@gmail.com>

Author: jzbahrai

.github/actions/waffles/requirements.txt

@@ -2,4 +2,4 @@ docopt==0.6.2
 Flask==2.3.3
 markupsafe==2.1.5
 setuptools==78.1.1 # required for distutils in Python 3.12
-git+https://github.com/cds-snc/notifier-utils.git@53.2.12#egg=notifications-utils
+git+https://github.com/cds-snc/notifier-utils.git@53.2.13#egg=notifications-utils

notifications_utils/clients/redis/annual_limit.py

@@ -1,15 +1,8 @@
 """
 This module stores daily notification counts and annual limit statuses for a service in Redis using a hash structure:
 
-# TODO: Remove the first key once all services have been migrated to the new Redis structure
 annual-limit: {
     {service_id}: {
-        notifications: {
-            sms_delivered: int,
-            email_delivered: int,
-            sms_failed: int,
-            email_failed: int
-        },
         status: {
             near_sms_limit: Datetime,
             near_email_limit: Datetime,
@@ -38,11 +31,6 @@
 
 from notifications_utils.clients.redis.redis_client import RedisClient
 
-# TODO: Remove the first 4 keys once all services have been migrated to the new Redis structure
-SMS_DELIVERED = "sms_delivered"
-EMAIL_DELIVERED = "email_delivered"
-SMS_FAILED = "sms_failed"
-EMAIL_FAILED = "email_failed"
 SMS_DELIVERED_TODAY = "sms_delivered_today"
 EMAIL_DELIVERED_TODAY = "email_delivered_today"
 SMS_FAILED_TODAY = "sms_failed_today"
@@ -51,7 +39,6 @@
 TOTAL_SMS_FISCAL_YEAR_TO_YESTERDAY = "total_sms_fiscal_year_to_yesterday"
 TOTAL_EMAIL_FISCAL_YEAR_TO_YESTERDAY = "total_email_fiscal_year_to_yesterday"
 
-NOTIFICATION_FIELDS = [SMS_DELIVERED, EMAIL_DELIVERED, SMS_FAILED, EMAIL_FAILED]
 NOTIFICATION_FIELDS_V2 = [
     SMS_DELIVERED_TODAY,
     EMAIL_DELIVERED_TODAY,
@@ -69,14 +56,6 @@
 STATUS_FIELDS = [NEAR_SMS_LIMIT, NEAR_EMAIL_LIMIT, OVER_SMS_LIMIT, OVER_EMAIL_LIMIT]
 
 
-# TODO: Remove this once all services have been migrated to the new Redis structure
-def annual_limit_notifications_key(service_id):
-    """
-    Generates the Redis hash key for storing daily metrics of a service.
-    """
-    return f"annual-limit:{service_id}:notifications"
-
-
 def annual_limit_notifications_v2_key(service_id):
     """
     Generates the Redis hash key for storing daily metrics of a service.
@@ -144,22 +123,14 @@ def increment_notification_count(self, service_id: str, field: str):
             service_id (str): _description_
             field (str): _description_
         """
-        # TODO: Remove the else
         if field in NOTIFICATION_FIELDS_V2:
             self._redis_client.increment_hash_value(annual_limit_notifications_v2_key(service_id), field)
-        else:
-            self._redis_client.increment_hash_value(annual_limit_notifications_key(service_id), field)
 
     def get_notification_count(self, service_id: str, field: str):
         """
         Retrieves the specified daily notification count for a service. (e.g. SMS_DELIVERED, EMAIL_FAILED, SMS_DELIVERED_TODAY etc.)
         """
         count = self._redis_client.get_hash_field(annual_limit_notifications_v2_key(service_id), field)
-        if count:
-            return int(count.decode("utf-8"))
-        # TODO: Remove this once all services have been migrated to the new Redis structure
-        # TODO: Change the above to return 0 if count is None
-        count = self._redis_client.get_hash_field(annual_limit_notifications_key(service_id), field)
         return 0 if not count else int(count.decode("utf-8"))
 
     def get_all_notification_counts(self, service_id: str):
@@ -177,10 +148,7 @@ def get_all_notification_counts(self, service_id: str):
                 int,
                 NOTIFICATION_FIELDS_V2,
             )
-        # TODO: Remove this once all services have been migrated to the new Redis structure
-        return prepare_byte_dict(
-            self._redis_client.get_all_from_hash(annual_limit_notifications_key(service_id)), int, NOTIFICATION_FIELDS
-        )
+        return {}
 
     def reset_all_notification_counts(self, service_ids=None):
         """Resets all daily notification metrics.
@@ -196,31 +164,14 @@ def reset_all_notification_counts(self, service_ids=None):
         )
         # We also want to remove the seeded_at field from the notifications_v2 hash
         self._redis_client.delete_hash_fields(hashes=hashes, fields=NOTIFICATION_FIELDS_V2 + [SEEDED_AT])
-        # TODO: Remove the else once all services have been migrated to the new Redis structure
-        hashes = (
-            annual_limit_notifications_key("*")
-            if not service_ids
-            else [annual_limit_notifications_key(service_id) for service_id in service_ids]
-        )
-        self._redis_client.delete_hash_fields(hashes=hashes, fields=NOTIFICATION_FIELDS)
 
     def seed_annual_limit_notifications(self, service_id: str, mapping: dict):
         """Seeds annual limit notifications for a service.
-        # TODO: Update the docstring once all services have been migrated to the new Redis structure
         Args:
             service_id (str): Service to seed annual limit notifications for.
             mapping (dict): A dict used to map notification counts to their respective fields formatted as follows
 
         Examples:
-            `mapping` format:
-
-                {
-                    "sms_delivered": int,
-                    "email_delivered": int,
-                    "sms_failed": int,
-                    "email_failed": int,
-                }
-            as we added notifications_v2, the mapping can also be:
                 {
                     "sms_delivered_today": int,
                     "email_delivered_today": int,
@@ -252,28 +203,6 @@ def seed_annual_limit_notifications(self, service_id: str, mapping: dict):
         v2_values = prepare_byte_dict(self._redis_client.get_all_from_hash(annual_limit_notifications_v2_key(service_id)), str)
         current_app.logger.info(f"[alimit-debug-redis] Finished setting values, result is: {v2_values}")
 
-        # Create a legacy mapping from either original legacy keys or mapped from V2 keys
-        legacy_mapping = {}
-
-        # First try to use any legacy fields directly present in the mapping
-        for k in NOTIFICATION_FIELDS:
-            if k in mapping:
-                legacy_mapping[k] = mapping[k]
-
-        # If legacy fields aren't present, map from V2 fields
-        if SMS_DELIVERED_TODAY in mapping and SMS_DELIVERED not in legacy_mapping:
-            legacy_mapping[SMS_DELIVERED] = mapping[SMS_DELIVERED_TODAY]
-        if EMAIL_DELIVERED_TODAY in mapping and EMAIL_DELIVERED not in legacy_mapping:
-            legacy_mapping[EMAIL_DELIVERED] = mapping[EMAIL_DELIVERED_TODAY]
-        if SMS_FAILED_TODAY in mapping and SMS_FAILED not in legacy_mapping:
-            legacy_mapping[SMS_FAILED] = mapping[SMS_FAILED_TODAY]
-        if EMAIL_FAILED_TODAY in mapping and EMAIL_FAILED not in legacy_mapping:
-            legacy_mapping[EMAIL_FAILED] = mapping[EMAIL_FAILED_TODAY]
-
-        # Store legacy fields if we have any
-        if legacy_mapping:
-            self._redis_client.bulk_set_hash_fields(key=annual_limit_notifications_key(service_id), mapping=legacy_mapping)
-
         current_app.logger.info(f"[alimit-debug-redis] Setting seeded_at for service {service_id}")
         self.set_seeded_at(service_id)
         current_app.logger.info(
@@ -293,16 +222,12 @@ def set_seeded_at(self, service_id):
         self._redis_client.set_hash_value(
             annual_limit_notifications_v2_key(service_id), SEEDED_AT, datetime.utcnow().strftime("%Y-%m-%d")
         )
-        # TODO: Remove the below once all services have been migrated to the new Redis structure
-        # Setting the seeded at in status for backward compatibility
-        self._redis_client.set_hash_value(annual_limit_status_key(service_id), SEEDED_AT, datetime.utcnow().strftime("%Y-%m-%d"))
 
     def clear_notification_counts(self, service_id: str):
         """
         Clears all daily notification metrics for a service.
         """
         self._redis_client.expire(annual_limit_notifications_v2_key(service_id), -1)
-        self._redis_client.expire(annual_limit_notifications_key(service_id), -1)
 
     def set_annual_limit_status(self, service_id: str, field: str, value: datetime):
         """
@@ -351,23 +276,15 @@ def clear_annual_limit_statuses(self, service_id: str):
     # Helper methods for daily metrics
     def increment_sms_delivered(self, service_id: str):
         self.increment_notification_count(service_id, SMS_DELIVERED_TODAY)
-        # TODO: remove the below line
-        self.increment_notification_count(service_id, SMS_DELIVERED)
 
     def increment_sms_failed(self, service_id: str):
         self.increment_notification_count(service_id, SMS_FAILED_TODAY)
-        # TODO: remove the below line
-        self.increment_notification_count(service_id, SMS_FAILED)
 
     def increment_email_delivered(self, service_id: str):
         self.increment_notification_count(service_id, EMAIL_DELIVERED_TODAY)
-        # TODO: remove the below line
-        self.increment_notification_count(service_id, EMAIL_DELIVERED)
 
     def increment_email_failed(self, service_id: str):
         self.increment_notification_count(service_id, EMAIL_FAILED_TODAY)
-        # TODO: remove the below line
-        self.increment_notification_count(service_id, EMAIL_FAILED)
 
     # Helper methods for annual limits statuses
     def set_nearing_sms_limit(self, service_id: str):
@@ -412,11 +329,8 @@ def delete_all_annual_limit_hashes(self, service_ids=None):
         """
         if not service_ids:
             self._redis_client.delete_cache_keys_by_pattern(annual_limit_notifications_v2_key("*"))
-            # TODO: Remove the line below
-            self._redis_client.delete_cache_keys_by_pattern(annual_limit_notifications_key("*"))
             self._redis_client.delete_cache_keys_by_pattern(annual_limit_status_key("*"))
         else:
             for service_id in service_ids:
                 self._redis_client.delete(annual_limit_notifications_v2_key(service_id))
-                self._redis_client.delete(annual_limit_notifications_key(service_id))
                 self._redis_client.delete(annual_limit_status_key(service_id))

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "notifications-utils"
-version = "53.2.12"
+version = "53.2.13"
 description = "Shared python code for Notification - Provides logging utils etc."
 authors = ["Canadian Digital Service"]
 license = "MIT license"