Merge pull request #202 from swishjam/supplemented-augment-event-user…

…-properties

supplement/augment event/user properties

backend/app/lib/ingestion/event_preparers/helpers/event_properties_augmentor.rb

@@ -0,0 +1,21 @@
+module Ingestion
+  module EventPreparers
+    module Helpers
+      class EventPropertiesAugmentor < Ingestion::EventPreparers::Base
+        URL_PROPERTIES_TO_NORMALIZE = [
+          Analytics::Event::ReservedPropertyNames.URL,
+          Analytics::Event::ReservedPropertyNames.PAGE_REFERRER,
+          Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_URL,
+          Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_URL,
+          Analytics::Event::ReservedPropertyNames.REFERRER, # deprecated, but including to account for older SDKs that dont use PAGE_REFERRER
+        ]
+
+        def augment_properties!
+          URL_PROPERTIES_TO_NORMALIZE.each do |url_property_name|
+            Ingestion::EventPreparers::Helpers::PropertyAugmentors::UrlNormalizer.add_normalized_url_properties!(parsed_event, url_property_name)
+          end
+        end
+      end
+    end
+  end
+end

backend/app/lib/ingestion/event_preparers/helpers/organization_properties_augmentor.rb

@@ -0,0 +1,53 @@
+module Ingestion
+  module EventPreparers
+    module Helpers
+      class OrganizationPropertiesAugmentor < Ingestion::EventPreparers::Base
+        attr_reader :org, :parsed_event, :user_profile_for_event
+
+        AUTO_APPLY_FROM_USER_PROPERTIES_DICT = [
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL_HOST,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL_PATH,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL_QUERY_PARAMS,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_BASE_URL,
+
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL_HOST,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL_PATH,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL_QUERY_PARAMS,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_BASE_URL,
+
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CAMPAIGN,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_SOURCE,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_MEDIUM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CONTENT,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_TERM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_GCLID,
+        ]
+
+        def initialize(org, parsed_event, user_profile_for_event)
+          @org = org
+          @parsed_event = parsed_event
+          @user_profile_for_event = user_profile_for_event
+        end
+
+        def augment_organization_properties!
+          return org if !user_profile_for_event.present? || org_has_any_initial_properties_set?
+          AUTO_APPLY_FROM_USER_PROPERTIES_DICT.each do |property_name|
+            next if !user_profile_for_event.metadata[property_name].present?
+            org.metadata[property_name] ||= user_profile_for_event.metadata[property_name] 
+          end
+          org.save! if org.changed?
+          org
+        end
+
+        private
+
+        def org_has_any_initial_properties_set?
+          AUTO_APPLY_FROM_USER_PROPERTIES_DICT.any?{ |property_name| org.metadata.key?(property_name) }
+        end
+
+      end
+    end
+  end
+end

backend/app/lib/ingestion/event_preparers/helpers/property_augmentors/url_normalizer.rb

@@ -0,0 +1,34 @@
+module Ingestion
+  module EventPreparers
+    module Helpers
+      module PropertyAugmentors
+        class UrlNormalizer
+          def self.add_normalized_url_properties!(parsed_event, property_name)
+            return if !parsed_event.properties[property_name].present?
+            return if parsed_event.properties[property_name] == 'direct'
+            parsed_url = URI.parse(parsed_event.properties[property_name])
+            parsed_event.set_property(new_property_name(property_name, 'url_path'), parsed_url.path.blank? ? '/' : parsed_url.path)
+            parsed_event.set_property(new_property_name(property_name, 'url_host'), parsed_url.host)
+            parsed_event.set_property(new_property_name(property_name, 'url_query_params'), parsed_url.query)
+            parsed_event.set_property(new_property_name(property_name, 'base_url'), "#{parsed_url.host}#{parsed_url.path == '/' ? '' : parsed_url.path}")
+            parsed_event
+          rescue URI::InvalidURIError => e
+            Sentry.capture_exception(e, extra: { attempted_url: parsed_event.properties[property_name] })
+            parsed_event
+          end
+
+          def self.new_property_name(old_property_name, augmented_proprety_name)
+            if old_property_name.include?('url')
+              # session_landing_page_url -> session_landing_page_url_path
+              old_property_name.gsub('url', augmented_proprety_name)
+            else
+              # referrer -> referrer_url_path
+              [old_property_name, augmented_proprety_name].join('_')
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

backend/app/lib/ingestion/event_preparers/helpers/swishjam_event_organization_attributor.rb

@@ -4,17 +4,6 @@ module Helpers
       class SwishjamEventOrganizationAttributor < Ingestion::EventPreparers::Base
         attr_reader :parsed_event, :user_profile_for_event
 
-        AUTO_APPLY_FROM_USER_PROPERTIES_DICT = [
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CAMPAIGN,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_SOURCE,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_MEDIUM,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CONTENT,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_TERM,
-          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_GCLID,
-        ]
-
         def initialize(parsed_event, user_profile_for_event)
           @parsed_event = parsed_event
           @user_profile_for_event = user_profile_for_event
@@ -30,15 +19,7 @@ def organization_for_event
             org.metadata ||= {}
             org.metadata = org.metadata.merge(sanitized_provided_org_properties)
             org.domain = org_attr('domain') if org_attr('domain').present?
-            if org.domain.nil? && user_profile_for_event&.email.present? && !GenericEmailDetector.is_generic_email?(user_profile_for_event.email)
-              domain_from_user_email = user_profile_for_event.email.split('@').last
-              org.domain = domain_from_user_email
-            end
-            if user_profile_for_event.present?
-              AUTO_APPLY_FROM_USER_PROPERTIES_DICT.each do |property_name|
-                org.metadata[property_name] ||= user_profile_for_event.metadata[property_name] if user_profile_for_event.metadata[property_name].present?
-              end
-            end
+            apply_domain_value_if_necessary!(org)
             org.save! if org.changed?
             if user_profile_for_event.present? && !org.analytics_organization_members.exists?(analytics_user_profile_id: user_profile_for_event.id)
               org.analytics_organization_members.create!(analytics_user_profile_id: user_profile_for_event.id)
@@ -65,6 +46,12 @@ def sanitized_provided_org_properties
           (provided_org_attributes['metadata'] || provided_org_attributes).except('id', 'identifier', 'organization_identifier', 'name', 'organization_name', 'domain', 'org_id', 'organization_id', 'orgIdentifier', 'organizationIdentifier', 'organization_identifier', *Analytics::Event::ReservedPropertyNames.all.map(&:to_s))
         end
 
+        def apply_domain_value_if_necessary!(org)
+          if org.domain.nil? && user_profile_for_event&.email.present? && !GenericEmailDetector.is_generic_email?(user_profile_for_event.email)
+            org.domain = user_profile_for_event.email.split('@').last
+          end
+        end
+
       end
     end
   end

backend/app/lib/ingestion/event_preparers/helpers/swishjam_event_user_attributor.rb

@@ -2,12 +2,6 @@ module Ingestion
   module EventPreparers
     module Helpers
       class SwishjamEventUserAttributor < Ingestion::EventPreparers::Base
-        attr_reader :parsed_event
-
-        def initialize(parsed_event)
-          @parsed_event = parsed_event
-        end
-
         def get_user_profile_and_associate_to_device_if_necessary!
           is_new_device_owner = user_profile_for_event.present? && device_for_event.present? && device_for_event.owner != user_profile_for_event
           return user_profile_for_event if !is_new_device_owner
@@ -37,7 +31,6 @@ def user_profile_for_event
             user_profile.email = provided_user_properties['email'] if !provided_user_properties['email'].blank?
             user_profile.metadata ||= {}
             user_profile.metadata = user_profile.metadata.merge(sanitized_user_properties.dig('metadata') || sanitized_user_properties)
-            Ingestion::EventPreparers::Helpers::AutomaticUserAttributeApplier.apply_user_attributes_if_necessary!(user_profile.metadata, parsed_event.properties)
             user_profile.last_seen_at_in_web_app = Time.current
             user_profile.first_seen_at_in_web_app ||= Time.current
             user_profile.save! if user_profile.changed?

backend/app/lib/ingestion/event_preparers/helpers/user_properties_augmentor.rb

@@ -0,0 +1,62 @@
+module Ingestion
+  module EventPreparers
+    module Helpers
+      class UserPropertiesAugmentor < Ingestion::EventPreparers::Base
+        attr_reader :user_profile, :parsed_event
+
+        AUTO_APPLY_USER_PROPERTIES_DICT = {
+          # .URL and .REFERRER are legacy property names but falling back to these values for backwards compatibility
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL => [Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_URL, Analytics::Event::ReservedPropertyNames.URL],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL_HOST => [Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_URL_HOST, Analytics::Event::ReservedPropertyNames.URL_HOST],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL_PATH => [Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_URL_PATH, Analytics::Event::ReservedPropertyNames.URL_PATH],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL_QUERY_PARAMS => [Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_URL_QUERY_PARAMS, Analytics::Event::ReservedPropertyNames.URL_QUERY_PARAMS],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_BASE_URL => [Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_BASE_URL, Analytics::Event::ReservedPropertyNames.BASE_URL],
+
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL => [Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_URL, Analytics::Event::ReservedPropertyNames.REFERRER],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL_HOST => [Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_URL_HOST, Analytics::Event::ReservedPropertyNames.REFERRER_URL_HOST],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL_PATH => [Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_URL_PATH, Analytics::Event::ReservedPropertyNames.REFERRER_URL_PATH],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL_QUERY_PARAMS => [Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_URL_QUERY_PARAMS, Analytics::Event::ReservedPropertyNames.REFERRER_URL_QUERY_PARAMS],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_BASE_URL => [Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_BASE_URL, Analytics::Event::ReservedPropertyNames.REFERRER],
+
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CAMPAIGN => Analytics::Event::ReservedPropertyNames.SESSION_UTM_CAMPAIGN,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_SOURCE => Analytics::Event::ReservedPropertyNames.SESSION_UTM_SOURCE,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_MEDIUM => Analytics::Event::ReservedPropertyNames.SESSION_UTM_MEDIUM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CONTENT => Analytics::Event::ReservedPropertyNames.SESSION_UTM_CONTENT,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_TERM => Analytics::Event::ReservedPropertyNames.SESSION_UTM_TERM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_GCLID => Analytics::Event::ReservedPropertyNames.SESSION_GCLID,
+        }
+
+        def initialize(user_profile, parsed_event)
+          @user_profile = user_profile
+          @parsed_event = parsed_event
+        end
+
+        def augment_user_properties!
+          # if we have already set the `initial_referrer_url`, but not the `initial_landing_page_url` then we should not continue
+          # this is really only for users who have been identified earlier than ~04/22/2024
+          return user_profile if user_already_has_any_initial_properties_set?
+          AUTO_APPLY_USER_PROPERTIES_DICT.each do |user_property_key, event_property_name|
+            next if user_profile.metadata.key?(user_property_key)
+            user_property_value = nil
+            if event_property_name.is_a?(Array)
+              user_property_value = event_property_name.map{ |prop_name| parsed_event.properties[prop_name] }.compact.first
+            else
+              user_property_value = parsed_event.properties[event_property_name]
+            end
+            next if user_property_value.nil?
+            user_profile.metadata[user_property_key] = user_property_value
+          end
+          user_profile.save! if user_profile.changed?
+          user_profile
+        end
+
+        private
+
+        def user_already_has_any_initial_properties_set?
+          AUTO_APPLY_USER_PROPERTIES_DICT.keys.any?{ |property_name| user_profile.metadata.key?(property_name) }
+        end
+
+      end
+    end
+  end
+end

backend/app/lib/ingestion/event_preparers/swishjam_event_handler.rb

@@ -2,8 +2,16 @@ module Ingestion
   module EventPreparers
     class SwishjamEventHandler < Base
       def handle_and_return_prepared_events!
+        # order is important here, need to augment the event properties before attributing the user and organization
+        # the user and organization attributors rely on the augmented properties
+        Ingestion::EventPreparers::Helpers::EventPropertiesAugmentor.new(parsed_event).augment_properties!
+
         user_profile_for_event = Ingestion::EventPreparers::Helpers::SwishjamEventUserAttributor.new(parsed_event).get_user_profile_and_associate_to_device_if_necessary!
+        Ingestion::EventPreparers::Helpers::UserPropertiesAugmentor.new(user_profile_for_event, parsed_event).augment_user_properties!
+
         organization_for_event = Ingestion::EventPreparers::Helpers::SwishjamEventOrganizationAttributor.new(parsed_event, user_profile_for_event).organization_for_event
+        Ingestion::EventPreparers::Helpers::OrganizationPropertiesAugmentor.new(organization_for_event, parsed_event, user_profile_for_event).augment_organization_properties!
+
         parsed_event.set_user_profile(user_profile_for_event) if user_profile_for_event.present?
         parsed_event.set_organization_profile(organization_for_event) if organization_for_event.present?
         parsed_event

backend/app/lib/ingestion/parsed_event_from_ingestion.rb

@@ -81,14 +81,35 @@ def properties
       safe_json(event_json['properties'])
     end
 
+    def set_property(key, value)
+      if !event_json['properties'].is_a?(Hash)
+        event_json['properties'] = safe_json(event_json['properties'])
+      end
+      event_json['properties'][key] = value
+    end
+
     def user_properties
       safe_json(event_json['user_properties'])
     end
 
+    def set_user_property(key, value)
+      if !event_json['user_properties'].is_a?(Hash)
+        event_json['user_properties'] = safe_json(event_json['user_properties'])
+      end
+      event_json['user_properties'][key] = value
+    end
+
     def organization_properties
       safe_json(event_json['organization_properties'])
     end
 
+    def set_organization_property(key, value)
+      if !event_json['organization_properties'].is_a?(Hash)
+        event_json['organization_properties'] = safe_json(event_json['organization_properties'])
+      end
+      event_json['organization_properties'][key] = value
+    end
+
     def occurred_at
       @occurred_at ||= begin
         case event_json['occurred_at'].class.to_s

backend/app/models/analytics/event.rb

@@ -36,13 +36,29 @@ class << self
           OS
           OS_VERSION
           PAGE_REFERRER
+          PAGE_REFERRER_URL_PATH
+          PAGE_REFERRER_URL_HOST
+          PAGE_REFERRER_URL_QUERY_PARAMS
+          PAGE_REFERRER_BASE_URL
           PAGE_VIEW_IDENTIFIER
           REFERRER
+          REFERRER_URL_PATH
+          REFERRER_URL_HOST
+          REFERRER_URL_QUERY_PARAMS
+          REFERRER_BASE_URL
           SDK_VERSION
           SESSION_GCLID
           SESSION_IDENTIFIER
           SESSION_LANDING_PAGE_URL
+          SESSION_LANDING_PAGE_URL_HOST
+          SESSION_LANDING_PAGE_URL_PATH
+          SESSION_LANDING_PAGE_URL_QUERY_PARAMS
+          SESSION_LANDING_PAGE_BASE_URL
           SESSION_REFERRER_URL
+          SESSION_REFERRER_URL_HOST
+          SESSION_REFERRER_URL_PATH
+          SESSION_REFERRER_URL_QUERY_PARAMS
+          SESSION_REFERRER_BASE_URL
           SESSION_UTM_CAMPAIGN
           SESSION_UTM_CONTENT
           SESSION_UTM_MEDIUM
@@ -52,6 +68,10 @@ class << self
           SYSTEM_LANGUAGE
           TIMEZONE
           URL
+          URL_HOST
+          URL_PATH
+          URL_QUERY_PARAMS
+          BASE_URL
           USER_AGENT
           USER_ATTRIBUTES
           USER_DEVICE_IDENTIFIER