Merge pull request #200 from swishjam/instrumentation-improvements

instrumentation improvements

backend/app/controllers/api/v1/events_controller.rb

@@ -3,6 +3,22 @@ module V1
     class EventsController < BaseController
       include TimeseriesHelper
 
+      # global events list for workspace
+      def index
+        params[:data_source] ||= 'all'
+        limit = (params[:limit] || 100).to_i
+        events = ClickHouseQueries::Events::List.new(
+          public_keys_for_requested_data_source,
+          start_time: start_timestamp,
+          end_time: end_timestamp,
+          limit: limit,
+          columns: ['uuid'],
+          user_profile_id: params[:user_id],
+          workspace_id: current_workspace.id,
+        ).get
+        render json: events, status: :ok
+      end
+
       def unique
         params[:data_source] ||= 'all'
         limit = (params[:limit] || 200).to_i

backend/app/controllers/api/v1/workspaces_controller.rb

@@ -1,6 +1,19 @@
 module Api
   module V1
     class WorkspacesController < BaseController
+      def index
+        render json: current_user.workspaces, status: :ok
+      end
+
+      def create
+        workspace = current_user.workspaces.new(workspace_params)
+        if workspace.save
+          render json: { workspace: workspace }, status: :ok
+        else
+          render json: { error: workspace.errors.full_messages.join('. ') }, status: :unprocessable_entity
+        end
+      end
+
       def update
         workspace = current_workspace
         if workspace.update(workspace_params)
@@ -25,7 +38,7 @@ def update_current_workspace
       private
 
       def workspace_params
-        params.require(:workspace).permit(:name)
+        params.require(:workspace).permit(:name, :company_url)
       end
     end
   end

backend/app/lib/click_house_queries/events/list.rb

@@ -16,6 +16,7 @@ def initialize(
         property: nil, 
         user_profile_id: nil, 
         organization_profile_id: nil, 
+        columns: nil,
         limit: 10
       )
         @public_keys = public_keys.is_a?(Array) ? public_keys : [public_keys]
@@ -27,6 +28,7 @@ def initialize(
         @user_profile_id = user_profile_id
         @organization_profile_id = organization_profile_id
         @limit = limit
+        @additional_columns = columns || []
         validate!
       end
 
@@ -50,6 +52,8 @@ def de_duped_events_query
         needed_columns = ['name', 'occurred_at', 'properties']
         needed_columns << 'user_profile_id' if @user_profile_id
         needed_columns << 'organization_profile_id' if @organization_profile_id
+        needed_columns << 'user_properties' if @additional_columns.include?('user_properties')
+        needed_columns << 'organization_properties' if @additional_columns.include?('organization_properties')
         <<~SQL
           SELECT uuid, #{needed_columns.map{ |column| "argMax(#{column}, ingested_at) AS #{column}" }.join(', ')}
           FROM events AS e
@@ -69,11 +73,11 @@ def select_clause
             CAST(COUNT(DISTINCT e.uuid) AS INT) AS count
           SQL
         else
-          <<~SQL
-            e.name AS name, 
-            e.occurred_at AS occurred_at,
-            e.properties AS properties
-          SQL
+          columns = ['name', 'occurred_at', 'properties']
+          columns << 'user_properties' if @additional_columns.include?('user_properties')
+          columns << 'organization_properties' if @additional_columns.include?('organization_properties')
+          columns << 'uuid' if @additional_columns.include?('uuid')
+          columns.map{ |column| "#{column} AS #{column}" }.join(', ')
         end
       end
 

backend/app/lib/click_house_queries/events/stacked_bar_chart.rb

@@ -25,7 +25,7 @@ def initialize(
         @start_time, @end_time = rounded_timestamps(start_ts: start_time, end_ts: end_time, group_by: @group_by)
         @select_function_formatter = select_function_formatter
         @select_function_argument = select_function_argument
-        @property_alias = property_alias || property
+        @property_alias = property_alias || property.to_s.gsub('.', '_')
         @count_distinct_property = count_distinct_property
         @max_ranking_to_not_be_considered_other = max_ranking_to_not_be_considered_other
         @exclude_empty_property_values = exclude_empty_property_values
@@ -86,7 +86,11 @@ def select_clause_property_portion
         # ie: extractURLParameter(JSONExtractString(e.properties, 'url'), 'utm_source')
         select = ""
         select += "#{@select_function_formatter}(" if @select_function_formatter
-        select += "JSONExtractString(e.properties, '#{@property}')"
+        if @property.starts_with?('user.')
+          select += "JSONExtractString(e.user_properties, '#{@property.gsub('user.', '')}')"
+        else
+          select += "JSONExtractString(e.properties, '#{@property}')"
+        end
         select += ", '#{@select_function_argument}'" if @select_function_argument && @select_function_formatter
         select += ")" if @select_function_formatter
         select

backend/app/lib/ingestion/event_preparers/helpers/automatic_user_attribute_applier.rb

@@ -0,0 +1,33 @@
+module Ingestion
+  module EventPreparers
+    module Helpers
+      class AutomaticUserAttributeApplier
+        AUTO_APPLY_USER_PROPERTIES_DICT = {
+          # .URL and .REFERRER are legacy property names but falling back to these values for backwards compatibility
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL => [Analytics::Event::ReservedPropertyNames.SESSION_LANDING_PAGE_URL, Analytics::Event::ReservedPropertyNames.URL],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL => [Analytics::Event::ReservedPropertyNames.SESSION_REFERRER_URL, Analytics::Event::ReservedPropertyNames.REFERRER],
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CAMPAIGN => Analytics::Event::ReservedPropertyNames.SESSION_UTM_CAMPAIGN,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_SOURCE => Analytics::Event::ReservedPropertyNames.SESSION_UTM_SOURCE,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_MEDIUM => Analytics::Event::ReservedPropertyNames.SESSION_UTM_MEDIUM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CONTENT => Analytics::Event::ReservedPropertyNames.SESSION_UTM_CONTENT,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_TERM => Analytics::Event::ReservedPropertyNames.SESSION_UTM_TERM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_GCLID => Analytics::Event::ReservedPropertyNames.SESSION_GCLID,
+        }
+
+        def self.apply_user_attributes_if_necessary!(user_properties, event_properties)
+          AUTO_APPLY_USER_PROPERTIES_DICT.each do |user_property_key, event_property|
+            next if user_properties.key?(user_property_key)
+            user_property_value = nil
+            if event_property.is_a?(Array)
+              user_property_value = event_property.map{ |prop_name| event_properties[prop_name] }.compact.first
+            else
+              user_property_value = event_properties[event_property]
+            end
+            next if user_property_value.nil?
+            user_properties[user_property_key] = user_property_value 
+          end
+        end
+      end
+    end
+  end
+end

backend/app/lib/ingestion/event_preparers/helpers/swishjam_event_organization_attributor.rb

@@ -0,0 +1,71 @@
+module Ingestion
+  module EventPreparers
+    module Helpers
+      class SwishjamEventOrganizationAttributor < Ingestion::EventPreparers::Base
+        attr_reader :parsed_event, :user_profile_for_event
+
+        AUTO_APPLY_FROM_USER_PROPERTIES_DICT = [
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_LANDING_PAGE_URL,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_REFERRER_URL,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CAMPAIGN,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_SOURCE,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_MEDIUM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_CONTENT,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_UTM_TERM,
+          AnalyticsUserProfile::ReservedMetadataProperties.INITIAL_GCLID,
+        ]
+
+        def initialize(parsed_event, user_profile_for_event)
+          @parsed_event = parsed_event
+          @user_profile_for_event = user_profile_for_event
+        end
+
+        def organization_for_event
+          @organization_for_event ||= begin
+            organization_identifier = org_attr('id', 'identifier', 'organization_identifier', 'org_id', 'organization_id', 'orgIdentifier', 'organizationIdentifier', 'organization_identifier')
+            return if organization_identifier.blank?
+            org = workspace.analytics_organization_profiles.find_by(organization_unique_identifier: organization_identifier) || workspace.analytics_organization_profiles.new(organization_unique_identifier: organization_identifier)
+            maybe_org_name = org_attr('organization_name', 'name', 'organizationName', 'org_name', 'orgName')
+            org.name = maybe_org_name if maybe_org_name.present?
+            org.metadata ||= {}
+            org.metadata = org.metadata.merge(sanitized_provided_org_properties)
+            org.domain = org_attr('domain') if org_attr('domain').present?
+            if org.domain.nil? && user_profile_for_event&.email.present? && !GenericEmailDetector.is_generic_email?(user_profile_for_event.email)
+              domain_from_user_email = user_profile_for_event.email.split('@').last
+              org.domain = domain_from_user_email
+            end
+            if user_profile_for_event.present?
+              AUTO_APPLY_FROM_USER_PROPERTIES_DICT.each do |property_name|
+                org.metadata[property_name] ||= user_profile_for_event.metadata[property_name] if user_profile_for_event.metadata[property_name].present?
+              end
+            end
+            org.save! if org.changed?
+            if user_profile_for_event.present? && !org.analytics_organization_members.exists?(analytics_user_profile_id: user_profile_for_event.id)
+              org.analytics_organization_members.create!(analytics_user_profile_id: user_profile_for_event.id)
+            end
+            org
+          end
+        end
+
+        def provided_org_attributes 
+          # legacy instrumentation sends `organization` events with the organization properties in the root of the event properties, and all other events with the organization properties in the `organization_attributes` key
+          parsed_event.properties['organization'] || parsed_event.properties['organization_attributes'] || (parsed_event.name == 'organization' ? parsed_event.properties : {}) || {}
+        end
+
+        def org_attr(*keys)
+          value = nil
+          keys.each do |key|
+            value = provided_org_attributes[key] || provided_org_attributes.dig('metadata', key)
+            break if value.present?
+          end
+          value
+        end
+
+        def sanitized_provided_org_properties
+          (provided_org_attributes['metadata'] || provided_org_attributes).except('id', 'identifier', 'organization_identifier', 'name', 'organization_name', 'domain', 'org_id', 'organization_id', 'orgIdentifier', 'organizationIdentifier', 'organization_identifier', *Analytics::Event::ReservedPropertyNames.all.map(&:to_s))
+        end
+
+      end
+    end
+  end
+end