Bulk (CSV) import for users (coopdevs#139)

* add import participants from cvs

* Refactor CSV importer and fix style offenses

* refactoring

* fix lint

* Return of a removed controller

* fix job

* add rspec for ParticipantsCsvImporter

* add rspec for ImportParticipantsCsvJob

* add spec for ImportParticipantsMailer

* fix tests

* fix tests

* add system spec

* fix tests

* perform_later

* checks (coopdevs#135)

* add check callout

* add specs

* add importer

* fix importer

* add settings check tests

* fix flaky

* add develop to actions

* fix already voted delegation (coopdevs#137)

* fix js & css ui bugs

* remove unneded js

* remove log

* user id relation (coopdevs#140)

* add user relation

* cops

* add spec cases

* change import existing participants, add new info to email

* add spec

* fix ponderation (float)

* add info to email

* add attached file with import errors

* fix missing template, change sending an attachment without rows

* change file name

* change file path

* add instructions to email

* change import when ponderation type is a string that does not exist

* refactoring

* refactoring

* fix tests

* fix lint

* fix locale

* add system tests

* remove extra code

* change tests

* remove extra code

* changes after review

* refactoring

* change controller name in spec

* fix locale

* remove transaction, add spec when phone alreaddy exists

* fix test

* add test

* change test

* change test

---------

Co-authored-by: Ivan Vergés <ivan@pokecode.net>

Gemfile.lock

@@ -500,6 +500,8 @@ GEM
     multi_xml (0.6.0)
     mustache (1.1.1)
     nio4r (2.5.8)
+    nokogiri (1.13.9-x86_64-darwin)
+      racc (~> 1.4)
     nokogiri (1.13.9-x86_64-linux)
       racc (~> 1.4)
     nori (2.6.0)
@@ -797,6 +799,7 @@ GEM
     zeitwerk (2.6.4)
 
 PLATFORMS
+  x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES

app/controllers/decidim/action_delegator/admin/manage_participants_controller.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Decidim
+  module ActionDelegator
+    module Admin
+      class ManageParticipantsController < ActionDelegator::Admin::ApplicationController
+        include NeedsPermission
+        include Decidim::Paginable
+
+        helper ::Decidim::ActionDelegator::Admin::DelegationHelper
+        helper_method :organization_settings, :current_setting
+
+        layout "decidim/admin/users"
+
+        def new
+          enforce_permission_to :create, :participant
+
+          @errors = []
+        end
+
+        def create
+          enforce_permission_to :create, :participant
+
+          @csv_file = params[:csv_file]
+          redirect_to seting_manage_participants_path && return if @csv_file.blank?
+
+          @import_summary = Decidim::ActionDelegator::Admin::ImportParticipantsCsvJob.perform_later(
+            current_user,
+            @csv_file.read.force_encoding("utf-8").encode("utf-8"),
+            current_setting
+          )
+
+          flash[:notice] = t(".success")
+
+          redirect_to decidim_admin_action_delegator.setting_participants_path(current_setting)
+        end
+
+        private
+
+        def current_setting
+          @current_setting ||= organization_settings.find_by(id: params[:setting_id])
+        end
+
+        def organization_settings
+          Decidim::ActionDelegator::OrganizationSettings.new(current_organization).query
+        end
+      end
+    end
+  end
+end

app/forms/decidim/action_delegator/admin/participant_form.rb

@@ -9,6 +9,7 @@ class ParticipantForm < Form
         attribute :email, String
         attribute :phone, String
         attribute :decidim_action_delegator_ponderation_id, Integer
+        attribute :weight, String
 
         validates :email, presence: true, if: ->(form) { form.authorization_method.in? %w(email both) }
         validates :phone, presence: true, if: ->(form) { form.authorization_method.in? %w(phone both) }

app/jobs/decidim/action_delegator/admin/import_participants_csv_job.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Decidim
+  module ActionDelegator
+    module Admin
+      class ImportParticipantsCsvJob < ApplicationJob
+        queue_as :exports
+
+        def perform(current_user, csv_file, current_setting)
+          importer = Decidim::ActionDelegator::ParticipantsCsvImporter.new(csv_file, current_user, current_setting)
+          import_summary = importer.import!
+
+          Decidim::ActionDelegator::ImportParticipantsMailer
+            .import(current_user, import_summary, import_summary[:details_csv_path])
+            .deliver_later
+
+          import_summary
+        end
+      end
+    end
+  end
+end

app/mailers/decidim/action_delegator/import_participants_mailer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Decidim
+  module ActionDelegator
+    # This mailer sends a notification email containing the result of importing a
+    # CSV of results.
+    class ImportParticipantsMailer < Decidim::ApplicationMailer
+      # Public: Sends a notification email with the result of a CSV import
+      # of results.
+      #
+      # user   - The user to be notified.
+      # errors - The list of errors generated by the import
+      #
+      # Returns nothing.
+      def import(user, import_summary, csv_file_path)
+        @user = user
+        @organization = user.organization
+        @import_summary = import_summary
+        @csv_file_path = csv_file_path
+
+        @csv_file_path = "" if @import_summary[:total_rows] == @import_summary[:imported_rows]
+
+        attachments["details.csv"] = File.read(@csv_file_path) if @csv_file_path.present? && File.exist?(@csv_file_path)
+
+        with_user(user) do
+          mail(to: "#{user.name} <#{user.email}>", subject: I18n.t("decidim.action_delegator.import_participants_mailer.import.subject"))
+        end
+      end
+    end
+  end
+end

app/services/decidim/action_delegator/participants_csv_importer.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Decidim
+  module ActionDelegator
+    class ParticipantsCsvImporter
+      include Decidim::FormFactory
+
+      def initialize(csv_file, current_user, current_setting)
+        @csv_file = csv_file
+        @current_user = current_user
+        @current_setting = current_setting
+      end
+
+      def import!
+        import_summary = {
+          total_rows: 0,
+          imported_rows: 0,
+          error_rows: [],
+          skipped_rows: [],
+          details_csv_path: nil
+        }
+
+        details_csv_file = File.join(File.dirname(@csv_file), "details.csv")
+
+        i = 1
+        csv = CSV.new(@csv_file, headers: true, col_sep: ",")
+
+        CSV.open(details_csv_file, "wb") do |details_csv|
+          headers(csv, details_csv)
+
+          csv.rewind
+
+          while (row = csv.shift).present?
+            i += 1
+
+            params = extract_params(row)
+            weight = ponderation_value(row["weight"].strip) if row["weight"].present?
+
+            @form = form(Decidim::ActionDelegator::Admin::ParticipantForm).from_params(params, setting: @current_setting)
+
+            next if row&.empty?
+
+            if participant_exists?(@form)
+              mismatch_fields = mismatched_fields(@form)
+              info_message = generate_info_message(mismatch_fields)
+              handle_skipped_row(row, details_csv, import_summary, i, info_message)
+
+              next
+            end
+
+            if phone_exists?(@form)
+              reason = I18n.t("phone_exists", scope: "decidim.action_delegator.participants_csv_importer.import")
+              handle_skipped_row(row, details_csv, import_summary, i, reason)
+
+              next
+            end
+
+            if weight.present? && find_ponderation(weight).nil?
+              reason = I18n.t("ponderation_not_found", scope: "decidim.action_delegator.participants_csv_importer.import")
+              handle_skipped_row(row, details_csv, import_summary, i, reason)
+
+              next
+            end
+
+            handle_form_validity(row, details_csv, import_summary, i)
+          end
+        end
+        import_summary[:total_rows] = i - 1
+        import_summary[:details_csv_path] = details_csv_file
+
+        import_summary
+      end
+
+      private
+
+      def authorization_method
+        @current_setting.authorization_method
+      end
+
+      def extract_params(row)
+        email, phone, weight = extract_contact_details(row, authorization_method)
+        weight = ponderation_value(row["weight"].strip) if row["weight"].present?
+
+        params = {
+          email: email,
+          phone: phone,
+          weight: weight,
+          decidim_action_delegator_ponderation_id: find_ponderation(weight)&.id
+        }
+
+        @form = form(Decidim::ActionDelegator::Admin::ParticipantForm).from_params(params, setting: @current_setting)
+
+        params
+      end
+
+      def extract_contact_details(row, authorization_method)
+        email = row["email"].to_s.strip.downcase
+        phone = row["phone"].to_s.strip
+
+        email = nil if %w(email both).include?(authorization_method.to_s) && invalid_email?(email)
+
+        phone = nil if %w(phone both).include?(authorization_method.to_s) && invalid_phone?(phone)
+
+        [email, phone]
+      end
+
+      def invalid_email?(email)
+        email.blank? || !email.match?(::Devise.email_regexp)
+      end
+
+      def invalid_phone?(phone)
+        phone.blank? || !phone.gsub(/[^+0-9]/, "").match?(Decidim::ActionDelegator.phone_regex)
+      end
+
+      def process_participant(form)
+        if find_ponderation(form.weight).present?
+          ponderation = find_ponderation(form.weight)
+          form.decidim_action_delegator_ponderation_id = ponderation.id
+        end
+
+        create_new_participant(form)
+      end
+
+      def participant_exists?(form)
+        check_exists?(:email, form)
+      end
+
+      def phone_exists?(form)
+        check_exists?(:phone, form)
+      end
+
+      def check_exists?(field, form)
+        @participant = Decidim::ActionDelegator::Participant.find_by(field => form.send(field), setting: @current_setting) if form.send(field).present?
+        @participant.present?
+      end
+
+      def handle_skipped_row(row, details_csv, import_summary, row_number, reason)
+        import_summary[:skipped_rows] << { row_number: row_number - 1 }
+        row["reason"] = reason
+        details_csv << row
+      end
+
+      def handle_import_error(row, details_csv, import_summary, row_number, error_messages)
+        import_summary[:error_rows] << { row_number: row_number - 1, error_messages: error_messages }
+        row["reason"] = error_messages
+        details_csv << row
+      end
+
+      def handle_form_validity(row, details_csv, import_summary, row_number)
+        if @form.valid?
+          process_participant(@form)
+          import_summary[:imported_rows] += 1
+        else
+          handle_import_error(row, details_csv, import_summary, row_number, @form.errors.full_messages.join(", "))
+        end
+      end
+
+      def mismatched_fields(form)
+        mismatch_fields = []
+        mismatch_fields << I18n.t("decidim.action_delegator.participants_csv_importer.import.field_name.phone") if form.phone != @participant.phone
+
+        if form.decidim_action_delegator_ponderation_id != @participant.decidim_action_delegator_ponderation_id
+          mismatch_fields << I18n.t("decidim.action_delegator.participants_csv_importer.import.field_name.weight")
+        end
+
+        mismatch_fields.empty? ? nil : mismatch_fields.join(", ")
+      end
+
+      def generate_info_message(mismatch_fields)
+        with_mismatched_fields = mismatch_fields.present? ? I18n.t("decidim.action_delegator.participants_csv_importer.import.with_mismatched_fields", fields: mismatch_fields) : ""
+        I18n.t("decidim.action_delegator.participants_csv_importer.import.skip_import_info", with_mismatched_fields: with_mismatched_fields)
+      end
+
+      def create_new_participant(form)
+        Decidim::ActionDelegator::Admin::CreateParticipant.call(form)
+      end
+
+      def find_ponderation(weight)
+        case weight
+        when String
+          @current_setting.ponderations.find_by(name: weight).presence
+        when Numeric
+          ponderation = @current_setting.ponderations.find_by(weight: weight)
+          ponderation.presence || @current_setting.ponderations.create(name: "weight-#{weight}", weight: weight)
+        end
+      end
+
+      def ponderation_value(value)
+        Float(value)
+      rescue StandardError
+        value
+      end
+
+      def headers(csv, details_csv)
+        headers = csv.first.headers
+        headers << I18n.t("decidim.action_delegator.participants_csv_importer.import.error_field")
+        details_csv << headers
+      end
+    end
+  end
+end

app/views/decidim/action_delegator/admin/manage_participants/new.html.erb

@@ -0,0 +1,33 @@
+<%= form_tag(setting_manage_participants_path, multipart: true, class: "form new_import") do %>
+  <%= hidden_field_tag :setting_id, current_setting.id %>
+  <div class="card">
+    <div class="card-divider">
+      <h2 class="card-title">
+        <%= t(".title") %>
+      </h2>
+    </div>
+  </div>
+  <p><%= t(".upload_instructions") %></p>
+  <p><%= t(".required_fields", authorization_method: t(".authorization_method.#{current_setting.authorization_method}")) %></p>
+  <p><%= t(".title_example") %></p>
+  <pre class="code-block">
+    email,phone,weight
+    foo@example.org,6660000,1.5
+    bar@example.org,6660001,1
+    baz@example.org,6660002,2
+    ...</pre>
+  <p><%= t(".describe") %></p>
+  <pre class="code-block">
+    email,phone,weight
+    foo@example.org,6660000,producer
+    bar@example.org,6660001,consumer
+    baz@example.org,6660002,associate
+    ...</pre>
+  <div class="row column">
+    <%= file_field_tag :csv_file %>
+  </div>
+
+  <div class="button--double form-general-submit">
+    <%= submit_tag t(".import"), class: "button" %>
+  </div>
+<% end %>