CSV Transaction Imports (#708)

Introduces a basic CSV import module for bulk-importing account transactions.

Changes include:

- User can load a CSV
- User can configure the column mappings for a CSV
- Imported CSV shows invalid cells
- User can clean up their data directly in the UI
- User can see a preview of the import rows and confirm import
- Layout refactor + Import nav stepper
- System test stability improvements

Gemfile

@@ -46,6 +46,7 @@ gem "octokit"
 gem "pagy"
 gem "rails-settings-cached"
 gem "tzinfo-data", platforms: %i[ windows jruby ]
+gem "csv"
 
 group :development, :test do
   gem "debug", platforms: %i[ mri windows ]

Gemfile.lock

@@ -156,6 +156,7 @@ GEM
       bigdecimal
       rexml
     crass (1.0.6)
+    csv (3.2.8)
     date (3.3.4)
     debug (1.9.2)
       irb (~> 1.10)
@@ -456,6 +457,7 @@ DEPENDENCIES
   brakeman
   capybara
   climate_control
+  csv
   debug
   dotenv-rails
   erb_lint

app/controllers/accounts_controller.rb

@@ -1,4 +1,6 @@
 class AccountsController < ApplicationController
+  layout "with_sidebar"
+
   include Filterable
   before_action :set_account, only: %i[ show update destroy sync ]
 
@@ -48,7 +50,7 @@ def update
         end
       end
     else
-      render "edit", status: :unprocessable_entity
+      render "show", status: :unprocessable_entity
     end
   end
 
@@ -84,11 +86,11 @@ def sync
 
   private
 
-  def set_account
-    @account = Current.family.accounts.find(params[:id])
-  end
+    def set_account
+      @account = Current.family.accounts.find(params[:id])
+    end
 
-  def account_params
-    params.require(:account).permit(:name, :accountable_type, :balance, :start_date, :currency, :subtype, :is_active)
-  end
+    def account_params
+      params.require(:account).permit(:name, :accountable_type, :balance, :start_date, :currency, :subtype, :is_active)
+    end
 end

app/controllers/imports_controller.rb

@@ -0,0 +1,102 @@
+require "ostruct"
+
+class ImportsController < ApplicationController
+  before_action :set_import, except: %i[ index new create ]
+
+  def index
+    @imports = Current.family.imports
+    render layout: "with_sidebar"
+  end
+
+  def new
+    @import = Import.new
+  end
+
+  def edit
+  end
+
+  def update
+    account = Current.family.accounts.find(params[:import][:account_id])
+
+    @import.update! account: account
+    redirect_to load_import_path(@import), notice: t(".import_updated")
+  end
+
+  def create
+    account = Current.family.accounts.find(params[:import][:account_id])
+    @import = Import.create!(account: account)
+
+    redirect_to load_import_path(@import), notice: t(".import_created")
+  end
+
+  def destroy
+    @import.destroy!
+    redirect_to imports_url, notice: t(".import_destroyed"), status: :see_other
+  end
+
+  def load
+  end
+
+  def load_csv
+    if @import.update(import_params)
+      redirect_to configure_import_path(@import), notice: t(".import_loaded")
+    else
+      flash.now[:error] = @import.errors.full_messages.to_sentence
+      render :load, status: :unprocessable_entity
+    end
+  end
+
+  def configure
+    unless @import.loaded?
+      redirect_to load_import_path(@import), alert: t(".invalid_csv")
+    end
+  end
+
+  def update_mappings
+    @import.update! import_params(@import.expected_fields.map(&:key))
+    redirect_to clean_import_path(@import), notice: t(".column_mappings_saved")
+  end
+
+  def clean
+    unless @import.loaded?
+      redirect_to load_import_path(@import), alert: t(".invalid_csv")
+    end
+  end
+
+  def update_csv
+    update_params = import_params[:csv_update]
+
+    @import.update_csv! \
+      row_idx: update_params[:row_idx],
+      col_idx: update_params[:col_idx],
+      value: update_params[:value]
+
+    render :clean
+  end
+
+  def confirm
+    unless @import.cleaned?
+      redirect_to clean_import_path(@import), alert: t(".invalid_data")
+    end
+  end
+
+  def publish
+    if @import.valid?
+      @import.publish_later
+      redirect_to imports_path, notice: t(".import_published")
+    else
+      flash.now[:error] = t(".invalid_data")
+      render :confirm, status: :unprocessable_entity
+    end
+  end
+
+  private
+
+    def set_import
+      @import = Current.family.imports.find(params[:id])
+    end
+
+    def import_params(permitted_mappings = nil)
+      params.require(:import).permit(:raw_csv_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])
+    end
+end

app/controllers/pages_controller.rb

@@ -1,4 +1,6 @@
 class PagesController < ApplicationController
+  layout "with_sidebar"
+
   include Filterable
 
   def dashboard

app/controllers/settings/billings_controller.rb

@@ -1,4 +1,4 @@
-class Settings::BillingsController < ApplicationController
+class Settings::BillingsController < SettingsController
   def edit
   end
 

app/controllers/settings/hostings_controller.rb

@@ -1,4 +1,4 @@
-class Settings::HostingsController < ApplicationController
+class Settings::HostingsController < SettingsController
   before_action :verify_hosting_mode
 
   def show

app/controllers/settings/notifications_controller.rb

@@ -1,4 +1,4 @@
-class Settings::NotificationsController < ApplicationController
+class Settings::NotificationsController < SettingsController
   def edit
   end
 

app/controllers/settings/preferences_controller.rb

@@ -1,4 +1,4 @@
-class Settings::PreferencesController < ApplicationController
+class Settings::PreferencesController < SettingsController
   def edit
   end
 
@@ -14,11 +14,12 @@ def update
       redirect_to settings_preferences_path, notice: t(".success")
     else
       redirect_to settings_preferences_path, notice: t(".success")
-      render :edit, status: :unprocessable_entity
+      render :show, status: :unprocessable_entity
     end
   end
 
   private
+
     def preference_params
       params.require(:user).permit(family_attributes: [ :id, :currency ])
     end

app/controllers/settings/profiles_controller.rb

@@ -1,4 +1,4 @@
-class Settings::ProfilesController < ApplicationController
+class Settings::ProfilesController < SettingsController
   def show
   end
 

app/controllers/settings/securities_controller.rb

@@ -1,4 +1,4 @@
-class Settings::SecuritiesController < ApplicationController
+class Settings::SecuritiesController < SettingsController
   def edit
   end
 

app/controllers/settings_controller.rb

@@ -0,0 +1,3 @@
+class SettingsController < ApplicationController
+  layout "with_sidebar"
+end

app/controllers/transactions/categories/deletions_controller.rb

@@ -1,4 +1,6 @@
 class Transactions::Categories::DeletionsController < ApplicationController
+  layout "with_sidebar"
+
   before_action :set_category
   before_action :set_replacement_category, only: :create
 

app/controllers/transactions/categories_controller.rb

@@ -1,4 +1,6 @@
 class Transactions::CategoriesController < ApplicationController
+  layout "with_sidebar"
+
   before_action :set_category, only: %i[ edit update ]
   before_action :set_transaction, only: :create
 

app/controllers/transactions/merchants_controller.rb

@@ -1,4 +1,6 @@
 class Transactions::MerchantsController < ApplicationController
+  layout "with_sidebar"
+
   before_action :set_merchant, only: %i[ edit update destroy ]
 
   def index

app/controllers/transactions/rules_controller.rb

@@ -1,4 +1,6 @@
 class Transactions::RulesController < ApplicationController
+  layout "with_sidebar"
+
   def index
   end
 end

app/controllers/transactions_controller.rb

@@ -1,4 +1,6 @@
 class TransactionsController < ApplicationController
+  layout "with_sidebar"
+
   before_action :set_transaction, only: %i[ show edit update destroy ]
 
   def index

app/helpers/application_helper.rb

@@ -51,6 +51,11 @@ def sidebar_link_to(name, path, options = {})
     end
   end
 
+  def return_to_path(params, fallback = root_path)
+    uri = URI.parse(params[:return_to] || fallback)
+    uri.relative? ? uri.path : root_path
+  end
+
   def trend_styles(trend)
     fallback = { bg_class: "bg-gray-500/5", text_class: "text-gray-500", symbol: "", icon: "minus" }
     return fallback if trend.nil? || trend.direction.flat?

app/helpers/imports_helper.rb

@@ -0,0 +1,19 @@
+module ImportsHelper
+  def table_corner_class(row_idx, col_idx, rows, cols)
+    return "rounded-tl-xl" if row_idx == 0 && col_idx == 0
+    return "rounded-tr-xl" if row_idx == 0 && col_idx == cols.size - 1
+    return "rounded-bl-xl" if row_idx == rows.size - 1 && col_idx == 0
+    return "rounded-br-xl" if row_idx == rows.size - 1 && col_idx == cols.size - 1
+    ""
+  end
+
+  def nav_steps(import = Import.new)
+    [
+      { name: "Select", complete: import.persisted?, path: import.persisted? ? edit_import_path(import) : new_import_path },
+      { name: "Import", complete: import.loaded?, path: import.persisted? ? load_import_path(import) : nil },
+      { name: "Setup", complete: import.configured?, path: import.persisted? ? configure_import_path(import) : nil },
+      { name: "Clean", complete: import.cleaned?, path: import.persisted? ? clean_import_path(import) : nil },
+      { name: "Confirm", complete: import.complete?, path: import.persisted? ? confirm_import_path(import) : nil }
+    ]
+  end
+end

app/jobs/import_job.rb

@@ -0,0 +1,7 @@
+class ImportJob < ApplicationJob
+  queue_as :default
+
+  def perform(import)
+    import.publish
+  end
+end

app/models/account.rb

@@ -9,6 +9,7 @@ class Account < ApplicationRecord
   has_many :balances, dependent: :destroy
   has_many :valuations, dependent: :destroy
   has_many :transactions, dependent: :destroy
+  has_many :imports, dependent: :destroy
 
   monetize :balance
 

app/models/family.rb

@@ -2,6 +2,7 @@ class Family < ApplicationRecord
   has_many :users, dependent: :destroy
   has_many :accounts, dependent: :destroy
   has_many :transactions, through: :accounts
+  has_many :imports, through: :accounts
   has_many :transaction_categories, dependent: :destroy, class_name: "Transaction::Category"
   has_many :transaction_merchants, dependent: :destroy, class_name: "Transaction::Merchant"