Master into rails-6-1-compability

.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: Test Suite
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  spec:
+    name: RSpec
+    runs-on: ubuntu-latest
+
+    container:
+      image: ruby:2.7.8
+      credentials:
+        username: ${{ secrets.ORG_DOCKERHUB_USERNAME }}
+        password: ${{ secrets.ORG_DOCKERHUB_TOKEN }}
+      env:
+        DB_HOST: db
+        DB_USERNAME: root
+
+    services: # versions here should match those used in docker-compose.yml
+      db:
+        image: mysql:8.0.34
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run RSpec
+        shell: script -q -e -c "bash {0}" # force colour output - see https://github.com/actions/runner/issues/241
+        run: |
+          set -euo pipefail
+
+          ./docker-entrypoint.sh
+
+          mkdir -p tmp
+          mkdir -p log
+
+          bin/rspec
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: "test.log"
+          path: log/test.log

.github/workflows/codeql.yml

@@ -42,11 +42,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -60,7 +60,7 @@ jobs:
     # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -73,6 +73,6 @@ jobs:
     #     ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/github-security-jira.yml

@@ -0,0 +1,22 @@
+name: GitHub Security Alerts for Jira
+
+on:
+  schedule:
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+
+jobs:
+  syncSecurityAlerts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Sync security alerts to Jira issues"
+        uses: reload/github-security-jira@v1.5.0
+        env:
+          GH_SECURITY_TOKEN: ${{ secrets.ORG_GITHUBSECURITYTOKEN }}
+          JIRA_TOKEN: ${{ secrets.ORG_JIRA_TOKEN }}
+          JIRA_HOST: https://360insights.atlassian.net/
+          JIRA_USER: ${{secrets.ORG_JIRA_USERNAME}}
+          JIRA_PROJECT: SB
+          JIRA_ISSUE_TYPE: Security Code Analysis
+          JIRA_ISSUE_LABELS: MYREWARDS_SECURITY_ALERT
+          JIRA_RESTRICTED_COMMENT_ROLE: Developer

app/controllers/concerns/csv2db/controller_helpers.rb

@@ -36,7 +36,7 @@ def enqueue_csv_import_and_redirect(klass, options = {}, &block)
   end
 
   def enqueue_csv_import(klass, options = {})
-    permitted_params = options.fetch(:params) do 
+    permitted_params = options.fetch(:params) do
       params.require(klass.model_name.param_key).permit(
         :file,
         *options[:extra_params]

app/models/concerns/csv2db/active_storage_adapter.rb

@@ -0,0 +1,67 @@
+module Csv2db::ActiveStorageAdapter
+  require 'active_support/all'
+
+  extend ActiveSupport::Concern
+  FILE_TYPE = 'text/csv'.freeze
+  LINK_MAX_EXPIRY = 7.days.to_s.freeze
+
+  included do
+    has_one_attached Csv2db.config.file_attachment_name
+
+    validate :check_file_extension
+
+    alias_method :file_attachment, Csv2db.config.file_attachment_name
+  end
+
+  def file=(file)
+    # Override Dragonfly setter method
+    return unless file.present?
+
+    filename = file.original_filename
+
+    file_attachment.attach(
+      io: File.open(file),
+      filename: filename,
+      content_type: file.content_type
+    )
+
+    self.file_name = filename
+  end
+
+  def download_link
+    return unless file_attachment.present? && file_attachment.attached?
+
+    set_current_host
+
+    file_attachment.service_url(
+      expires_in: LINK_MAX_EXPIRY.to_i,
+      disposition: 'attachment'
+    )
+  end
+
+  private
+
+  def set_current_host
+    return unless %i[test local].include?(Rails.application.config.active_storage.service)
+
+    ActiveStorage::Current.host = Csv2db.config.local_storage_host
+  end
+
+  def check_file_extension
+    # very basic check of file extension
+    errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file_attachment.blob.content_type == FILE_TYPE
+  end
+
+  def file_data
+    return @file_data if @file_data.present?
+
+    file_attachment.blob.open do |blob|
+      @file_data = str_to_utf8(blob.read)
+    end
+
+    byte_order_mark = Csv2db::Import::BYTE_ORDER_MARK
+    @file_data.sub!(byte_order_mark, '') if @file_data.starts_with?(byte_order_mark)
+
+    @file_data
+  end
+end

app/models/concerns/csv2db/dragonfly_adapter.rb

@@ -0,0 +1,31 @@
+module Csv2db::DragonflyAdapter
+  extend ActiveSupport::Concern
+  require 'dragonfly'
+
+  included do
+    extend Dragonfly::Model
+
+    dragonfly_accessor :file
+
+    validates :file, presence: true
+    validate :check_file_extension
+  end
+
+  def download_link
+    file.url
+  end
+
+  private
+
+  def check_file_extension
+    # very basic check of file extension
+    errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file.ext == 'csv'
+  end
+
+  def file_data
+    file_data = str_to_utf8(file.data)
+    byte_order_mark = Csv2db::Import::BYTE_ORDER_MARK
+    file_data.sub!(byte_order_mark, '') if file_data.starts_with?(byte_order_mark)
+    file_data
+  end
+end

app/models/concerns/csv2db/import.rb

@@ -27,13 +27,9 @@ def around_process(*args, &block)
     end
 
     included do
-      extend Dragonfly::Model
+      include Module.const_get("Csv2db::#{Csv2db.config.storage_adapter.camelize.constantize}Adapter")
 
-      validates :file, presence: true
       validate :required_params_are_present
-      validate :check_file_extension
-
-      dragonfly_accessor :file
 
       after_initialize :set_default_values, :set_required_params
 
@@ -128,10 +124,14 @@ def method_missing(method, *args, &block)
       end
     end
 
+    def respond_to_missing?(method, include_private = false)
+      method.to_s.start_with?('param_') || super
+    end
+
     private
 
     def check_file_contains_data
-      error(I18n.t('shared.file_processor.insufficient_rows')) unless file.data.present? && csv.count > 0
+      error(I18n.t('shared.file_processor.insufficient_rows')) unless csv.headers.present? && csv.count.positive?
       stop if errors?
     end
 
@@ -165,12 +165,6 @@ def csv
       @csv ||= CSV.parse(file_data, headers: true)
     end
 
-    def file_data
-      file_data = str_to_utf_8(file.data)
-      file_data.sub!(BYTE_ORDER_MARK, '') if file_data.starts_with?(BYTE_ORDER_MARK)
-      file_data
-    end
-
     def required_params_are_present
       return if @required_params.empty?
 
@@ -183,7 +177,7 @@ def required_params_are_present
     end
 
     def log(message, level = :info)
-      log_messages << { message: str_to_utf_8(message), level: level, time: Time.now }
+      log_messages << { message: str_to_utf8(message), level: level, time: Time.now }
     end
 
     def error(message)
@@ -203,16 +197,16 @@ def set_default_values
       self.status ||= Status::PENDING
     end
 
-    def str_to_utf_8(str)
-      CharlockHolmes::Converter.convert(str, str.detect_encoding[:encoding], 'UTF-8')
+    def str_to_utf8(str)
+      CharlockHolmes::Converter.convert(str, str_encoding(str), 'UTF-8')
     end
 
-    def set_required_params
-      @required_params = []
+    def str_encoding(str)
+      str.detect_encoding[:encoding]
     end
 
-    def check_file_extension
-      errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file.ext == 'csv'
+    def set_required_params
+      @required_params = []
     end
   end
 end

app/views/csv2db/_csv_import_table.html.haml

@@ -15,7 +15,7 @@
     %tr
       %td= import.id
       %td= import.created_at
-      %td= link_to import.file.name, import.file.url, target: '_blank'
+      %td= link_to(import.file_name, import.download_link)
       %td
         .badge{ class: "badge-upload-#{import.status}" }= import.status
       %td

csv2db.gemspec

@@ -33,17 +33,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activerecord', '>= 4.2', '< 6.2'
-  spec.add_dependency 'activesupport', '>= 4.2', '< 6.2'
+  spec.add_dependency 'activerecord', '>= 4.2', '< 7.1'
+  spec.add_dependency 'activesupport', '>= 4.2', '< 7.1'
   spec.add_dependency 'charlock_holmes', '~> 0.7.3'
   spec.add_dependency 'dragonfly', '~> 1'
-  spec.add_dependency 'railties', '>= 4.2', '< 6.2'
+  spec.add_dependency 'railties', '>= 4.2', '< 7.1'
   spec.add_dependency 'sidekiq', '>= 3'
 
   spec.add_development_dependency 'bundler', '>= 2.2.18', '< 3'
   spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'mysql2', '~> 0.5.3'
   spec.add_development_dependency 'rake', '~> 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4'
 end

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   app:
-    image: ruby:${RUBY_VERSION:-2.6.10}
+    image: ruby:${RUBY_VERSION:-2.7.8}
     working_dir: /app
     command: >
       bash -eu -c '
@@ -18,7 +18,7 @@ services:
       - .:/app:cached
 
   db:
-    image: mysql:5.7
+    image: mysql:8.0.34
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=yes
     expose: