diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 8775da4..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,32 +0,0 @@ -steps: - - label: ':rspec:' - key: spec - plugins: - - docker-compose#v3.9.0: - run: app - env: - # used for names of JUnit XML files - - BUILDKITE_JOB_ID - timeout_in_minutes: 5 - commands: - - './docker-entrypoint.sh' - - '.buildkite/test.sh' - env: - BYEBUG: '0' - DEBUGGER: '0' - artifact_paths: - - log/*.log - - tmp/rspec-junit-*.xml - - tmp/rspec/*.txt - - tmp/capybara/* - - tmp/screenshots/* - - - wait: ~ - continue_on_failure: true - - - label: ':junit:' - plugins: - - junit-annotate#v1.9.0: - artifacts: tmp/rspec-junit-*.xml - job-uuid-file-pattern: rspec-junit-([^.]+)\.xml - failure-format: file diff --git a/.buildkite/test.sh b/.buildkite/test.sh deleted file mode 100755 index 08e258c..0000000 --- a/.buildkite/test.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env sh -set -eu - -echo "~~~ Waiting for MySQL" -retries=5 - -until ruby -rsocket -e 'Socket.tcp(ENV["DB_HOST"], 3306).close' 2>/dev/null; do - retries="$(("$retries" - 1))" - - if [ "$retries" -eq 0 ]; then - echo "Failed to reach MySQL" >&2 - exit 1 - fi - - sleep 5 - echo "Waiting for MySQL ($retries retries left)" -done - -echo "+++ :rspec: Running specs" -mkdir -p tmp -mkdir -p log - -bin/rspec \ - --format RspecJunitFormatter \ - --out "tmp/rspec-junit-$BUILDKITE_JOB_ID.xml" \ - --format documentation diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5b4cd1c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: Test Suite + +on: + push: + workflow_dispatch: + +jobs: + spec: + name: RSpec + runs-on: ubuntu-latest + + container: + image: ruby:2.7.8 + credentials: + username: ${{ secrets.ORG_DOCKERHUB_USERNAME }} + password: ${{ secrets.ORG_DOCKERHUB_TOKEN }} + env: + DB_HOST: db + DB_USERNAME: root + + services: # versions here should match those used in docker-compose.yml + db: + image: mysql:8.0.34 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=3 + + steps: + - uses: actions/checkout@v4 + + - name: Run RSpec + shell: script -q -e -c "bash {0}" # force colour output - see https://github.com/actions/runner/issues/241 + run: | + set -euo pipefail + + ./docker-entrypoint.sh + + mkdir -p tmp + mkdir -p log + + bin/rspec + + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: "test.log" + path: log/test.log diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7756a87..f52af9a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -42,11 +42,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -60,7 +60,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # â„šī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -73,6 +73,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/github-security-jira.yml b/.github/workflows/github-security-jira.yml new file mode 100644 index 0000000..c0e05a8 --- /dev/null +++ b/.github/workflows/github-security-jira.yml @@ -0,0 +1,22 @@ +name: GitHub Security Alerts for Jira + +on: + schedule: + - cron: '0 4 * * *' + workflow_dispatch: + +jobs: + syncSecurityAlerts: + runs-on: ubuntu-latest + steps: + - name: "Sync security alerts to Jira issues" + uses: reload/github-security-jira@v1.5.0 + env: + GH_SECURITY_TOKEN: ${{ secrets.ORG_GITHUBSECURITYTOKEN }} + JIRA_TOKEN: ${{ secrets.ORG_JIRA_TOKEN }} + JIRA_HOST: https://360insights.atlassian.net/ + JIRA_USER: ${{secrets.ORG_JIRA_USERNAME}} + JIRA_PROJECT: SB + JIRA_ISSUE_TYPE: Security Code Analysis + JIRA_ISSUE_LABELS: MYREWARDS_SECURITY_ALERT + JIRA_RESTRICTED_COMMENT_ROLE: Developer diff --git a/app/controllers/concerns/csv2db/controller_helpers.rb b/app/controllers/concerns/csv2db/controller_helpers.rb index 4c9b07a..ad34ebb 100644 --- a/app/controllers/concerns/csv2db/controller_helpers.rb +++ b/app/controllers/concerns/csv2db/controller_helpers.rb @@ -36,7 +36,7 @@ def enqueue_csv_import_and_redirect(klass, options = {}, &block) end def enqueue_csv_import(klass, options = {}) - permitted_params = options.fetch(:params) do + permitted_params = options.fetch(:params) do params.require(klass.model_name.param_key).permit( :file, *options[:extra_params] diff --git a/app/models/concerns/csv2db/active_storage_adapter.rb b/app/models/concerns/csv2db/active_storage_adapter.rb new file mode 100644 index 0000000..993ccfe --- /dev/null +++ b/app/models/concerns/csv2db/active_storage_adapter.rb @@ -0,0 +1,67 @@ +module Csv2db::ActiveStorageAdapter + require 'active_support/all' + + extend ActiveSupport::Concern + FILE_TYPE = 'text/csv'.freeze + LINK_MAX_EXPIRY = 7.days.to_s.freeze + + included do + has_one_attached Csv2db.config.file_attachment_name + + validate :check_file_extension + + alias_method :file_attachment, Csv2db.config.file_attachment_name + end + + def file=(file) + # Override Dragonfly setter method + return unless file.present? + + filename = file.original_filename + + file_attachment.attach( + io: File.open(file), + filename: filename, + content_type: file.content_type + ) + + self.file_name = filename + end + + def download_link + return unless file_attachment.present? && file_attachment.attached? + + set_current_host + + file_attachment.service_url( + expires_in: LINK_MAX_EXPIRY.to_i, + disposition: 'attachment' + ) + end + + private + + def set_current_host + return unless %i[test local].include?(Rails.application.config.active_storage.service) + + ActiveStorage::Current.host = Csv2db.config.local_storage_host + end + + def check_file_extension + # very basic check of file extension + errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file_attachment.blob.content_type == FILE_TYPE + end + + def file_data + return @file_data if @file_data.present? + + file_attachment.blob.open do |blob| + @file_data = str_to_utf8(blob.read) + end + + byte_order_mark = Csv2db::Import::BYTE_ORDER_MARK + @file_data.sub!(byte_order_mark, '') if @file_data.starts_with?(byte_order_mark) + + @file_data + end +end diff --git a/app/models/concerns/csv2db/dragonfly_adapter.rb b/app/models/concerns/csv2db/dragonfly_adapter.rb new file mode 100644 index 0000000..f17fc71 --- /dev/null +++ b/app/models/concerns/csv2db/dragonfly_adapter.rb @@ -0,0 +1,31 @@ +module Csv2db::DragonflyAdapter + extend ActiveSupport::Concern + require 'dragonfly' + + included do + extend Dragonfly::Model + + dragonfly_accessor :file + + validates :file, presence: true + validate :check_file_extension + end + + def download_link + file.url + end + + private + + def check_file_extension + # very basic check of file extension + errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file.ext == 'csv' + end + + def file_data + file_data = str_to_utf8(file.data) + byte_order_mark = Csv2db::Import::BYTE_ORDER_MARK + file_data.sub!(byte_order_mark, '') if file_data.starts_with?(byte_order_mark) + file_data + end +end diff --git a/app/models/concerns/csv2db/import.rb b/app/models/concerns/csv2db/import.rb index e831e3f..5d4f26a 100644 --- a/app/models/concerns/csv2db/import.rb +++ b/app/models/concerns/csv2db/import.rb @@ -27,13 +27,9 @@ def around_process(*args, &block) end included do - extend Dragonfly::Model + include Module.const_get("Csv2db::#{Csv2db.config.storage_adapter.camelize.constantize}Adapter") - validates :file, presence: true validate :required_params_are_present - validate :check_file_extension - - dragonfly_accessor :file after_initialize :set_default_values, :set_required_params @@ -128,10 +124,14 @@ def method_missing(method, *args, &block) end end + def respond_to_missing?(method, include_private = false) + method.to_s.start_with?('param_') || super + end + private def check_file_contains_data - error(I18n.t('shared.file_processor.insufficient_rows')) unless file.data.present? && csv.count > 0 + error(I18n.t('shared.file_processor.insufficient_rows')) unless csv.headers.present? && csv.count.positive? stop if errors? end @@ -165,12 +165,6 @@ def csv @csv ||= CSV.parse(file_data, headers: true) end - def file_data - file_data = str_to_utf_8(file.data) - file_data.sub!(BYTE_ORDER_MARK, '') if file_data.starts_with?(BYTE_ORDER_MARK) - file_data - end - def required_params_are_present return if @required_params.empty? @@ -183,7 +177,7 @@ def required_params_are_present end def log(message, level = :info) - log_messages << { message: str_to_utf_8(message), level: level, time: Time.now } + log_messages << { message: str_to_utf8(message), level: level, time: Time.now } end def error(message) @@ -203,16 +197,16 @@ def set_default_values self.status ||= Status::PENDING end - def str_to_utf_8(str) - CharlockHolmes::Converter.convert(str, str.detect_encoding[:encoding], 'UTF-8') + def str_to_utf8(str) + CharlockHolmes::Converter.convert(str, str_encoding(str), 'UTF-8') end - def set_required_params - @required_params = [] + def str_encoding(str) + str.detect_encoding[:encoding] end - def check_file_extension - errors.add(:file, I18n.t('shared.file_processor.incorrect_file_type')) unless file.ext == 'csv' + def set_required_params + @required_params = [] end end end diff --git a/app/views/csv2db/_csv_import_table.html.haml b/app/views/csv2db/_csv_import_table.html.haml index df87f42..9db07f4 100644 --- a/app/views/csv2db/_csv_import_table.html.haml +++ b/app/views/csv2db/_csv_import_table.html.haml @@ -15,7 +15,7 @@ %tr %td= import.id %td= import.created_at - %td= link_to import.file.name, import.file.url, target: '_blank' + %td= link_to(import.file_name, import.download_link) %td .badge{ class: "badge-upload-#{import.status}" }= import.status %td diff --git a/csv2db.gemspec b/csv2db.gemspec index a2df913..087c5ce 100644 --- a/csv2db.gemspec +++ b/csv2db.gemspec @@ -33,11 +33,11 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activerecord', '>= 4.2', '< 6.2' - spec.add_dependency 'activesupport', '>= 4.2', '< 6.2' + spec.add_dependency 'activerecord', '>= 4.2', '< 7.1' + spec.add_dependency 'activesupport', '>= 4.2', '< 7.1' spec.add_dependency 'charlock_holmes', '~> 0.7.3' spec.add_dependency 'dragonfly', '~> 1' - spec.add_dependency 'railties', '>= 4.2', '< 6.2' + spec.add_dependency 'railties', '>= 4.2', '< 7.1' spec.add_dependency 'sidekiq', '>= 3' spec.add_development_dependency 'bundler', '>= 2.2.18', '< 3' @@ -45,5 +45,4 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'mysql2', '~> 0.5.3' spec.add_development_dependency 'rake', '~> 12.3.3' spec.add_development_dependency 'rspec', '~> 3.0' - spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4' end diff --git a/docker-compose.yml b/docker-compose.yml index 2c039bc..f3a1ed4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: app: - image: ruby:${RUBY_VERSION:-2.6.10} + image: ruby:${RUBY_VERSION:-2.7.8} working_dir: /app command: > bash -eu -c ' @@ -18,7 +18,7 @@ services: - .:/app:cached db: - image: mysql:5.7 + image: mysql:8.0.34 environment: - MYSQL_ALLOW_EMPTY_PASSWORD=yes expose: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e6f0441..a430d43 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,11 +1,27 @@ #!/usr/bin/env sh set -eu -echo "~~~ update RubyGems and Bundler" +# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#grouping-log-lines +group() { + echo "::group::${1}" +} + +endgroup() { + echo "::endgroup::" +} + +group "update RubyGems and Bundler" + +# latest supported versions for Ruby 2.x gem install bundler -v "~> 2.4.22" gem update --system 3.4.22 >/dev/null -echo "~~~ bundle install" +endgroup + +group "bundle install" + bundle install \ --jobs "$(getconf _NPROCESSORS_ONLN)" \ --retry 2 + +endgroup diff --git a/lib/csv2db.rb b/lib/csv2db.rb index 727c232..07a90d6 100644 --- a/lib/csv2db.rb +++ b/lib/csv2db.rb @@ -1,8 +1,18 @@ require 'csv2db/version' +require 'csv2db/config' module Csv2db class Error < StandardError; end - # Your code goes here... + + class << self + def config + Config.instance + end + + def configure + yield(config) + end + end end require 'csv2db/rails' if defined?(Rails) diff --git a/lib/csv2db/config.rb b/lib/csv2db/config.rb new file mode 100644 index 0000000..960c78a --- /dev/null +++ b/lib/csv2db/config.rb @@ -0,0 +1,22 @@ +require 'singleton' + +module Csv2db + class Config + include Singleton + + attr_writer :storage_adapter, :local_storage_host, :file_attachment_name + + def storage_adapter + @storage_adapter ||= :dragonfly + ActiveSupport::StringInquirer.new(@storage_adapter.to_s) + end + + def local_storage_host + @local_storage_host ||= '' + end + + def file_attachment_name + @file_attachment_name ||= :file_attachment + end + end +end diff --git a/lib/csv2db/rails.rb b/lib/csv2db/rails.rb index 0434d4c..2207c1b 100644 --- a/lib/csv2db/rails.rb +++ b/lib/csv2db/rails.rb @@ -13,6 +13,8 @@ class Engine < Rails::Engine initializer 'csv2db.add_controller_helpers' do ActiveSupport.on_load(:action_controller) do + require_relative '../../app/controllers/concerns/csv2db/controller_helpers' + include Csv2db::ControllerHelpers end end diff --git a/spec/models/concerns/csv2db/import_spec.rb b/spec/models/concerns/csv2db/import_spec.rb index 64a43ed..2b43bbd 100644 --- a/spec/models/concerns/csv2db/import_spec.rb +++ b/spec/models/concerns/csv2db/import_spec.rb @@ -104,4 +104,44 @@ class TestModel < ActiveRecord::Base expect(subject.errors?).to be_truthy end end + + context 'ActiveStorageAdapter' do + let(:file) do + Rack::Test::UploadedFile.new(Tempfile.new) + end + + let(:attachment_spy) do + spy('file_attachment') + end + + subject do + TestModel.new + end + + before do + allow(TestModel).to receive(:has_one_attached) + allow(TestModel).to receive(:alias_method) + .with(:file_attachment, :file_attachment).and_return(nil) + TestModel.include(Csv2db::ActiveStorageAdapter) + allow(subject).to receive(:file_attachment).and_return(attachment_spy) + end + + it 'calls correct attach methods' do + expect(file).to receive(:original_filename) + expect(file).to receive(:content_type) + expect(attachment_spy).to receive(:attach) + + subject.file = file + end + + it 'sets the file_name on the model' do + subject.file = file + + expect(subject.file_name).to eq(file.original_filename) + end + + it 'returns nil if no file passed' do + expect(subject.file = nil).to eq(nil) + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a74a278..0050175 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,10 +1,13 @@ require 'bundler/setup' require 'byebug' require 'mysql2' +require 'logger' require 'rails/all' require 'csv2db' require_relative '../app/models/concerns/csv2db/import' require_relative '../app/workers/csv2db/import_worker' +require_relative '../app/models/concerns/csv2db/dragonfly_adapter' +require_relative '../app/models/concerns/csv2db/active_storage_adapter' ENV['RAILS_ENV'] ||= 'test' @@ -21,7 +24,7 @@ end db_config = { - database: "csv2db_test#{ENV['CIRCLE_NODE_INDEX']}", + database: 'csv2db_test', adapter: 'mysql2', encoding: 'utf8mb4', pool: 5,