From 92be05daacd3b9fda17313eef387045a387d3636 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 19:05:14 -0600 Subject: [PATCH 01/13] rip out all the delayed_job infra --- Gemfile | 5 ++++- Gemfile.lock | 16 ++++++++++------ Procfile | 2 +- config/application.rb | 4 ++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index 84b963636..d08fd6f0d 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,10 @@ gem 'skylight' # Forum gem 'thredded' gem 'rails-ujs' -gem 'delayed_job_active_record' + +# Workers +gem 'sidekiq' +gem 'redis' # Exports gem 'csv' diff --git a/Gemfile.lock b/Gemfile.lock index f89e44e3e..656dd9024 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -120,6 +120,7 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.1.3) + connection_pool (2.2.2) crack (0.4.3) safe_yaml (~> 1.0.0) crass (1.0.4) @@ -152,11 +153,6 @@ GEM db_text_search (0.3.0) activerecord (>= 4.1.15, < 6.0) debug_inspector (0.0.3) - delayed_job (4.1.5) - activesupport (>= 3.0, < 5.3) - delayed_job_active_record (4.1.3) - activerecord (>= 3.0, < 5.3) - delayed_job (>= 3.0, < 5) devise (4.4.3) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -319,6 +315,8 @@ GEM rack-pjax (1.0.0) nokogiri (~> 1.5) rack (>= 1.1) + rack-protection (2.0.4) + rack rack-test (1.1.0) rack (>= 1.0, < 3) rack-timeout (0.5.1) @@ -380,6 +378,7 @@ GEM rb-inotify (0.9.10) ffi (>= 0.5.0, < 2) redcarpet (3.4.0) + redis (4.0.3) remotipart (1.4.2) responders (2.4.0) actionpack (>= 4.2.0, < 5.3) @@ -453,6 +452,10 @@ GEM shellany (0.0.1) shoulda-matchers (3.1.2) activesupport (>= 4.0.0) + sidekiq (5.2.3) + connection_pool (~> 2.2, >= 2.2.2) + rack-protection (>= 1.5.0) + redis (>= 3.3.5, < 5) simplecov (0.13.0) docile (~> 1.1.0) json (>= 1.8, < 3) @@ -553,7 +556,6 @@ DEPENDENCIES cucumber-rails database_cleaner dateslices - delayed_job_active_record devise factory_bot_rails filesize @@ -585,6 +587,7 @@ DEPENDENCIES rails_12factor rails_admin (~> 1.3) redcarpet + redis rmagick rspec-prof rspec-rails @@ -594,6 +597,7 @@ DEPENDENCIES selenium-webdriver serendipitous! shoulda-matchers (~> 3.1) + sidekiq simplecov skylight slack-notifier diff --git a/Procfile b/Procfile index f787d4ed7..b18e4b6be 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ web: bundle exec puma -C config/puma.rb -worker: bundle exec rake jobs:work +worker: bundle exec sidekiq diff --git a/config/application.rb b/config/application.rb index 2d2ca5acb..85c932417 100644 --- a/config/application.rb +++ b/config/application.rb @@ -23,7 +23,7 @@ class Application < Rails::Application # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] # config.i18n.default_locale = :de - # Use delayed_job to process jobs - config.active_job.queue_adapter = :delayed_job + # Use sidekiq to process jobs + config.active_job.queue_adapter = :sidekiq end end From e552aa6f915839c624512565b8e002dc9ae156ee Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 19:09:29 -0600 Subject: [PATCH 02/13] disable all forum emails for now --- config/initializers/thredded.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/thredded.rb b/config/initializers/thredded.rb index 821908819..19817b97d 100644 --- a/config/initializers/thredded.rb +++ b/config/initializers/thredded.rb @@ -152,7 +152,7 @@ # Thredded.notifiers = [Thredded::EmailNotifier.new] # # none: -#Thredded.notifiers = [] +Thredded.notifiers = [] # # add in (must install separate gem (under development) as well): # Thredded.notifiers = [Thredded::EmailNotifier.new, Thredded::PushoverNotifier.new(ENV['PUSHOVER_APP_ID'])] From 5163d4a50f4fc8e3aa6afd75dd177c4c77609fa7 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 19:15:16 -0600 Subject: [PATCH 03/13] rack timeout 20 -> 10 seconds --- config/initializers/rack_timeout.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb index 4a318e20d..f0fefa7d0 100644 --- a/config/initializers/rack_timeout.rb +++ b/config/initializers/rack_timeout.rb @@ -1 +1 @@ -Rails.application.config.middleware.insert_before Rack::Runtime, Rack::Timeout, service_timeout: 20 # seconds +Rails.application.config.middleware.insert_before Rack::Runtime, Rack::Timeout, service_timeout: 10 # seconds From 61fa7032249afc4f1a326ab4f1ca5d70dbab25c5 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 19:23:14 -0600 Subject: [PATCH 04/13] update latest partial from thredded --- app/views/thredded/users/_link.html.erb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/views/thredded/users/_link.html.erb b/app/views/thredded/users/_link.html.erb index 27dd6fcee..c5a1bb53f 100644 --- a/app/views/thredded/users/_link.html.erb +++ b/app/views/thredded/users/_link.html.erb @@ -1,10 +1,14 @@ <% if user %> <% if !user.thredded_anonymous? %> - <%= user.thredded_display_name %> - <%= render partial: 'thredded/users/badge', locals: { user: user } %> + <% user_path = user_path(user) %> + <% if user_path.blank? %> + <%= user.thredded_display_name %> + <% else %> + <%= user.thredded_display_name %> + <%= render partial: 'thredded/users/badge', locals: { user: user } %> + <% end %> <% else %> <%= user.thredded_display_name %> - <%= render partial: 'thredded/users/badge', locals: { user: user } %> <% end %> <% else %> <%= t 'thredded.null_user_name' %> From a5aaa39a152ef75713853b221d7a56b413415beb Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 19:31:41 -0600 Subject: [PATCH 05/13] add sidekiq config --- Procfile | 2 +- config/sidekiq.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 config/sidekiq.rb diff --git a/Procfile b/Procfile index b18e4b6be..5d34b934b 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ web: bundle exec puma -C config/puma.rb -worker: bundle exec sidekiq +worker: bundle exec sidekiq -e production -C config/sidekiq.yml diff --git a/config/sidekiq.rb b/config/sidekiq.rb new file mode 100644 index 000000000..0515ae218 --- /dev/null +++ b/config/sidekiq.rb @@ -0,0 +1 @@ +:concurrency: 1 From a0ebac0439c91a572647ad41412b5fa7b496e12a Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 22:19:50 -0600 Subject: [PATCH 06/13] remove rack-timeout gem --- Gemfile | 1 - Gemfile.lock | 2 -- config/initializers/rack_timeout.rb | 1 - 3 files changed, 4 deletions(-) delete mode 100644 config/initializers/rack_timeout.rb diff --git a/Gemfile b/Gemfile index d08fd6f0d..bce726522 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,6 @@ ruby "~> 2.5" gem 'rails', '~> 5.2' gem 'puma', '~> 3.7' gem 'puma-heroku' -gem 'rack-timeout' # Storage gem 'aws-sdk', '~> 1.5' diff --git a/Gemfile.lock b/Gemfile.lock index 656dd9024..fdbb96b61 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -319,7 +319,6 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rack-timeout (0.5.1) rails (5.2.1.1) actioncable (= 5.2.1.1) actionmailer (= 5.2.1.1) @@ -578,7 +577,6 @@ DEPENDENCIES puma (~> 3.7) puma-heroku rack-mini-profiler - rack-timeout rails (~> 5.2) rails-controller-testing rails-jquery-autocomplete diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb deleted file mode 100644 index f0fefa7d0..000000000 --- a/config/initializers/rack_timeout.rb +++ /dev/null @@ -1 +0,0 @@ -Rails.application.config.middleware.insert_before Rack::Runtime, Rack::Timeout, service_timeout: 10 # seconds From 0739fdc11e9311c8acbc401465387f5a793e2462 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 22:29:09 -0600 Subject: [PATCH 07/13] todo --- config/initializers/paperclip.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 508de07a3..749fb7dac 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -1 +1,3 @@ -Paperclip::Attachment.default_options[:url] = ':s3_domain_url' \ No newline at end of file +# TODO we can probably remove this. Need to figure out what we're using for uploads again. + +Paperclip::Attachment.default_options[:url] = ':s3_domain_url' From 92d52177c836112b4991c4fec9a849293b181cb2 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 22:29:59 -0600 Subject: [PATCH 08/13] mount sidekiq at /sidekiq --- config/routes.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/routes.rb b/config/routes.rb index ad86570f1..7d152c4d2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -267,6 +267,9 @@ # get '/forum/:wildcard/:another', to: 'emergency#temporarily_disabled' mount Thredded::Engine => '/forum' mount StripeEvent::Engine, at: '/webhooks/stripe' + + require 'sidekiq/web' + mount Sidekiq::Web => '/sidekiq' end # rubocop:enable LineLength From d0ce86a731d62ee685fbbdd184745a254751953b Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 22:31:11 -0600 Subject: [PATCH 09/13] lock /sidekiq down to site administrators --- config/routes.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/routes.rb b/config/routes.rb index 7d152c4d2..e2bf15a71 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -269,7 +269,9 @@ mount StripeEvent::Engine, at: '/webhooks/stripe' require 'sidekiq/web' - mount Sidekiq::Web => '/sidekiq' + authenticate :user, lambda { |u| u.site_administrator? } do + mount Sidekiq::Web => '/sidekiq' + end end # rubocop:enable LineLength From 9a15679f3c7a1fe248ba8ca752bbfc36522465fc Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 22:32:57 -0600 Subject: [PATCH 10/13] lock /admin/* to site administrators also --- config/routes.rb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/config/routes.rb b/config/routes.rb index e2bf15a71..0b047efcd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -179,15 +179,17 @@ end get 'search/', to: 'search#results' - scope 'admin' do - get '/stats', to: 'admin#dashboard', as: :admin_dashboard - get '/content_type/:type', to: 'admin#content_type', as: :admin_content_type - get '/attributes', to: 'admin#attributes', as: :admin_attributes - get '/masquerade/:user_id', to: 'admin#masquerade', as: :masquerade - get '/unsubscribe', to: 'admin#unsubscribe', as: :mass_unsubscribe - post '/perform_unsubscribe', to: 'admin#perform_unsubscribe', as: :perform_unsubscribe + authenticate :user, lambda { |u| u.site_administrator? } do + scope 'admin' do + get '/stats', to: 'admin#dashboard', as: :admin_dashboard + get '/content_type/:type', to: 'admin#content_type', as: :admin_content_type + get '/attributes', to: 'admin#attributes', as: :admin_attributes + get '/masquerade/:user_id', to: 'admin#masquerade', as: :masquerade + get '/unsubscribe', to: 'admin#unsubscribe', as: :mass_unsubscribe + post '/perform_unsubscribe', to: 'admin#perform_unsubscribe', as: :perform_unsubscribe + end + mount RailsAdmin::Engine => '/admin', as: 'rails_admin' end - mount RailsAdmin::Engine => '/admin', as: 'rails_admin' scope 'export' do get '/', to: 'export#index', as: :notebook_export From 3a7adb6e59c8e1f581a71d9c745a159254d0ff1c Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 22:55:24 -0600 Subject: [PATCH 11/13] links are being duplicated. no longer! --- app/models/serializers/content_serializer.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/models/serializers/content_serializer.rb b/app/models/serializers/content_serializer.rb index 6963f3e99..832f19333 100644 --- a/app/models/serializers/content_serializer.rb +++ b/app/models/serializers/content_serializer.rb @@ -104,6 +104,9 @@ def initialize(content) # ] # } def old_style_link_fields + # TODO I think we can remove this method + return [] + categories = Hash[YAML.load_file(Rails.root.join('config', 'attributes', "#{self.class_name.downcase}.yml")).map do |category_name, details| [ category_name.to_s, From c16d5e8e15747415c6294f00a9c86812b9a9a182 Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 23:12:39 -0600 Subject: [PATCH 12/13] whoops --- app/models/serializers/content_serializer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/serializers/content_serializer.rb b/app/models/serializers/content_serializer.rb index 832f19333..b476d5117 100644 --- a/app/models/serializers/content_serializer.rb +++ b/app/models/serializers/content_serializer.rb @@ -105,7 +105,7 @@ def initialize(content) # } def old_style_link_fields # TODO I think we can remove this method - return [] + return {} categories = Hash[YAML.load_file(Rails.root.join('config', 'attributes', "#{self.class_name.downcase}.yml")).map do |category_name, details| [ From 3f5ada6f62596cac37b0e85a6fc68b0625a8da7a Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 3 Dec 2018 23:26:35 -0600 Subject: [PATCH 13/13] typo on field type prevented links from showing --- app/views/content/display/_sidelinks.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/content/display/_sidelinks.html.erb b/app/views/content/display/_sidelinks.html.erb index 4f1ddc543..eb2fee27d 100644 --- a/app/views/content/display/_sidelinks.html.erb +++ b/app/views/content/display/_sidelinks.html.erb @@ -48,7 +48,7 @@ next unless category[:fields].any? do |field| ( # This is a link field AND it has existing links - field[:field_type] == 'link' && field[:old_column_source].present? && raw_model.send(field[:old_column_source]).any? + field[:type] == 'link' && field[:old_column_source].present? && raw_model.send(field[:old_column_source]).any? ) || field[:value].present? # or: this is a text field with text in it end