From 60bcbc7e1601aafb41afc08c674ececf6b70c949 Mon Sep 17 00:00:00 2001 From: thesecretmaster Date: Sun, 23 Aug 2020 12:17:04 -0500 Subject: [PATCH 1/5] Further migrations from threads to jobs (cc #746), create graphql query queue --- Gemfile | 3 + Gemfile.lock | 30 ++++++++++ Procfile | 4 +- Rakefile | 2 + app/channels/topbar_channel.rb | 4 +- app/controllers/authentication_controller.rb | 14 ++--- app/controllers/flag_conditions_controller.rb | 4 +- app/controllers/flag_settings_controller.rb | 4 +- app/controllers/graphql_controller.rb | 58 ++++++++++++++++--- app/controllers/review_queues_controller.rb | 6 +- app/controllers/status_controller.rb | 10 ++-- app/jobs/queue_graphql_query_job.rb | 21 +++++++ app/jobs/revalidate_flag_conditions_job.rb | 16 +++++ app/jobs/review_queue_recheck_items_job.rb | 10 ++++ app/jobs/update_chat_ids_job.rb | 9 +++ app/jobs/update_moderator_sites_job.rb | 7 +++ .../validate_flag_condition_for_user_job.rb | 4 +- app/models/flag_condition.rb | 13 ----- app/views/graphql/pending_job.html.erb | 7 +++ config/application.rb | 2 + config/initializers/backburner.rb | 18 ++++++ config/routes.rb | 5 ++ config/schedule.rb | 6 +- test/jobs/queue_graphql_query_job_test.rb | 7 +++ .../revalidate_flag_conditions_job_test.rb | 7 +++ .../review_queue_recheck_items_job_test.rb | 7 +++ test/jobs/update_chat_ids_job_test.rb | 7 +++ test/jobs/update_moderator_sites_job_test.rb | 7 +++ 28 files changed, 234 insertions(+), 58 deletions(-) create mode 100644 app/jobs/queue_graphql_query_job.rb create mode 100644 app/jobs/revalidate_flag_conditions_job.rb create mode 100644 app/jobs/review_queue_recheck_items_job.rb create mode 100644 app/jobs/update_chat_ids_job.rb create mode 100644 app/jobs/update_moderator_sites_job.rb create mode 100644 app/views/graphql/pending_job.html.erb create mode 100644 config/initializers/backburner.rb create mode 100644 test/jobs/queue_graphql_query_job_test.rb create mode 100644 test/jobs/revalidate_flag_conditions_job_test.rb create mode 100644 test/jobs/review_queue_recheck_items_job_test.rb create mode 100644 test/jobs/update_chat_ids_job_test.rb create mode 100644 test/jobs/update_moderator_sites_job_test.rb diff --git a/Gemfile b/Gemfile index efc9849b7..87feeae70 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,9 @@ gem 'grape', '1.3.0' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' gem 'rack', '>= 2.0.6' gem 'rails', '~> 5.2.0' +gem 'backburner' + +gem 'beanstalkd_view', group: :development # Use sqlite3 as the database for Active Record gem 'sqlite3' # Use SCSS for stylesheets diff --git a/Gemfile.lock b/Gemfile.lock index 833250133..b0332bc7a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -103,8 +103,19 @@ GEM aws-sigv4 (~> 1.1) aws-sigv4 (1.1.0) aws-eventstream (~> 1.0, >= 1.0.2) + backburner (1.5.0) + beaneater (~> 1.0) + concurrent-ruby (~> 1.0, >= 1.0.1) + dante (> 0.1.5) backports (3.15.0) bcrypt (3.1.13) + beaneater (1.0.0) + beanstalkd_view (2.0.0) + beaneater (~> 1.0.0) + json + sinatra (>= 1.3.0) + sinatra-contrib (>= 1.3.0) + vegas (~> 0.1.2) bindex (0.8.1) builder (3.2.3) byebug (11.0.1) @@ -148,6 +159,7 @@ GEM crack (0.4.3) safe_yaml (~> 1.0.0) crass (1.0.5) + dante (0.2.0) devise (4.7.2) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -291,6 +303,8 @@ GEM rack-cors (0.4.1) rack-mini-profiler (1.1.0) rack (>= 1.2.0) + rack-protection (2.0.8.1) + rack rack-test (1.1.0) rack (>= 1.0, < 3) rails (5.2.3) @@ -368,6 +382,18 @@ GEM json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.2) + sinatra (2.0.8.1) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.8.1) + tilt (~> 2.0) + sinatra-contrib (2.0.8.1) + backports (>= 2.8.2) + multi_json + mustermann (~> 1.0) + rack-protection (= 2.0.8.1) + sinatra (= 2.0.8.1) + tilt (~> 2.0) spring (2.1.0) sprockets (4.0.0) concurrent-ruby (~> 1.0) @@ -405,6 +431,8 @@ GEM uglifier (4.2.0) execjs (>= 0.3.0, < 3) unicode-display_width (1.6.0) + vegas (0.1.11) + rack (>= 1.0.0) warden (1.2.8) rack (>= 2.0.6) web-console (3.7.0) @@ -436,6 +464,8 @@ DEPENDENCIES audited (~> 4.4) awesome_print aws-sdk-s3 + backburner + beanstalkd_view blazer (= 2.2.1.charcoal)! byebug capistrano diff --git a/Procfile b/Procfile index ed24ee4cf..d080429e9 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,5 @@ -web: bundle exec rails s +web: bundle exec bin/rails s webpacker: ./bin/webpack-dev-server redis: ./redis-git/src/redis-server --port 6378 --loadmodule ./zhregex/module.so --dbfilename dump.rdb +beanstalkd: beanstalkd -l 127.0.0.1 -V +backburner: bundle exec rake backburner:work QUEUE=default,graphql_queries diff --git a/Rakefile b/Rakefile index da4efd575..fa0d905d9 100644 --- a/Rakefile +++ b/Rakefile @@ -4,5 +4,7 @@ # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. require File.expand_path('../config/application', __FILE__) +require 'backburner/tasks' +require_relative 'config/initializers/backburner.rb' Rails.application.load_tasks diff --git a/app/channels/topbar_channel.rb b/app/channels/topbar_channel.rb index 9a917d975..c7a1bff8f 100644 --- a/app/channels/topbar_channel.rb +++ b/app/channels/topbar_channel.rb @@ -4,9 +4,7 @@ class TopbarChannel < ApplicationCable::Channel def subscribed stream_from 'topbar' - Thread.new do - ActionCable.server.broadcast 'topbar', commit: CurrentCommit - end + ActionCable.server.broadcast 'topbar', commit: CurrentCommit end def unsubscribed diff --git a/app/controllers/authentication_controller.rb b/app/controllers/authentication_controller.rb index 6e041a255..e5f2edf82 100644 --- a/app/controllers/authentication_controller.rb +++ b/app/controllers/authentication_controller.rb @@ -27,11 +27,8 @@ def redirect_target ActiveRecord::Base.logger = old_logger if current_user.write_authenticated - u = current_user - Thread.new do - # Do this in the background to keep the page load fast. - u.update_moderator_sites - end + # Do this in the background to keep the page load fast. + UpdateModeratorSitesJob.perform_later(current_user.id) end flash[:success] = 'Successfully registered token' @@ -74,11 +71,8 @@ def login_redirect_target user.save! - Thread.new do - # Do this in the background to keep the page load fast. - user.update_chat_ids - user.save! - end + # Do this in the background to keep the page load fast. + UpdateChatIdsJob.perform_later(user.id) flash[:success] = "New account created for #{user.username}. Have fun!" end diff --git a/app/controllers/flag_conditions_controller.rb b/app/controllers/flag_conditions_controller.rb index 66300e82c..77b2a0e4e 100644 --- a/app/controllers/flag_conditions_controller.rb +++ b/app/controllers/flag_conditions_controller.rb @@ -131,9 +131,7 @@ def user_overview def validate_user @user = User.find params[:user] - Thread.new do - FlagCondition.validate_for_user @user, current_user - end + ValidateFlagConditionForUserJob.perform_later(@user.id, current_user.id) flash[:info] = 'Validation launched in background.' redirect_back fallback_location: root_path end diff --git a/app/controllers/flag_settings_controller.rb b/app/controllers/flag_settings_controller.rb index e40e3ec22..0667e589d 100644 --- a/app/controllers/flag_settings_controller.rb +++ b/app/controllers/flag_settings_controller.rb @@ -55,9 +55,7 @@ def update # we want to re-validate all existing FlagConditions # and disable them if they aren't in compliance with the # new settings - Thread.new do - FlagCondition.revalidate_all - end + RevalidateFlagConditionsJob.perform_later end format.html { redirect_to flag_settings_path, notice: 'Flag setting was successfully updated.' } diff --git a/app/controllers/graphql_controller.rb b/app/controllers/graphql_controller.rb index 385c71922..2dfdaaaef 100644 --- a/app/controllers/graphql_controller.rb +++ b/app/controllers/graphql_controller.rb @@ -2,7 +2,6 @@ class GraphqlController < ApplicationController skip_before_action :verify_authenticity_token, only: [:execute] - before_action :verify_authorization def execute variables = ensure_hash(params[:variables]) @@ -10,16 +9,56 @@ def execute operation_name = params[:operationName] context = { # Query context goes here, for example: - current_user: current_user + # current_user: current_user } query_params = { variables: variables, context: context, operation_name: operation_name } if user_signed_in? && current_user.has_role?(:core) # query_params.merge!({max_depth: 8, max_complexity:20}) end - @results = MetasmokeSchema.execute(query, **query_params) - respond_to do |format| - format.json { render json: @results } - format.html { @results = JSON.pretty_generate(@results.to_hash) } + + api_key = APIKey.find_by(key: params[:key]) + if (user_signed_in? && current_user.has_role?(:core)) || (!api_key.nil? && api_key.api_tokens.exists?(token: params[:token])) + @results = MetasmokeSchema.execute(query, **query_params) + respond_to do |format| + format.json { render json: @results } + format.html { @results = JSON.pretty_generate(@results.to_hash) } + end + elsif user_signed_in? || !api_key.nil? + @job_id = QueueGraphqlQueryJob.perform_later(query, **query_params).job_id + respond_to do |format| + format.json { render json: {job_id: @job_id} } + format.html { redirect_to view_graphql_job_path(job_id: @job_id) } + end + else + return head :forbidden + end + end + + def view_job + k = "graphql_queries/#{params[:job_id]}" + if redis.sismember "pending_graphql_queries", params[:job_id] + respond_to do |format| + format.json { render json: {complete: false, pending: true} } + format.html { render action: :pending_job } + end + elsif redis.exists k + v = redis.get k + time_remaining = redis.ttl(k).to_i + @time_elapsed = 600 - time_remaining + if v == "" + respond_to do |format| + format.json { render json: {complete: false, time_elapsed: @time_elapsed} } + format.html { render action: :pending_job } + end + else + @results = JSON.parse(v) + respond_to do |format| + format.json { render json: @results } + format.html { @results = JSON.pretty_generate(@results.to_hash); render action: :execute } + end + end + else + return head 404 end end @@ -27,9 +66,10 @@ def query; end private - def verify_authorization - return head 403 unless !APIKey.find_by(key: params[:key]).nil? || (user_signed_in? && current_user.has_role?(:core)) - end + # def valid_api_token + # key = APIKey.find_by(key: params[:key]) + # !key.nil? && key.api_tokens.exists?(token: params[:token]) + # end # Handle form data, JSON body, or a blank value def ensure_hash(ambiguous_param) diff --git a/app/controllers/review_queues_controller.rb b/app/controllers/review_queues_controller.rb index 1b6145ded..88da56ece 100644 --- a/app/controllers/review_queues_controller.rb +++ b/app/controllers/review_queues_controller.rb @@ -83,11 +83,7 @@ def reviews end def recheck_items - Thread.new do - @queue.items.includes(:reviewable).each do |i| - i.update(completed: true) if i.reviewable.should_dq?(@queue) - end - end + ReviewQueueRecheckItemsJob.perform_later(@queue.id) flash[:info] = 'Checking started in background.' redirect_back fallback_location: review_queues_path end diff --git a/app/controllers/status_controller.rb b/app/controllers/status_controller.rb index a85d4020a..b46ba9110 100644 --- a/app/controllers/status_controller.rb +++ b/app/controllers/status_controller.rb @@ -25,12 +25,10 @@ def status_update @smoke_detector.save! - Thread.new do - ActionCable.server.broadcast 'status', status_channel_data - ActionCable.server.broadcast 'status_blacklist_manager', status_channel_data.merge(failover_link: failover_link) - ActionCable.server.broadcast 'topbar', last_ping: @smoke_detector.last_ping.to_f - ActionCable.server.broadcast 'smokey_pings', smokey: @smoke_detector.as_json - end + ActionCable.server.broadcast 'status', status_channel_data + ActionCable.server.broadcast 'status_blacklist_manager', status_channel_data.merge(failover_link: failover_link) + ActionCable.server.broadcast 'topbar', last_ping: @smoke_detector.last_ping.to_f + ActionCable.server.broadcast 'smokey_pings', smokey: @smoke_detector.as_json respond_to do |format| format.json do diff --git a/app/jobs/queue_graphql_query_job.rb b/app/jobs/queue_graphql_query_job.rb new file mode 100644 index 000000000..e8479dc82 --- /dev/null +++ b/app/jobs/queue_graphql_query_job.rb @@ -0,0 +1,21 @@ +class QueueGraphqlQueryJob < ApplicationJob + queue_as :graphql_queries + + def perform(query, **query_params) + puts "PERF" + @results = MetasmokeSchema.execute(query, **query_params) + end + + before_enqueue do + redis.sadd "pending_graphql_queries", job_id + end + + before_perform do + redis.set "graphql_queries/#{job_id}", "", ex: 600 + redis.srem "pending_graphql_queries", job_id + end + + after_perform do + redis.set "graphql_queries/#{job_id}", JSON.generate(@results), ex: 300 + end +end diff --git a/app/jobs/revalidate_flag_conditions_job.rb b/app/jobs/revalidate_flag_conditions_job.rb new file mode 100644 index 000000000..d3163fc9a --- /dev/null +++ b/app/jobs/revalidate_flag_conditions_job.rb @@ -0,0 +1,16 @@ +class RevalidateFlagConditionsJob < ApplicationJob + queue_as :default + + def perform + FlagCondition.where(flags_enabled: true).find_each do |fc| + unless fc.validate + failures = fc.errors.full_messages + fc.flags_enabled = false + fc.save(validate: false) + ActionCable.server.broadcast 'smokedetector_messages', + message: "@#{fc.user&.username&.tr(' ', '')} " \ + "Your flag condition was disabled: #{failures.join(',')}" + end + end + end +end diff --git a/app/jobs/review_queue_recheck_items_job.rb b/app/jobs/review_queue_recheck_items_job.rb new file mode 100644 index 000000000..84ee5f9aa --- /dev/null +++ b/app/jobs/review_queue_recheck_items_job.rb @@ -0,0 +1,10 @@ +class ReviewQueueRecheckItemsJob < ApplicationJob + queue_as :default + + def perform(queue_id) + queue = Queue.find(queue_id) + queue.items.includes(:reviewable).each do |i| + i.update(completed: true) if i.reviewable.should_dq?(queue) + end + end +end diff --git a/app/jobs/update_chat_ids_job.rb b/app/jobs/update_chat_ids_job.rb new file mode 100644 index 000000000..174ea6d52 --- /dev/null +++ b/app/jobs/update_chat_ids_job.rb @@ -0,0 +1,9 @@ +class UpdateChatIdsJob < ApplicationJob + queue_as :default + + def perform(user_id) + u = User.find(user_id) + u.update_chat_ids + u.save! + end +end diff --git a/app/jobs/update_moderator_sites_job.rb b/app/jobs/update_moderator_sites_job.rb new file mode 100644 index 000000000..2c70bb443 --- /dev/null +++ b/app/jobs/update_moderator_sites_job.rb @@ -0,0 +1,7 @@ +class UpdateModeratorSitesJob < ApplicationJob + queue_as :default + + def perform(user_id) + User.find(user_id).update_moderator_sites + end +end diff --git a/app/jobs/validate_flag_condition_for_user_job.rb b/app/jobs/validate_flag_condition_for_user_job.rb index a0d280a6f..2616637b4 100644 --- a/app/jobs/validate_flag_condition_for_user_job.rb +++ b/app/jobs/validate_flag_condition_for_user_job.rb @@ -3,8 +3,8 @@ class ValidateFlagConditionForUserJob < ApplicationJob queue_as :default - def perform(user_id) - FlagCondition.validate_for_user(User.find(user_id), User.find(-1)) + def perform(user_id, other_user_id = -1) + FlagCondition.validate_for_user(User.find(user_id), User.find(other_user_id)) # Do something later end end diff --git a/app/models/flag_condition.rb b/app/models/flag_condition.rb index 3254e4dfb..6e31e9e50 100644 --- a/app/models/flag_condition.rb +++ b/app/models/flag_condition.rb @@ -30,19 +30,6 @@ def accuracy true_positive_count.to_f * 100 / post_feedback_results.count.to_f end - def self.revalidate_all - FlagCondition.where(flags_enabled: true).find_each do |fc| - unless fc.validate - failures = fc.errors.full_messages - fc.flags_enabled = false - fc.save(validate: false) - ActionCable.server.broadcast 'smokedetector_messages', - message: "@#{fc.user&.username&.tr(' ', '')} " \ - "Your flag condition was disabled: #{failures.join(',')}" - end - end - end - def self.overall_accuracy(user) query = File.read(Rails.root.join('lib/queries/overall_accuracy.sql')) sanitized = ActiveRecord::Base.sanitize_sql([query, user_id: user.id]) diff --git a/app/views/graphql/pending_job.html.erb b/app/views/graphql/pending_job.html.erb new file mode 100644 index 000000000..a2dd77ba2 --- /dev/null +++ b/app/views/graphql/pending_job.html.erb @@ -0,0 +1,7 @@ +<% if @time_elapsed.nil? %> +

Your job is in the queue. Keep refreshing this page to see once it starts running. +<% else %> +

Your job is still pending, it has been running for <%= @time_elapsed %> seconds and will terminate after 600 seconds

+

Reload the page to see if your job has completed. If you get a 404, it has expired.

+<% end %> +

After your job has completed, it will be available for 5 minutes

diff --git a/config/application.rb b/config/application.rb index 618b1a050..99b0a23b4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -25,6 +25,8 @@ class Application < Rails::Application # Prevent SE write codes (used to get a token) from showing up in the logs config.filter_parameters += [:code] + config.active_job.queue_adapter = :backburner + config.middleware.insert_before 0, Rack::Cors do allow do origins '*' diff --git a/config/initializers/backburner.rb b/config/initializers/backburner.rb new file mode 100644 index 000000000..54bdccf1a --- /dev/null +++ b/config/initializers/backburner.rb @@ -0,0 +1,18 @@ +Backburner.configure do |config| + config.beanstalk_url = "beanstalk://127.0.0.1" + config.tube_namespace = "metasmoke.production" + config.namespace_separator = "." + config.on_error = lambda { |e| puts e } + # config.max_job_retries = 3 # default 0 retries + # config.retry_delay = 2 # default 5 seconds + # config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } + config.default_priority = 65536 + config.respond_timeout = 600 + config.default_worker = Backburner::Workers::Simple + config.logger = Logger.new(STDOUT) + config.primary_queue = "default" + # config.priority_labels = { :custom => 50, :useless => 1000 } + config.reserve_timeout = nil + config.job_serializer_proc = lambda { |body| JSON.dump(body) } + config.job_parser_proc = lambda { |body| JSON.parse(body) } +end diff --git a/config/routes.rb b/config/routes.rb index fa002a90f..6e27eb2c1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -18,6 +18,10 @@ resources :dumps + authenticate(:user, ->(user) { user.has_role?(:developer) }) do + mount BeanstalkdView::Server, :at => "/beanstalkd" if Rails.env.development? + end + get 'sites/dash', to: 'dashboard#site_dash', as: :site_dash get 'search', to: 'search#index' @@ -160,6 +164,7 @@ post 'graphql', to: 'graphql#execute', as: :graphql get 'graphql', to: 'graphql#query', as: :query_graphql + get 'graphql/view_job/:job_id', to: 'graphql#view_job', as: :view_graphql_job end mount GraphiQL::Rails::Engine, at: '/api/graphiql', graphql_path: '/api/graphql', query_params: true diff --git a/config/schedule.rb b/config/schedule.rb index 802779ba1..43791e3da 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -15,9 +15,9 @@ every 1.day, at: '2:00 am' do runner 'User.where.not(stack_exchange_account_id: nil).each { |u| u.update_moderator_sites; sleep(5) }' - runner 'FlagCondition.revalidate_all' + runner 'RevalidateFlagConditionsJob.perform_now' runner 'AbuseReport.update_stale_reports' - runner 'ConflictingFeedbackJob.perform_later' + runner 'ConflictingFeedbackJob.perform_now' end # every 1.day, at: '3:10am' do @@ -25,5 +25,5 @@ # end every 1.day, at: '3:00am' do - runner 'ScheduledMailJob.perform_later' + runner 'ScheduledMailJob.perform_now' end diff --git a/test/jobs/queue_graphql_query_job_test.rb b/test/jobs/queue_graphql_query_job_test.rb new file mode 100644 index 000000000..e751cf44f --- /dev/null +++ b/test/jobs/queue_graphql_query_job_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class QueueGraphqlQueryJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/jobs/revalidate_flag_conditions_job_test.rb b/test/jobs/revalidate_flag_conditions_job_test.rb new file mode 100644 index 000000000..8d2748f59 --- /dev/null +++ b/test/jobs/revalidate_flag_conditions_job_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class RevalidateFlagConditionsJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/jobs/review_queue_recheck_items_job_test.rb b/test/jobs/review_queue_recheck_items_job_test.rb new file mode 100644 index 000000000..f595eec3d --- /dev/null +++ b/test/jobs/review_queue_recheck_items_job_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class ReviewQueueRecheckItemsJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/jobs/update_chat_ids_job_test.rb b/test/jobs/update_chat_ids_job_test.rb new file mode 100644 index 000000000..4c676219c --- /dev/null +++ b/test/jobs/update_chat_ids_job_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class UpdateChatIdsJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/jobs/update_moderator_sites_job_test.rb b/test/jobs/update_moderator_sites_job_test.rb new file mode 100644 index 000000000..a328fb58d --- /dev/null +++ b/test/jobs/update_moderator_sites_job_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class UpdateModeratorSitesJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end From 372a91686d9585f1d9b38ffe289c5f19f8411b90 Mon Sep 17 00:00:00 2001 From: thesecretmaster Date: Mon, 24 Aug 2020 10:53:07 -0500 Subject: [PATCH 2/5] Create search queue --- app/controllers/search_controller.rb | 50 +++++++++++ app/jobs/search_extend_job.rb | 33 +++++++ app/jobs/search_helpers.rb | 110 +++++++++++++++++++++++ app/jobs/search_job.rb | 53 +++++++++++ app/views/search/new_search.html.erb | 93 +++++++++++++++++++ app/views/search/search_pending.html.erb | 5 ++ app/views/search/search_results.html.erb | 38 ++++++++ config/initializers/backburner.rb | 2 +- config/routes.rb | 5 ++ 9 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 app/jobs/search_extend_job.rb create mode 100644 app/jobs/search_helpers.rb create mode 100644 app/jobs/search_job.rb create mode 100644 app/views/search/new_search.html.erb create mode 100644 app/views/search/search_pending.html.erb create mode 100644 app/views/search/search_results.html.erb diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 2aeb49872..cee3487ab 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -1,6 +1,34 @@ # frozen_string_literal: true class SearchController < ApplicationController + def new_search; end + + def create_search + ops = %i[title body why username].map do |s| + SearchHelper.parse_search_params(params, s, current_user) + end.flatten + job_id = SearchJob.perform_later(ops, params.permit(VALID_SEARCH_PARAMS)).job_id + redirect_to search_pending_path(job_id: job_id) + end + + def search_pending + return head :not_found unless redis.exists "search_jobs/#{params[:job_id]}/sid" + @sid = redis.get "search_jobs/#{params[:job_id]}/sid" + @sid_ttl = redis.ttl("search_jobs/#{params[:job_id]}/sid").to_i + @be_page = redis.get("search_jobs/#{params[:job_id]}/be_page").to_i + if redis.exists "searches/#{@sid}/results/#{@be_page}" + per_page = 100 + redirect_to search_results_path(sid: @sid, page_num: (@be_page * SearchJob.search_page_length) / per_page, per_page: per_page) + end + end + + def search_results + @results = search_get_page(params[:page_num].to_i, params[:per_page].to_i, sid: params[:sid].to_i) + if @results.is_a? String + redirect_to search_pending_path(job_id: @results) + end + end + def index # This might be ugly, but it's better than the alternative. # @@ -319,4 +347,26 @@ def index_fast end redis.del final_key end + + private + + def search_get_page(page_num, per_page = 100, **hsh) + total_offset = per_page * page_num + per_page_real = [per_page, SearchJob.search_page_length].min + be_page_offset = total_offset % SearchJob.search_page_length + be_page = (total_offset / SearchJob.search_page_length).floor + if redis.exists "searches/#{hsh[:sid]}/results/#{be_page}" + @counts_by_accuracy_group = JSON.parse(redis.get("searches/#{hsh[:sid]}/counts_by_accuracy_group")).symbolize_keys + @counts_by_feedback = JSON.parse(redis.get("searches/#{hsh[:sid]}/counts_by_feedback")).symbolize_keys + return Post.where(id: redis.get("searches/#{hsh[:sid]}/results/#{be_page}").unpack("I!*")).order(created_at: :desc).offset(be_page_offset).limit(per_page_real).includes_for_post_row + elsif hsh.has_key?(:sid) + if redis.set("searches/#{hsh[:sid]}/results/#{be_page}/job_id", "", nx: true) + job_id = SearchExtendJob.perform_later(hsh[:sid], be_page).job_id + redis.set "searches/#{hsh[:sid]}/results/#{be_page}/job_id", job_id + return job_id + else + return redis.get("searches/#{hsh[:sid]}/results/#{be_page}/job_id") + end + end + end end diff --git a/app/jobs/search_extend_job.rb b/app/jobs/search_extend_job.rb new file mode 100644 index 000000000..2d5f34cfc --- /dev/null +++ b/app/jobs/search_extend_job.rb @@ -0,0 +1,33 @@ +require "#{Rails.root}/app/jobs/search_helpers.rb" + +class SearchExtendJob < ApplicationJob + queue_as :searches + include Backburner::Queue + queue_respond_timeout SEARCH_JOB_TIMEOUT + + include SearchJobQueryBuilder + + before_enqueue do |job| + sid, be_page = job.arguments + job_id = job.job_id + redis.set "search_jobs/#{job_id}/sid", sid + redis.set "search_jobs/#{job_id}/be_page", be_page + end + + def perform(sid, be_page) + redis.expire "search_jobs/#{job_id}/sid", SEARCH_JOB_TIMEOUT + redis.expire "search_jobs/#{job_id}/be_page", SEARCH_JOB_TIMEOUT + wrapped_params = JSON.parse(redis.get("searches/#{sid}/params")).symbolize_keys + ops = redis.lrange "searches/#{sid}/ops", 0, -1 + created_at = redis.get "searches/#{sid}/created_at" + query = build_query(ops, wrapped_params) + query = query.where('posts.created_at < ?', Time.at(created_at.to_i)) + redis.set "searches/#{sid}/results/#{be_page}", query.offset(SEARCH_PAGE_LENGTH * be_page).limit(SEARCH_PAGE_LENGTH).select(:id).map(&:id).pack("I!*") + redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION + end + + after_perform do |job| + sid, be_page = job.arguments + redis.del "searches/#{sid}/results/#{be_page}/job_id" + end +end diff --git a/app/jobs/search_helpers.rb b/app/jobs/search_helpers.rb new file mode 100644 index 000000000..0f62c08a5 --- /dev/null +++ b/app/jobs/search_helpers.rb @@ -0,0 +1,110 @@ +SEARCH_PAGE_LENGTH = 10000 +VALID_SEARCH_PARAMS = %i[post_type_include_unmatched autoflagged post_type user_rep_direction site edited or_search option body_is_like user_reputation reason feedback].freeze +SEARCH_JOB_TIMEOUT = (60*60*3) +SEACH_PAGE_EXPIRATION = 1.hour +module SearchJobQueryBuilder + def build_query(ops, wrapped_params) + title, title_operation, + body, body_operation, + why, why_operation, + username, username_operation = ops + + user_reputation = wrapped_params[:user_reputation].to_i || 0 + + case wrapped_params[:feedback] + when /true/ + feedback = :is_tp + when /false/ + feedback = :is_fp + when /NAA/ + feedback = :is_naa + end + + results = if wrapped_params[:reason].present? + Reason.find(wrapped_params[:reason]).posts + else + Post.all + end#.includes_for_post_row + + search_string = [] + search_params = {} + [[:username, username, username_operation], [:title, title, title_operation], + [:why, why, why_operation]].each do |si| + if si[1].present? && si[1] != '%%' + search_string << "IFNULL(`posts`.`#{si[0]}`, '') #{si[2]} :#{si[0]}" + search_params[si[0]] = si[1] + end + end + + if body.present? + if ['LIKE', 'NOT LIKE'].include?(body_operation) && wrapped_params[:body_is_like] != '1' + # If the operation would be LIKE, hijack it and use our fulltext index for a search instead. + # UNLESS... params[:body_is_like] is set, in which case the user has explicitly specified a LIKE query. + results = results.match_search(body, with_search_score: false, posts: :body) + else + if wrapped_params[:body_is_like] == '1' && !user_signed_in? + flash[:warning] = 'Unregistered users cannot use LIKE searches on the body field. Please sign in.' + redirect_to(search_path) && return + end + # Otherwise, it's REGEX or NOT REGEX, which fulltext won't do - fall back on search_string and params + search_string << "IFNULL(`posts`.`body`, '') #{body_operation} :body" + search_params[:body] = body.present? ? body : '%%' + end + end + + results = results.where(search_string.join(wrapped_params[:or_search].present? ? ' OR ' : ' AND '), **search_params) + + # results = results.includes(:reasons).includes(:feedbacks) if wrapped_params[:option].nil? + results = results.joins(:reasons).joins(:feedbacks) if wrapped_params[:option].nil? + + if feedback.present? + results = results.where(feedback => true) + elsif wrapped_params[:feedback] == 'conflicted' + results = results.where(is_tp: true, is_fp: true) + end + + results = case wrapped_params[:user_rep_direction] + when '>=' + if user_reputation > 0 + results.where('IFNULL(user_reputation, 0) >= :rep', rep: user_reputation) + end + when '==' + results.where('IFNULL(user_reputation, 0) = :rep', rep: user_reputation) + when '<=' + results.where('IFNULL(user_reputation, 0) <= :rep', rep: user_reputation) + else + results + end + + results = results.where(site_id: wrapped_params[:site]) if wrapped_params[:site].present? + + results = results.where('revision_count > 1') if wrapped_params[:edited].present? + + # results = results.includes(feedbacks: [:user]) + + case wrapped_params[:autoflagged].try(:downcase) + when 'yes' + results = results.autoflagged + when 'no' + results = results.not_autoflagged + end + + post_type = case wrapped_params[:post_type].try(:downcase).try(:[], 0) + when 'q' + 'questions' + when 'a' + 'a' + end + + if post_type.present? + unmatched = results.where.not("link LIKE '%/questions/%' OR link LIKE '%/a/%'") + results = if wrapped_params[:post_type_include_unmatched] + results.where('link like ?', "%/#{post_type}/%").or(unmatched) + else + results.where('link like ?', "%/#{post_type}/%") + end + end + + return results.distinct.order(Arel.sql('`posts`.`created_at` DESC')) + end +end diff --git a/app/jobs/search_job.rb b/app/jobs/search_job.rb new file mode 100644 index 000000000..2928d7061 --- /dev/null +++ b/app/jobs/search_job.rb @@ -0,0 +1,53 @@ +require "#{Rails.root}/app/jobs/search_helpers.rb" + +class SearchJob < ApplicationJob + queue_as :searches + include Backburner::Queue + queue_respond_timeout SEARCH_JOB_TIMEOUT + + include SearchJobQueryBuilder + + def self.search_page_length + SEARCH_PAGE_LENGTH + end + + def self.search_job_timeout + SEARCH_JOB_TIMEOUT + end + + attr_accessor :search_id + + before_enqueue do |job| + ops, params = job.arguments + job_id = job.job_id + wrapped_params = VALID_SEARCH_PARAMS.map { |k| [k, params[k]] }.to_h + sid = redis.incr "search_counter" + redis.set "search_jobs/#{job_id}/sid", sid + redis.set "search_jobs/#{job_id}/be_page", 0 + redis.set "searches/#{sid}/params", JSON.generate(wrapped_params) + redis.rpush "searches/#{sid}/ops", ops + end + + + def perform(ops, params) + # sid = @sid + redis.expire "search_jobs/#{job_id}/sid", SEARCH_JOB_TIMEOUT + redis.expire "search_jobs/#{job_id}/be_page", SEARCH_JOB_TIMEOUT + sid = redis.get("search_jobs/#{job_id}/sid").to_i + wrapped_params = VALID_SEARCH_PARAMS.map { |k| [k, params[k]] }.to_h + created_at = Time.now.to_i + redis.set "searches/#{sid}/created_at", created_at + + results = build_query(ops, wrapped_params) + results = results.where('posts.created_at < ?', Time.at(created_at)) + + counts_by_accuracy_group = results.group(:is_tp, :is_fp, :is_naa).count + counts_by_feedback = %i[is_tp is_fp is_naa].each_with_index.map do |symbol, i| + [symbol, counts_by_accuracy_group.select { |k, _v| k[i] }.values.sum] + end.to_h + redis.set "searches/#{sid}/results/0", results.limit(SEARCH_PAGE_LENGTH).map(&:id).pack("I!*") + redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION + redis.set "searches/#{sid}/counts_by_accuracy_group", JSON.generate(counts_by_accuracy_group) + redis.set "searches/#{sid}/counts_by_feedback", JSON.generate(counts_by_feedback) + end +end diff --git a/app/views/search/new_search.html.erb b/app/views/search/new_search.html.erb new file mode 100644 index 000000000..4c7d6ec76 --- /dev/null +++ b/app/views/search/new_search.html.erb @@ -0,0 +1,93 @@ +<% title "Search" %> + +<%= form_tag (params[:option] == 'graphs' ? create_search_path(anchor: "graphs") : create_search_path), method: "get" do |f| %> +
+ <%= label_tag :title %> + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :title_is_regex, 1, params[:title_is_regex] %> regex + <%= check_box_tag :title_is_inverse_regex, 1, params[:title_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :title, params[:title], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :body %> + <%= check_box_tag :body_is_like, 1, params[:body_is_like] %> + + LIKE (?) + + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :body_is_regex, 1, params[:body_is_regex] %> regex + <%= check_box_tag :body_is_inverse_regex, 1, params[:body_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :body, params[:body], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :username %> + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :username_is_regex, 1, params[:username_is_regex] %> regex + <%= check_box_tag :username_is_inverse_regex, 1, params[:username_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :username, params[:username], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :why %> + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :why_is_regex, 1, params[:why_is_regex] %> regex + <%= check_box_tag :why_is_inverse_regex, 1, params[:why_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :why, params[:why], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :or_search, "Use OR for text search (uses AND by default)" %> + <%= check_box_tag :or_search, 1, params[:or_search] %> +
+
+ <% h = HTMLEntities.new %> + <%= label_tag :site %> + <%= select_tag :site, options_for_select(Site.select(:site_name, :id).all.order(:site_name).map {|s| [h.decode(s.site_name), s.id] }, params[:site]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :post_type %> + <%= check_box_tag :post_type_include_unmatched, 1, params[:post_type_include_unmatched] %> include unmatched + <%= select_tag :post_type, options_for_select(%i[question answer], params[:post_type]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :feedback %> + <%= select_tag :feedback, options_for_select(["true positive", "false positive", "NAA", "conflicted"], params[:feedback]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :autoflagged %> + <%= select_tag :autoflagged, options_for_select(["Yes", "No"], params[:autoflagged]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :reason %> + <%= select_tag :reason, options_from_collection_for_select(Reason.all.order(:reason_name), "id", "reason_name", params[:reason]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :user_reputation %> +
+
+ <%= select_tag :user_rep_direction, options_for_select(%w[== <= >=], params[:user_rep_direction]), include_blank: true, + class: "form-control" %> +
+ +
+ <%= number_field_tag :user_reputation, params[:user_reputation], class: "form-control" %> +
+
+
+ +
+ <%= label_tag :edited, "Edited" %> + <%= check_box_tag :edited, 1, params[:edited] %> +
+ + <%= hidden_field_tag :option, params[:option] if params[:option] == 'graphs' %> + + <%= submit_tag "Search", class: "btn btn-primary" %> +<% end %> diff --git a/app/views/search/search_pending.html.erb b/app/views/search/search_pending.html.erb new file mode 100644 index 000000000..60549256a --- /dev/null +++ b/app/views/search/search_pending.html.erb @@ -0,0 +1,5 @@ +<% if @sid_ttl == -1 %> +

Your search is pending. Please keep reloading and eventually it'll work :)

+<% else %> +

Your search has been running for <%= SearchJob.search_job_timeout - @sid_ttl %> seconds...

+<% end %> diff --git a/app/views/search/search_results.html.erb b/app/views/search/search_results.html.erb new file mode 100644 index 000000000..bfc09bad6 --- /dev/null +++ b/app/views/search/search_results.html.erb @@ -0,0 +1,38 @@ + + +<% if params[:option].nil? %> + + <%= render partial: "posts/post", collection: @results, as: :post %> +
+ <% if false %> +
+ <%= will_paginate @results, renderer: BootstrapPagination::Rails %> +

+ <%= link_to "JSON (still subject to paging limits)", search_path(params: request.query_parameters, format: :json) %> +

+
+ <% end %> +<% else %> +
+ <% h = HTMLEntities.new %> + <%= pie_chart @results.group(:site).count.map{ |k,v| {(k.nil? ? "Unknown" : h.decode(k.site_name))=>v} }.reduce(:merge).select{|k,v| k != "Unknown"}.sort_by {|k,v| v}.reverse %> + + <%= line_chart [{name: "True positives", data: @results.where(is_tp: true).group_by_day('`posts`.`created_at`', range: 1.month.ago.to_date..Time.now).count}, {name: "False positives", data: @results.where(is_fp: true).group_by_day('`posts`.`created_at`', range: 1.month.ago.to_date..Time.now).count}], colors: ["green", "red"], library: {"tooltip" => {'shared' => 'true'} } %> +
+<% end %> diff --git a/config/initializers/backburner.rb b/config/initializers/backburner.rb index 54bdccf1a..3cb52212c 100644 --- a/config/initializers/backburner.rb +++ b/config/initializers/backburner.rb @@ -10,8 +10,8 @@ config.respond_timeout = 600 config.default_worker = Backburner::Workers::Simple config.logger = Logger.new(STDOUT) - config.primary_queue = "default" # config.priority_labels = { :custom => 50, :useless => 1000 } + # config.default_worker = Backburner::Workers::ThreadsOnFork config.reserve_timeout = nil config.job_serializer_proc = lambda { |body| JSON.dump(body) } config.job_parser_proc = lambda { |body| JSON.parse(body) } diff --git a/config/routes.rb b/config/routes.rb index 6e27eb2c1..38e6a494c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -24,6 +24,11 @@ get 'sites/dash', to: 'dashboard#site_dash', as: :site_dash + + get 'search_job', to: 'search#new_search', as: :new_search_job + get 'search_job/create', to: 'search#create_search', as: :create_search + get 'search_job/pending/:job_id', to: 'search#search_pending', as: :search_pending + get 'search_job/:sid', to: 'search#search_results', as: :search_results get 'search', to: 'search#index' get 'search_fast', to: 'search#index_fast' get 'reasons', to: 'dashboard#index', as: :reasons From 50371010a7ada9f0f17ea0ee8699b6c93ef53abc Mon Sep 17 00:00:00 2001 From: thesecretmaster Date: Mon, 24 Aug 2020 11:15:41 -0500 Subject: [PATCH 3/5] Rubocop --- Gemfile | 2 +- app/controllers/graphql_controller.rb | 15 +++-- app/controllers/search_controller.rb | 28 +++++----- app/jobs/queue_graphql_query_job.rb | 10 ++-- app/jobs/revalidate_flag_conditions_job.rb | 2 + app/jobs/review_queue_recheck_items_job.rb | 2 + app/jobs/search_extend_job.rb | 5 +- app/jobs/search_helpers.rb | 65 ++++++++++++---------- app/jobs/search_job.rb | 7 ++- app/jobs/update_chat_ids_job.rb | 2 + app/jobs/update_moderator_sites_job.rb | 2 + config/initializers/backburner.rb | 16 +++--- 12 files changed, 90 insertions(+), 66 deletions(-) diff --git a/Gemfile b/Gemfile index 87feeae70..3832b2c21 100644 --- a/Gemfile +++ b/Gemfile @@ -50,9 +50,9 @@ gem 'grape', '1.3.0' # gem 'puma_worker_killer' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' +gem 'backburner' gem 'rack', '>= 2.0.6' gem 'rails', '~> 5.2.0' -gem 'backburner' gem 'beanstalkd_view', group: :development # Use sqlite3 as the database for Active Record diff --git a/app/controllers/graphql_controller.rb b/app/controllers/graphql_controller.rb index 2dfdaaaef..35964fb32 100644 --- a/app/controllers/graphql_controller.rb +++ b/app/controllers/graphql_controller.rb @@ -26,7 +26,7 @@ def execute elsif user_signed_in? || !api_key.nil? @job_id = QueueGraphqlQueryJob.perform_later(query, **query_params).job_id respond_to do |format| - format.json { render json: {job_id: @job_id} } + format.json { render json: { job_id: @job_id } } format.html { redirect_to view_graphql_job_path(job_id: @job_id) } end else @@ -36,25 +36,28 @@ def execute def view_job k = "graphql_queries/#{params[:job_id]}" - if redis.sismember "pending_graphql_queries", params[:job_id] + if redis.sismember 'pending_graphql_queries', params[:job_id] respond_to do |format| - format.json { render json: {complete: false, pending: true} } + format.json { render json: { complete: false, pending: true } } format.html { render action: :pending_job } end elsif redis.exists k v = redis.get k time_remaining = redis.ttl(k).to_i @time_elapsed = 600 - time_remaining - if v == "" + if v == '' respond_to do |format| - format.json { render json: {complete: false, time_elapsed: @time_elapsed} } + format.json { render json: { complete: false, time_elapsed: @time_elapsed } } format.html { render action: :pending_job } end else @results = JSON.parse(v) respond_to do |format| format.json { render json: @results } - format.html { @results = JSON.pretty_generate(@results.to_hash); render action: :execute } + format.html do + @results = JSON.pretty_generate(@results.to_hash) + render action: :execute + end end end else diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index cee3487ab..4a2475ef1 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -16,17 +16,14 @@ def search_pending @sid = redis.get "search_jobs/#{params[:job_id]}/sid" @sid_ttl = redis.ttl("search_jobs/#{params[:job_id]}/sid").to_i @be_page = redis.get("search_jobs/#{params[:job_id]}/be_page").to_i - if redis.exists "searches/#{@sid}/results/#{@be_page}" - per_page = 100 - redirect_to search_results_path(sid: @sid, page_num: (@be_page * SearchJob.search_page_length) / per_page, per_page: per_page) - end + return unless redis.exists "searches/#{@sid}/results/#{@be_page}" + per_page = 100 + redirect_to search_results_path(sid: @sid, page_num: (@be_page * SearchJob.search_page_length) / per_page, per_page: per_page) end def search_results @results = search_get_page(params[:page_num].to_i, params[:per_page].to_i, sid: params[:sid].to_i) - if @results.is_a? String - redirect_to search_pending_path(job_id: @results) - end + redirect_to search_pending_path(job_id: @results) if @results.is_a? String end def index @@ -358,15 +355,18 @@ def search_get_page(page_num, per_page = 100, **hsh) if redis.exists "searches/#{hsh[:sid]}/results/#{be_page}" @counts_by_accuracy_group = JSON.parse(redis.get("searches/#{hsh[:sid]}/counts_by_accuracy_group")).symbolize_keys @counts_by_feedback = JSON.parse(redis.get("searches/#{hsh[:sid]}/counts_by_feedback")).symbolize_keys - return Post.where(id: redis.get("searches/#{hsh[:sid]}/results/#{be_page}").unpack("I!*")).order(created_at: :desc).offset(be_page_offset).limit(per_page_real).includes_for_post_row - elsif hsh.has_key?(:sid) - if redis.set("searches/#{hsh[:sid]}/results/#{be_page}/job_id", "", nx: true) - job_id = SearchExtendJob.perform_later(hsh[:sid], be_page).job_id - redis.set "searches/#{hsh[:sid]}/results/#{be_page}/job_id", job_id - return job_id - else + return Post.where(id: redis.get("searches/#{hsh[:sid]}/results/#{be_page}").unpack('I!*')) + .order(created_at: :desc) + .offset(be_page_offset) + .limit(per_page_real) + .includes_for_post_row + elsif hsh.key?(:sid) + unless redis.set("searches/#{hsh[:sid]}/results/#{be_page}/job_id", '', nx: true) return redis.get("searches/#{hsh[:sid]}/results/#{be_page}/job_id") end + job_id = SearchExtendJob.perform_later(hsh[:sid], be_page).job_id + redis.set "searches/#{hsh[:sid]}/results/#{be_page}/job_id", job_id + return job_id end end end diff --git a/app/jobs/queue_graphql_query_job.rb b/app/jobs/queue_graphql_query_job.rb index e8479dc82..d1a62ed6a 100644 --- a/app/jobs/queue_graphql_query_job.rb +++ b/app/jobs/queue_graphql_query_job.rb @@ -1,18 +1,20 @@ +# frozen_string_literal: true + class QueueGraphqlQueryJob < ApplicationJob queue_as :graphql_queries def perform(query, **query_params) - puts "PERF" + puts 'PERF' @results = MetasmokeSchema.execute(query, **query_params) end before_enqueue do - redis.sadd "pending_graphql_queries", job_id + redis.sadd 'pending_graphql_queries', job_id end before_perform do - redis.set "graphql_queries/#{job_id}", "", ex: 600 - redis.srem "pending_graphql_queries", job_id + redis.set "graphql_queries/#{job_id}", '', ex: 600 + redis.srem 'pending_graphql_queries', job_id end after_perform do diff --git a/app/jobs/revalidate_flag_conditions_job.rb b/app/jobs/revalidate_flag_conditions_job.rb index d3163fc9a..346027b11 100644 --- a/app/jobs/revalidate_flag_conditions_job.rb +++ b/app/jobs/revalidate_flag_conditions_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class RevalidateFlagConditionsJob < ApplicationJob queue_as :default diff --git a/app/jobs/review_queue_recheck_items_job.rb b/app/jobs/review_queue_recheck_items_job.rb index 84ee5f9aa..14c2c3458 100644 --- a/app/jobs/review_queue_recheck_items_job.rb +++ b/app/jobs/review_queue_recheck_items_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ReviewQueueRecheckItemsJob < ApplicationJob queue_as :default diff --git a/app/jobs/search_extend_job.rb b/app/jobs/search_extend_job.rb index 2d5f34cfc..2f1f0c391 100644 --- a/app/jobs/search_extend_job.rb +++ b/app/jobs/search_extend_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "#{Rails.root}/app/jobs/search_helpers.rb" class SearchExtendJob < ApplicationJob @@ -22,7 +24,8 @@ def perform(sid, be_page) created_at = redis.get "searches/#{sid}/created_at" query = build_query(ops, wrapped_params) query = query.where('posts.created_at < ?', Time.at(created_at.to_i)) - redis.set "searches/#{sid}/results/#{be_page}", query.offset(SEARCH_PAGE_LENGTH * be_page).limit(SEARCH_PAGE_LENGTH).select(:id).map(&:id).pack("I!*") + search_results = query.offset(SEARCH_PAGE_LENGTH * be_page).limit(SEARCH_PAGE_LENGTH).select(:id).map(&:id).pack('I!*') + redis.set "searches/#{sid}/results/#{be_page}", search_results redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION end diff --git a/app/jobs/search_helpers.rb b/app/jobs/search_helpers.rb index 0f62c08a5..91aae6e6f 100644 --- a/app/jobs/search_helpers.rb +++ b/app/jobs/search_helpers.rb @@ -1,6 +1,11 @@ -SEARCH_PAGE_LENGTH = 10000 -VALID_SEARCH_PARAMS = %i[post_type_include_unmatched autoflagged post_type user_rep_direction site edited or_search option body_is_like user_reputation reason feedback].freeze -SEARCH_JOB_TIMEOUT = (60*60*3) +# frozen_string_literal: true + +SEARCH_PAGE_LENGTH = 10_000 +VALID_SEARCH_PARAMS = %i[ + post_type_include_unmatched autoflagged post_type user_rep_direction + site edited or_search option body_is_like user_reputation reason feedback +].freeze +SEARCH_JOB_TIMEOUT = (60 * 60 * 3) SEACH_PAGE_EXPIRATION = 1.hour module SearchJobQueryBuilder def build_query(ops, wrapped_params) @@ -21,15 +26,15 @@ def build_query(ops, wrapped_params) end results = if wrapped_params[:reason].present? - Reason.find(wrapped_params[:reason]).posts - else - Post.all - end#.includes_for_post_row + Reason.find(wrapped_params[:reason]).posts + else + Post.all + end search_string = [] search_params = {} [[:username, username, username_operation], [:title, title, title_operation], - [:why, why, why_operation]].each do |si| + [:why, why, why_operation]].each do |si| if si[1].present? && si[1] != '%%' search_string << "IFNULL(`posts`.`#{si[0]}`, '') #{si[2]} :#{si[0]}" search_params[si[0]] = si[1] @@ -64,17 +69,17 @@ def build_query(ops, wrapped_params) end results = case wrapped_params[:user_rep_direction] - when '>=' - if user_reputation > 0 - results.where('IFNULL(user_reputation, 0) >= :rep', rep: user_reputation) - end - when '==' - results.where('IFNULL(user_reputation, 0) = :rep', rep: user_reputation) - when '<=' - results.where('IFNULL(user_reputation, 0) <= :rep', rep: user_reputation) - else - results - end + when '>=' + if user_reputation > 0 + results.where('IFNULL(user_reputation, 0) >= :rep', rep: user_reputation) + end + when '==' + results.where('IFNULL(user_reputation, 0) = :rep', rep: user_reputation) + when '<=' + results.where('IFNULL(user_reputation, 0) <= :rep', rep: user_reputation) + else + results + end results = results.where(site_id: wrapped_params[:site]) if wrapped_params[:site].present? @@ -90,21 +95,21 @@ def build_query(ops, wrapped_params) end post_type = case wrapped_params[:post_type].try(:downcase).try(:[], 0) - when 'q' - 'questions' - when 'a' - 'a' - end + when 'q' + 'questions' + when 'a' + 'a' + end if post_type.present? unmatched = results.where.not("link LIKE '%/questions/%' OR link LIKE '%/a/%'") - results = if wrapped_params[:post_type_include_unmatched] - results.where('link like ?', "%/#{post_type}/%").or(unmatched) - else - results.where('link like ?', "%/#{post_type}/%") - end + results = if wrapped_params[:post_type_include_unmatched] + results.where('link like ?', "%/#{post_type}/%").or(unmatched) + else + results.where('link like ?', "%/#{post_type}/%") + end end - return results.distinct.order(Arel.sql('`posts`.`created_at` DESC')) + results.distinct.order(Arel.sql('`posts`.`created_at` DESC')) end end diff --git a/app/jobs/search_job.rb b/app/jobs/search_job.rb index 2928d7061..a1781d575 100644 --- a/app/jobs/search_job.rb +++ b/app/jobs/search_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "#{Rails.root}/app/jobs/search_helpers.rb" class SearchJob < ApplicationJob @@ -21,14 +23,13 @@ def self.search_job_timeout ops, params = job.arguments job_id = job.job_id wrapped_params = VALID_SEARCH_PARAMS.map { |k| [k, params[k]] }.to_h - sid = redis.incr "search_counter" + sid = redis.incr 'search_counter' redis.set "search_jobs/#{job_id}/sid", sid redis.set "search_jobs/#{job_id}/be_page", 0 redis.set "searches/#{sid}/params", JSON.generate(wrapped_params) redis.rpush "searches/#{sid}/ops", ops end - def perform(ops, params) # sid = @sid redis.expire "search_jobs/#{job_id}/sid", SEARCH_JOB_TIMEOUT @@ -45,7 +46,7 @@ def perform(ops, params) counts_by_feedback = %i[is_tp is_fp is_naa].each_with_index.map do |symbol, i| [symbol, counts_by_accuracy_group.select { |k, _v| k[i] }.values.sum] end.to_h - redis.set "searches/#{sid}/results/0", results.limit(SEARCH_PAGE_LENGTH).map(&:id).pack("I!*") + redis.set "searches/#{sid}/results/0", results.limit(SEARCH_PAGE_LENGTH).map(&:id).pack('I!*') redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION redis.set "searches/#{sid}/counts_by_accuracy_group", JSON.generate(counts_by_accuracy_group) redis.set "searches/#{sid}/counts_by_feedback", JSON.generate(counts_by_feedback) diff --git a/app/jobs/update_chat_ids_job.rb b/app/jobs/update_chat_ids_job.rb index 174ea6d52..e8b4ca7d4 100644 --- a/app/jobs/update_chat_ids_job.rb +++ b/app/jobs/update_chat_ids_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class UpdateChatIdsJob < ApplicationJob queue_as :default diff --git a/app/jobs/update_moderator_sites_job.rb b/app/jobs/update_moderator_sites_job.rb index 2c70bb443..809f0fb21 100644 --- a/app/jobs/update_moderator_sites_job.rb +++ b/app/jobs/update_moderator_sites_job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class UpdateModeratorSitesJob < ApplicationJob queue_as :default diff --git a/config/initializers/backburner.rb b/config/initializers/backburner.rb index 3cb52212c..399b3ebef 100644 --- a/config/initializers/backburner.rb +++ b/config/initializers/backburner.rb @@ -1,18 +1,20 @@ +# frozen_string_literal: true + Backburner.configure do |config| - config.beanstalk_url = "beanstalk://127.0.0.1" - config.tube_namespace = "metasmoke.production" - config.namespace_separator = "." - config.on_error = lambda { |e| puts e } + config.beanstalk_url = 'beanstalk://127.0.0.1' + config.tube_namespace = 'metasmoke.production' + config.namespace_separator = '.' + config.on_error = ->(e) { puts e } # config.max_job_retries = 3 # default 0 retries # config.retry_delay = 2 # default 5 seconds # config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } - config.default_priority = 65536 + config.default_priority = 65_536 config.respond_timeout = 600 config.default_worker = Backburner::Workers::Simple config.logger = Logger.new(STDOUT) # config.priority_labels = { :custom => 50, :useless => 1000 } # config.default_worker = Backburner::Workers::ThreadsOnFork config.reserve_timeout = nil - config.job_serializer_proc = lambda { |body| JSON.dump(body) } - config.job_parser_proc = lambda { |body| JSON.parse(body) } + config.job_serializer_proc = ->(body) { JSON.dump(body) } + config.job_parser_proc = ->(body) { JSON.parse(body) } end From 2d2065bdad98fb63393265e1ae923dfc7e9ade4b Mon Sep 17 00:00:00 2001 From: thesecretmaster Date: Mon, 24 Aug 2020 11:31:35 -0500 Subject: [PATCH 4/5] Rubocop, again --- test/jobs/queue_graphql_query_job_test.rb | 2 ++ test/jobs/revalidate_flag_conditions_job_test.rb | 2 ++ test/jobs/review_queue_recheck_items_job_test.rb | 2 ++ test/jobs/search_job_test.rb | 9 +++++++++ test/jobs/update_chat_ids_job_test.rb | 2 ++ test/jobs/update_moderator_sites_job_test.rb | 2 ++ 6 files changed, 19 insertions(+) create mode 100644 test/jobs/search_job_test.rb diff --git a/test/jobs/queue_graphql_query_job_test.rb b/test/jobs/queue_graphql_query_job_test.rb index e751cf44f..6183191a0 100644 --- a/test/jobs/queue_graphql_query_job_test.rb +++ b/test/jobs/queue_graphql_query_job_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class QueueGraphqlQueryJobTest < ActiveJob::TestCase diff --git a/test/jobs/revalidate_flag_conditions_job_test.rb b/test/jobs/revalidate_flag_conditions_job_test.rb index 8d2748f59..6f16b6def 100644 --- a/test/jobs/revalidate_flag_conditions_job_test.rb +++ b/test/jobs/revalidate_flag_conditions_job_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class RevalidateFlagConditionsJobTest < ActiveJob::TestCase diff --git a/test/jobs/review_queue_recheck_items_job_test.rb b/test/jobs/review_queue_recheck_items_job_test.rb index f595eec3d..65d332fac 100644 --- a/test/jobs/review_queue_recheck_items_job_test.rb +++ b/test/jobs/review_queue_recheck_items_job_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class ReviewQueueRecheckItemsJobTest < ActiveJob::TestCase diff --git a/test/jobs/search_job_test.rb b/test/jobs/search_job_test.rb new file mode 100644 index 000000000..49c1679ec --- /dev/null +++ b/test/jobs/search_job_test.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'test_helper' + +class SearchJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/jobs/update_chat_ids_job_test.rb b/test/jobs/update_chat_ids_job_test.rb index 4c676219c..66a4b9080 100644 --- a/test/jobs/update_chat_ids_job_test.rb +++ b/test/jobs/update_chat_ids_job_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class UpdateChatIdsJobTest < ActiveJob::TestCase diff --git a/test/jobs/update_moderator_sites_job_test.rb b/test/jobs/update_moderator_sites_job_test.rb index a328fb58d..8ee548bc2 100644 --- a/test/jobs/update_moderator_sites_job_test.rb +++ b/test/jobs/update_moderator_sites_job_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class UpdateModeratorSitesJobTest < ActiveJob::TestCase From 8753c084afb09ee20cfed4f037004638f0454836 Mon Sep 17 00:00:00 2001 From: thesecretmaster Date: Mon, 31 Aug 2020 09:57:10 -0500 Subject: [PATCH 5/5] Pagination and visible search params --- app/controllers/search_controller.rb | 10 ++- app/jobs/search_helpers.rb | 6 +- app/jobs/search_job.rb | 5 +- app/views/search/search_results.html.erb | 91 +++++++++++++++++++++++- 4 files changed, 105 insertions(+), 7 deletions(-) diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 4a2475ef1..7ffc437db 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -18,12 +18,18 @@ def search_pending @be_page = redis.get("search_jobs/#{params[:job_id]}/be_page").to_i return unless redis.exists "searches/#{@sid}/results/#{@be_page}" per_page = 100 - redirect_to search_results_path(sid: @sid, page_num: (@be_page * SearchJob.search_page_length) / per_page, per_page: per_page) + redirect_to search_results_path(sid: @sid, page: (@be_page * SearchJob.search_page_length) / per_page, per_page: per_page) end def search_results - @results = search_get_page(params[:page_num].to_i, params[:per_page].to_i, sid: params[:sid].to_i) + @search_params = JSON.parse(redis.get("searches/#{params[:sid]}/params")).symbolize_keys + @result_count = redis.get("searches/#{params[:sid].to_i}/result_count").to_i + @results = search_get_page(params[:page].to_i, params[:per_page].to_i, sid: params[:sid].to_i) redirect_to search_pending_path(job_id: @results) if @results.is_a? String + total_pages = (@result_count / params[:per_page].to_i) + current_page = params[:page].to_i + @results.define_singleton_method(:total_pages) { total_pages } + @results.define_singleton_method(:current_page) { current_page } end def index diff --git a/app/jobs/search_helpers.rb b/app/jobs/search_helpers.rb index 91aae6e6f..14eb9de02 100644 --- a/app/jobs/search_helpers.rb +++ b/app/jobs/search_helpers.rb @@ -3,7 +3,11 @@ SEARCH_PAGE_LENGTH = 10_000 VALID_SEARCH_PARAMS = %i[ post_type_include_unmatched autoflagged post_type user_rep_direction - site edited or_search option body_is_like user_reputation reason feedback + site edited or_search option user_reputation reason feedback + title title_is_regex title_is_inverse_regex + body body_is_like body_is_regex body_is_inverse_regex + why why_is_regex why_is_inverse_regex + username username_is_regex username_is_inverse_regex ].freeze SEARCH_JOB_TIMEOUT = (60 * 60 * 3) SEACH_PAGE_EXPIRATION = 1.hour diff --git a/app/jobs/search_job.rb b/app/jobs/search_job.rb index a1781d575..4c5d1d705 100644 --- a/app/jobs/search_job.rb +++ b/app/jobs/search_job.rb @@ -46,9 +46,10 @@ def perform(ops, params) counts_by_feedback = %i[is_tp is_fp is_naa].each_with_index.map do |symbol, i| [symbol, counts_by_accuracy_group.select { |k, _v| k[i] }.values.sum] end.to_h - redis.set "searches/#{sid}/results/0", results.limit(SEARCH_PAGE_LENGTH).map(&:id).pack('I!*') - redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION + redis.set "searches/#{sid}/result_count", results.count redis.set "searches/#{sid}/counts_by_accuracy_group", JSON.generate(counts_by_accuracy_group) redis.set "searches/#{sid}/counts_by_feedback", JSON.generate(counts_by_feedback) + redis.set "searches/#{sid}/results/0", results.limit(SEARCH_PAGE_LENGTH).map(&:id).pack('I!*') + redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION end end diff --git a/app/views/search/search_results.html.erb b/app/views/search/search_results.html.erb index bfc09bad6..b83a348bc 100644 --- a/app/views/search/search_results.html.erb +++ b/app/views/search/search_results.html.erb @@ -1,7 +1,95 @@ +
+
+ <%= label_tag :title %> + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :title_is_regex, 1, @search_params[:title_is_regex] %> regex + <%= check_box_tag :title_is_inverse_regex, 1, @search_params[:title_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :title, @search_params[:title], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :body %> + <%= check_box_tag :body_is_like, 1, @search_params[:body_is_like] %> + + LIKE (?) + + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :body_is_regex, 1, @search_params[:body_is_regex] %> regex + <%= check_box_tag :body_is_inverse_regex, 1, @search_params[:body_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :body, @search_params[:body], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :username %> + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :username_is_regex, 1, @search_params[:username_is_regex] %> regex + <%= check_box_tag :username_is_inverse_regex, 1, @search_params[:username_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :username, @search_params[:username], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :why %> + <% if current_user&.can_use_regex_search? %> + <%= check_box_tag :why_is_regex, 1, @search_params[:why_is_regex] %> regex + <%= check_box_tag :why_is_inverse_regex, 1, @search_params[:why_is_inverse_regex] %> invert + <% end %> + <%= text_field_tag :why, @search_params[:why], placeholder: "*", class: "form-control" %> +
+
+ <%= label_tag :or_search, "Use OR for text search (uses AND by default)" %> + <%= check_box_tag :or_search, 1, @search_params[:or_search] %> +
+
+ <% h = HTMLEntities.new %> + <%= label_tag :site %> + <%= select_tag :site, options_for_select(Site.select(:site_name, :id).all.order(:site_name).map {|s| [h.decode(s.site_name), s.id] }, @search_params[:site]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :post_type %> + <%= check_box_tag :post_type_include_unmatched, 1, @search_params[:post_type_include_unmatched] %> include unmatched + <%= select_tag :post_type, options_for_select(%i[question answer], @search_params[:post_type]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :feedback %> + <%= select_tag :feedback, options_for_select(["true positive", "false positive", "NAA", "conflicted"], @search_params[:feedback]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :autoflagged %> + <%= select_tag :autoflagged, options_for_select(["Yes", "No"], @search_params[:autoflagged]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :reason %> + <%= select_tag :reason, options_from_collection_for_select(Reason.all.order(:reason_name), "id", "reason_name", @search_params[:reason]), class: "form-control", include_blank: "All" %> +
+ +
+ <%= label_tag :user_reputation %> +
+
+ <%= select_tag :user_rep_direction, options_for_select(%w[== <= >=], @search_params[:user_rep_direction]), include_blank: true, + class: "form-control" %> +
+ +
+ <%= number_field_tag :user_reputation, @search_params[:user_reputation], class: "form-control" %> +
+
+
+ +
+ <%= label_tag :edited, "Edited" %> + <%= check_box_tag :edited, 1, @search_params[:edited] %> +
+
    +
<% if params[:option].nil? %> <%= render partial: "posts/post", collection: @results, as: :post %>
- <% if false %>
<%= will_paginate @results, renderer: BootstrapPagination::Rails %>

<%= link_to "JSON (still subject to paging limits)", search_path(params: request.query_parameters, format: :json) %>

- <% end %> <% else %>
<% h = HTMLEntities.new %>