diff --git a/Gemfile b/Gemfile
index efc9849b7..3832b2c21 100644
--- a/Gemfile
+++ b/Gemfile
@@ -50,8 +50,11 @@ gem 'grape', '1.3.0'
# gem 'puma_worker_killer'
# Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
+gem 'backburner'
gem 'rack', '>= 2.0.6'
gem 'rails', '~> 5.2.0'
+
+gem 'beanstalkd_view', group: :development
# Use sqlite3 as the database for Active Record
gem 'sqlite3'
# Use SCSS for stylesheets
diff --git a/Gemfile.lock b/Gemfile.lock
index 833250133..b0332bc7a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -103,8 +103,19 @@ GEM
aws-sigv4 (~> 1.1)
aws-sigv4 (1.1.0)
aws-eventstream (~> 1.0, >= 1.0.2)
+ backburner (1.5.0)
+ beaneater (~> 1.0)
+ concurrent-ruby (~> 1.0, >= 1.0.1)
+ dante (> 0.1.5)
backports (3.15.0)
bcrypt (3.1.13)
+ beaneater (1.0.0)
+ beanstalkd_view (2.0.0)
+ beaneater (~> 1.0.0)
+ json
+ sinatra (>= 1.3.0)
+ sinatra-contrib (>= 1.3.0)
+ vegas (~> 0.1.2)
bindex (0.8.1)
builder (3.2.3)
byebug (11.0.1)
@@ -148,6 +159,7 @@ GEM
crack (0.4.3)
safe_yaml (~> 1.0.0)
crass (1.0.5)
+ dante (0.2.0)
devise (4.7.2)
bcrypt (~> 3.0)
orm_adapter (~> 0.1)
@@ -291,6 +303,8 @@ GEM
rack-cors (0.4.1)
rack-mini-profiler (1.1.0)
rack (>= 1.2.0)
+ rack-protection (2.0.8.1)
+ rack
rack-test (1.1.0)
rack (>= 1.0, < 3)
rails (5.2.3)
@@ -368,6 +382,18 @@ GEM
json (>= 1.8, < 3)
simplecov-html (~> 0.10.0)
simplecov-html (0.10.2)
+ sinatra (2.0.8.1)
+ mustermann (~> 1.0)
+ rack (~> 2.0)
+ rack-protection (= 2.0.8.1)
+ tilt (~> 2.0)
+ sinatra-contrib (2.0.8.1)
+ backports (>= 2.8.2)
+ multi_json
+ mustermann (~> 1.0)
+ rack-protection (= 2.0.8.1)
+ sinatra (= 2.0.8.1)
+ tilt (~> 2.0)
spring (2.1.0)
sprockets (4.0.0)
concurrent-ruby (~> 1.0)
@@ -405,6 +431,8 @@ GEM
uglifier (4.2.0)
execjs (>= 0.3.0, < 3)
unicode-display_width (1.6.0)
+ vegas (0.1.11)
+ rack (>= 1.0.0)
warden (1.2.8)
rack (>= 2.0.6)
web-console (3.7.0)
@@ -436,6 +464,8 @@ DEPENDENCIES
audited (~> 4.4)
awesome_print
aws-sdk-s3
+ backburner
+ beanstalkd_view
blazer (= 2.2.1.charcoal)!
byebug
capistrano
diff --git a/Procfile b/Procfile
index ed24ee4cf..d080429e9 100644
--- a/Procfile
+++ b/Procfile
@@ -1,3 +1,5 @@
-web: bundle exec rails s
+web: bundle exec bin/rails s
webpacker: ./bin/webpack-dev-server
redis: ./redis-git/src/redis-server --port 6378 --loadmodule ./zhregex/module.so --dbfilename dump.rdb
+beanstalkd: beanstalkd -l 127.0.0.1 -V
+backburner: bundle exec rake backburner:work QUEUE=default,graphql_queries
diff --git a/Rakefile b/Rakefile
index da4efd575..fa0d905d9 100644
--- a/Rakefile
+++ b/Rakefile
@@ -4,5 +4,7 @@
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
require File.expand_path('../config/application', __FILE__)
+require 'backburner/tasks'
+require_relative 'config/initializers/backburner.rb'
Rails.application.load_tasks
diff --git a/app/channels/topbar_channel.rb b/app/channels/topbar_channel.rb
index 9a917d975..c7a1bff8f 100644
--- a/app/channels/topbar_channel.rb
+++ b/app/channels/topbar_channel.rb
@@ -4,9 +4,7 @@
class TopbarChannel < ApplicationCable::Channel
def subscribed
stream_from 'topbar'
- Thread.new do
- ActionCable.server.broadcast 'topbar', commit: CurrentCommit
- end
+ ActionCable.server.broadcast 'topbar', commit: CurrentCommit
end
def unsubscribed
diff --git a/app/controllers/authentication_controller.rb b/app/controllers/authentication_controller.rb
index 6e041a255..e5f2edf82 100644
--- a/app/controllers/authentication_controller.rb
+++ b/app/controllers/authentication_controller.rb
@@ -27,11 +27,8 @@ def redirect_target
ActiveRecord::Base.logger = old_logger
if current_user.write_authenticated
- u = current_user
- Thread.new do
- # Do this in the background to keep the page load fast.
- u.update_moderator_sites
- end
+ # Do this in the background to keep the page load fast.
+ UpdateModeratorSitesJob.perform_later(current_user.id)
end
flash[:success] = 'Successfully registered token'
@@ -74,11 +71,8 @@ def login_redirect_target
user.save!
- Thread.new do
- # Do this in the background to keep the page load fast.
- user.update_chat_ids
- user.save!
- end
+ # Do this in the background to keep the page load fast.
+ UpdateChatIdsJob.perform_later(user.id)
flash[:success] = "New account created for #{user.username}. Have fun!"
end
diff --git a/app/controllers/flag_conditions_controller.rb b/app/controllers/flag_conditions_controller.rb
index 66300e82c..77b2a0e4e 100644
--- a/app/controllers/flag_conditions_controller.rb
+++ b/app/controllers/flag_conditions_controller.rb
@@ -131,9 +131,7 @@ def user_overview
def validate_user
@user = User.find params[:user]
- Thread.new do
- FlagCondition.validate_for_user @user, current_user
- end
+ ValidateFlagConditionForUserJob.perform_later(@user.id, current_user.id)
flash[:info] = 'Validation launched in background.'
redirect_back fallback_location: root_path
end
diff --git a/app/controllers/flag_settings_controller.rb b/app/controllers/flag_settings_controller.rb
index e40e3ec22..0667e589d 100644
--- a/app/controllers/flag_settings_controller.rb
+++ b/app/controllers/flag_settings_controller.rb
@@ -55,9 +55,7 @@ def update
# we want to re-validate all existing FlagConditions
# and disable them if they aren't in compliance with the
# new settings
- Thread.new do
- FlagCondition.revalidate_all
- end
+ RevalidateFlagConditionsJob.perform_later
end
format.html { redirect_to flag_settings_path, notice: 'Flag setting was successfully updated.' }
diff --git a/app/controllers/graphql_controller.rb b/app/controllers/graphql_controller.rb
index 385c71922..35964fb32 100644
--- a/app/controllers/graphql_controller.rb
+++ b/app/controllers/graphql_controller.rb
@@ -2,7 +2,6 @@
class GraphqlController < ApplicationController
skip_before_action :verify_authenticity_token, only: [:execute]
- before_action :verify_authorization
def execute
variables = ensure_hash(params[:variables])
@@ -10,16 +9,59 @@ def execute
operation_name = params[:operationName]
context = {
# Query context goes here, for example:
- current_user: current_user
+ # current_user: current_user
}
query_params = { variables: variables, context: context, operation_name: operation_name }
if user_signed_in? && current_user.has_role?(:core)
# query_params.merge!({max_depth: 8, max_complexity:20})
end
- @results = MetasmokeSchema.execute(query, **query_params)
- respond_to do |format|
- format.json { render json: @results }
- format.html { @results = JSON.pretty_generate(@results.to_hash) }
+
+ api_key = APIKey.find_by(key: params[:key])
+ if (user_signed_in? && current_user.has_role?(:core)) || (!api_key.nil? && api_key.api_tokens.exists?(token: params[:token]))
+ @results = MetasmokeSchema.execute(query, **query_params)
+ respond_to do |format|
+ format.json { render json: @results }
+ format.html { @results = JSON.pretty_generate(@results.to_hash) }
+ end
+ elsif user_signed_in? || !api_key.nil?
+ @job_id = QueueGraphqlQueryJob.perform_later(query, **query_params).job_id
+ respond_to do |format|
+ format.json { render json: { job_id: @job_id } }
+ format.html { redirect_to view_graphql_job_path(job_id: @job_id) }
+ end
+ else
+ return head :forbidden
+ end
+ end
+
+ def view_job
+ k = "graphql_queries/#{params[:job_id]}"
+ if redis.sismember 'pending_graphql_queries', params[:job_id]
+ respond_to do |format|
+ format.json { render json: { complete: false, pending: true } }
+ format.html { render action: :pending_job }
+ end
+ elsif redis.exists k
+ v = redis.get k
+ time_remaining = redis.ttl(k).to_i
+ @time_elapsed = 600 - time_remaining
+ if v == ''
+ respond_to do |format|
+ format.json { render json: { complete: false, time_elapsed: @time_elapsed } }
+ format.html { render action: :pending_job }
+ end
+ else
+ @results = JSON.parse(v)
+ respond_to do |format|
+ format.json { render json: @results }
+ format.html do
+ @results = JSON.pretty_generate(@results.to_hash)
+ render action: :execute
+ end
+ end
+ end
+ else
+ return head 404
end
end
@@ -27,9 +69,10 @@ def query; end
private
- def verify_authorization
- return head 403 unless !APIKey.find_by(key: params[:key]).nil? || (user_signed_in? && current_user.has_role?(:core))
- end
+ # def valid_api_token
+ # key = APIKey.find_by(key: params[:key])
+ # !key.nil? && key.api_tokens.exists?(token: params[:token])
+ # end
# Handle form data, JSON body, or a blank value
def ensure_hash(ambiguous_param)
diff --git a/app/controllers/review_queues_controller.rb b/app/controllers/review_queues_controller.rb
index 1b6145ded..88da56ece 100644
--- a/app/controllers/review_queues_controller.rb
+++ b/app/controllers/review_queues_controller.rb
@@ -83,11 +83,7 @@ def reviews
end
def recheck_items
- Thread.new do
- @queue.items.includes(:reviewable).each do |i|
- i.update(completed: true) if i.reviewable.should_dq?(@queue)
- end
- end
+ ReviewQueueRecheckItemsJob.perform_later(@queue.id)
flash[:info] = 'Checking started in background.'
redirect_back fallback_location: review_queues_path
end
diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb
index 2aeb49872..7ffc437db 100644
--- a/app/controllers/search_controller.rb
+++ b/app/controllers/search_controller.rb
@@ -1,6 +1,37 @@
# frozen_string_literal: true
class SearchController < ApplicationController
+ def new_search; end
+
+ def create_search
+ ops = %i[title body why username].map do |s|
+ SearchHelper.parse_search_params(params, s, current_user)
+ end.flatten
+ job_id = SearchJob.perform_later(ops, params.permit(VALID_SEARCH_PARAMS)).job_id
+ redirect_to search_pending_path(job_id: job_id)
+ end
+
+ def search_pending
+ return head :not_found unless redis.exists "search_jobs/#{params[:job_id]}/sid"
+ @sid = redis.get "search_jobs/#{params[:job_id]}/sid"
+ @sid_ttl = redis.ttl("search_jobs/#{params[:job_id]}/sid").to_i
+ @be_page = redis.get("search_jobs/#{params[:job_id]}/be_page").to_i
+ return unless redis.exists "searches/#{@sid}/results/#{@be_page}"
+ per_page = 100
+ redirect_to search_results_path(sid: @sid, page: (@be_page * SearchJob.search_page_length) / per_page, per_page: per_page)
+ end
+
+ def search_results
+ @search_params = JSON.parse(redis.get("searches/#{params[:sid]}/params")).symbolize_keys
+ @result_count = redis.get("searches/#{params[:sid].to_i}/result_count").to_i
+ @results = search_get_page(params[:page].to_i, params[:per_page].to_i, sid: params[:sid].to_i)
+ redirect_to search_pending_path(job_id: @results) if @results.is_a? String
+ total_pages = (@result_count / params[:per_page].to_i)
+ current_page = params[:page].to_i
+ @results.define_singleton_method(:total_pages) { total_pages }
+ @results.define_singleton_method(:current_page) { current_page }
+ end
+
def index
# This might be ugly, but it's better than the alternative.
#
@@ -319,4 +350,29 @@ def index_fast
end
redis.del final_key
end
+
+ private
+
+ def search_get_page(page_num, per_page = 100, **hsh)
+ total_offset = per_page * page_num
+ per_page_real = [per_page, SearchJob.search_page_length].min
+ be_page_offset = total_offset % SearchJob.search_page_length
+ be_page = (total_offset / SearchJob.search_page_length).floor
+ if redis.exists "searches/#{hsh[:sid]}/results/#{be_page}"
+ @counts_by_accuracy_group = JSON.parse(redis.get("searches/#{hsh[:sid]}/counts_by_accuracy_group")).symbolize_keys
+ @counts_by_feedback = JSON.parse(redis.get("searches/#{hsh[:sid]}/counts_by_feedback")).symbolize_keys
+ return Post.where(id: redis.get("searches/#{hsh[:sid]}/results/#{be_page}").unpack('I!*'))
+ .order(created_at: :desc)
+ .offset(be_page_offset)
+ .limit(per_page_real)
+ .includes_for_post_row
+ elsif hsh.key?(:sid)
+ unless redis.set("searches/#{hsh[:sid]}/results/#{be_page}/job_id", '', nx: true)
+ return redis.get("searches/#{hsh[:sid]}/results/#{be_page}/job_id")
+ end
+ job_id = SearchExtendJob.perform_later(hsh[:sid], be_page).job_id
+ redis.set "searches/#{hsh[:sid]}/results/#{be_page}/job_id", job_id
+ return job_id
+ end
+ end
end
diff --git a/app/controllers/status_controller.rb b/app/controllers/status_controller.rb
index a85d4020a..b46ba9110 100644
--- a/app/controllers/status_controller.rb
+++ b/app/controllers/status_controller.rb
@@ -25,12 +25,10 @@ def status_update
@smoke_detector.save!
- Thread.new do
- ActionCable.server.broadcast 'status', status_channel_data
- ActionCable.server.broadcast 'status_blacklist_manager', status_channel_data.merge(failover_link: failover_link)
- ActionCable.server.broadcast 'topbar', last_ping: @smoke_detector.last_ping.to_f
- ActionCable.server.broadcast 'smokey_pings', smokey: @smoke_detector.as_json
- end
+ ActionCable.server.broadcast 'status', status_channel_data
+ ActionCable.server.broadcast 'status_blacklist_manager', status_channel_data.merge(failover_link: failover_link)
+ ActionCable.server.broadcast 'topbar', last_ping: @smoke_detector.last_ping.to_f
+ ActionCable.server.broadcast 'smokey_pings', smokey: @smoke_detector.as_json
respond_to do |format|
format.json do
diff --git a/app/jobs/queue_graphql_query_job.rb b/app/jobs/queue_graphql_query_job.rb
new file mode 100644
index 000000000..d1a62ed6a
--- /dev/null
+++ b/app/jobs/queue_graphql_query_job.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class QueueGraphqlQueryJob < ApplicationJob
+ queue_as :graphql_queries
+
+ def perform(query, **query_params)
+ puts 'PERF'
+ @results = MetasmokeSchema.execute(query, **query_params)
+ end
+
+ before_enqueue do
+ redis.sadd 'pending_graphql_queries', job_id
+ end
+
+ before_perform do
+ redis.set "graphql_queries/#{job_id}", '', ex: 600
+ redis.srem 'pending_graphql_queries', job_id
+ end
+
+ after_perform do
+ redis.set "graphql_queries/#{job_id}", JSON.generate(@results), ex: 300
+ end
+end
diff --git a/app/jobs/revalidate_flag_conditions_job.rb b/app/jobs/revalidate_flag_conditions_job.rb
new file mode 100644
index 000000000..346027b11
--- /dev/null
+++ b/app/jobs/revalidate_flag_conditions_job.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class RevalidateFlagConditionsJob < ApplicationJob
+ queue_as :default
+
+ def perform
+ FlagCondition.where(flags_enabled: true).find_each do |fc|
+ unless fc.validate
+ failures = fc.errors.full_messages
+ fc.flags_enabled = false
+ fc.save(validate: false)
+ ActionCable.server.broadcast 'smokedetector_messages',
+ message: "@#{fc.user&.username&.tr(' ', '')} " \
+ "Your flag condition was disabled: #{failures.join(',')}"
+ end
+ end
+ end
+end
diff --git a/app/jobs/review_queue_recheck_items_job.rb b/app/jobs/review_queue_recheck_items_job.rb
new file mode 100644
index 000000000..14c2c3458
--- /dev/null
+++ b/app/jobs/review_queue_recheck_items_job.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class ReviewQueueRecheckItemsJob < ApplicationJob
+ queue_as :default
+
+ def perform(queue_id)
+ queue = Queue.find(queue_id)
+ queue.items.includes(:reviewable).each do |i|
+ i.update(completed: true) if i.reviewable.should_dq?(queue)
+ end
+ end
+end
diff --git a/app/jobs/search_extend_job.rb b/app/jobs/search_extend_job.rb
new file mode 100644
index 000000000..2f1f0c391
--- /dev/null
+++ b/app/jobs/search_extend_job.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "#{Rails.root}/app/jobs/search_helpers.rb"
+
+class SearchExtendJob < ApplicationJob
+ queue_as :searches
+ include Backburner::Queue
+ queue_respond_timeout SEARCH_JOB_TIMEOUT
+
+ include SearchJobQueryBuilder
+
+ before_enqueue do |job|
+ sid, be_page = job.arguments
+ job_id = job.job_id
+ redis.set "search_jobs/#{job_id}/sid", sid
+ redis.set "search_jobs/#{job_id}/be_page", be_page
+ end
+
+ def perform(sid, be_page)
+ redis.expire "search_jobs/#{job_id}/sid", SEARCH_JOB_TIMEOUT
+ redis.expire "search_jobs/#{job_id}/be_page", SEARCH_JOB_TIMEOUT
+ wrapped_params = JSON.parse(redis.get("searches/#{sid}/params")).symbolize_keys
+ ops = redis.lrange "searches/#{sid}/ops", 0, -1
+ created_at = redis.get "searches/#{sid}/created_at"
+ query = build_query(ops, wrapped_params)
+ query = query.where('posts.created_at < ?', Time.at(created_at.to_i))
+ search_results = query.offset(SEARCH_PAGE_LENGTH * be_page).limit(SEARCH_PAGE_LENGTH).select(:id).map(&:id).pack('I!*')
+ redis.set "searches/#{sid}/results/#{be_page}", search_results
+ redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION
+ end
+
+ after_perform do |job|
+ sid, be_page = job.arguments
+ redis.del "searches/#{sid}/results/#{be_page}/job_id"
+ end
+end
diff --git a/app/jobs/search_helpers.rb b/app/jobs/search_helpers.rb
new file mode 100644
index 000000000..14eb9de02
--- /dev/null
+++ b/app/jobs/search_helpers.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+SEARCH_PAGE_LENGTH = 10_000
+VALID_SEARCH_PARAMS = %i[
+ post_type_include_unmatched autoflagged post_type user_rep_direction
+ site edited or_search option user_reputation reason feedback
+ title title_is_regex title_is_inverse_regex
+ body body_is_like body_is_regex body_is_inverse_regex
+ why why_is_regex why_is_inverse_regex
+ username username_is_regex username_is_inverse_regex
+].freeze
+SEARCH_JOB_TIMEOUT = (60 * 60 * 3)
+SEACH_PAGE_EXPIRATION = 1.hour
+module SearchJobQueryBuilder
+ def build_query(ops, wrapped_params)
+ title, title_operation,
+ body, body_operation,
+ why, why_operation,
+ username, username_operation = ops
+
+ user_reputation = wrapped_params[:user_reputation].to_i || 0
+
+ case wrapped_params[:feedback]
+ when /true/
+ feedback = :is_tp
+ when /false/
+ feedback = :is_fp
+ when /NAA/
+ feedback = :is_naa
+ end
+
+ results = if wrapped_params[:reason].present?
+ Reason.find(wrapped_params[:reason]).posts
+ else
+ Post.all
+ end
+
+ search_string = []
+ search_params = {}
+ [[:username, username, username_operation], [:title, title, title_operation],
+ [:why, why, why_operation]].each do |si|
+ if si[1].present? && si[1] != '%%'
+ search_string << "IFNULL(`posts`.`#{si[0]}`, '') #{si[2]} :#{si[0]}"
+ search_params[si[0]] = si[1]
+ end
+ end
+
+ if body.present?
+ if ['LIKE', 'NOT LIKE'].include?(body_operation) && wrapped_params[:body_is_like] != '1'
+ # If the operation would be LIKE, hijack it and use our fulltext index for a search instead.
+ # UNLESS... params[:body_is_like] is set, in which case the user has explicitly specified a LIKE query.
+ results = results.match_search(body, with_search_score: false, posts: :body)
+ else
+ if wrapped_params[:body_is_like] == '1' && !user_signed_in?
+ flash[:warning] = 'Unregistered users cannot use LIKE searches on the body field. Please sign in.'
+ redirect_to(search_path) && return
+ end
+ # Otherwise, it's REGEX or NOT REGEX, which fulltext won't do - fall back on search_string and params
+ search_string << "IFNULL(`posts`.`body`, '') #{body_operation} :body"
+ search_params[:body] = body.present? ? body : '%%'
+ end
+ end
+
+ results = results.where(search_string.join(wrapped_params[:or_search].present? ? ' OR ' : ' AND '), **search_params)
+
+ # results = results.includes(:reasons).includes(:feedbacks) if wrapped_params[:option].nil?
+ results = results.joins(:reasons).joins(:feedbacks) if wrapped_params[:option].nil?
+
+ if feedback.present?
+ results = results.where(feedback => true)
+ elsif wrapped_params[:feedback] == 'conflicted'
+ results = results.where(is_tp: true, is_fp: true)
+ end
+
+ results = case wrapped_params[:user_rep_direction]
+ when '>='
+ if user_reputation > 0
+ results.where('IFNULL(user_reputation, 0) >= :rep', rep: user_reputation)
+ end
+ when '=='
+ results.where('IFNULL(user_reputation, 0) = :rep', rep: user_reputation)
+ when '<='
+ results.where('IFNULL(user_reputation, 0) <= :rep', rep: user_reputation)
+ else
+ results
+ end
+
+ results = results.where(site_id: wrapped_params[:site]) if wrapped_params[:site].present?
+
+ results = results.where('revision_count > 1') if wrapped_params[:edited].present?
+
+ # results = results.includes(feedbacks: [:user])
+
+ case wrapped_params[:autoflagged].try(:downcase)
+ when 'yes'
+ results = results.autoflagged
+ when 'no'
+ results = results.not_autoflagged
+ end
+
+ post_type = case wrapped_params[:post_type].try(:downcase).try(:[], 0)
+ when 'q'
+ 'questions'
+ when 'a'
+ 'a'
+ end
+
+ if post_type.present?
+ unmatched = results.where.not("link LIKE '%/questions/%' OR link LIKE '%/a/%'")
+ results = if wrapped_params[:post_type_include_unmatched]
+ results.where('link like ?', "%/#{post_type}/%").or(unmatched)
+ else
+ results.where('link like ?', "%/#{post_type}/%")
+ end
+ end
+
+ results.distinct.order(Arel.sql('`posts`.`created_at` DESC'))
+ end
+end
diff --git a/app/jobs/search_job.rb b/app/jobs/search_job.rb
new file mode 100644
index 000000000..4c5d1d705
--- /dev/null
+++ b/app/jobs/search_job.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "#{Rails.root}/app/jobs/search_helpers.rb"
+
+class SearchJob < ApplicationJob
+ queue_as :searches
+ include Backburner::Queue
+ queue_respond_timeout SEARCH_JOB_TIMEOUT
+
+ include SearchJobQueryBuilder
+
+ def self.search_page_length
+ SEARCH_PAGE_LENGTH
+ end
+
+ def self.search_job_timeout
+ SEARCH_JOB_TIMEOUT
+ end
+
+ attr_accessor :search_id
+
+ before_enqueue do |job|
+ ops, params = job.arguments
+ job_id = job.job_id
+ wrapped_params = VALID_SEARCH_PARAMS.map { |k| [k, params[k]] }.to_h
+ sid = redis.incr 'search_counter'
+ redis.set "search_jobs/#{job_id}/sid", sid
+ redis.set "search_jobs/#{job_id}/be_page", 0
+ redis.set "searches/#{sid}/params", JSON.generate(wrapped_params)
+ redis.rpush "searches/#{sid}/ops", ops
+ end
+
+ def perform(ops, params)
+ # sid = @sid
+ redis.expire "search_jobs/#{job_id}/sid", SEARCH_JOB_TIMEOUT
+ redis.expire "search_jobs/#{job_id}/be_page", SEARCH_JOB_TIMEOUT
+ sid = redis.get("search_jobs/#{job_id}/sid").to_i
+ wrapped_params = VALID_SEARCH_PARAMS.map { |k| [k, params[k]] }.to_h
+ created_at = Time.now.to_i
+ redis.set "searches/#{sid}/created_at", created_at
+
+ results = build_query(ops, wrapped_params)
+ results = results.where('posts.created_at < ?', Time.at(created_at))
+
+ counts_by_accuracy_group = results.group(:is_tp, :is_fp, :is_naa).count
+ counts_by_feedback = %i[is_tp is_fp is_naa].each_with_index.map do |symbol, i|
+ [symbol, counts_by_accuracy_group.select { |k, _v| k[i] }.values.sum]
+ end.to_h
+ redis.set "searches/#{sid}/result_count", results.count
+ redis.set "searches/#{sid}/counts_by_accuracy_group", JSON.generate(counts_by_accuracy_group)
+ redis.set "searches/#{sid}/counts_by_feedback", JSON.generate(counts_by_feedback)
+ redis.set "searches/#{sid}/results/0", results.limit(SEARCH_PAGE_LENGTH).map(&:id).pack('I!*')
+ redis.expire "searches/#{sid}/results/0", SEACH_PAGE_EXPIRATION
+ end
+end
diff --git a/app/jobs/update_chat_ids_job.rb b/app/jobs/update_chat_ids_job.rb
new file mode 100644
index 000000000..e8b4ca7d4
--- /dev/null
+++ b/app/jobs/update_chat_ids_job.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class UpdateChatIdsJob < ApplicationJob
+ queue_as :default
+
+ def perform(user_id)
+ u = User.find(user_id)
+ u.update_chat_ids
+ u.save!
+ end
+end
diff --git a/app/jobs/update_moderator_sites_job.rb b/app/jobs/update_moderator_sites_job.rb
new file mode 100644
index 000000000..809f0fb21
--- /dev/null
+++ b/app/jobs/update_moderator_sites_job.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class UpdateModeratorSitesJob < ApplicationJob
+ queue_as :default
+
+ def perform(user_id)
+ User.find(user_id).update_moderator_sites
+ end
+end
diff --git a/app/jobs/validate_flag_condition_for_user_job.rb b/app/jobs/validate_flag_condition_for_user_job.rb
index a0d280a6f..2616637b4 100644
--- a/app/jobs/validate_flag_condition_for_user_job.rb
+++ b/app/jobs/validate_flag_condition_for_user_job.rb
@@ -3,8 +3,8 @@
class ValidateFlagConditionForUserJob < ApplicationJob
queue_as :default
- def perform(user_id)
- FlagCondition.validate_for_user(User.find(user_id), User.find(-1))
+ def perform(user_id, other_user_id = -1)
+ FlagCondition.validate_for_user(User.find(user_id), User.find(other_user_id))
# Do something later
end
end
diff --git a/app/models/flag_condition.rb b/app/models/flag_condition.rb
index 3254e4dfb..6e31e9e50 100644
--- a/app/models/flag_condition.rb
+++ b/app/models/flag_condition.rb
@@ -30,19 +30,6 @@ def accuracy
true_positive_count.to_f * 100 / post_feedback_results.count.to_f
end
- def self.revalidate_all
- FlagCondition.where(flags_enabled: true).find_each do |fc|
- unless fc.validate
- failures = fc.errors.full_messages
- fc.flags_enabled = false
- fc.save(validate: false)
- ActionCable.server.broadcast 'smokedetector_messages',
- message: "@#{fc.user&.username&.tr(' ', '')} " \
- "Your flag condition was disabled: #{failures.join(',')}"
- end
- end
- end
-
def self.overall_accuracy(user)
query = File.read(Rails.root.join('lib/queries/overall_accuracy.sql'))
sanitized = ActiveRecord::Base.sanitize_sql([query, user_id: user.id])
diff --git a/app/views/graphql/pending_job.html.erb b/app/views/graphql/pending_job.html.erb
new file mode 100644
index 000000000..a2dd77ba2
--- /dev/null
+++ b/app/views/graphql/pending_job.html.erb
@@ -0,0 +1,7 @@
+<% if @time_elapsed.nil? %>
+Your job is in the queue. Keep refreshing this page to see once it starts running.
+<% else %>
+
Your job is still pending, it has been running for <%= @time_elapsed %> seconds and will terminate after 600 seconds
+Reload the page to see if your job has completed. If you get a 404, it has expired.
+<% end %>
+After your job has completed, it will be available for 5 minutes
diff --git a/app/views/search/new_search.html.erb b/app/views/search/new_search.html.erb
new file mode 100644
index 000000000..4c7d6ec76
--- /dev/null
+++ b/app/views/search/new_search.html.erb
@@ -0,0 +1,93 @@
+<% title "Search" %>
+
+<%= form_tag (params[:option] == 'graphs' ? create_search_path(anchor: "graphs") : create_search_path), method: "get" do |f| %>
+
+ <%= label_tag :title %>
+ <% if current_user&.can_use_regex_search? %>
+ <%= check_box_tag :title_is_regex, 1, params[:title_is_regex] %> regex
+ <%= check_box_tag :title_is_inverse_regex, 1, params[:title_is_inverse_regex] %> invert
+ <% end %>
+ <%= text_field_tag :title, params[:title], placeholder: "*", class: "form-control" %>
+
+
+
+ <%= label_tag :username %>
+ <% if current_user&.can_use_regex_search? %>
+ <%= check_box_tag :username_is_regex, 1, params[:username_is_regex] %> regex
+ <%= check_box_tag :username_is_inverse_regex, 1, params[:username_is_inverse_regex] %> invert
+ <% end %>
+ <%= text_field_tag :username, params[:username], placeholder: "*", class: "form-control" %>
+
+
+ <%= label_tag :why %>
+ <% if current_user&.can_use_regex_search? %>
+ <%= check_box_tag :why_is_regex, 1, params[:why_is_regex] %> regex
+ <%= check_box_tag :why_is_inverse_regex, 1, params[:why_is_inverse_regex] %> invert
+ <% end %>
+ <%= text_field_tag :why, params[:why], placeholder: "*", class: "form-control" %>
+
+
+ <%= label_tag :or_search, "Use OR for text search (uses AND by default)" %>
+ <%= check_box_tag :or_search, 1, params[:or_search] %>
+
+
+ <% h = HTMLEntities.new %>
+ <%= label_tag :site %>
+ <%= select_tag :site, options_for_select(Site.select(:site_name, :id).all.order(:site_name).map {|s| [h.decode(s.site_name), s.id] }, params[:site]), class: "form-control", include_blank: "All" %>
+
+
+
+ <%= label_tag :post_type %>
+ <%= check_box_tag :post_type_include_unmatched, 1, params[:post_type_include_unmatched] %> include unmatched
+ <%= select_tag :post_type, options_for_select(%i[question answer], params[:post_type]), class: "form-control", include_blank: "All" %>
+
+
+
+ <%= label_tag :feedback %>
+ <%= select_tag :feedback, options_for_select(["true positive", "false positive", "NAA", "conflicted"], params[:feedback]), class: "form-control", include_blank: "All" %>
+
+
+
+ <%= label_tag :autoflagged %>
+ <%= select_tag :autoflagged, options_for_select(["Yes", "No"], params[:autoflagged]), class: "form-control", include_blank: "All" %>
+
+
+
+ <%= label_tag :reason %>
+ <%= select_tag :reason, options_from_collection_for_select(Reason.all.order(:reason_name), "id", "reason_name", params[:reason]), class: "form-control", include_blank: "All" %>
+
+
+
+
+
+ <%= label_tag :edited, "Edited" %>
+ <%= check_box_tag :edited, 1, params[:edited] %>
+
+
+ <%= hidden_field_tag :option, params[:option] if params[:option] == 'graphs' %>
+
+ <%= submit_tag "Search", class: "btn btn-primary" %>
+<% end %>
diff --git a/app/views/search/search_pending.html.erb b/app/views/search/search_pending.html.erb
new file mode 100644
index 000000000..60549256a
--- /dev/null
+++ b/app/views/search/search_pending.html.erb
@@ -0,0 +1,5 @@
+<% if @sid_ttl == -1 %>
+Your search is pending. Please keep reloading and eventually it'll work :)
+<% else %>
+
Your search has been running for <%= SearchJob.search_job_timeout - @sid_ttl %> seconds...
+<% end %>
diff --git a/app/views/search/search_results.html.erb b/app/views/search/search_results.html.erb
new file mode 100644
index 000000000..b83a348bc
--- /dev/null
+++ b/app/views/search/search_results.html.erb
@@ -0,0 +1,125 @@
+
+
+ - ">
+ <%= link_to "Results (#{@counts_by_accuracy_group.values.sum})", search_path(request.query_parameters.merge!({option: nil, feedback_filter: nil})) %>
+
+
+
+
+<% if params[:option].nil? %>
+
+ <%= render partial: "posts/post", collection: @results, as: :post %>
+
+
+ <%= will_paginate @results, renderer: BootstrapPagination::Rails %>
+
+ <%= link_to "JSON (still subject to paging limits)", search_path(params: request.query_parameters, format: :json) %>
+
+
+ <% h = HTMLEntities.new %>
+ <%= pie_chart @results.group(:site).count.map{ |k,v| {(k.nil? ? "Unknown" : h.decode(k.site_name))=>v} }.reduce(:merge).select{|k,v| k != "Unknown"}.sort_by {|k,v| v}.reverse %>
+
+ <%= line_chart [{name: "True positives", data: @results.where(is_tp: true).group_by_day('`posts`.`created_at`', range: 1.month.ago.to_date..Time.now).count}, {name: "False positives", data: @results.where(is_fp: true).group_by_day('`posts`.`created_at`', range: 1.month.ago.to_date..Time.now).count}], colors: ["green", "red"], library: {"tooltip" => {'shared' => 'true'} } %>
+
+<% end %>
diff --git a/config/application.rb b/config/application.rb
index f798db97e..1a6e5552c 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -22,6 +22,8 @@ class Application < Rails::Application
# config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
# config.i18n.default_locale = :de
+ config.active_job.queue_adapter = :backburner
+
config.middleware.insert_before 0, Rack::Cors do
allow do
origins '*'
diff --git a/config/initializers/backburner.rb b/config/initializers/backburner.rb
new file mode 100644
index 000000000..399b3ebef
--- /dev/null
+++ b/config/initializers/backburner.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+Backburner.configure do |config|
+ config.beanstalk_url = 'beanstalk://127.0.0.1'
+ config.tube_namespace = 'metasmoke.production'
+ config.namespace_separator = '.'
+ config.on_error = ->(e) { puts e }
+ # config.max_job_retries = 3 # default 0 retries
+ # config.retry_delay = 2 # default 5 seconds
+ # config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+ config.default_priority = 65_536
+ config.respond_timeout = 600
+ config.default_worker = Backburner::Workers::Simple
+ config.logger = Logger.new(STDOUT)
+ # config.priority_labels = { :custom => 50, :useless => 1000 }
+ # config.default_worker = Backburner::Workers::ThreadsOnFork
+ config.reserve_timeout = nil
+ config.job_serializer_proc = ->(body) { JSON.dump(body) }
+ config.job_parser_proc = ->(body) { JSON.parse(body) }
+end
diff --git a/config/routes.rb b/config/routes.rb
index fa002a90f..38e6a494c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -18,8 +18,17 @@
resources :dumps
+ authenticate(:user, ->(user) { user.has_role?(:developer) }) do
+ mount BeanstalkdView::Server, :at => "/beanstalkd" if Rails.env.development?
+ end
+
get 'sites/dash', to: 'dashboard#site_dash', as: :site_dash
+
+ get 'search_job', to: 'search#new_search', as: :new_search_job
+ get 'search_job/create', to: 'search#create_search', as: :create_search
+ get 'search_job/pending/:job_id', to: 'search#search_pending', as: :search_pending
+ get 'search_job/:sid', to: 'search#search_results', as: :search_results
get 'search', to: 'search#index'
get 'search_fast', to: 'search#index_fast'
get 'reasons', to: 'dashboard#index', as: :reasons
@@ -160,6 +169,7 @@
post 'graphql', to: 'graphql#execute', as: :graphql
get 'graphql', to: 'graphql#query', as: :query_graphql
+ get 'graphql/view_job/:job_id', to: 'graphql#view_job', as: :view_graphql_job
end
mount GraphiQL::Rails::Engine, at: '/api/graphiql', graphql_path: '/api/graphql', query_params: true
diff --git a/config/schedule.rb b/config/schedule.rb
index 802779ba1..43791e3da 100644
--- a/config/schedule.rb
+++ b/config/schedule.rb
@@ -15,9 +15,9 @@
every 1.day, at: '2:00 am' do
runner 'User.where.not(stack_exchange_account_id: nil).each { |u| u.update_moderator_sites; sleep(5) }'
- runner 'FlagCondition.revalidate_all'
+ runner 'RevalidateFlagConditionsJob.perform_now'
runner 'AbuseReport.update_stale_reports'
- runner 'ConflictingFeedbackJob.perform_later'
+ runner 'ConflictingFeedbackJob.perform_now'
end
# every 1.day, at: '3:10am' do
@@ -25,5 +25,5 @@
# end
every 1.day, at: '3:00am' do
- runner 'ScheduledMailJob.perform_later'
+ runner 'ScheduledMailJob.perform_now'
end
diff --git a/test/jobs/queue_graphql_query_job_test.rb b/test/jobs/queue_graphql_query_job_test.rb
new file mode 100644
index 000000000..6183191a0
--- /dev/null
+++ b/test/jobs/queue_graphql_query_job_test.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class QueueGraphqlQueryJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/jobs/revalidate_flag_conditions_job_test.rb b/test/jobs/revalidate_flag_conditions_job_test.rb
new file mode 100644
index 000000000..6f16b6def
--- /dev/null
+++ b/test/jobs/revalidate_flag_conditions_job_test.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class RevalidateFlagConditionsJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/jobs/review_queue_recheck_items_job_test.rb b/test/jobs/review_queue_recheck_items_job_test.rb
new file mode 100644
index 000000000..65d332fac
--- /dev/null
+++ b/test/jobs/review_queue_recheck_items_job_test.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class ReviewQueueRecheckItemsJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/jobs/search_job_test.rb b/test/jobs/search_job_test.rb
new file mode 100644
index 000000000..49c1679ec
--- /dev/null
+++ b/test/jobs/search_job_test.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class SearchJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/jobs/update_chat_ids_job_test.rb b/test/jobs/update_chat_ids_job_test.rb
new file mode 100644
index 000000000..66a4b9080
--- /dev/null
+++ b/test/jobs/update_chat_ids_job_test.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class UpdateChatIdsJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end
diff --git a/test/jobs/update_moderator_sites_job_test.rb b/test/jobs/update_moderator_sites_job_test.rb
new file mode 100644
index 000000000..8ee548bc2
--- /dev/null
+++ b/test/jobs/update_moderator_sites_job_test.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class UpdateModeratorSitesJobTest < ActiveJob::TestCase
+ # test "the truth" do
+ # assert true
+ # end
+end