diff --git a/.ruby-version b/.ruby-version index f01bc44..a831048 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.1.4 +ruby-2.3.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fffe67..846603a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] +### Added +- Accept `expired_at` from the client in order to do our own cleaning (and not rely on getting delete requests) + ## [1.1.0] - 2016-03-21 ### Added - Re-raise error if job worker runs into one. Makes errors more visible. diff --git a/MIT-LICENSE b/MIT-LICENSE index b8aa94e..c47b622 100644 --- a/MIT-LICENSE +++ b/MIT-LICENSE @@ -1,4 +1,4 @@ -Copyright 2015 G5 +Copyright 2016 G5 MIT License diff --git a/README.md b/README.md index 34dbed6..714c638 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,37 @@ If you use `protected_attributes`, in an initializer: Asyncapi::Server::Job.attr_accessible :status, :callback_url, :class_name, :params, :secret ``` +## Usage without Asyncapi::Client + +If you want to use this without asyncapi client, you need to prepare two things: the endpoint that asyncapi-server will reply to. + +Create the job by POSTing the following to CreateSomething above: + +```json +{ + "job": { + "callback_url": "https://myclient.com/jobs_callback", + "params": { + "name": "Something's name", + "approved": true + }, + "secret": "A secret unique to this job, so that you know what job the server is referring to" + } +} +``` + +When the server is done processing, it will post something to your client. Your endpoint must accept the following json as the body: + +``` +{ + "job": { + "status": "success", + "message": "The output of the Runner class (i.e. `CreateSomething`)", + "secret": "The secret you had sent earlier (this is how you can be sure it's not someone else updating your endpoint)", + } +} +``` + ### RSpec If you want to create an integration spec for you Asyncapi server endpoint, make sure you require the helper: @@ -70,9 +101,15 @@ asyncapi_post("/api/v1/long_running_job", name: "Compute") This helper calls `post` underneath but builds the request in a way that Asyncapi server understands. +## Development + +- Run `rake db:migrate && rake db:migrate RAILS_ENV=test` +- Make changes +- `rspec` + ## License -Copyright (c) 2015 G5 +Copyright (c) 2016 G5 MIT License diff --git a/app/models/asyncapi/server/job.rb b/app/models/asyncapi/server/job.rb index a251c3b..95fd20b 100644 --- a/app/models/asyncapi/server/job.rb +++ b/app/models/asyncapi/server/job.rb @@ -4,6 +4,10 @@ class Job < ActiveRecord::Base self.table_name = "asyncapi_server_jobs" enum status: %i[queued success error] + scope :expired, -> do + expired_at = arel_table[:expired_at] + where.not(expired_at: nil).where(expired_at.lt(Time.now)) + end def url Engine.routes.url_helpers.v1_job_url(self) diff --git a/app/serializers/asyncapi/server/job_serializer.rb b/app/serializers/asyncapi/server/job_serializer.rb index 667d70f..9eca372 100644 --- a/app/serializers/asyncapi/server/job_serializer.rb +++ b/app/serializers/asyncapi/server/job_serializer.rb @@ -2,7 +2,7 @@ module Asyncapi module Server class JobSerializer < ActiveModel::Serializer - attributes :id, :url, :secret + attributes :id, :url, :secret, :expired_at end end diff --git a/app/workers/asyncapi/server/cleaner_worker.rb b/app/workers/asyncapi/server/cleaner_worker.rb new file mode 100644 index 0000000..0fdb749 --- /dev/null +++ b/app/workers/asyncapi/server/cleaner_worker.rb @@ -0,0 +1,22 @@ +module Asyncapi + module Server + class CleanerWorker + + include Sidekiq::Worker + sidekiq_options retry: false + + def perform + Job.expired.find_each(&:destroy) + end + + end + end +end + +if Sidekiq.server? + Sidekiq::Cron::Job.create({ + name: "Delete expired jobs", + cron: Asyncapi::Server.clean_job_cron, + klass: Asyncapi::Server::CleanerWorker.name, + }) +end diff --git a/asyncapi_server.gemspec b/asyncapi_server.gemspec index a7fe333..08d23bb 100644 --- a/asyncapi_server.gemspec +++ b/asyncapi_server.gemspec @@ -19,10 +19,12 @@ Gem::Specification.new do |s| s.add_dependency "kaminari" s.add_dependency "api-pagination" - s.add_dependency "rails" - s.add_dependency "active_model_serializers" + s.add_dependency "rails", "~> 4.0" + s.add_dependency "active_model_serializers", "~> 0.9.0" s.add_dependency "typhoeus" s.add_dependency "sidekiq" + s.add_dependency "sidekiq-cron" + s.add_dependency "responders", "~> 2.0" s.add_development_dependency "sqlite3", "~> 1.3.10" s.add_development_dependency "rspec-rails", "~> 3.1.0" diff --git a/db/migrate/20150130062520_add_expired_at_to_asyncapi_server_job.rb b/db/migrate/20150130062520_add_expired_at_to_asyncapi_server_job.rb deleted file mode 100644 index 68502dc..0000000 --- a/db/migrate/20150130062520_add_expired_at_to_asyncapi_server_job.rb +++ /dev/null @@ -1,11 +0,0 @@ -class AddExpiredAtToAsyncapiServerJob < ActiveRecord::Migration - def change - add_column :asyncapi_server_jobs, :expired_at, :datetime - - Asyncapi::Server::Job.reset_column_information - - Asyncapi::Server::Job. - where(Asyncapi::Server::Job.arel_table[:expired_at].eq(nil)). - update_all(expired_at: Asyncapi::Server.expiry_threshold.from_now) - end -end diff --git a/db/migrate/20150201231018_drop_expired_at_from_asyncapi_server_jobs.rb b/db/migrate/20150201231018_drop_expired_at_from_asyncapi_server_jobs.rb deleted file mode 100644 index 911d4d2..0000000 --- a/db/migrate/20150201231018_drop_expired_at_from_asyncapi_server_jobs.rb +++ /dev/null @@ -1,9 +0,0 @@ -class DropExpiredAtFromAsyncapiServerJobs < ActiveRecord::Migration - def up - remove_column :asyncapi_server_jobs, :expired_at - end - - def down - add_column :asyncapi_server_jobs, :expired_at, :datetime - end -end diff --git a/db/migrate/20160721002627_add_expired_at_to_asyncapi_server_jobs.rb b/db/migrate/20160721002627_add_expired_at_to_asyncapi_server_jobs.rb new file mode 100644 index 0000000..bc8f75e --- /dev/null +++ b/db/migrate/20160721002627_add_expired_at_to_asyncapi_server_jobs.rb @@ -0,0 +1,5 @@ +class AddExpiredAtToAsyncapiServerJobs < ActiveRecord::Migration + def change + add_column :asyncapi_server_jobs, :expired_at, :datetime + end +end diff --git a/lib/asyncapi/server.rb b/lib/asyncapi/server.rb index b05a74b..c67f46b 100644 --- a/lib/asyncapi/server.rb +++ b/lib/asyncapi/server.rb @@ -1,10 +1,12 @@ +require 'active_model_serializers' +require 'responders' require "asyncapi/server/engine" module Asyncapi module Server CONFIGURATION = { - expiry_threshold: 10.days, + clean_job_cron: "0 * * * *", } CONFIGURATION.each do |var, default| diff --git a/lib/asyncapi/server/rails_ext/controller.rb b/lib/asyncapi/server/rails_ext/controller.rb index c26d069..d34bff4 100644 --- a/lib/asyncapi/server/rails_ext/controller.rb +++ b/lib/asyncapi/server/rails_ext/controller.rb @@ -23,6 +23,7 @@ def job_params_with(class_name) :class_name, :params, :secret, + :expired_at, ) end diff --git a/spec/dummy/db/migrate/20141212065005_create_asyncapi_server_jobs.asyncapi_server.rb b/spec/dummy/db/migrate/20141212065005_create_asyncapi_server_jobs.asyncapi_server.rb deleted file mode 100644 index 2d56856..0000000 --- a/spec/dummy/db/migrate/20141212065005_create_asyncapi_server_jobs.asyncapi_server.rb +++ /dev/null @@ -1,11 +0,0 @@ -# This migration comes from asyncapi_server (originally 20141112034324) -class CreateAsyncapiServerJobs < ActiveRecord::Migration - def change - create_table :asyncapi_server_jobs do |t| - t.integer :status - t.string :callback_url - t.string :class_name - t.text :params - end - end -end diff --git a/spec/dummy/db/migrate/20141212065006_add_secret_to_asyncapi_server_job.asyncapi_server.rb b/spec/dummy/db/migrate/20141212065006_add_secret_to_asyncapi_server_job.asyncapi_server.rb deleted file mode 100644 index 5682b3c..0000000 --- a/spec/dummy/db/migrate/20141212065006_add_secret_to_asyncapi_server_job.asyncapi_server.rb +++ /dev/null @@ -1,6 +0,0 @@ -# This migration comes from asyncapi_server (originally 20141212064931) -class AddSecretToAsyncapiServerJob < ActiveRecord::Migration - def change - add_column :asyncapi_server_jobs, :secret, :string - end -end diff --git a/spec/dummy/db/migrate/20150130062901_add_expired_at_to_asyncapi_server_job.asyncapi_server.rb b/spec/dummy/db/migrate/20150130062901_add_expired_at_to_asyncapi_server_job.asyncapi_server.rb deleted file mode 100644 index 8fb063c..0000000 --- a/spec/dummy/db/migrate/20150130062901_add_expired_at_to_asyncapi_server_job.asyncapi_server.rb +++ /dev/null @@ -1,12 +0,0 @@ -# This migration comes from asyncapi_server (originally 20150130062520) -class AddExpiredAtToAsyncapiServerJob < ActiveRecord::Migration - def change - add_column :asyncapi_server_jobs, :expired_at, :datetime - - Asyncapi::Server::Job.reset_column_information - - Asyncapi::Server::Job. - where(Asyncapi::Server::Job.arel_table[:expired_at].eq(nil)). - update_all(expired_at: Asyncapi::Server.expiry_threshold.from_now) - end -end diff --git a/spec/dummy/db/migrate/20150201231329_drop_expired_at_from_asyncapi_server_jobs.asyncapi_server.rb b/spec/dummy/db/migrate/20150201231329_drop_expired_at_from_asyncapi_server_jobs.asyncapi_server.rb deleted file mode 100644 index 6095090..0000000 --- a/spec/dummy/db/migrate/20150201231329_drop_expired_at_from_asyncapi_server_jobs.asyncapi_server.rb +++ /dev/null @@ -1,10 +0,0 @@ -# This migration comes from asyncapi_server (originally 20150201231018) -class DropExpiredAtFromAsyncapiServerJobs < ActiveRecord::Migration - def up - remove_column :asyncapi_server_jobs, :expired_at - end - - def down - add_column :asyncapi_server_jobs, :expired_at, :datetime - end -end diff --git a/spec/dummy/db/schema.rb b/spec/dummy/db/schema.rb index 83a013b..ca68d92 100644 --- a/spec/dummy/db/schema.rb +++ b/spec/dummy/db/schema.rb @@ -11,14 +11,15 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150201231329) do +ActiveRecord::Schema.define(version: 20160721002627) do - create_table "asyncapi_server_jobs", force: true do |t| - t.integer "status" - t.string "callback_url" - t.string "class_name" - t.text "params" - t.string "secret" + create_table "asyncapi_server_jobs", force: :cascade do |t| + t.integer "status" + t.string "callback_url" + t.string "class_name" + t.text "params" + t.string "secret" + t.datetime "expired_at" end end diff --git a/spec/models/job_spec.rb b/spec/models/job_spec.rb index 69eef9e..86f5529 100644 --- a/spec/models/job_spec.rb +++ b/spec/models/job_spec.rb @@ -39,6 +39,25 @@ module Server end end + describe ".expired" do + let!(:job_1) do + create(:asyncapi_server_job, expired_at: 2.minutes.ago) + end + let!(:job_2) do + create(:asyncapi_server_job, expired_at: 2.minutes.from_now) + end + let!(:job_3) do + create(:asyncapi_server_job, expired_at: 2.minutes.ago) + end + let!(:job_4) do + create(:asyncapi_server_job, expired_at: nil) + end + + it "returns expired job" do + expect(Job.expired).to match_array([job_1, job_3]) + end + end + end end end diff --git a/spec/requests/enqueueing_jobs_spec.rb b/spec/requests/enqueueing_jobs_spec.rb index 8e9c84e..ca2961d 100644 --- a/spec/requests/enqueueing_jobs_spec.rb +++ b/spec/requests/enqueueing_jobs_spec.rb @@ -2,7 +2,10 @@ describe "Enqueueing jobs", type: :request do - let(:job) { build_stubbed(:asyncapi_server_job, secret: "secret") } + let(:job) do + build_stubbed(:asyncapi_server_job, secret: "secret", expired_at: expired_at) + end + let(:expired_at) { 2.days.from_now } before do allow(Asyncapi::Server::Job).to receive(:create).with( @@ -10,24 +13,27 @@ callback_url: "callback_url", params: {client: "params"}.to_json, secret: "secret", + expired_at: expired_at.to_s, ).and_return(job) allow(job).to receive(:url).and_return("server_job_url") end - it "allows asynchronous handing of http requests and cleans up old jobs" do + it "allows asynchronous handling of http requests and cleans up old jobs" do expect(Asyncapi::Server::JobWorker).to receive(:perform_async).with(job.id) - post("tests", job: { + post("/tests", job: { callback_url: "callback_url", params: {client: "params"}.to_json, secret: "secret", + expired_at: expired_at, }) expect(response).to be_successful parsed_response = indifferent_hash(response.body)[:job] expect(parsed_response[:url]).to eq "server_job_url" expect(parsed_response[:secret]).to eq "secret" + expect(Time.parse(parsed_response[:expired_at]).to_i).to eq expired_at.to_i end end diff --git a/spec/serializers/job_serializer_spec.rb b/spec/serializers/job_serializer_spec.rb index 8710ae6..90d1962 100644 --- a/spec/serializers/job_serializer_spec.rb +++ b/spec/serializers/job_serializer_spec.rb @@ -3,11 +3,13 @@ module Asyncapi::Server describe JobSerializer do - let(:job) { build_stubbed(:asyncapi_server_job) } + let(:expired_at) { 2.days.from_now } + let(:job) { build_stubbed(:asyncapi_server_job, expired_at: expired_at) } let(:serializer) { described_class.new(job) } subject(:serialized_hash) { serializer.attributes } its([:id]) { is_expected.to eq job.id } + its([:expired_at]) { is_expected.to eq expired_at } it "has a url" do allow(job).to receive(:url).and_return("url") diff --git a/spec/server_spec.rb b/spec/server_spec.rb index 3d8129c..46b2e50 100644 --- a/spec/server_spec.rb +++ b/spec/server_spec.rb @@ -3,20 +3,5 @@ module Asyncapi describe Server do - describe ".expiry_threshold=" do - let!(:original_expiry_threshold) { described_class.expiry_threshold } - after { described_class.expiry_threshold = original_expiry_threshold } - it "sets the threshold for expiring old jobs" do - described_class.expiry_threshold = 2.days - expect(described_class.expiry_threshold).to eq 2.days - end - end - - describe ".expiry_threshold" do - it "defaults to 10 days" do - expect(described_class.expiry_threshold).to eq 10.days - end - end - end end diff --git a/spec/workers/cleaner_worker_spec.rb b/spec/workers/cleaner_worker_spec.rb new file mode 100644 index 0000000..6ecbc9e --- /dev/null +++ b/spec/workers/cleaner_worker_spec.rb @@ -0,0 +1,33 @@ +require 'spec_helper' + +module Asyncapi + module Server + RSpec.describe CleanerWorker do + + it "does not retry" do + expect(described_class.sidekiq_options_hash['retry']).to be false + end + + describe "#perform" do + let!(:job_1) do + create(:asyncapi_server_job, expired_at: 2.minutes.ago) + end + let!(:job_2) do + create(:asyncapi_server_job, expired_at: 2.minutes.from_now) + end + let!(:job_3) do + create(:asyncapi_server_job, expired_at: 2.minutes.ago) + end + let!(:job_4) do + create(:asyncapi_server_job, expired_at: nil) + end + + it "deletes jobs that are expired" do + described_class.new.perform + expect(Job.all).to match_array([job_2, job_4]) + end + end + + end + end +end