From 0a0536bd84b3cd60bac23e82423f55f6b8047506 Mon Sep 17 00:00:00 2001
From: Daan Davidsz <daan@roqua.nl>
Date: Wed, 15 Aug 2018 07:14:34 +0000
Subject: [PATCH] Delayed Job monitoring
---
Gemfile | 1 +
Gemfile.lock | 4 +-
README.md | 8 ++
.../delayed_job/activity_monitoring.rb | 32 ++++++++
.../delayed_job/activity_monitoring_spec.rb | 75 +++++++++++++++++++
5 files changed, 119 insertions(+), 1 deletion(-)
create mode 100644 lib/roqua/core_ext/delayed_job/activity_monitoring.rb
create mode 100644 spec/roqua/core_ext/delayed_job/activity_monitoring_spec.rb
diff --git a/Gemfile b/Gemfile
index 280e35d..8544661 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,6 +11,7 @@ group :test do
gem 'active_interaction', '~> 3.0'
gem 'appsignal'
gem 'combustion', '~> 0.5.2'
+ gem 'fakefs', require: 'fakefs/safe'
gem 'guard-rspec', '~> 4.2.6'
gem 'responders'
gem 'rspec-instrumentation-matcher'
diff --git a/Gemfile.lock b/Gemfile.lock
index 6532b6c..25e647b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -65,6 +65,7 @@ GEM
delayed_job (>= 3.0, < 5)
diff-lcs (1.3)
erubis (2.7.0)
+ fakefs (0.15.0)
ffi (1.9.17)
formatador (0.2.5)
guard (2.14.0)
@@ -183,6 +184,7 @@ DEPENDENCIES
bundler (~> 1.0)
combustion (~> 0.5.2)
delayed_job_active_record
+ fakefs
guard-rspec (~> 4.2.6)
rake
responders
@@ -195,4 +197,4 @@ DEPENDENCIES
timecop
BUNDLED WITH
- 1.16.1
+ 1.16.2
diff --git a/README.md b/README.md
index de2b5d2..ced8891 100644
--- a/README.md
+++ b/README.md
@@ -100,6 +100,14 @@ class ApiAreaController < ApplicationController
...
```
+# Delayed Job activity monitoring
+
+To add monitoring of whether Delayed Job keeps picking up jobs or checking for new jobs, add the following line to an initializer:
+
+```ruby
+require 'roqua/core_ext/delayed_job/activity_monitoring'
+```
+
# AppSignal Probes
roqua-support contains one generic probe to monitor the current delayed job backlog count: `Roqua::Probes::DelayedJobProbe`
diff --git a/lib/roqua/core_ext/delayed_job/activity_monitoring.rb b/lib/roqua/core_ext/delayed_job/activity_monitoring.rb
new file mode 100644
index 0000000..49b2fe2
--- /dev/null
+++ b/lib/roqua/core_ext/delayed_job/activity_monitoring.rb
@@ -0,0 +1,32 @@
+require 'delayed_job'
+
+#
+# Updates the modification time of the file $RAILS_ROOT/tmp/delayed_job_activity when
+# relevant worker activity occurs. To be used for monitoring whether a DelayedJob
+# worker is still picking up new work in a timely fashion.
+#
+# A file based approach is used so it is easy to add a health check to a Docker container
+# with the following command:
+# `find /app/tmp -mmin -$MAXIMUM_AGE_OF_FILE_IN_MINUTES -type f -print | grep delayed_job_activity`
+#
+module DelayedJobActivityMonitoring
+ def work_off(num = 100)
+ FileUtils.touch(Rails.root.join('tmp', 'delayed_job_activity'))
+ super(num)
+ end
+
+ protected
+
+ def reserve_job
+ super.tap do |job|
+ if job
+ FileUtils.touch(
+ Rails.root.join('tmp', 'delayed_job_activity'),
+ mtime: job.max_run_time.from_now
+ )
+ end
+ end
+ end
+end
+
+Delayed::Worker.send(:prepend, DelayedJobActivityMonitoring) if defined?(Delayed) && defined?(Delayed::Worker)
diff --git a/spec/roqua/core_ext/delayed_job/activity_monitoring_spec.rb b/spec/roqua/core_ext/delayed_job/activity_monitoring_spec.rb
new file mode 100644
index 0000000..1bd1e4a
--- /dev/null
+++ b/spec/roqua/core_ext/delayed_job/activity_monitoring_spec.rb
@@ -0,0 +1,75 @@
+require 'spec_helper'
+require 'pathname'
+require 'timecop'
+require 'fakefs/safe'
+
+require 'roqua/core_ext/delayed_job/activity_monitoring'
+
+describe DelayedJobActivityMonitoring do
+ after do
+ Timecop.return
+ end
+
+ def initialize_fake_rails_root
+ FileUtils.mkdir_p(File.join('/app', 'tmp'))
+ allow(Rails).to receive(:root).and_return(Pathname.new('/app'))
+ end
+
+ let(:monitoring_filename) { Rails.root.join('tmp', 'delayed_job_activity').to_s }
+
+ it 'creates a file' do
+ FakeFS.with_fresh do
+ initialize_fake_rails_root
+ expect { Delayed::Worker.new.work_off }
+ .to change { File.exist?(monitoring_filename) }
+
+ expect(File.mtime(monitoring_filename))
+ .to be_within(1.second).of(Time.now)
+ end
+ end
+
+ it 'updates the file modification time' do
+ FakeFS.with_fresh do
+ initialize_fake_rails_root
+
+ Delayed::Worker.new.work_off
+
+ Timecop.travel(1.minute)
+
+ expect { Delayed::Worker.new.work_off }
+ .to change { File.mtime(monitoring_filename) }
+
+ expect(File.mtime(monitoring_filename))
+ .to be_within(1.second).of(Time.now)
+ end
+ end
+
+ describe 'when a job is reserved' do
+ let(:max_run_time) { 2.hours }
+ let(:job) do
+ double(
+ name: 'name',
+ id: 123,
+ queue: 'queue',
+ max_run_time: max_run_time,
+ invoke_job: nil,
+ destroy: nil
+ )
+ end
+
+ it 'creates a file with a modification time in the future' do
+ FakeFS.with_fresh do
+ initialize_fake_rails_root
+
+ worker = Delayed::Worker.new
+ expect(Delayed::Job).to receive(:reserve).and_return(job)
+
+ expect { worker.send(:reserve_and_run_one_job) }
+ .to change { File.exist?(monitoring_filename) }
+
+ expect(File.mtime(monitoring_filename))
+ .to be_within(1.second).of(max_run_time.from_now)
+ end
+ end
+ end
+end
--
GitLab