Delayed Job monitoring

Gemfile

@@ -11,6 +11,7 @@ group :test do
   gem 'active_interaction', '~> 3.0'
   gem 'appsignal'
   gem 'combustion', '~> 0.5.2'
+  gem 'fakefs', require: 'fakefs/safe'
   gem 'guard-rspec', '~> 4.2.6'
   gem 'responders'
   gem 'rspec-instrumentation-matcher'

Gemfile.lock

@@ -65,6 +65,7 @@ GEM
       delayed_job (>= 3.0, < 5)
     diff-lcs (1.3)
     erubis (2.7.0)
+    fakefs (0.15.0)
     ffi (1.9.17)
     formatador (0.2.5)
     guard (2.14.0)
@@ -183,6 +184,7 @@ DEPENDENCIES
   bundler (~> 1.0)
   combustion (~> 0.5.2)
   delayed_job_active_record
+  fakefs
   guard-rspec (~> 4.2.6)
   rake
   responders
@@ -195,4 +197,4 @@ DEPENDENCIES
   timecop
 
 BUNDLED WITH
-   1.16.1
+   1.16.2

README.md

@@ -100,6 +100,14 @@ class ApiAreaController < ApplicationController
   ...
 ```
 
+# Delayed Job activity monitoring
+
+To add monitoring of whether Delayed Job keeps picking up jobs or checking for new jobs, add the following line to an initializer:
+
+```ruby
+require 'roqua/core_ext/delayed_job/activity_monitoring'
+```
+
 # AppSignal Probes
 
 roqua-support contains one generic probe to monitor the current delayed job backlog count: `Roqua::Probes::DelayedJobProbe`

lib/roqua/core_ext/delayed_job/activity_monitoring.rb

+require 'delayed_job'
+
+#
+# Updates the modification time of the file $RAILS_ROOT/tmp/delayed_job_activity when
+# relevant worker activity occurs. To be used for monitoring whether a DelayedJob
+# worker is still picking up new work in a timely fashion.
+#
+# A file based approach is used so it is easy to add a health check to a Docker container
+# with the following command:
+# `find /app/tmp -mmin -$MAXIMUM_AGE_OF_FILE_IN_MINUTES -type f -print | grep delayed_job_activity`
+#
+module DelayedJobActivityMonitoring
+  def work_off(num = 100)
+    FileUtils.touch(Rails.root.join('tmp', 'delayed_job_activity'))
+    super(num)
+  end
+
+  protected
+
+  def reserve_job
+    super.tap do |job|
+      if job
+        FileUtils.touch(
+          Rails.root.join('tmp', 'delayed_job_activity'),
+          mtime: job.max_run_time.from_now
+        )
+      end
+    end
+  end
+end
+
+Delayed::Worker.send(:prepend, DelayedJobActivityMonitoring) if defined?(Delayed) && defined?(Delayed::Worker)

spec/roqua/core_ext/delayed_job/activity_monitoring_spec.rb

+require 'spec_helper'
+require 'pathname'
+require 'timecop'
+require 'fakefs/safe'
+
+require 'roqua/core_ext/delayed_job/activity_monitoring'
+
+describe DelayedJobActivityMonitoring do
+  after do
+    Timecop.return
+  end
+
+  def initialize_fake_rails_root
+    FileUtils.mkdir_p(File.join('/app', 'tmp'))
+    allow(Rails).to receive(:root).and_return(Pathname.new('/app'))
+  end
+
+  let(:monitoring_filename) { Rails.root.join('tmp', 'delayed_job_activity').to_s }
+
+  it 'creates a file' do
+    FakeFS.with_fresh do
+      initialize_fake_rails_root
+      expect { Delayed::Worker.new.work_off }
+        .to change { File.exist?(monitoring_filename) }
+
+      expect(File.mtime(monitoring_filename))
+        .to be_within(1.second).of(Time.now)
+    end
+  end
+
+  it 'updates the file modification time' do
+    FakeFS.with_fresh do
+      initialize_fake_rails_root
+
+      Delayed::Worker.new.work_off
+
+      Timecop.travel(1.minute)
+
+      expect { Delayed::Worker.new.work_off }
+        .to change { File.mtime(monitoring_filename) }
+
+      expect(File.mtime(monitoring_filename))
+        .to be_within(1.second).of(Time.now)
+    end
+  end
+
+  describe 'when a job is reserved' do
+    let(:max_run_time) { 2.hours }
+    let(:job) do
+      double(
+        name: 'name',
+        id: 123,
+        queue: 'queue',
+        max_run_time: max_run_time,
+        invoke_job: nil,
+        destroy: nil
+      )
+    end
+
+    it 'creates a file with a modification time in the future' do
+      FakeFS.with_fresh do
+        initialize_fake_rails_root
+
+        worker = Delayed::Worker.new
+        expect(Delayed::Job).to receive(:reserve).and_return(job)
+
+        expect { worker.send(:reserve_and_run_one_job) }
+          .to change { File.exist?(monitoring_filename) }
+
+        expect(File.mtime(monitoring_filename))
+          .to be_within(1.second).of(max_run_time.from_now)
+      end
+    end
+  end
+end