[25573] Implement health check with okcomputer

Gemfile

@@ -103,6 +103,9 @@ gem 'rack-protection', '~> 2.0.0'
 # https://github.com/kickstarter/rack-attack
 gem 'rack-attack', '~> 5.0.1'
 
+# Providing health checks
+gem 'okcomputer', '~> 1.16.0'
+
 # Patch Rails HTML whitelisting for Angular curly braces
 gem 'rails-angular-xss', git: 'https://github.com/opf/rails-angular-xss', ref: 'a45267d5'
 

Gemfile.lock

@@ -377,6 +377,7 @@ GEM
     nokogiri (1.7.2)
       mini_portile2 (~> 2.1.0)
     oj (3.0.6)
+    okcomputer (1.16.0)
     openproject-token (1.0.0)
       activemodel (~> 5.0)
     parallel (1.11.1)
@@ -668,6 +669,7 @@ DEPENDENCIES
   newrelic_rpm
   nokogiri (~> 1.7.2)
   oj (~> 3.0.6)
+  okcomputer (~> 1.16.0)
   omniauth!
   openproject-token (~> 1.0.0)
   openproject-translations!

config/initializers/health_checks.rb

@@ -0,0 +1,39 @@
+class DelayedJobNeverRanCheck < OkComputer::Check
+  attr_reader :threshold
+
+  def initialize(minute_threshold)
+    @threshold = minute_threshold.to_i
+  end
+
+  def check
+    never_ran = Delayed::Job.where('run_at < ?', threshold.minutes.ago).count
+
+    if never_ran.zero?
+      mark_message "All previous jobs have completed within the past #{threshold} minutes."
+    else
+      mark_failure
+      mark_message "#{never_ran} jobs waiting to be executed for more than #{threshold} minutes"
+    end
+  end
+end
+
+# Mount at /health_checks
+OkComputer.mount_at = 'health_checks'
+
+# Register delayed_job backed up test
+dj_max = OpenProject::Configuration.health_checks_jobs_queue_count_threshold
+OkComputer::Registry.register "delayed_jobs_backed_up",
+                              OkComputer::DelayedJobBackedUpCheck.new(0, dj_max)
+
+dj_never_ran_max = OpenProject::Configuration.health_checks_jobs_never_ran_minutes_ago
+OkComputer::Registry.register "delayed_jobs_never_ran",
+                              DelayedJobNeverRanCheck.new(dj_never_ran_max)
+
+# Make dj backed up optional due to bursts
+OkComputer.make_optional %w(delayed_jobs_backed_up)
+
+# Check if authentication required
+authentication_password = OpenProject::Configuration.health_checks_authentication_password
+if authentication_password.present?
+  OkComputer.require_authentication('health_checks', authentication_password)
+end

lib/open_project/configuration.rb

@@ -96,7 +96,15 @@ module OpenProject
       'onboarding_video_url' => 'https://player.vimeo.com/video/163426858?autoplay=1',
       'onboarding_enabled' => true,
 
-      'ee_manager_visible' => true
+      'ee_manager_visible' => true,
+
+      # Health check configuration
+      'health_checks_authentication_password' => nil,
+      # Maximum number of backed up jobs (that are not yet executed)
+      # before health check fails
+      'health_checks_jobs_queue_count_threshold' => 50,
+      # Maximum number of minutes that jobs have not yet run after their designated 'run_at' time
+      'health_checks_jobs_never_ran_minutes_ago' => 5,
     }
 
     @config = nil