Merge pull request #8254 from opf/feature/delayed-cron-jobs

Add cron job functionality to delayed_job

.pkgr.yml

@@ -28,11 +28,6 @@ targets:
       - sqlite3-devel
 before_precompile: "packaging/setup"
 after_precompile: "packaging/teardown"
-crons:
-  - packaging/cron/openproject-hourly-tasks
-  - packaging/cron/openproject-clear-old-sessions
-  - packaging/cron/openproject-clear-uploaded-files
-  - packaging/cron/openproject-clear-tmp-cache
 services:
   - postgres
 installer: https://github.com/pkgr/installer.git

Gemfile

@@ -111,6 +111,7 @@ gem 'oj', '~> 3.10.2'
 
 gem 'daemons'
 gem 'delayed_job_active_record', '~> 4.1.4'
+gem 'delayed_cron_job', '~> 0.7.2'
 
 gem 'rack-protection', '~> 2.0.8'
 

Gemfile.lock

@@ -418,6 +418,8 @@ GEM
     declarative-builder (0.1.0)
       declarative-option (< 0.2.0)
     declarative-option (0.1.0)
+    delayed_cron_job (0.7.2)
+      delayed_job (>= 4.1)
     delayed_job (4.1.8)
       activesupport (>= 3.0, < 6.1)
     delayed_job_active_record (4.1.4)
@@ -984,6 +986,7 @@ DEPENDENCIES
   database_cleaner (~> 1.8)
   date_validator (~> 0.9.0)
   deckar01-task_list (~> 2.3.1)
+  delayed_cron_job (~> 0.7.2)
   delayed_job_active_record (~> 4.1.4)
   doorkeeper (~> 5.3.1)
   equivalent-xml (~> 0.6)

app/workers/cron/clear_old_sessions_job.rb

@@ -0,0 +1,42 @@
+#-- encoding: UTF-8
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) 2012-2020 the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2017 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See docs/COPYRIGHT.rdoc for more details.
+#++
+
+module Cron
+  class ClearOldSessionsJob < CronJob
+    include ::RakeJob
+
+    # runs at 1:15 nightly
+    self.cron_expression = '15 1 * * *'
+
+    def perform
+      super 'db:sessions:expire', 7
+    end
+  end
+end

app/workers/cron/clear_tmp_cache_job.rb

@@ -0,0 +1,42 @@
+#-- encoding: UTF-8
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) 2012-2020 the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2017 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See docs/COPYRIGHT.rdoc for more details.
+#++
+
+module Cron
+  class ClearTmpCacheJob < CronJob
+    include ::RakeJob
+
+    # runs at 02:45 sundays
+    self.cron_expression = '45 2 * * 7'
+
+    def perform
+      super 'tmp:cache:clear'
+    end
+  end
+end

app/workers/cron/clear_uploaded_files_job.rb

@@ -0,0 +1,42 @@
+#-- encoding: UTF-8
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) 2012-2020 the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2017 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See docs/COPYRIGHT.rdoc for more details.
+#++
+
+module Cron
+  class ClearUploadedFilesJob < CronJob
+    include ::RakeJob
+
+    # Runs 23pm fridays
+    self.cron_expression = '0 23 * * 5'
+
+    def perform
+      super 'attachments:clear'
+    end
+  end
+end

app/workers/cron/cron_job.rb

@@ -0,0 +1,78 @@
+#-- encoding: UTF-8
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) 2012-2020 the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2017 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See docs/COPYRIGHT.rdoc for more details.
+#++
+
+module Cron
+  class CronJob < ApplicationJob
+    class_attribute :cron_expression
+
+    # List of registered jobs, requires eager load in dev(!)
+    class_attribute :registered_jobs, default: []
+
+    class << self
+      ##
+      # Register new job class(es)
+      def register!(*job_classes)
+        Array(job_classes).each do |clz|
+          raise ArgumentError, "Needs to be subclass of ::Cron::CronJob" unless clz.ancestors.include?(self)
+
+          registered_jobs << clz
+        end
+      end
+
+      ##
+      # Ensure the job is scheduled unless it is already
+      def ensure_scheduled!
+        # Ensure scheduled only onced
+        return if scheduled?
+
+        Rails.logger.info { "Scheduling #{name} recurrent background job." }
+        set(cron: cron_expression).perform_later
+      end
+
+      ##
+      # Remove the scheduled job, if any
+      def remove
+        delayed_job&.destroy
+      end
+
+      ##
+      # Is there a job scheduled?
+      def scheduled?
+        delayed_job.present?
+      end
+
+      def delayed_job
+        Delayed::Job
+          .where('handler LIKE ?', "%job_class: #{name}%")
+          .first
+      end
+    end
+  end
+end

app/workers/oauth/cleanup_job.rb

@@ -29,10 +29,12 @@
 #++
 
 module OAuth
-  class CleanupJob < ::RakeJob
+  class CleanupJob < ::ApplicationJob
+    include ::RakeJob
+
     queue_with_priority :low
 
-    def initialize
+    def perform
       super 'doorkeeper:db:cleanup'
     end
   end

app/workers/rake_job.rb

@@ -30,11 +30,13 @@ require 'rake'
 ##
 # Invoke a rake task while safely loading the tasks only once
 # to ensure they are neither loaded nor executed twice.
-class RakeJob < ApplicationJob
+module RakeJob
   attr_reader :task_name
+  attr_reader :args
 
-  def perform(task_name)
+  def perform(task_name, *args)
     @task_name = task_name
+    @args = args
 
     Rails.logger.info { "Invoking Rake task #{task_name}." }
     invoke
@@ -44,17 +46,13 @@ class RakeJob < ApplicationJob
 
   def invoke
     load_tasks!
-    task.invoke
+    task.invoke *args
   end
 
-  private
-
   ##
   # Load tasks if there are none. This should only be run once in an environment
   def load_tasks!
-    raise unless tasks_loaded?
-  rescue StandardError
-    OpenProject::Application.load_rake_tasks
+    OpenProject::Application.load_rake_tasks unless tasks_loaded?
   end
 
   ##

config/initializers/cronjobs.rb

@@ -0,0 +1,9 @@
+# Register "Cron-like jobs"
+
+OpenProject::Application.configure do |application|
+  application.config.to_prepare do
+    ::Cron::CronJob.register! ::Cron::ClearOldSessionsJob,
+                              ::Cron::ClearTmpCacheJob,
+                              ::Cron::ClearUploadedFilesJob
+  end
+end

db/migrate/20200403105252_add_cron_to_delayed_jobs.rb

@@ -0,0 +1,9 @@
+class AddCronToDelayedJobs < ActiveRecord::Migration[6.0]
+  def self.up
+    add_column :delayed_jobs, :cron, :string
+  end
+
+  def self.down
+    remove_column :delayed_jobs, :cron
+  end
+end

lib/open_project/plugins/acts_as_op_engine.rb

@@ -290,6 +290,16 @@ module OpenProject::Plugins
         end
       end
 
+      ##
+      # Register a "cron"-like background job
+      def add_cron_jobs(&block)
+        config.to_prepare do
+          Array(block.call).each do |clz|
+            ::Cron::CronJob.register!(clz.is_a?(Class) ? clz : clz.to_s.constantize)
+          end
+        end
+      end
+
       # Add custom inflection for file name to class name mapping. Otherwise, the default zeitwerk
       # #camelize method will be utilized.
       #

lib/tasks/cron.rake

@@ -31,4 +31,9 @@ namespace 'openproject:cron' do
   task :hourly do
     # Does nothing by default
   end
+
+  desc 'Ensure the cron-like background jobs are actively scheduled'
+  task schedule: [:environment] do
+    ::Cron::CronJob.registered_jobs.each(&:ensure_scheduled!)
+  end
 end

lib/tasks/delayed_job.rake

@@ -37,3 +37,9 @@ end
 Rake::Task['jobs:environment_options']
   .clear_prerequisites
   .enhance(['environment:full'])
+
+# Enhance delayed job workers to use cron
+load 'lib/tasks/cron.rake'
+Rake::Task["jobs:work"].enhance [:"openproject:cron:schedule"]
+Rake::Task["jobs:workoff"].enhance [:"openproject:cron:schedule"]
+

lib/tasks/environment.rake

@@ -36,4 +36,9 @@ namespace 'environment' do
     # Require the environment, bypassing the default :environment flag
     Rails.application.require_environment!
   end
+
+  desc 'Eager load the application (only applicable in dev mode)'
+  task eager_load: :environment do
+    ::Zeitwerk::Loader.eager_load_all if Rails.env.development?
+  end
 end

modules/costs/app/workers/ldap_groups/synchronization_job.rb

@@ -0,0 +1,50 @@
+#-- encoding: UTF-8
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) 2012-2020 the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2017 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See docs/COPYRIGHT.rdoc for more details.
+#++
+
+module LdapGroups
+  class SynchronizationJob < ::Cron::CronJob
+    # Run every 30 minutes
+    self.cron_expression = '*/30 * * * *'
+
+    def perform
+      return unless EnterpriseToken.allows_to?(:ldap_groups)
+
+      begin
+        LdapAuthSource.find_each do |ldap|
+          Rails.logger.info { "[LDAP groups] Start synchronization for ldap auth source #{ldap.name}" }
+          OpenProject::LdapGroups::Synchronization.new(ldap)
+        end
+      rescue StandardError => e
+        msg = "[LDAP groups] Failed to run LDAP group synchronization. #{e.class.name}: #{e.message}"
+        Rails.logger.error msg
+      end
+    end
+  end
+end

modules/ldap_groups/lib/open_project/ldap_groups/engine.rb

@@ -20,6 +20,8 @@ module OpenProject::LdapGroups
            caption: ->(*) { I18n.t('ldap_groups.label_menu_item') }
     end
 
+    add_cron_jobs { LdapGroups::SynchronizationJob }
+
     patches %i[AuthSource Group]
   end
 end

modules/ldap_groups/lib/open_project/ldap_groups/synchronization.rb

@@ -8,13 +8,10 @@ module OpenProject::LdapGroups
       # Get current synced groups in OP
       @synced_groups = ::LdapGroups::SynchronizedGroup.where(auth_source: ldap)
 
-      begin
-        synchronize!
-      rescue => e
-        error = "Failed to perform LDAP group synchronization: #{e.class}: #{e.message}"
-        Rails.logger.error(error)
-        warn error
-      end
+      synchronize!
+    rescue StandardError => e
+      error = "[LDAP groups] Failed to perform LDAP group synchronization: #{e.class}: #{e.message}"
+      Rails.logger.error(error)
     end
 
     def synchronize!
@@ -72,11 +69,11 @@ module OpenProject::LdapGroups
     # Add new users to the synced group
     def add_memberships!(new_users, sync)
       if new_users.empty?
-        puts "No new users to add for #{sync.entry}"
+        Rails.logger.info "[LDAP groups] No new users to add for #{sync.entry}"
         return
       end
 
-      puts "Adding users #{new_users.pluck(:login)} to #{sync.entry}"
+      Rails.logger.info { "[LDAP groups] Adding users #{new_users.pluck(:login)} to #{sync.entry}" }
       sync.users << new_users.map {|user| ::LdapGroups::Membership.new(group: sync, user: user)}
       sync.group.users << new_users
     end
@@ -85,11 +82,11 @@ module OpenProject::LdapGroups
     # Remove a set of memberships
     def remove_memberships!(memberships, sync)
       if memberships.empty?
-        puts "No users to remove for #{sync.entry}"
+        Rails.logger.info "[LDAP groups] No users to remove for #{sync.entry}"
         return
       end
 
-      puts "Removing users #{memberships.pluck(:user_id)} from #{sync.entry}"
+      Rails.logger.info "[LDAP groups] Removing users #{memberships.pluck(:user_id)} from #{sync.entry}"
       sync.remove_members!(memberships)
     end
   end

modules/ldap_groups/lib/tasks/ldap_groups.rake

@@ -31,20 +31,7 @@ namespace :ldap_groups do
   desc 'Synchronize groups and their users from the LDAP auth source.' \
        'Will only synchronize for those users already present in the application.'
   task synchronize: :environment do
-
-    next unless EnterpriseToken.allows_to?(:ldap_groups)
-
-    begin
-      LdapAuthSource.find_each do |ldap|
-        puts ("-" * 20)
-        puts "Synchronizing for ldap auth source #{ldap.name}"
-        OpenProject::LdapGroups::Synchronization.new(ldap)
-      end
-    rescue =>  e
-      msg = "Failed to run LDAP group synchronization. #{e.class.name}: #{e.message}"
-      Rails.logger.error msg
-      warn msg
-    end
+    ::LdapGroups::SynchronizationJob.perform_now
   end
 
   namespace :development do
@@ -93,9 +80,3 @@ namespace :ldap_groups do
     end
   end
 end
-
-# Ensure core cron task is loaded
-load 'lib/tasks/cron.rake'
-Rake::Task["openproject:cron:hourly"].enhance do
-  Rake::Task["ldap_groups:synchronize"].invoke
-end

modules/ldap_groups/spec/lib/synchronization_spec.rb

@@ -176,7 +176,10 @@ describe OpenProject::LdapGroups::Synchronization, with_ee: %i[ldap_groups] do
         group_foo.users << user_aa729
 
         # Outputs that nothing was added to sync group
-        expect { subject }.to output("No users to remove for foo\nNo new users to add for foo\n").to_stdout
+        expect(Rails.logger).to receive(:info).with("[LDAP groups] No users to remove for foo")
+        expect(Rails.logger).to receive(:info).with("[LDAP groups] No new users to add for foo")
+
+        subject
 
         # Does not add to synced group since added manually
         expect(synced_foo.users.count).to eq(0)
@@ -212,7 +215,8 @@ describe OpenProject::LdapGroups::Synchronization, with_ee: %i[ldap_groups] do
     end
 
     it 'does not raise, but print to stderr' do
-      expect { subject }.to output(/Failed to perform LDAP group synchronization/).to_stderr
+      expect(Rails.logger).to receive(:error).with(/Failed to perform LDAP group synchronization/)
+      subject
     end
   end
 

packaging/cron/openproject-clear-old-sessions

@@ -1,3 +0,0 @@
-APP_NAME="_APP_NAME_"
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-15 1 * * * root ${APP_NAME} run rake -s db:sessions:expire[7] >> /var/log/${APP_NAME}/cron-clear-old-sessions.log 2>&1

packaging/cron/openproject-clear-tmp-cache

@@ -1,3 +0,0 @@
-APP_NAME="_APP_NAME_"
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-45 2 * * 7 root ${APP_NAME} run rake -s tmp:cache:clear >> /var/log/${APP_NAME}/cron-clear-tmp-files.log 2>&1

packaging/cron/openproject-clear-uploaded-files

@@ -1,3 +0,0 @@
-APP_NAME="_APP_NAME_"
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-0 23 * * 5 root ${APP_NAME} run rake -s attachments:clear >> /var/log/${APP_NAME}/cron-clear-uploaded-files.log 2>&1

packaging/cron/openproject-hourly-tasks

@@ -1,3 +0,0 @@
-APP_NAME="_APP_NAME_"
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-55 * * * * root ${APP_NAME} run rake -s openproject:cron:hourly >> /var/log/${APP_NAME}/cron-hourly.log 2>&1