Merge pull request #3725 from opf/plugin-parallel-testing

Parallel testing for plugins

doc/RUNNING_TESTS.md

@@ -100,7 +100,7 @@ You can run the specs with the following commands:
 
 ### Cucumber
 
-NOTE: *We do not write new cucumber features. The current plan is to move away from
+**Note:** *We do not write new cucumber features. The current plan is to move away from
 cucumber towards regular specs using Capybara. For the time being however, please keep the existing
 cucumber features green or write your feature specs in Capybara for any  code that is not already
 covered by cucumber.*
@@ -178,6 +178,7 @@ If you need Firebug and Firepath while debugging a scenario, just replace
 
 Running tests in parallel makes usage of all available cores of the machine.
 Functionality is being provided by [parallel_tests](https://github.com/grosser/parallel_tests) gem.
+See the github page for any options like number of cpus used.
 
 #### Prepare
 
@@ -195,28 +196,50 @@ Prepare all databases:
 
 `RAILS_ENV=test parallel_test -e "rake db:drop db:create db:migrate"`
 
-**Note: Until `rake db:schema:load` we have to use the command above. Then we
+**Note: Until `rake db:schema:load` works we have to use the command above. Then we
 can use `rake parallel:prepare`**
 
+You may also just dump your current schema with `rake db:schema:dump` (db/schema.rb)
+is not part of the repository. Then you can just use `rake parallel:prepare` to prepare
+test databases.
 
 #### RSpec legacy specs
 
-Run all legacy specs in parallel:
+Run all legacy specs in parallel with `rake parallel:spec_legacy`
 
-`parallel_test -t rspec -o '-I spec_legacy' spec_legacy`
+Or run them manually with `parallel_test -t rspec -o '-I spec_legacy' spec_legacy`
 
 #### RSpec specs
 
-Run all specs in parallel:
+Run all specs in parallel with `rake parallel:spec`
 
-`parallel_test -t rspec spec`
+Or run them manually with `parallel_test -t rspec spec`.
 
 #### Cucumber
 
-Run all cucumber features in parallel:
+Run all cucumber features in parallel with `rake parallel:cucumber`.
 
-`parallel_test -t cucumber -o '-r features' features`
+Or run them manually with `parallel_test -t cucumber -o '-r features' features`.
 
+**Note:** there is also a official rake task to run cucumber features but the OpenProject cucumber
+test suite requires `-r features` to run correctly. This needs to be passed to the command
+thus it looks not very handy `rake parallel:features\[,,"-r features"\]`
+(this is zsh compatible, command takes three arguments but we just want to pass the last one here.)
+
+#### Plugins
+
+Run specs for all activated plugins with `rake parallel:plugins:spec`.
+
+Run cucumber features for all activated plugins with `rake parallel:plugins:cucumber`.
+
+#### Full test suite
+
+You may run all existing parts of OpenProject test suite in parallel with
+`rake parallel:all`
+
+**Note:** This will run core specs, core cucumber features, core legacy specs,
+plugin specs and plugin cucumber features. This task will take around 40 minutes
+on a machine with 8 parallel instances.
 
 ## For the fancy programmer
 

lib/plugins/load_path_helper.rb

@@ -0,0 +1,50 @@
+#-- copyright
+# OpenProject is a project management system.
+# Copyright (C) 2012-2015 the OpenProject Foundation (OPF)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See doc/COPYRIGHT.rdoc for more details.
+#++
+
+module Plugins
+  module LoadPathHelper
+    def self.spec_load_paths
+      plugin_load_paths.map { |path|
+        File.join(path, 'spec')
+      }.keep_if{ |path| File.directory?(path) }
+    end
+
+    def self.cucumber_load_paths
+      plugin_load_paths.map { |path|
+        File.join(path, 'features')
+      }.keep_if{ |path| File.directory?(path) }
+    end
+
+    private
+
+    # fetch load paths for available plugins
+    def self.plugin_load_paths
+      Rails.application.config.plugins_to_test_paths.map(&:to_s)
+    end
+  end
+end

lib/tasks/cucumber.rake

@@ -50,28 +50,17 @@ unless ARGV.any? { |a| a =~ /\Agems/ } # Don't load anything when running the ge
         ::CodeStatistics::TEST_TYPES << 'Cucumber features' if File.exist?('features')
       end
 
-      def get_plugin_features(prefix = nil)
-        features = []
-        Rails.application.config.plugins_to_test_paths.each do |dir|
-          feature_dir = Shellwords.escape(File.join(dir, 'features'))
-          if File.directory?(feature_dir)
-            features << prefix unless prefix.nil?
-            features << feature_dir
-          end
-        end
-        features
-      end
-
       def define_cucumber_task(name, description, arguments = [])
         desc description
-        task name, arguments => ['db:test:prepare', 'assets:webpack'] do |_t, args|
+        # task name, arguments => ['db:test:prepare', 'assets:webpack'] do |_t, args|
+        task name, arguments => ['db:test:prepare'] do |_t, args|
           if name == :custom
             if not args[:features]
               raise 'Please provide :features argument, e.g. rake cucumber:custom[features/my_feature.feature]'
             end
             features = args[:features].split(/\s+/)
           else
-            features = get_plugin_features
+            features = Plugins::LoadPathHelper.cucumber_load_paths
 
             if name == :plugins
               # in case we want to run cucumber plugins and there are none
@@ -85,7 +74,7 @@ unless ARGV.any? { |a| a =~ /\Agems/ } # Don't load anything when running the ge
             end
 
             if name == :all
-              features += [File.join(Rails.root, 'features')]
+              features << File.join(Rails.root, 'features')
             end
           end
 
@@ -94,9 +83,10 @@ unless ARGV.any? { |a| a =~ /\Agems/ } # Don't load anything when running the ge
             ENV.delete('CUCUMBER_OPTS')
             opts += args[:options].split(/\s+/) if args[:options]
 
-            # load feature support files from Rails root
-            support_files = ['-r', Shellwords.escape(File.join(Rails.root, 'features'))]
-            support_files += get_plugin_features(prefix = '-r')
+            support_files = [Rails.root.join('features').to_s] + Plugins::LoadPathHelper.cucumber_load_paths
+            support_files =  support_files.map { |path|
+              ['-r', Shellwords.escape(path)]
+            }.flatten
 
             t.cucumber_opts = opts + support_files + features
 

lib/tasks/parallel_testing.rake

@@ -0,0 +1,98 @@
+#-- encoding: UTF-8
+#-- copyright
+# OpenProject is a project management system.
+# Copyright (C) 2012-2015 the OpenProject Foundation (OPF)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See doc/COPYRIGHT.rdoc for more details.
+#++
+
+namespace :parallel do
+  namespace :plugins do
+
+    desc 'Run plugin specs in parallel'
+    task spec: [:environment] do
+      ParallelTests::Tasks.check_for_pending_migrations
+
+      num_cpus       = ENV['GROUP_SIZE']
+      group          = ENV['GROUP']
+      runtime_log    = ENV['RUNTIME_LOG'] || 'tmp/parallel_runtime_plugins_rspec.log'
+
+      group_options       = num_cpus ? "-n #{num_cpus}" : ''
+      group_options      += " --only-group #{group}" if group
+      runtime_log_option  = "--runtime-log #{runtime_log}"
+
+      spec_folders = Plugins::LoadPathHelper.spec_load_paths.join(' ')
+
+      # Change this if changed in spec/support/rspec_failures.rb
+      if File.exist? 'tmp/rspec-examples.txt'
+        sh 'rm tmp/rspec-examples.txt'
+      end
+
+      cmd = "bundle exec parallel_test --type rspec #{runtime_log_option} #{group_options} #{spec_folders} || \
+             bundle exec rspec --only-failures"
+
+      sh cmd
+    end
+
+    desc 'Run plugin cucumber features in parallel'
+    task cucumber: [:environment] do
+      ParallelTests::Tasks.check_for_pending_migrations
+
+      num_cpus       = ENV['GROUP_SIZE']
+      group          = ENV['GROUP']
+      runtime_log    = ENV['RUNTIME_LOG'] || 'tmp/parallel_runtime_plugins_cucumber.log'
+
+      group_options       = num_cpus ? "-n #{num_cpus}" : ''
+      group_options      += " --only-group #{group}" if group
+      runtime_log_option  = "--runtime-log #{runtime_log}"
+
+      support_files = [Rails.root.join('features').to_s] + Plugins::LoadPathHelper.cucumber_load_paths
+      support_files = support_files.map { |path|
+        ['-r', Shellwords.escape(path)]
+      }.flatten.join(' ')
+
+      feature_folders  = Plugins::LoadPathHelper.cucumber_load_paths.join(' ')
+      cucumber_options = "-o '#{support_files}'"
+
+      cmd = "bundle exec parallel_test --type cucumber #{cucumber_options} #{runtime_log_option} #{group_options} #{feature_folders}"
+
+      sh cmd
+    end
+  end
+
+  desc 'Run all suites in parallel (one after another)'
+  task all: [:spec, :cucumber, :spec_legacy, 'parallel:plugins:spec', 'parallel:plugins:cucumber']
+
+
+  desc 'Run legacy specs in parallel'
+  task :spec_legacy do
+    sh "bundle exec parallel_test --type rspec -o '-I spec_legacy' spec_legacy || \
+        bundle exec rspec -I spec_legacy spec_legacy"
+  end
+
+  desc 'Run cucumber features in parallel (custom task)'
+  task :cucumber do
+    sh "bundle exec parallel_test --type cucumber -o '-r features' features"
+  end
+end

lib/tasks/plugin_specs.rake

@@ -48,47 +48,22 @@ begin
   namespace :spec do
     desc 'Run core and plugin specs'
     RSpec::Core::RakeTask.new(all: [:environment, 'assets:webpack']) do |t|
-      pattern = []
-      dirs = get_plugins_to_test
-      dirs << File.join(Rails.root).to_s
-      dirs.each do |dir|
-        if File.directory?(dir)
-          pattern << File.join(dir, 'spec', '**', '*_spec.rb').to_s
-        end
-      end
-      t.pattern = pattern
+      t.pattern = [Rails.root.join('spec').to_s] + Plugins::LoadPathHelper.spec_load_paths
     end
 
     desc 'Run plugin specs'
     RSpec::Core::RakeTask.new(plugins: [:environment, 'assets:webpack']) do |t|
-      pattern = []
-      get_plugins_to_test.each do |dir|
-        if File.directory?(dir)
-          pattern << File.join(dir, 'spec', '**', '*_spec.rb').to_s
-        end
-      end
-      t.pattern = pattern
+      t.pattern = Plugins::LoadPathHelper.spec_load_paths
 
       # in case we want to run plugins' specs and there are none
       # we exit with positive message
-      if pattern.empty?
+      if t.pattern.empty?
         puts
         puts '##### There are no specs for OpenProject plugins to be run.'
         puts
         exit(0)
       end
-
     end
   end
 rescue LoadError
 end
-
-def get_plugins_to_test
-  plugin_paths = []
-  Rails.application.config.plugins_to_test_paths.each do |dir|
-    if File.directory?(dir)
-      plugin_paths << File.join(dir).to_s
-    end
-  end
-  plugin_paths
-end

spec/support/rspec_failures.rb

@@ -1,5 +1,6 @@
 
 RSpec.configure do |c|
+  # If the filename is being changed change it in lib/tasks/parallel_testing.rake
   c.example_status_persistence_file_path = "tmp/rspec-examples.txt"
   c.run_all_when_everything_filtered = true
 end