openproject/script/bulk_run_rspec

#!/usr/bin/env ruby

# Runs the given specs multiple times to check if they are reliable.
#
# Tests to run are fetched from tmp/spec_examples.txt matching /features/ pattern.
# Results are stored in tmp/bulk_run
#     results.txt   summary
#     logs/*.log    one log file per test run

# rubocop:disable Rails
require 'ostruct'
require 'pathname'
require 'fileutils'

RAILS_ROOT = Pathname.new File.expand_path('../', __dir__)
RSPEC_EXAMPLE_STATUS_PERSISTENCE_FILE_PATH = RAILS_ROOT.join('tmp/spec_examples.txt')
RUN_COUNT = ENV['BULK_RUN_COUNT'] ? ENV['BULK_RUN_COUNT'].to_i : 5
PATTERN = ENV['BULK_RUN_PATTERN'] ? Regexp.new(ENV['BULK_RUN_PATTERN']) : /features/

def normalized(path)
  path.sub(/^\.\//, '').tr('/', '__').tr('[]:', '_')
end

class String
  # Colors, colors everywhere
  {
    red: 31,
    green: 32,
    brown: 33
  }.each do |color, ansi_code|
    define_method(color) { "\e[#{ansi_code}m#{self}\e[0m" }
  end
end

class Test
  attr_accessor :path, :runs

  def initialize(path)
    @path = path
    @runs = []
  end

  def to_line
    [path, normalized_name, runs.count, runs.select(&:passed?).count, runs.select(&:failed?).count].join(' | ')
  end

  def normalized_name
    normalized(path)
  end
end

class Run
  attr_accessor :output
  attr_reader :result, :path, :start, :end

  def initialize(path)
    @path = path
    @result = nil
    @start = Time.now
    @end = nil
    @output = nil
  end

  def failed!
    self.result = :failed
  end

  def failed?
    result == :failed
  end

  def passed!
    self.result = :passed
  end

  def passed?
    result == :passed
  end

  def result=(result)
    @result = result
    @end = Time.now
  end
end

class BulkRunner
  def found_tests
    @found_tests ||=
      if ARGV.any?
        ARGV
      else
        lines = File.readlines(RSPEC_EXAMPLE_STATUS_PERSISTENCE_FILE_PATH)
        lines = lines.grep(PATTERN)
        lines.map { |line| line.split('|').first.strip }
      end
  end

  def tests
    @tests ||= found_tests.map { |test_path| Test.new(test_path) }
  end

  # rubocop:disable Metrics/AbcSize
  def run
    FileUtils.rm_rf(bulk_run_dir('logs')) if bulk_run_dir('logs').exist?
    `mkdir -p #{bulk_run_dir('logs')}`
    puts "#{tests.count} tests to run"
    tests.each do |test|
      puts '=' * 80
      puts "Running #{test.path} #{RUN_COUNT} times"
      puts '=' * 80
      RUN_COUNT.times do |i|
        puts " Run #{i} ".center(80, '-')
        run = Run.new(test.path)
        run.output = `CI=true bundle exec rspec '#{run.path}' 2>&1`
        if $?.success?
          puts 'ok'.green
          run.passed!
        else
          puts 'ko'.red
          run.failed!
        end
        File.write(bulk_run_dir("logs/#{test.normalized_name}.#{run.result}.#{i}.log"), run.output)
        test.runs << run
      end

      if test.runs.all?(&:passed?)
        puts 'PASSED'.green
      else
        puts 'FAILED'.red
      end
    end
  end
  # rubocop:enable Metrics/AbcSize

  def bulk_run_dir(path)
    RAILS_ROOT.join('tmp/bulk_run').join(path)
  end

  def save
    puts "Saving"
    File.open(bulk_run_dir('results.txt'), 'w') do |results|
      results.puts('test_path | log_prefix | count | passed | failed')
      tests.each do |test|
        results.puts(test.to_line)
      end
    end
    puts "Done"
  end
end

bulk_runner = BulkRunner.new
begin
  bulk_runner.run
rescue Interrupt
  puts "Interrupted. Stopping and Saving.".brown
ensure
  bulk_runner.save
end
puts "results in #{bulk_runner.bulk_run_dir('')}"

# rubocop:enable Rails