remove mysql specific code throughout the application

pull/7367/head
ulferts 6 years ago
parent 9ee183098a
commit 9c3206220b
No known key found for this signature in database
GPG Key ID: A205708DE1284017
  1. 6
      app/helpers/warning_bar_helper.rb
  2. 18
      app/models/custom_field/order_statements.rb
  3. 11
      app/models/journal.rb
  4. 50
      app/models/journal/aggregated_journal.rb
  5. 6
      app/models/principal.rb
  6. 31
      app/models/user.rb
  7. 34
      app/views/warning_bar/_mysql_database.html.erb
  8. 1
      app/views/warning_bar/_warning_bar.html.erb
  9. 7
      config/locales/en.yml
  10. 13
      db/migrate/20190502102512_ensure_postgres_index_names.rb
  11. 4
      db/migrate/migration_utils/utils.rb
  12. 31
      lib/api/v3/work_packages/eager_loading/checksum.rb
  13. 18
      lib/open_project/database.rb
  14. 24
      lib/open_project/text_formatting/formats/markdown/textile_converter.rb
  15. 108
      lib/tasks/backup.rake
  16. 10
      lib/tasks/shared/attachment_migration.rb
  17. 17
      modules/reporting/app/models/cost_query/custom_field_mixin.rb
  18. 87
      modules/reporting_engine/lib/report/query_utils.rb
  19. 2
      modules/reporting_engine/lib/report/sql_statement.rb
  20. 22
      modules/reporting_engine/lib/reporting_engine/engine.rb
  21. 61
      spec/features/mysql/deprecation_spec.rb
  22. 115
      spec/features/work_packages/table/hierarchy/hierarchy_spec.rb
  23. 9
      spec/lib/database_spec.rb
  24. 3
      spec/models/custom_field_spec.rb
  25. 2
      spec/models/queries/queries/query_query_spec.rb
  26. 8
      spec/models/query/results_spec.rb
  27. 89
      spec/tasks/backup_specs.rb

@ -35,12 +35,6 @@ module WarningBarHelper
OpenProject::Database.migrations_pending?
end
def render_mysql_deprecation_warning?
current_user.admin? &&
OpenProject::Database.mysql? &&
current_layout == 'admin'
end
##
# By default, never show a warning bar in the
# test mode due to overshadowing other elements.

@ -108,15 +108,8 @@ module CustomField::OrderStatements
end
def select_custom_values_as_group
aggr_sql =
if OpenProject::Database.mysql?
"GROUP_CONCAT(cv_sort.value SEPARATOR '.')"
else
"string_agg(cv_sort.value, '.')"
end
<<-SQL
COALESCE((SELECT #{aggr_sql} FROM #{CustomValue.table_name} cv_sort
COALESCE((SELECT string_agg(cv_sort.value, '.') FROM #{CustomValue.table_name} cv_sort
WHERE cv_sort.customized_type='#{self.class.customized_class.name}'
AND cv_sort.customized_id=#{self.class.customized_class.table_name}.id
AND cv_sort.custom_field_id=#{id}
@ -125,15 +118,8 @@ module CustomField::OrderStatements
end
def select_custom_values_joined_options_as_group
aggr_sql =
if OpenProject::Database.mysql?
"GROUP_CONCAT(co_sort.value SEPARATOR '.')"
else
"string_agg(co_sort.value, '.' ORDER BY co_sort.position ASC)"
end
<<-SQL
COALESCE((SELECT #{aggr_sql} FROM #{CustomOption.table_name} co_sort
COALESCE((SELECT string_agg(co_sort.value, '.' ORDER BY co_sort.position ASC) FROM #{CustomOption.table_name} co_sort
LEFT JOIN #{CustomValue.table_name} cv_sort
ON co_sort.id = CAST(cv_sort.value AS decimal(60,3))
WHERE cv_sort.customized_type='#{self.class.customized_class.name}'

@ -56,17 +56,10 @@ class Journal < ActiveRecord::Base
# Ensure that no INSERT/UPDATE/DELETE statements as well as other code inside :with_write_lock
# is run concurrently to the code inside this block, by using database locking.
# Note for PostgreSQL: If this is called from inside a transaction, the lock will last until the
# Note: If this is called from inside a transaction, the lock will last until the
# end of that transaction.
# Note for MySQL: THis method does not currently change anything (no locking at all)
def self.with_write_lock(journable)
lock_name =
if OpenProject::Database.mysql?
# MySQL only supports a single lock
"journals.write_lock"
else
"journal.#{journable.class}.#{journable.id}"
end
lock_name = "journal.#{journable.class}.#{journable.id}"
result = Journal.with_advisory_lock_result(lock_name, timeout_seconds: 60) do
yield

@ -100,10 +100,10 @@ class Journal::AggregatedJournal
# that our own row (master) would not already have been merged by its predecessor. If it is
# (that means if we can find a valid predecessor), we drop our current row, because it will
# already be present (in a merged form) in the row of our predecessor.
Journal.from("(#{sql_rough_group(1, journable, until_version, journal_id)}) #{table_name}")
.joins(Arel.sql("LEFT OUTER JOIN (#{sql_rough_group(2, journable, until_version, journal_id)}) addition
Journal.from("(#{sql_rough_group(journable, until_version, journal_id)}) #{table_name}")
.joins(Arel.sql("LEFT OUTER JOIN (#{sql_rough_group(journable, until_version, journal_id)}) addition
ON #{sql_on_groups_belong_condition(table_name, 'addition')}"))
.joins(Arel.sql("LEFT OUTER JOIN (#{sql_rough_group(3, journable, until_version, journal_id)}) predecessor
.joins(Arel.sql("LEFT OUTER JOIN (#{sql_rough_group(journable, until_version, journal_id)}) predecessor
ON #{sql_on_groups_belong_condition('predecessor', table_name)}"))
.where(Arel.sql('predecessor.id IS NULL'))
.order(Arel.sql("COALESCE(addition.created_at, #{table_name}.created_at) ASC"))
@ -166,7 +166,7 @@ class Journal::AggregatedJournal
# To be able to self-join results of this statement, we add an additional column called
# "group_number" to the result. This allows to compare a group resulting from this query with
# its predecessor and successor.
def sql_rough_group(uid, journable, until_version, journal_id)
def sql_rough_group(journable, until_version, journal_id)
if until_version && !journable
raise 'need to provide a journable, when specifying a version limit'
elsif journable && journable.id.nil?
@ -175,8 +175,8 @@ class Journal::AggregatedJournal
conditions = additional_conditions(journable, until_version, journal_id)
"SELECT predecessor.*, #{sql_group_counter(uid)} AS group_number
FROM #{sql_rough_group_from_clause(uid)}
"SELECT predecessor.*, #{sql_group_counter} AS group_number
FROM journals predecessor
#{sql_rough_group_join(conditions[:join_conditions])}
#{sql_rough_group_where(conditions[:where_conditions])}
#{sql_rough_group_order}"
@ -229,33 +229,10 @@ class Journal::AggregatedJournal
"ORDER BY predecessor.created_at"
end
# The "group_number" required in :sql_rough_group has to be generated differently depending on
# the DBMS used. This method returns the appropriate statement to be used inside a SELECT to
# This method returns the appropriate statement to be used inside a SELECT to
# obtain the current group number.
# The :uid parameter allows to define non-conflicting variable names (for MySQL).
def sql_group_counter(uid)
if OpenProject::Database.mysql?
group_counter = mysql_group_count_variable(uid)
"(#{group_counter} := #{group_counter} + 1)"
else
'row_number() OVER (ORDER BY predecessor.version ASC)'
end
end
# MySQL requires some initialization to be performed before being able to count the groups.
# This method allows to inject further FROM sources to achieve that in a single SQL statement.
# Sadly MySQL requires the whole statement to be wrapped in parenthesis, while PostgreSQL
# prohibits that.
def sql_rough_group_from_clause(uid)
if OpenProject::Database.mysql?
"(journals predecessor, (SELECT #{mysql_group_count_variable(uid)}:=0) number_initializer)"
else
'journals predecessor'
end
end
def mysql_group_count_variable(uid)
"@aggregated_journal_row_counter_#{uid}"
def sql_group_counter
'row_number() OVER (ORDER BY predecessor.version ASC)'
end
# Similar to the WHERE statement used in :sql_rough_group. However, this condition will
@ -281,13 +258,8 @@ class Journal::AggregatedJournal
return '(true = true)'
end
if OpenProject::Database.mysql?
difference = "TIMESTAMPDIFF(second, #{predecessor}.created_at, #{successor}.created_at)"
threshold = aggregation_time_seconds
else
difference = "(#{successor}.created_at - #{predecessor}.created_at)"
threshold = "interval '#{aggregation_time_seconds} second'"
end
difference = "(#{successor}.created_at - #{predecessor}.created_at)"
threshold = "interval '#{aggregation_time_seconds} second'"
"(#{difference} > #{threshold})"
end

@ -82,12 +82,6 @@ class Principal < ActiveRecord::Base
firstnamelastname = "((firstname || ' ') || lastname)"
lastnamefirstname = "((lastname || ' ') || firstname)"
# special concat for mysql
if OpenProject::Database.mysql?
firstnamelastname = "CONCAT(CONCAT(firstname, ' '), lastname)"
lastnamefirstname = "CONCAT(CONCAT(lastname, ' '), firstname)"
end
s = "%#{q.to_s.downcase.strip.tr(',', '')}%"
where(['LOWER(login) LIKE :s OR ' +

@ -238,7 +238,7 @@ class User < Principal
def self.activate_user!(user, session)
if session[:invitation_token]
token = Token::Invitation.find_by_plaintext_value session[:invitation_token]
invited_id = token && token.user.id
invited_id = token&.user&.id
if user.id == invited_id
user.activate!
@ -476,28 +476,28 @@ class User < Principal
# Find a user account by matching the exact login and then a case-insensitive
# version. Exact matches will be given priority.
def self.find_by_login(login)
# force string comparison to be case sensitive on MySQL
type_cast = (OpenProject::Database.mysql?) ? 'BINARY' : ''
# First look for an exact match
user = where(["#{type_cast} login = ?", login]).first
user = find_by(login: login)
# Fail over to case-insensitive if none was found
user ||= where(["#{type_cast} LOWER(login) = ?", login.to_s.downcase]).first
user || where(["LOWER(login) = ?", login.to_s.downcase]).first
end
def self.find_by_rss_key(key)
return nil unless Setting.feeds_enabled?
token = Token::Rss.find_by(value: key)
if token && token.user.active?
if token&.user&.active?
token.user
end
end
def self.find_by_api_key(key)
return nil unless Setting.rest_api_enabled?
token = Token::Api.find_by_plaintext_value(key)
if token && token.user.active?
if token&.user&.active?
token.user
end
end
@ -513,16 +513,11 @@ class User < Principal
skip_suffix_check, regexp = mail_regexp(mail)
# If the recipient part already contains a suffix, don't expand
return where("LOWER(mail) = ?", mail) if skip_suffix_check
command =
if OpenProject::Database.mysql?
'REGEXP'
else
'~*'
end
where("LOWER(mail) #{command} ?", regexp)
if skip_suffix_check
where("LOWER(mail) = ?", mail)
else
where("LOWER(mail) ~* ?", regexp)
end
end
##
@ -563,7 +558,7 @@ class User < Principal
roles = []
# No role on archived projects
return roles unless project && project.active?
return roles unless project&.active?
# Return all roles if user is admin
return Role.givable.to_a if admin?

@ -1,34 +0,0 @@
<div id="mysql-db-warning" class="warning-bar--item">
<span class="icon3 icon-warning warning-bar--disable-on-hover"></span>
<p>
<strong><%= t('mysql_deprecation.notice') %></strong>
<br/>
<%= static_link_to :postgres_migration %>
</p>
</div>
<%= nonced_javascript_tag do %>
(function() {
// Hide initially
var message = document.getElementById('mysql-db-warning');
try {
var hidden = window.localStorage.getItem('mysql-db-warning-ignore') === '1';
if (hidden) {
message.style.display = 'none';
}
} catch (e) {
console.error('Failed to access your browsers local storage.');
}
// Click handler to hide
var span = message.querySelector('.warning-bar--disable-on-hover');
span.onclick = function() {
message.style.display = 'none';
try {
window.localStorage.setItem('mysql-db-warning-ignore', '1');
} catch (e) {
console.error('Failed to access your browsers local storage.');
}
};
})();
<% end %>

@ -2,6 +2,5 @@
<div class="warning-bar--wrapper">
<%= render partial: 'warning_bar/pending_migrations' if render_pending_migrations_warning? %>
<%= render partial: 'warning_bar/unsupported_browser' if unsupported_browser? %>
<%= render partial: 'warning_bar/mysql_database' if render_mysql_deprecation_warning? %>
</div>
<% end %>

@ -2608,13 +2608,6 @@ en:
menu_item: "Menu item"
menu_item_setting: "Visibility"
mysql_deprecation:
notice: >
Future versions of OpenProject will likely drop or reduce support for MySQL and MariaDB databases.
Your installation is still running on MySQL and we suggest you migrate your installation to PostgreSQL.
This process is easy when following our migration guides:
wiki_menu_item_for: "Menu item for wikipage \"%{title}\""
wiki_menu_item_setting: "Visibility"
wiki_menu_item_new_main_item_explanation: >

@ -2,11 +2,6 @@ class EnsurePostgresIndexNames < ActiveRecord::Migration[5.2]
disable_ddl_transaction!
def up
if OpenProject::Database.mysql?
warn "You're using MySQL, skipping index renaming. You will need to re-run this when switching to PostgreSQL"
return
end
sql = <<~SQL
SELECT
FORMAT('%s_pkey', table_name) as new_name,
@ -23,11 +18,9 @@ class EnsurePostgresIndexNames < ActiveRecord::Migration[5.2]
new_name = entry['new_name']
ActiveRecord::Base.transaction do
begin
execute %(ALTER INDEX "#{old_name}" RENAME TO #{new_name};)
rescue StandardError => e
warn "Failed to rename index #{old_name} to #{new_name}: #{e.message}. Skipping"
end
execute %(ALTER INDEX "#{old_name}" RENAME TO #{new_name};)
rescue StandardError => e
warn "Failed to rename index #{old_name} to #{new_name}: #{e.message}. Skipping"
end
end
end

@ -48,9 +48,5 @@ module Migration
def postgres?
ActiveRecord::Base.connection.instance_values['config'][:adapter] == 'postgresql'
end
def mysql?
ActiveRecord::Base.connection.instance_values['config'][:adapter] == 'mysql2'
end
end
end

@ -47,22 +47,16 @@ module API
end
def fetch_checksums_for(work_packages)
concat = if OpenProject::Database.mysql?
md5_concat_mysql
else
md5_concat_postgresql
end
WorkPackage
.where(id: work_packages.map(&:id).uniq)
.left_joins(:status, :author, :responsible, :assigned_to, :fixed_version, :priority, :category, :type)
.pluck('work_packages.id', Arel.sql(concat.squish))
.pluck('work_packages.id', Arel.sql(md5_concat.squish))
.to_h
end
protected
def md5_concat_postgresql
def md5_concat
<<-SQL
MD5(CONCAT(statuses.id,
statuses.updated_at,
@ -82,27 +76,6 @@ module API
categories.updated_at))
SQL
end
def md5_concat_mysql
<<-SQL
MD5(CONCAT(COALESCE(statuses.id, 0),
COALESCE(statuses.updated_at, '-'),
COALESCE(users.id, '-'),
COALESCE(users.updated_on, '-'),
COALESCE(responsibles_work_packages.id, '-'),
COALESCE(responsibles_work_packages.updated_on, '-'),
COALESCE(assigned_tos_work_packages.id, '-'),
COALESCE(assigned_tos_work_packages.updated_on, '-'),
COALESCE(versions.id, '-'),
COALESCE(versions.updated_on, '-'),
COALESCE(types.id, '-'),
COALESCE(types.updated_at, '-'),
COALESCE(enumerations.id, '-'),
COALESCE(enumerations.updated_at, '-'),
COALESCE(categories.id, '-'),
COALESCE(categories.updated_at, '-')))
SQL
end
end
private

@ -87,12 +87,13 @@ module OpenProject
# Raises an +InsufficientVersionError+ when the version is incompatible
def self.check!
if !postgresql?
message = "Database server is not PostgreSql." \
message = "Database server is not PostgreSql. " \
"As OpenProject uses non standard ANSI-SQL for performance optimizations, using a different DBMS will " \
"break and is thus prevented."
if adapter_name =~ /mysql/i
message << " As MySql used to be supported, there is a migration script to ease the transition (https://www.openproject.org/deprecating-mysql-support/)."
if adapter_name.match?(/mysql/i)
message << " As MySql used to be supported, there is a migration script to ease the transition " \
"(https://www.openproject.org/deprecating-mysql-support/)."
end
raise UnsupportedDatabaseError.new message
@ -127,9 +128,9 @@ module OpenProject
# returns the identifier of the specified connection
# (defaults to ActiveRecord::Base.connection)
def self.name(connection = self.connection)
supported_adapters.find(proc { [:unknown, //] }) { |_adapter, regex|
supported_adapters.find(proc { [:unknown, //] }) do |_adapter, regex|
adapter_name(connection) =~ regex
}[0]
end[0]
end
# Provide helper methods to quickly check the database type
@ -144,8 +145,9 @@ module OpenProject
end
end
def self.mysql?(_arg)
ActiveSupport::Deprecation.warn ".mysql? is no longer supported and will always return false. Remove the call.", caller
def self.mysql?(_arg = nil)
message = ".mysql? is no longer supported and will always return false. Remove the call."
ActiveSupport::Deprecation.warn message, caller
false
end
@ -158,7 +160,7 @@ module OpenProject
raw ? @version : @version.match(/\APostgreSQL (\S+)/i)[1]
end
def self.semantic_version(version_string = self.version)
def self.semantic_version(version_string = version)
Semantic::Version.new version_string
rescue ArgumentError
# Cut anything behind the -

@ -303,14 +303,6 @@ module OpenProject::TextFormatting::Formats
end
def batch_update_statement(klass, attributes, values)
if OpenProject::Database.mysql?
batch_update_statement_mysql(klass, attributes, values)
else
batch_update_statement_postgresql(klass, attributes, values)
end
end
def batch_update_statement_postgresql(klass, attributes, values)
table_name = klass.table_name
sets = attributes.map { |a| "#{a} = new_values.#{a}" }.join(', ')
new_values = values.map do |value_hash|
@ -330,22 +322,6 @@ module OpenProject::TextFormatting::Formats
SQL
end
def batch_update_statement_mysql(klass, attributes, values)
table_name = klass.table_name
sets = attributes.map { |a| "#{table_name}.#{a} = new_values.#{a}" }.join(', ')
new_values_union = values.map do |value_hash|
text_values = value_hash.except(:id).map { |k, v| "#{ActiveRecord::Base.connection.quote(v)} AS #{k}" }.join(', ')
"SELECT #{value_hash[:id]} AS id, #{text_values}"
end.join(' UNION ')
<<-SQL
UPDATE #{table_name}, (#{new_values_union}) AS new_values
SET
#{sets}
WHERE #{table_name}.id = new_values.id
SQL
end
def concatenate_textile(textiles)
textiles.join("\n\n#{DOCUMENT_BOUNDARY}\n\n")
end

@ -38,38 +38,24 @@ namespace :backup do
FileUtils.mkdir_p(Pathname.new(args[:path_to_backup]).dirname)
config = database_configuration
case config['adapter']
when /PostgreSQL/i
with_pg_config(config) do |config_file|
pg_dump_call = ['pg_dump',
'--clean',
"--file=#{args[:path_to_backup]}",
'--format=custom',
'--no-owner']
pg_dump_call << "--host=#{config['host']}" if config['host']
pg_dump_call << "--port=#{config['port']}" if config['port']
user = config.values_at('user', 'username').compact.first
pg_dump_call << "--username=#{user}" if user
pg_dump_call << "#{config['database']}"
if config['password']
Kernel.system({ 'PGPASSFILE' => config_file }, *pg_dump_call)
else
Kernel.system(*pg_dump_call)
end
end
when /MySQL2/i
with_mysql_config(config) do |config_file|
Kernel.system 'mysqldump',
"--defaults-file=#{config_file}",
'--single-transaction',
'--add-drop-table',
'--add-locks',
"--result-file=#{args[:path_to_backup]}",
"#{config['database']}"
with_config_file(config) do |config_file|
pg_dump_call = ['pg_dump',
'--clean',
"--file=#{args[:path_to_backup]}",
'--format=custom',
'--no-owner']
pg_dump_call << "--host=#{config['host']}" if config['host']
pg_dump_call << "--port=#{config['port']}" if config['port']
user = config.values_at('user', 'username').compact.first
pg_dump_call << "--username=#{user}" if user
pg_dump_call << config['database'].to_s
if config['password']
Kernel.system({ 'PGPASSFILE' => config_file }, *pg_dump_call)
else
Kernel.system(*pg_dump_call)
end
else
raise "Database '#{config['adapter']}' not supported."
end
end
@ -79,32 +65,24 @@ namespace :backup do
raise "File '#{args[:path_to_backup]}' is not readable" unless File.readable?(args[:path_to_backup])
config = database_configuration
case config['adapter']
when /PostgreSQL/i
with_pg_config(config) do |config_file|
pg_restore_call = ['pg_restore',
'--clean',
'--no-owner',
'--single-transaction',
"--dbname=#{config['database']}"]
pg_restore_call << "--host=#{config['host']}" if config['host']
pg_restore_call << "--port=#{config['port']}" if config['port']
user = config.values_at('user', 'username').compact.first
pg_restore_call << "--username=#{user}" if user
pg_restore_call << "#{args[:path_to_backup]}"
if config['password']
Kernel.system({ 'PGPASSFILE' => config_file }, *pg_restore_call)
else
Kernel.system(*pg_restore_call)
end
end
when /MySQL2/i
with_mysql_config(config) do |config_file|
Kernel.system "mysql --defaults-file=\"#{config_file}\" \"#{config['database']}\" < \"#{args[:path_to_backup]}\""
with_config_file(config) do |config_file|
pg_restore_call = ['pg_restore',
'--clean',
'--no-owner',
'--single-transaction',
"--dbname=#{config['database']}"]
pg_restore_call << "--host=#{config['host']}" if config['host']
pg_restore_call << "--port=#{config['port']}" if config['port']
user = config.values_at('user', 'username').compact.first
pg_restore_call << "--username=#{user}" if user
pg_restore_call << args[:path_to_backup].to_s
if config['password']
Kernel.system({ 'PGPASSFILE' => config_file }, *pg_restore_call)
else
Kernel.system(*pg_restore_call)
end
else
raise "Database '#{config['adapter']}' not supported."
end
end
@ -114,7 +92,7 @@ namespace :backup do
ActiveRecord::Base.configurations[Rails.env] || Rails.application.config.database_configuration[Rails.env]
end
def with_pg_config(config, &blk)
def with_config_file(config, &blk)
file = Tempfile.new('op_pg_config')
file.write "*:*:*:*:#{config['password']}"
file.close
@ -122,14 +100,6 @@ namespace :backup do
file.unlink
end
def with_mysql_config(config, &blk)
file = Tempfile.new('op_mysql_config')
file.write sql_dump_tempfile(config)
file.close
blk.yield file.path
file.unlink
end
def sql_dump_tempfile(config)
t = "[client]\n"
t << "password=\"#{config['password']}\"\n"
@ -143,13 +113,7 @@ namespace :backup do
end
def default_db_filename
filename = "openproject-#{Rails.env}-db-#{date_string}"
case database_configuration['adapter']
when /PostgreSQL/i
filename << '.backup'
else
filename << '.sql'
end
filename = "openproject-#{Rails.env}-db-#{date_string}.backup"
Rails.root.join('backup', sanitize_filename(filename))
end

@ -32,15 +32,9 @@ module Tasks
def reset_journal_id_sequence!
con = ActiveRecord::Base.connection
if OpenProject::Database.mysql?
max_id = con.execute("SELECT MAX(id) FROM legacy_journals").to_a.first.first
max_id = con.execute("SELECT MAX(id) FROM legacy_journals").to_a.first["max"]
con.execute "ALTER TABLE journals AUTO_INCREMENT = #{max_id + 1}"
else # Postgres
max_id = con.execute("SELECT MAX(id) FROM legacy_journals").to_a.first["max"]
con.execute "ALTER SEQUENCE journals_id_seq RESTART WITH #{max_id + 1}"
end
con.execute "ALTER SEQUENCE journals_id_seq RESTART WITH #{max_id + 1}"
end
def move_project_attachments_to_wiki!

@ -22,13 +22,14 @@ module CostQuery::CustomFieldMixin
attr_reader :custom_field
SQL_TYPES = {
'string' => mysql? ? 'char' : 'varchar',
'list' => mysql? ? 'char' : 'varchar',
'text' => mysql? ? 'char' : 'text',
'bool' => mysql? ? 'unsigned' : 'boolean',
'date' => 'date',
'int' => 'decimal(60,3)',
'float' => 'decimal(60,3)' }
'string' => 'varchar',
'list' => 'varchar',
'text' => 'text',
'bool' => 'boolean',
'date' => 'date',
'int' => 'decimal(60,3)',
'float' => 'decimal(60,3)'
}.freeze
def self.extended(base)
base.inherited_attribute :factory
@ -101,7 +102,7 @@ module CostQuery::CustomFieldMixin
# contained invalid values.
def all_values_int?(field)
field.custom_values.pluck(:value).all? { |val| val.to_i > 0 }
rescue
rescue StandardError
false
end

@ -189,11 +189,6 @@ module Report::QueryUtils
}.join(', ')}\n\t\tELSE #{field_name_for else_part}\n\tEND"
end
def iso_year_week(field, default_table = nil)
field = field_name_for(field, default_table)
"-- code specific for #{adapter_name}\n\t" << super(field)
end
##
# Converts value with a given behavior, but treats nil differently.
# Params
@ -248,85 +243,19 @@ module Report::QueryUtils
second.size > first.size ? -1 : 0
end
def mysql?
[:mysql, :mysql2].include? adapter_name.to_s.downcase.to_sym
end
def sqlite?
adapter_name == :sqlite
end
def postgresql?
adapter_name == :postgresql
end
module SQL
def typed(_type, value, escape = true)
escape ? "'#{quote_string value}'" : value
end
def typed(type, value, escape = true)
safe_value = escape ? "'#{quote_string value}'" : value
"#{safe_value}::#{type}"
end
module MySql
include SQL
def iso_year_week(field)
"yearweek(#{field}, 1)"
end
end
module Sqlite
include SQL
def iso_year_week(field)
# enjoy
<<-EOS
case
when strftime('%W', strftime('%Y-01-04', #{field})) = '00' then
-- 01/01 is in week 1 of the current year => %W == week - 1
case
when strftime('%W', #{field}) = '52' and strftime('%W', (strftime('%Y', #{field}) + 1) || '-01-04') = '00' then
-- we are at the end of the year, and it's the first week of the next year
(strftime('%Y', #{field}) + 1) || '01'
when strftime('%W', #{field}) < '08' then
-- we are in week 1 to 9
strftime('%Y0', #{field}) || (strftime('%W', #{field}) + 1)
else
-- we are in week 10 or later
strftime('%Y', #{field}) || (strftime('%W', #{field}) + 1)
end
else
-- 01/01 is in week 53 of the last year
case
when strftime('%W', #{field}) = '52' and strftime('%W', (strftime('%Y', #{field}) + 1) || '-01-01') = '00' then
-- we are at the end of the year, and it's the first week of the next year
(strftime('%Y', #{field}) + 1) || '01'
when strftime('%W', #{field}) = '00' then
-- we are in the week belonging to last year
(strftime('%Y', #{field}) - 1) || '53'
else
-- everything is fine
strftime('%Y%W', #{field})
end
end
EOS
end
end
module Postres
include SQL
def typed(type, value, escape = true)
"#{super}::#{type}"
end
def iso_year_week(field, default_table = nil)
field_name = field_name_for(field, default_table)
def iso_year_week(field)
"(EXTRACT(isoyear from #{field})*100 + \n\t\t" \
"EXTRACT(week from #{field} - \n\t\t" \
"(EXTRACT(dow FROM #{field})::int+6)%7))"
end
"(EXTRACT(isoyear from #{field_name})*100 + \n\t\t" \
"EXTRACT(week from #{field_name} - \n\t\t" \
"(EXTRACT(dow FROM #{field_name})::int+6)%7))"
end
include MySql if mysql?
include Sqlite if sqlite?
include Postres if postgresql?
def self.cache
@cache ||= Hash.new { |h, k| h[k] = {} }
end

@ -100,7 +100,6 @@ class Report::SqlStatement
"\nWHERE #{where.join ' AND '}\n"
sql << "GROUP BY #{group_by.join ', '}\nORDER BY #{group_by.join ', '}\n" if group_by?
sql << "-- END #{desc}\n"
sql.gsub!('--', '#') if mysql?
sql # << " LIMIT 100"
end
end
@ -115,6 +114,7 @@ class Report::SqlStatement
# @param [#to_s] From part
def from(table = nil)
return @from unless table
@sql = nil
@from = table
end

@ -29,28 +29,6 @@ module ReportingEngine
Rails.application.config.assets.precompile += %w(reporting_engine.css reporting_engine.js)
end
initializer 'check mysql version' do
connection = ActiveRecord::Base.connection
adapter_name = connection.adapter_name.to_s.downcase.to_sym
if [:mysql, :mysql2].include?(adapter_name)
# The reporting engine is incompatible with the
# following mysql versions due to a bug in MySQL itself:
# 5.6.0 - 5.6.12
# 5.7.0 - 5.7.1
# see https://www.openproject.org/issues/967 for details.
required_patch_levels = { '5.6' => 13, '5.7' => 2 }
mysql_version = connection.show_variable('VERSION')
release_version, patch_level = mysql_version.match(/(\d*\.\d*)\.(\d*)/).captures
required_patch_level = required_patch_levels[release_version]
if required_patch_level && (patch_level.to_i < required_patch_level)
raise "MySQL #{mysql_version} is not supported. Version #{release_version} \
requires patch level >= #{required_patch_level}."
end
end
end
config.to_prepare do
require 'reporting_engine/patches'
require 'reporting_engine/patches/big_decimal_patch'

@ -1,61 +0,0 @@
#-- copyright
# OpenProject is a project management system.
# Copyright (C) 2012-2018 the OpenProject Foundation (OPF)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License version 3.
#
# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
# Copyright (C) 2006-2017 Jean-Philippe Lang
# Copyright (C) 2010-2013 the ChiliProject Team
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# See docs/COPYRIGHT.rdoc for more details.
#++
require 'spec_helper'
feature 'MySQL deprecation spec', js: true do
let(:user) { FactoryBot.create :admin }
before do
Capybara.reset!
login_as user
end
it 'renders a warning in admin areas', with_config: { show_warning_bars: true } do
if OpenProject::Database.postgresql?
# Does not render
visit info_admin_index_path
expect(page).to have_no_selector('#mysql-db-warning')
else
visit home_path
expect(page).to have_no_selector('#mysql-db-warning')
visit info_admin_index_path
expect(page).to have_selector('#mysql-db-warning')
# Hides in localstorage
find('.warning-bar--disable-on-hover').click
expect(page).to have_no_selector('#mysql-db-warning')
visit info_admin_index_path
expect(page).to have_no_selector('#mysql-db-warning')
end
end
end

@ -209,26 +209,14 @@ describe 'Work Package table hierarchy', js: true do
wp_table.expect_work_package_listed(leaf, inter, root)
wp_table.expect_work_package_listed(leaf_assigned, inter_assigned, root_assigned)
if OpenProject::Database.mysql?
# MySQL returns empty first before assigned
wp_table.expect_work_package_order(
root,
inter,
leaf,
root_assigned,
inter_assigned,
leaf_assigned
)
else
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
leaf_assigned,
root,
inter,
leaf
)
end
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
leaf_assigned,
root,
inter,
leaf
)
# Hierarchy should be disabled
hierarchy.expect_no_hierarchies
@ -238,34 +226,21 @@ describe 'Work Package table hierarchy', js: true do
hierarchy.expect_hierarchy_at(root_assigned, inter)
hierarchy.expect_leaf_at(root, leaf, leaf_assigned, inter_assigned)
# When ascending, psql order should be:
# MySQL orders empty values before assigned ones
# When ascending, order should be:
# ├──root_assigned
# | ├─ inter_assigned
# | ├─ inter
# | | ├─ leaf_assigned
# | | ├─ leaf
# ├──root
if OpenProject::Database.mysql?
# MySQL returns empty first before assigned
wp_table.expect_work_package_order(
root,
root_assigned,
inter,
leaf,
leaf_assigned,
inter_assigned
)
else
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
inter,
leaf_assigned,
leaf,
root
)
end
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
inter,
leaf_assigned,
leaf,
root
)
# Test collapsing of rows
hierarchy.toggle_row(root_assigned)
@ -285,50 +260,26 @@ describe 'Work Package table hierarchy', js: true do
# | | ├─ leaf
# | | ├─ leaf_assigned
# | ├─ inter_assigned
if OpenProject::Database.mysql?
# MySQL returns empty first before assigned
wp_table.expect_work_package_order(
root,
root_assigned,
inter,
leaf,
leaf_assigned,
inter_assigned
)
else
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
inter,
leaf_assigned,
leaf,
root
)
end
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
inter,
leaf_assigned,
leaf,
root
)
# Disable hierarchy mode
hierarchy.disable_hierarchy
if OpenProject::Database.mysql?
# MySQL returns empty first before assigned
wp_table.expect_work_package_order(
root,
inter,
leaf,
root_assigned,
inter_assigned,
leaf_assigned
)
else
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
leaf_assigned,
root,
inter,
leaf
)
end
wp_table.expect_work_package_order(
root_assigned,
inter_assigned,
leaf_assigned,
root,
inter,
leaf
)
end
end
end

@ -62,7 +62,6 @@ describe OpenProject::Database do
it 'should be able to use the helper methods' do
allow(OpenProject::Database).to receive(:adapter_name).and_return 'PostgresQL'
expect(OpenProject::Database.mysql?).to equal(false)
expect(OpenProject::Database.postgresql?).to equal(true)
end
@ -74,12 +73,4 @@ describe OpenProject::Database do
expect(OpenProject::Database.version).to eq('8.3.11')
expect(OpenProject::Database.version(true)).to eq(raw_version)
end
it 'should return a version string for MySQL' do
allow(OpenProject::Database).to receive(:adapter_name).and_return 'MySQL'
allow(ActiveRecord::Base.connection).to receive(:select_value).and_return '5.1.2'
expect(OpenProject::Database.version).to eq('5.1.2')
expect(OpenProject::Database.version(true)).to eq('5.1.2')
end
end

@ -339,9 +339,8 @@ describe CustomField, type: :model do
shared_examples_for 'saving updates field\'s updated_at' do
it 'updates updated_at' do
# mysql does not store milliseconds so we have to slow down the tests by orders of magnitude
timestamp_before = field.updated_at
sleep 1
sleep 0.001
field.save
expect(field.updated_at).not_to eql(timestamp_before)
end

@ -49,7 +49,7 @@ describe Queries::Queries::QueryQuery, type: :model do
describe '#results' do
it 'is the same as handwriting the query' do
# apparently, strings are accepted to be compared to
# integers in the dbs (mysql, postgresql)
# integers in the postgresql
expected = base_scope
.merge(Query
.where("queries.project_id IN ('1','2')"))

@ -588,14 +588,6 @@ describe ::Query::Results, type: :model do
end
end
# Introduced to ensure being able to group by custom fields
# when running on a MySQL server.
# When upgrading to rails 5, the sql_mode passed on with the connection
# does include the "only_full_group_by" flag by default which causes our queries to become
# invalid because (mysql error):
# "SELECT list is not in GROUP BY clause and contains nonaggregated column
# 'config_myproject_test.work_packages.id' which is not functionally
# dependent on columns in GROUP BY clause"
context 'when grouping by custom field' do
let!(:custom_field) do
FactoryBot.create(:int_wp_custom_field, is_for_all: true, is_filter: true)

@ -28,95 +28,6 @@
require 'spec_helper'
describe 'mysql' do
let(:database_config) do
{ 'adapter' => 'mysql2',
'database' => 'openproject-database',
'username' => 'testuser',
'password' => 'testpassword' }
end
before do
expect(ActiveRecord::Base).to receive(:configurations).at_least(:once).and_return('test' => database_config)
allow(FileUtils).to receive(:mkdir_p).and_return(nil)
end
describe 'backup:database:create' do
include_context 'rake'
it 'calls the mysqldump binary' do
expect(Kernel).to receive(:system) do |*args|
expect(args.first).to eql('mysqldump')
end
subject.invoke
end
it 'writes the mysql config file' do
expect(Kernel).to receive(:system) do |*args|
defaults_file = args.find { |s| s.starts_with? '--defaults-file=' }
defaults_file = defaults_file[('--defaults-file='.length)..-1]
expect(File.readable?(defaults_file)).to be true
file_contents = File.read defaults_file
expect(file_contents).to include('testuser')
expect(file_contents).to include('testpassword')
end
subject.invoke
end
it 'uses the first task parameter as the target filename' do
custom_file_path = './foo/bar/testfile.sql'
expect(Kernel).to receive(:system) do |*args|
result_file = args.find { |s| s.starts_with? '--result-file=' }
expect(result_file).to include(custom_file_path)
end
subject.invoke custom_file_path
end
end
describe 'backup:database:restore' do
include_context 'rake'
let(:backup_file) do
Tempfile.new('test_backup')
end
after do
backup_file.unlink
end
it 'calls the mysql binary' do
expect(Kernel).to receive(:system) do |*args|
expect(args.first).to start_with('mysql')
end
subject.invoke backup_file.path
end
it 'writes the mysql config file' do
expect(Kernel).to receive(:system) do |*args|
defaults_file = args.first[/--defaults-file="(?<file>[^"]+)"/, :file]
expect(File.readable?(defaults_file)).to be true
file_contents = File.read defaults_file
expect(file_contents).to include('testuser')
expect(file_contents).to include('testpassword')
end
subject.invoke backup_file.path
end
it 'uses the first task parameter as the target filename' do
expect(Kernel).to receive(:system) do |*args|
expect(args.first).to include(backup_file.path)
end
subject.invoke backup_file.path
end
it 'throws an error when called without a parameter' do
expect { subject.invoke }.to raise_error
end
end
end
describe 'postgresql' do
let(:database_config) do
{ 'adapter' => 'postgresql',

Loading…
Cancel
Save