From be9f033306bcae8813937ad8873848992b54ce46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=BCnther?= Date: Thu, 2 May 2019 10:34:55 +0200 Subject: [PATCH 1/5] Extend docker guides to notify about hostname [ci skip] --- .../migrating/docker/postgresql-migration.md | 15 +++++++++------ .../migrating/manual/postgresql-migration.md | 16 ++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/operations/migrating/docker/postgresql-migration.md b/docs/operations/migrating/docker/postgresql-migration.md index 2ff9a6d43e..b4017f1864 100644 --- a/docs/operations/migrating/docker/postgresql-migration.md +++ b/docs/operations/migrating/docker/postgresql-migration.md @@ -82,7 +82,6 @@ MYSQL_DATABSAE_URL="mysql://user:password@localhost:3306/dbname" **Please note:** Ensure that the URL starts with `mysql://` , not with ` mysql2://` ! - ### The PostgreSQL DATABASE_URL Pass in `DATABASE_URL` pointing to your new PostgreSQL database. This is either the default `postgres://openproject:openproject@127.0.0.1/openproject` or if you set up a PostgreSQL installation above, use credentials for your installation you set up above. @@ -92,20 +91,24 @@ POSTGRES_DATABASE_URL="postgresql://:@/" ``` +### Adapting the hostname + +**Note:** Depending on your docker installation and networking, you may need to replace the hostname `localhost` in the database URLs +with `host.docker.internal` to access the docker host. On Mac for example, localhost will refer to the docker client. + ### Running the migration To run the migration script within the container, now simply run the following command, replacing the content of the environment variables with your actual values. - ```bash -docker run \ - -it openproject/community:latest +docker run -it \ -e MYSQL_DATABASE_URL="mysql://user:password@localhost:3306/dbname" \ - -e DATABASE_URL="postgresql://openproject:@localhost:5432/openproject" + -e DATABASE_URL="postgresql://openproject:@localhost:5432/openproject" \ + openproject/community:latest ``` -This will perform all necessary steps to perform the migration. Afterwards, simply remove the `MYSQL_DATABASE_URL`environment variable again and start your container as usual. \ No newline at end of file +This will perform all necessary steps to perform the migration. Afterwards, simply remove the `MYSQL_DATABASE_URL`environment variable again and start your container as usual. diff --git a/docs/operations/migrating/manual/postgresql-migration.md b/docs/operations/migrating/manual/postgresql-migration.md index b37a151cea..a4b43d910a 100644 --- a/docs/operations/migrating/manual/postgresql-migration.md +++ b/docs/operations/migrating/manual/postgresql-migration.md @@ -71,10 +71,6 @@ Pass in `DATABASE_URL` pointing to your new PostgreSQL database. Fill the templa export POSTGRES_DATABASE_URL="postgresql://openproject:@localhost/openproject ``` - - - - ## Running the migration via Docker OpenProject provides a simple conversion script that you can run as a single command via Docker. @@ -82,16 +78,20 @@ OpenProject provides a simple conversion script that you can run as a single com To run the migration script within the container, simply run the following command, replacing the content of the environment variables with your actual values. +### Adapting the hostname + +**Note:** Depending on your docker installation and networking, you may need to replace the hostname `localhost` in the database URLs +with `host.docker.internal` to access the docker host. On Mac for example, localhost will refer to the docker client. + ```bash -docker run \ - -it openproject/community:latest +docker run -it \ -e MYSQL_DATABASE_URL=$MYSQL_DATABASE_URL \ - -e DATABASE_URL=$POSTGRES_DATABASE_URL + -e DATABASE_URL=$POSTGRES_DATABASE_URL \ + openproject/community:latest ``` - This will perform all necessary steps to perform the migration. Afterwards, simply remove the `MYSQL_DATABASE_URL`environment variable again and start your container as usual. From 6253d8639e8bb36862cb25bc3c29799d07efc328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=BCnther?= Date: Thu, 2 May 2019 10:50:05 +0200 Subject: [PATCH 2/5] Ensure static version in BCF plugin Avoids problems with the Gemfile.lock upon bumping versions --- Gemfile.lock | 2 +- modules/bcf/lib/open_project/bcf/version.rb | 7 ------- modules/bcf/openproject-bcf.gemspec | 7 +------ 3 files changed, 2 insertions(+), 14 deletions(-) delete mode 100644 modules/bcf/lib/open_project/bcf/version.rb diff --git a/Gemfile.lock b/Gemfile.lock index 945b367d14..b745f2960a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -119,7 +119,7 @@ PATH PATH remote: modules/bcf specs: - openproject-bcf (9.0.0) + openproject-bcf (1.0.0) activerecord-import rails (~> 5) rubyzip (~> 1.2) diff --git a/modules/bcf/lib/open_project/bcf/version.rb b/modules/bcf/lib/open_project/bcf/version.rb deleted file mode 100644 index bbdebba05b..0000000000 --- a/modules/bcf/lib/open_project/bcf/version.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'open_project/version' - -module OpenProject - module Bcf - VERSION = ::OpenProject::VERSION.to_semver - end -end diff --git a/modules/bcf/openproject-bcf.gemspec b/modules/bcf/openproject-bcf.gemspec index 528b7e0a31..ad05791616 100644 --- a/modules/bcf/openproject-bcf.gemspec +++ b/modules/bcf/openproject-bcf.gemspec @@ -1,14 +1,9 @@ # encoding: UTF-8 -$:.push File.expand_path("../lib", __FILE__) -$:.push File.expand_path("../../lib", __dir__) - -require "open_project/bcf/version" - # Describe your gem and declare its dependencies: Gem::Specification.new do |s| s.name = "openproject-bcf" - s.version = OpenProject::Bcf::VERSION + s.version = "1.0.0" s.authors = "OpenProject GmbH" s.email = "info@openproject.com" s.homepage = "https://community.openproject.org/" From 8da0cf24db32d250b5095e6d7873eef413d846ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=BCnther?= Date: Thu, 2 May 2019 10:50:44 +0200 Subject: [PATCH 3/5] Remove unused utils --- db/migrate/migration_utils/utils.rb | 100 +--------------------------- 1 file changed, 3 insertions(+), 97 deletions(-) diff --git a/db/migrate/migration_utils/utils.rb b/db/migrate/migration_utils/utils.rb index cc00e2c785..2f34486976 100644 --- a/db/migrate/migration_utils/utils.rb +++ b/db/migrate/migration_utils/utils.rb @@ -39,39 +39,10 @@ module Migration end end - def filter(columns, terms) - column_filters = [] - - columns.each do |column| - filters = terms.map { |term| "#{column} LIKE '%#{term}%'" } - - column_filters << "(#{filters.join(' OR ')})" + def remove_index_if_exists(table_name, index_name) + if index_exists? table_name, index_name + remove_index table_name, index_name end - - column_filters.join(' OR ') - end - - def update_column_values(table, column_list, updater, conditions) - update_column_values_and_journals(table, column_list, updater, false, conditions) - end - - def update_column_values_and_journals(table, column_list, updater, update_journal, conditions) - processed_rows = [] - - select_rows_from_database(table, column_list, conditions).each do |row| - processed_rows << updater.call(row) - end - - updated_rows = processed_rows.select(&:updated) - - update_rows_in_database(table, column_list, updated_rows.map(&:row)) - - update_journals(table, updated_rows) if update_journal - end - - def reset_public_key_sequence_in_postgres(table) - return unless ActiveRecord::Base.connection.instance_values['config'][:adapter] == 'postgresql' - ActiveRecord::Base.connection.reset_pk_sequence!(table) end def postgres? @@ -81,70 +52,5 @@ module Migration def mysql? ActiveRecord::Base.connection.instance_values['config'][:adapter] == 'mysql2' end - - private - - def select_rows_from_database(table, column_list, conditions) - columns = (column_list.nil?) ? '' : ', ' + column_list.join(', ') - from_clause = table - where_clause = conditions.nil? ? '1 = 1' : conditions - - select_all <<-SQL - SELECT id#{columns} - FROM #{from_clause} - WHERE #{where_clause} - SQL - end - - def update_rows_in_database(table, column_list, updated_rows) - columns = (column_list.nil?) ? '' : column_list.join(', ') - - updated_rows.each do |row| - values = column_list.map { |c| "#{c}=#{quote(row[c])}" } - .join(', ') - - update <<-SQL - UPDATE #{table} - SET #{values} - WHERE id = #{row['id']} - SQL - end - end - - def update_journals(table, updated_rows) - created_journals = {} - - updated_ids = updated_rows.map { |r| r.row['id'] } - journal_table = "#{table.singularize}_journals" - journable_type = table.classify - - updated_ids.each do |id| - created_journals[id] = insert <<-SQL - INSERT INTO journals (journable_id, journable_type, user_id, created_at, version, activity_type) - SELECT journable_id, journable_type, #{system_user_id}, NOW(), MAX(version) + 1, activity_type - FROM journals - WHERE journable_type = '#{journable_type}' AND journable_id = #{id} - GROUP BY journable_id, journable_type, activity_type - SQL - end - - journal_table_columns = journal_table_columns(journal_table) - - insert <<-SQL - INSERT INTO #{journal_table} (journal_id, #{journal_table_columns.join(', ')}) - SELECT j.id AS journal_id, #{journal_table_columns.map { |c| "w.#{c}" }.join(', ')} - FROM journals AS j JOIN #{table} AS w ON (j.journable_id = w.id) - WHERE journable_type = '#{journable_type}' - AND j.id NOT IN (SELECT journal_id FROM work_package_journals) - SQL - end - - def system_user_id - @system_user_id ||= User.system.id - end - - def journal_table_columns(table) - "Journal::#{table.classify}".constantize.column_names - ['id', 'journal_id'] - end end end From 612900e8e74e2301c69ad691f128df7c11d53f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=BCnther?= Date: Thu, 2 May 2019 10:50:53 +0200 Subject: [PATCH 4/5] Fix more mysql migration issues --- db/migrate/20180105130053_rebuild_dag.rb | 26 ++++++++----------- ...e_planning_elemnt_type_colors_to_colors.rb | 9 ------- db/migrate/migration_utils/utils.rb | 10 +++++-- .../bin/migrate-mysql-to-postgres | 2 +- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/db/migrate/20180105130053_rebuild_dag.rb b/db/migrate/20180105130053_rebuild_dag.rb index 8a5e583bd7..40ecccd050 100644 --- a/db/migrate/20180105130053_rebuild_dag.rb +++ b/db/migrate/20180105130053_rebuild_dag.rb @@ -25,8 +25,11 @@ # # See docs/COPYRIGHT.rdoc for more details. #++ +require_relative './migration_utils/utils' class RebuildDag < ActiveRecord::Migration[5.0] + include ::Migration::Utils + def up truncate_closure_entries remove_duplicate_relations @@ -58,18 +61,11 @@ class RebuildDag < ActiveRecord::Migration[5.0] def down remove_column :relations, :count - if index_exists? :relations, 'index_relations_hierarchy_follows_scheduling' - remove_index :relations, - name: 'index_relations_hierarchy_follows_scheduling' - remove_index :relations, - name: 'index_relations_only_hierarchy' - remove_index :relations, - name: 'index_relations_to_from_only_follows' - remove_index :relations, - name: 'index_relations_direct_non_hierarchy' - remove_index :relations, - name: 'index_relations_on_type_columns' - end + remove_index_if_exists :relations, 'index_relations_hierarchy_follows_scheduling' + remove_index_if_exists :relations, 'index_relations_only_hierarchy' + remove_index_if_exists :relations, 'index_relations_to_from_only_follows' + remove_index_if_exists :relations, 'index_relations_direct_non_hierarchy' + remove_index_if_exists :relations, 'index_relations_on_type_columns' truncate_closure_entries end @@ -94,7 +90,7 @@ class RebuildDag < ActiveRecord::Migration[5.0] AND relations.includes = 0 AND relations.requires = 0 AND (hierarchy + relates + duplicates + follows + blocks + includes + requires > 0) - SQL + SQL add_index :relations, %i(from_id to_id hierarchy), @@ -106,7 +102,7 @@ class RebuildDag < ActiveRecord::Migration[5.0] AND relations.blocks = 0 AND relations.includes = 0 AND relations.requires = 0 - SQL + SQL add_index :relations, %i(to_id follows from_id), @@ -118,7 +114,7 @@ class RebuildDag < ActiveRecord::Migration[5.0] AND blocks = 0 AND includes = 0 AND requires = 0 - SQL + SQL end def add_non_hierarchy_index diff --git a/db/migrate/20180510184732_rename_planning_elemnt_type_colors_to_colors.rb b/db/migrate/20180510184732_rename_planning_elemnt_type_colors_to_colors.rb index f6a63b21d9..1762b5f2e5 100644 --- a/db/migrate/20180510184732_rename_planning_elemnt_type_colors_to_colors.rb +++ b/db/migrate/20180510184732_rename_planning_elemnt_type_colors_to_colors.rb @@ -6,15 +6,6 @@ class RenamePlanningElemntTypeColorsToColors < ActiveRecord::Migration[5.1] rename_index :planning_element_type_colors, :timelines_colors_pkey, :planning_element_type_colors_pkey end - if ActiveRecord::Base.connection.execute("SELECT 1 as value FROM pg_class c WHERE c.relkind = 'S' and c.relname = 'planning_element_type_colors_id_seq'").to_a.present? || - begin - puts "Renaming id_seq to pkey which seems to be required by rename_table" - rename_index :planning_element_type_colors, :planning_element_type_colors_id_seq, :planning_element_type_colors_pkey - rescue => e - raise e unless e.message.match? /planning_element_type_colors_pkey.+?already exists/ - end - end - rename_table :planning_element_type_colors, :colors remove_column :colors, :position end diff --git a/db/migrate/migration_utils/utils.rb b/db/migrate/migration_utils/utils.rb index 2f34486976..71293a987f 100644 --- a/db/migrate/migration_utils/utils.rb +++ b/db/migrate/migration_utils/utils.rb @@ -39,9 +39,15 @@ module Migration end end + def index_exists_by_name?(table_name, index_name) + ActiveRecord::Base.connection + .indexes(table_name) + .detect { |index| index.name == index_name} + end + def remove_index_if_exists(table_name, index_name) - if index_exists? table_name, index_name - remove_index table_name, index_name + if index_exists_by_name? table_name, index_name + remove_index table_name, name: index_name end end diff --git a/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres b/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres index 67fa73367d..5921f52e23 100755 --- a/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres +++ b/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres @@ -130,7 +130,7 @@ end # add the postgres specific columns. needs_fulltext_migration = check_output.include?("true") -puts "Migrating database ..." +puts "Running OpenProject migrations ..." _, migrate_status = run "bundle exec rake db:migrate", record_output: false From a77d1210fd47ca970ab37da239051e77efdafec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20G=C3=BCnther?= Date: Thu, 2 May 2019 11:26:18 +0200 Subject: [PATCH 5/5] Correct URL if mysql2 scheme --- docker/mysql-to-postgres/bin/migrate-mysql-to-postgres | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres b/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres index 5921f52e23..acdc2563d1 100755 --- a/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres +++ b/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres @@ -23,6 +23,11 @@ if db_url.scheme.start_with?("postgres") && mysql_url.nil? exit 0 end +# Correct the URL scheme +if mysql_url.scheme == 'mysql2' + mysql_url.scheme = 'mysql' +end + filtered_mysql_url = mysql_url.dup.tap{|url| url.password = "REDACTED"} filtered_db_url = db_url.dup.tap{|url| url.password = "REDACTED"}