Merge remote-tracking branch 'origin/dev' into release/9.0

Gemfile.lock

@@ -119,7 +119,7 @@ PATH
 PATH
   remote: modules/bcf
   specs:
-    openproject-bcf (9.0.0)
+    openproject-bcf (1.0.0)
       activerecord-import
       rails (~> 5)
       rubyzip (~> 1.2)

db/migrate/20180105130053_rebuild_dag.rb

@@ -25,8 +25,11 @@
 #
 # See docs/COPYRIGHT.rdoc for more details.
 #++
+require_relative './migration_utils/utils'
 
 class RebuildDag < ActiveRecord::Migration[5.0]
+  include ::Migration::Utils
+
   def up
     truncate_closure_entries
     remove_duplicate_relations
@@ -58,18 +61,11 @@ class RebuildDag < ActiveRecord::Migration[5.0]
   def down
     remove_column :relations, :count
 
-    if index_exists? :relations, 'index_relations_hierarchy_follows_scheduling'
+    remove_index_if_exists :relations, 'index_relations_hierarchy_follows_scheduling'
-      remove_index :relations,
+    remove_index_if_exists :relations, 'index_relations_only_hierarchy'
-                   name: 'index_relations_hierarchy_follows_scheduling'
+    remove_index_if_exists :relations, 'index_relations_to_from_only_follows'
-      remove_index :relations,
+    remove_index_if_exists :relations, 'index_relations_direct_non_hierarchy'
-                   name: 'index_relations_only_hierarchy'
+    remove_index_if_exists :relations, 'index_relations_on_type_columns'
-      remove_index :relations,
-                   name: 'index_relations_to_from_only_follows'
-      remove_index :relations,
-                   name: 'index_relations_direct_non_hierarchy'
-      remove_index :relations,
-                   name: 'index_relations_on_type_columns'
-    end
 
     truncate_closure_entries
   end
@@ -94,7 +90,7 @@ class RebuildDag < ActiveRecord::Migration[5.0]
                 AND relations.includes = 0
                 AND relations.requires = 0
                 AND (hierarchy + relates + duplicates + follows + blocks + includes + requires > 0)
-              SQL
+    SQL
 
     add_index :relations,
               %i(from_id to_id hierarchy),
@@ -106,7 +102,7 @@ class RebuildDag < ActiveRecord::Migration[5.0]
                 AND relations.blocks = 0
                 AND relations.includes = 0
                 AND relations.requires = 0
-              SQL
+    SQL
 
     add_index :relations,
               %i(to_id follows from_id),
@@ -118,7 +114,7 @@ class RebuildDag < ActiveRecord::Migration[5.0]
                 AND blocks = 0
                 AND includes = 0
                 AND requires = 0
-              SQL
+    SQL
   end
 
   def add_non_hierarchy_index

db/migrate/20180510184732_rename_planning_elemnt_type_colors_to_colors.rb

@@ -6,15 +6,6 @@ class RenamePlanningElemntTypeColorsToColors < ActiveRecord::Migration[5.1]
       rename_index :planning_element_type_colors, :timelines_colors_pkey, :planning_element_type_colors_pkey
     end
 
-    if ActiveRecord::Base.connection.execute("SELECT 1 as value FROM pg_class c WHERE c.relkind = 'S' and c.relname = 'planning_element_type_colors_id_seq'").to_a.present? ||
-      begin
-        puts "Renaming id_seq to pkey which seems to be required by rename_table"
-        rename_index :planning_element_type_colors, :planning_element_type_colors_id_seq, :planning_element_type_colors_pkey
-      rescue => e
-        raise e unless e.message.match? /planning_element_type_colors_pkey.+?already exists/
-      end
-    end
-
     rename_table :planning_element_type_colors, :colors
     remove_column :colors, :position
   end

db/migrate/migration_utils/utils.rb

@@ -39,39 +39,16 @@ module Migration
       end
     end
 
-    def filter(columns, terms)
+    def index_exists_by_name?(table_name, index_name)
-      column_filters = []
+      ActiveRecord::Base.connection
-
+        .indexes(table_name)
-      columns.each do |column|
+        .detect { |index| index.name == index_name}
-        filters = terms.map { |term| "#{column} LIKE '%#{term}%'" }
-
-        column_filters << "(#{filters.join(' OR ')})"
-      end
-
-      column_filters.join(' OR ')
     end
 
-    def update_column_values(table, column_list, updater, conditions)
+    def remove_index_if_exists(table_name, index_name)
-      update_column_values_and_journals(table, column_list, updater, false, conditions)
+      if index_exists_by_name? table_name, index_name
-    end
+        remove_index table_name, name: index_name
-
-    def update_column_values_and_journals(table, column_list, updater, update_journal, conditions)
-      processed_rows = []
-
-      select_rows_from_database(table, column_list, conditions).each do |row|
-        processed_rows << updater.call(row)
       end
-
-      updated_rows = processed_rows.select(&:updated)
-
-      update_rows_in_database(table, column_list, updated_rows.map(&:row))
-
-      update_journals(table, updated_rows) if update_journal
-    end
-
-    def reset_public_key_sequence_in_postgres(table)
-      return unless ActiveRecord::Base.connection.instance_values['config'][:adapter] == 'postgresql'
-      ActiveRecord::Base.connection.reset_pk_sequence!(table)
     end
 
     def postgres?
@@ -81,70 +58,5 @@ module Migration
     def mysql?
       ActiveRecord::Base.connection.instance_values['config'][:adapter] == 'mysql2'
     end
-
-    private
-
-    def select_rows_from_database(table, column_list, conditions)
-      columns = (column_list.nil?) ? '' : ', ' + column_list.join(', ')
-      from_clause = table
-      where_clause =  conditions.nil? ? '1 = 1' : conditions
-
-      select_all <<-SQL
-        SELECT id#{columns}
-        FROM #{from_clause}
-        WHERE #{where_clause}
-      SQL
-    end
-
-    def update_rows_in_database(table, column_list, updated_rows)
-      columns = (column_list.nil?) ? '' : column_list.join(', ')
-
-      updated_rows.each do |row|
-        values = column_list.map { |c| "#{c}=#{quote(row[c])}" }
-                 .join(', ')
-
-        update <<-SQL
-          UPDATE #{table}
-          SET #{values}
-          WHERE id = #{row['id']}
-        SQL
-      end
-    end
-
-    def update_journals(table, updated_rows)
-      created_journals = {}
-
-      updated_ids = updated_rows.map { |r| r.row['id'] }
-      journal_table = "#{table.singularize}_journals"
-      journable_type = table.classify
-
-      updated_ids.each do |id|
-        created_journals[id] = insert <<-SQL
-          INSERT INTO journals (journable_id, journable_type, user_id, created_at, version, activity_type)
-          SELECT journable_id, journable_type, #{system_user_id}, NOW(), MAX(version) + 1, activity_type
-          FROM journals
-          WHERE journable_type = '#{journable_type}' AND journable_id = #{id}
-          GROUP BY journable_id, journable_type, activity_type
-        SQL
-      end
-
-      journal_table_columns = journal_table_columns(journal_table)
-
-      insert <<-SQL
-        INSERT INTO #{journal_table} (journal_id, #{journal_table_columns.join(', ')})
-        SELECT j.id AS journal_id, #{journal_table_columns.map { |c| "w.#{c}" }.join(', ')}
-        FROM journals AS j JOIN #{table} AS w ON (j.journable_id = w.id)
-        WHERE journable_type = '#{journable_type}'
-          AND j.id NOT IN (SELECT journal_id FROM work_package_journals)
-      SQL
-    end
-
-    def system_user_id
-      @system_user_id ||= User.system.id
-    end
-
-    def journal_table_columns(table)
-      "Journal::#{table.classify}".constantize.column_names - ['id', 'journal_id']
-    end
   end
 end

docker/mysql-to-postgres/bin/migrate-mysql-to-postgres

@@ -23,6 +23,11 @@ if db_url.scheme.start_with?("postgres") && mysql_url.nil?
   exit 0
 end
 
+# Correct the URL scheme
+if mysql_url.scheme == 'mysql2'
+  mysql_url.scheme = 'mysql'
+end
+
 filtered_mysql_url = mysql_url.dup.tap{|url| url.password = "REDACTED"}
 filtered_db_url = db_url.dup.tap{|url| url.password = "REDACTED"}
 
@@ -130,7 +135,7 @@ end
 # add the postgres specific columns.
 needs_fulltext_migration = check_output.include?("true")
 
-puts "Migrating database ..."
+puts "Running OpenProject migrations ..."
 
 _, migrate_status = run "bundle exec rake db:migrate", record_output: false
 

docs/operations/migrating/docker/postgresql-migration.md

@@ -82,7 +82,6 @@ MYSQL_DATABSAE_URL="mysql://user:password@localhost:3306/dbname"
 **Please note:** Ensure that the URL starts with `mysql://` , not with ` mysql2://` !
 
 
-
 ### The PostgreSQL DATABASE_URL
 
 Pass in `DATABASE_URL` pointing to your new PostgreSQL database. This is either the default `postgres://openproject:openproject@127.0.0.1/openproject` or if you set up a PostgreSQL installation above, use credentials for your installation you set up above.
@@ -92,18 +91,22 @@ POSTGRES_DATABASE_URL="postgresql://<USER>:<PASSWORD>@<HOST>/<Database name>"
 ```
 
 
+### Adapting the hostname
+
+**Note:** Depending on your docker installation and networking, you may need to replace the hostname `localhost` in the database URLs
+with `host.docker.internal` to access the docker host. On Mac for example, localhost will refer to the docker client.
+
 
 ### Running the migration
 
 To run the migration script within the container, now simply run the following command, replacing the content of the environment variables with your actual values.
 
 
-
 ```bash
-docker run \
+docker run -it \
-  -it openproject/community:latest
   -e MYSQL_DATABASE_URL="mysql://user:password@localhost:3306/dbname" \
-  -e DATABASE_URL="postgresql://openproject:<PASSWORD>@localhost:5432/openproject" 
+  -e DATABASE_URL="postgresql://openproject:<PASSWORD>@localhost:5432/openproject" \
+  openproject/community:latest
 ```
 
 

docs/operations/migrating/manual/postgresql-migration.md

@@ -71,10 +71,6 @@ Pass in `DATABASE_URL` pointing to your new PostgreSQL database. Fill the templa
 export POSTGRES_DATABASE_URL="postgresql://openproject:<PASSWORD>@localhost/openproject
 ```
 
-
-
-
-
 ## Running the migration via Docker
 
 OpenProject provides a simple conversion script that you can run as a single command via Docker.
@@ -82,16 +78,20 @@ OpenProject provides a simple conversion script that you can run as a single com
 To run the migration script within the container, simply run the following command, replacing the content of the environment variables with your actual values.
 
 
+### Adapting the hostname
+
+**Note:** Depending on your docker installation and networking, you may need to replace the hostname `localhost` in the database URLs
+with `host.docker.internal` to access the docker host. On Mac for example, localhost will refer to the docker client.
+
 
 ```bash
-docker run \
+docker run -it \
-  -it openproject/community:latest
   -e MYSQL_DATABASE_URL=$MYSQL_DATABASE_URL \
-  -e DATABASE_URL=$POSTGRES_DATABASE_URL
+  -e DATABASE_URL=$POSTGRES_DATABASE_URL \
+  openproject/community:latest
 ```
 
 
-
 This will perform all necessary steps to perform the migration. Afterwards, simply remove the `MYSQL_DATABASE_URL`environment variable again and start your container as usual.
 
 

modules/bcf/lib/open_project/bcf/version.rb

@@ -1,7 +0,0 @@
-require 'open_project/version'
-
-module OpenProject
-  module Bcf
-    VERSION = ::OpenProject::VERSION.to_semver
-  end
-end

modules/bcf/openproject-bcf.gemspec

@@ -1,14 +1,9 @@
 # encoding: UTF-8
 
-$:.push File.expand_path("../lib", __FILE__)
-$:.push File.expand_path("../../lib", __dir__)
-
-require "open_project/bcf/version"
-
 # Describe your gem and declare its dependencies:
 Gem::Specification.new do |s|
   s.name        = "openproject-bcf"
-  s.version     = OpenProject::Bcf::VERSION
+  s.version     = "1.0.0"
   s.authors     = "OpenProject GmbH"
   s.email       = "info@openproject.com"
   s.homepage    = "https://community.openproject.org/"