openproject/docker/mysql-to-postgres/bin/migrate-mysql-to-postgres

#!/usr/bin/env ruby
require 'uri'

db_url = URI(ENV.fetch("DATABASE_URL"))
mysql_url = URI(ENV.fetch("MYSQL_DATABASE_URL")) if ENV.has_key?("MYSQL_DATABASE_URL")

if db_url.scheme.start_with?("mysql")
  puts "WARNING: You are running MySQL. OpenProject will drop support for MySQL in 10.0."
  puts "         We can convert it to Postgres for you. Please setup postgres and "
  puts "         rerun this with MYSQL_DATABASE_URL pointing to your original database "
  puts "         and DATABASE_URL pointing to a new Postgres database."
  puts ""
  puts "EXAMPLE: docker run \\"
  puts "           -e MYSQL_DATABASE_URL=mysql://openproject:<PASSWORD>@localhost:3306/openproject \\"
  puts "           -e DATABASE_URL=postgresql://openproject:<PASSWORD>@localhost:5432/openproject \\"
  puts "           -it openproject/community:latest"

  exit 0 # MySQL still supported until 9.x - with 10.0 we must make this an error (exit 1)
end

if db_url.scheme.start_with?("postgres") && mysql_url.nil?
  # nothing to do
  exit 0
end

filtered_mysql_url = mysql_url.dup.tap{|url| url.password = "REDACTED"}
filtered_db_url = db_url.dup.tap{|url| url.password = "REDACTED"}

puts "Migrating database from MySQL to PostgreSQL."
puts
puts "Import"
puts "  MySQL database"
puts "    #{filtered_mysql_url.to_s}"
puts "  into Postgres database "
puts "    #{filtered_db_url.to_s}"
puts "  ?"
puts "WARNING: This resets the given Postgres database."
puts
print "Y/n "

answer = ENV.fetch("FORCE_YES") { gets.chomp }

if answer.downcase == "n"
  exit 0
end

db_user, db_password, db_host, db_port, db_name = db_url.user, db_url.password, db_url.host, db_url.port, db_url.path.sub("/", "")
db_port ||= 5432

if [db_host, db_name, db_port].any?{|value| [nil, ""].include?(value)}
  puts "ERROR: Could not parse database URL (#{filtered_db_url})"

  exit 1
end

require 'pty'

# Memcached is not yet running at this point so we use the file cache store.
# This will and is supposed to affect the spawned ruby processes.
ENV["RAILS_CACHE_STORE"] = "file_store"

def run(cmd, silent: false, record_output: true)
  output = []

  PTY.spawn(cmd) do |stdout, stdin, pid|
    begin
      stdout.each do |line|
        output << line if record_output
        puts line unless silent
      end
    rescue Errno::EIO
      # raised when stdout is closed after process finished
    ensure
      stdout.close
      stdin.close
    end

    Process.wait pid
  end

  [output.join("\n").strip, $?.to_i]
end

puts "Resetting target database..."
drop_cmd = "PGPASSWORD=#{db_password} psql -U #{db_user} -h #{db_host} -p #{db_port} -d postgres -c 'DROP DATABASE #{db_name}'"
drop_output, _ = run drop_cmd, silent: true

if drop_output.include?("database \"#{db_name}\" does not exist") || drop_output == "DROP DATABASE"
  # that's ok then we don't have to drop it or dropped database successfully
  puts "Database dropped"
else
  puts drop_output
  exit 1 # something went wrong
end

puts "Creating database..."
create_cmd = "PGPASSWORD=#{db_password} psql -U #{db_user} -h #{db_host} -p #{db_port} -d postgres -c 'CREATE DATABASE #{db_name}'"
create_output, _ = run create_cmd, silent: true

if create_output == "CREATE DATABASE"
  # created database successfully
else
  puts create_output
  exit 1 # something went wrong
end

puts "Importing database ..."

mysql_url.query = nil
mysql_url.scheme = "mysql"
_, pgloader_status = run "pgloader-ccl --with \"preserve index names\" --verbose #{mysql_url} #{db_url}", record_output: false

if pgloader_status != 0
  puts "\nFailed to import MySQL database into Postgres. See above."

  exit 1
end

check_fulltext_cmd = "PGPASSWORD=#{db_password} psql -o /dev/stdout -U #{db_user} -h #{db_host} -p #{db_port} -d #{db_name} -c \"select 'true' from schema_migrations where version = '20180122135443'\""
check_output, check_status = run check_fulltext_cmd, silent: true

if check_status != 0
  puts check_output
  puts "Failed to check full-text status of database. See above."

  exit 1
end

# if the version was present on MySQL already we need to redo it to
# add the postgres specific columns.
needs_fulltext_migration = check_output.include?("true")

puts "Migrating database ..."

_, migrate_status = run "bundle exec rake db:migrate", record_output: false

if migrate_status != 0
  puts "\nMigration failed. See above."
end

if needs_fulltext_migration
  drop_version_cmd = "PGPASSWORD=#{db_password} psql -U #{db_user} -h #{db_host} -p #{db_port} -d #{db_name} -c \"delete from schema_migrations where version = '20180122135443'\""
  drop_version_output, drop_version_status = run drop_version_cmd, silent: true

  if drop_version_status != 0
    puts drop_version_output
    puts "Failed to drop schema_migrations entry. See above."

    exit 1
  end

  puts "Running full-text search migration"

  _, redo_status = run "bundle exec rake db:migrate:up VERSION=20180122135443", record_output: false

  if redo_status != 0
    puts "\nFull-text search migration failed. See above."

    exit 1
  end

  _, extract_status = run "bundle exec rake attachments:extract_fulltext_where_missing", record_output: false

  if extract_status != 0
    puts "\nFull-text extraction failed. See above."

    exit 1
  end
end

puts "\nRe-building DAG to create Postgres-specific indices"
_, dag_status = run "bundle exec rake db:migrate:redo VERSION=20180105130053", record_output: false

if dag_status != 0
  puts "\nRebuild DAG migration failed. See above."

  exit 1
end

puts "Migration from MySQL to Postgres completed successfully!"