Oliver Günther
6 years ago
commit
e850b2fc60
@ -0,0 +1,22 @@ |
||||
FROM ruby:2.6-stretch |
||||
MAINTAINER operations@openproject.com |
||||
|
||||
ENV PGLOADER_DEPENDENCIES "libsqlite3-dev make curl gawk freetds-dev libzip-dev" |
||||
|
||||
# Install |
||||
# |
||||
# 1) mysql and postgres clients |
||||
# 2) pgloader dependencies minus SBCL since we use CCL |
||||
RUN apt-get update -qq && \ |
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \ |
||||
mysql-client postgresql-client \ |
||||
$PGLOADER_DEPENDENCIES && \ |
||||
apt-get clean && rm -rf /var/lib/apt/lists/* |
||||
|
||||
# pgloader |
||||
ENV CCL_DEFAULT_DIRECTORY /opt/ccl |
||||
COPY docker/mysql-to-postgres/bin/build /tmp/build-pgloader |
||||
RUN /tmp/build-pgloader && rm /tmp/build-pgloader |
||||
COPY docker/mysql-to-postgres/bin/migrate-mysql-to-postgres /usr/local/bin/ |
||||
|
||||
CMD ["migrate-mysql-to-postgres"] |
||||
@ -0,0 +1,21 @@ |
||||
#!/bin/bash |
||||
|
||||
set -e |
||||
|
||||
cd /opt |
||||
|
||||
# install Clozure CL to avoid memory issues with the standard SBCL |
||||
wget -q https://github.com/Clozure/ccl/releases/download/v1.11.5/ccl-1.11.5-linuxx86.tar.gz |
||||
tar -xzf ccl-1.11.5-linuxx86.tar.gz |
||||
rm ccl-1.11.5-linuxx86.tar.gz |
||||
ln -s /opt/ccl/scripts/ccl64 /usr/local/bin/ccl |
||||
|
||||
export CCL_DEFAULT_DIRECTORY=/opt/ccl |
||||
|
||||
# build pgloader from source using CCL as the lisp runtime |
||||
git clone https://github.com/dimitri/pgloader.git |
||||
cd pgloader |
||||
git checkout v3.6.1 |
||||
make CL=ccl pgloader >& /tmp/pgloader-compile.log || (cat /tmp/pgloader-compile.log && exit 1) |
||||
mv /opt/pgloader/build/bin/pgloader /usr/local/bin/pgloader-ccl |
||||
rm -rf /opt/pgloader |
||||
@ -0,0 +1,180 @@ |
||||
#!/usr/bin/env ruby |
||||
require 'uri' |
||||
|
||||
db_url = URI(ENV.fetch("DATABASE_URL")) |
||||
mysql_url = URI(ENV.fetch("MYSQL_DATABASE_URL")) if ENV.has_key?("MYSQL_DATABASE_URL") |
||||
|
||||
if db_url.scheme.start_with?("mysql") |
||||
puts "WARNING: You are running MySQL. OpenProject will drop support for MySQL in 10.0." |
||||
puts " We can convert it to Postgres for you. Please setup postgres and " |
||||
puts " rerun this with MYSQL_DATABASE_URL pointing to your original database " |
||||
puts " and DATABASE_URL pointing to a new Postgres database." |
||||
puts "" |
||||
puts "EXAMPLE: docker run \\" |
||||
puts " -e MYSQL_DATABASE_URL=mysql://openproject:<PASSWORD>@localhost:3306/openproject \\" |
||||
puts " -e DATABASE_URL=postgresql://openproject:<PASSWORD>@localhost:5432/openproject \\" |
||||
puts " -it openproject/community:latest" |
||||
|
||||
exit 0 # MySQL still supported until 9.x - with 10.0 we must make this an error (exit 1) |
||||
end |
||||
|
||||
if db_url.scheme.start_with?("postgresql") && mysql_url.nil? |
||||
# nothing to do |
||||
exit 0 |
||||
end |
||||
|
||||
filtered_mysql_url = mysql_url.dup.tap{|url| url.password = "REDACTED"} |
||||
filtered_db_url = db_url.dup.tap{|url| url.password = "REDACTED"} |
||||
|
||||
puts "Migrating database from MySQL to PostgreSQL." |
||||
puts |
||||
puts "Import" |
||||
puts " MySQL database" |
||||
puts " #{filtered_mysql_url.to_s}" |
||||
puts " into Postgres database " |
||||
puts " #{filtered_db_url.to_s}" |
||||
puts " ?" |
||||
puts "WARNING: This resets the given Postgres database." |
||||
puts |
||||
print "Y/n " |
||||
|
||||
answer = ENV.fetch("FORCE_YES") { gets.chomp } |
||||
|
||||
if answer.downcase == "n" |
||||
exit 0 |
||||
end |
||||
|
||||
db_user, db_password, db_host, db_port, db_name = db_url.user, db_url.password, db_url.host, db_url.port, db_url.path.sub("/", "") |
||||
db_port ||= 5432 |
||||
|
||||
if [db_host, db_name, db_port].any?{|value| [nil, ""].include?(value)} |
||||
puts "ERROR: Could not parse database URL (#{filtered_db_url})" |
||||
|
||||
exit 1 |
||||
end |
||||
|
||||
require 'pty' |
||||
|
||||
# Memcached is not yet running at this point so we use the file cache store. |
||||
# This will and is supposed to affect the spawned ruby processes. |
||||
ENV["RAILS_CACHE_STORE"] = "file_store" |
||||
|
||||
def run(cmd, silent: false, record_output: true) |
||||
output = [] |
||||
|
||||
PTY.spawn(cmd) do |stdout, stdin, pid| |
||||
begin |
||||
stdout.each do |line| |
||||
output << line if record_output |
||||
puts line unless silent |
||||
end |
||||
rescue Errno::EIO |
||||
# raised when stdout is closed after process finished |
||||
ensure |
||||
stdout.close |
||||
stdin.close |
||||
end |
||||
|
||||
Process.wait pid |
||||
end |
||||
|
||||
[output.join("\n").strip, $?.to_i] |
||||
end |
||||
|
||||
puts "Resetting target database..." |
||||
drop_cmd = "PGPASSWORD=#{db_password} psql -U #{db_user} -h #{db_host} -p #{db_port} -d postgres -c 'DROP DATABASE #{db_name}'" |
||||
drop_output, _ = run drop_cmd, silent: true |
||||
|
||||
if drop_output.include?("database \"#{db_name}\" does not exist") || drop_output == "DROP DATABASE" |
||||
# that's ok then we don't have to drop it or dropped database successfully |
||||
puts "Database dropped" |
||||
else |
||||
puts drop_output |
||||
exit 1 # something went wrong |
||||
end |
||||
|
||||
puts "Creating database..." |
||||
create_cmd = "PGPASSWORD=#{db_password} psql -U #{db_user} -h #{db_host} -p #{db_port} -d postgres -c 'CREATE DATABASE #{db_name}'" |
||||
create_output, _ = run create_cmd, silent: true |
||||
|
||||
if create_output == "CREATE DATABASE" |
||||
# created database successfully |
||||
else |
||||
puts create_output |
||||
exit 1 # something went wrong |
||||
end |
||||
|
||||
puts "Importing database ..." |
||||
|
||||
mysql_url.query = nil |
||||
mysql_url.scheme = "mysql" |
||||
_, pgloader_status = run "pgloader-ccl --verbose #{mysql_url} #{db_url}", record_output: false |
||||
|
||||
if pgloader_status != 0 |
||||
puts "\nFailed to import MySQL database into Postgres. See above." |
||||
|
||||
exit 1 |
||||
end |
||||
|
||||
check_fulltext_cmd = "PGPASSWORD=#{db_password} psql -o /dev/stdout -U #{db_user} -h #{db_host} -p #{db_port} -d #{db_name} -c \"select 'true' from schema_migrations where version = '20180122135443'\"" |
||||
check_output, check_status = run check_fulltext_cmd, silent: true |
||||
|
||||
if check_status != 0 |
||||
puts check_output |
||||
puts "Failed to check full-text status of database. See above." |
||||
|
||||
exit 1 |
||||
end |
||||
|
||||
# if the version was present on MySQL already we need to redo it to |
||||
# add the postgres specific columns. |
||||
needs_fulltext_migration = check_output.include?("true") |
||||
|
||||
puts "Migrating database ..." |
||||
|
||||
_, migrate_status = run "bundle exec rake db:migrate", record_output: false |
||||
|
||||
if migrate_status != 0 |
||||
puts "\nMigration failed. See above." |
||||
end |
||||
|
||||
if needs_fulltext_migration |
||||
drop_version_cmd = "PGPASSWORD=#{db_password} psql -U #{db_user} -h #{db_host} -p #{db_port} -d #{db_name} -c \"delete from schema_migrations where version = '20180122135443'\"" |
||||
drop_version_output, drop_version_status = run drop_version_cmd, silent: true |
||||
|
||||
if drop_version_status != 0 |
||||
puts drop_version_output |
||||
puts "Failed to drop schema_migrations entry. See above." |
||||
|
||||
exit 1 |
||||
end |
||||
|
||||
puts "Running full-text search migration" |
||||
|
||||
_, redo_status = run "bundle exec rake db:migrate:up VERSION=20180122135443", record_output: false |
||||
|
||||
if redo_status != 0 |
||||
puts "\nFull-text search migration failed. See above." |
||||
|
||||
exit 1 |
||||
end |
||||
|
||||
_, extract_status = run "bundle exec rake attachments:extract_fulltext_where_missing", record_output: false |
||||
|
||||
if extract_status != 0 |
||||
puts "\nFull-text extraction failed. See above." |
||||
|
||||
exit 1 |
||||
end |
||||
end |
||||
|
||||
puts "\nRe-building DAG to create Postgres-specific indices" |
||||
_, dag_status = run "bundle exec rake db:migrate:redo VERSION=20180105130053", record_output: false |
||||
|
||||
if dag_status != 0 |
||||
puts "\nRebuild DAG migration failed. See above." |
||||
|
||||
exit 1 |
||||
end |
||||
|
||||
puts "Migration from MySQL to Postgres completed successfully!" |
||||
Loading…
Reference in new issue