Remote managed repositories

This commit starts the work to provide functionality with remote service hooks for managing repositories.

The main driver behind this is packager, for which we want apache to be the sole owner of the repositories it delivers, and OpenProject only as a reader, however it benefits other installations as well.

Gemfile

@@ -160,6 +160,7 @@ group :test do
   gem 'capybara-ng', '~> 0.2.1'
   gem 'selenium-webdriver', '~> 2.47.1'
   gem 'timecop', '~> 0.7.1'
+  gem 'webmock', '~> 1.21.0', require: false
 
   gem 'rb-readline', '~> 0.5.1' # ruby on CI needs this
   # why in Gemfile? see: https://github.com/guard/guard-test

Gemfile.lock

@@ -181,6 +181,8 @@ GEM
       descendants_tracker (~> 0.0.1)
     color-tools (1.3.0)
     columnize (0.9.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
     crowdin-api (0.2.8)
       rest-client (~> 1.6.8)
     cucumber (1.3.19)
@@ -453,6 +455,7 @@ GEM
       json (>= 1.7.5)
       structured_warnings (>= 0.1.3)
     rubyzip (1.1.7)
+    safe_yaml (1.0.4)
     sass (3.4.13)
     sass-rails (5.0.3)
       railties (>= 4.0.0, < 5.0)
@@ -511,6 +514,9 @@ GEM
       rack (>= 1.0)
     warden-basic_auth (0.2.1)
       warden (~> 1.2)
+    webmock (1.21.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
     websocket (1.2.2)
     will_paginate (3.0.7)
     xpath (2.0.0)
@@ -622,6 +628,7 @@ DEPENDENCIES
   unicorn
   warden (~> 1.2)
   warden-basic_auth (~> 0.2.1)
+  webmock (~> 1.21.0)
   will_paginate (~> 3.0)
 
 BUNDLED WITH

app/models/repository.rb

@@ -435,13 +435,10 @@ class Repository < ActiveRecord::Base
   # is managed by OpenProject
   def delete_managed_repository
     service = Scm::DeleteManagedRepositoryService.new(self)
-    # Even if the service can't remove the physical repository,
-    # we should continue removing the associated instance.
     if service.call
       true
     else
-      errors.add(:base, I18n.t('repositories.errors.filesystem_access_failed',
-                               message: I18n.t('repositories.errors.deletion_failed')))
+      errors.add(:base, service.localized_rejected_reason)
       raise ActiveRecord::Rollback
     end
   end

app/services/scm/create_managed_repository_service.rb

@@ -38,9 +38,6 @@ Scm::CreateManagedRepositoryService = Struct.new :repository do
   def call
     if repository.managed? && repository.manageable?
 
-      # Cowardly refusing to override existing local repository
-      return false if repository_exists?
-
       ##
       # We want to move this functionality in a Delayed Job,
       # but this heavily interferes with the error handling of the whole
@@ -48,7 +45,11 @@ Scm::CreateManagedRepositoryService = Struct.new :repository do
       # Instead, this will be refactored into a single service wrapper for
       # creating and deleting repositories, which provides transactional DB access
       # as well as filesystem access.
-      Scm::CreateRepositoryJob.new(repository).perform
+      if repository.class.manages_remote?
+        Scm::CreateRemoteRepositoryJob.new(repository).perform
+      else
+        Scm::CreateLocalRepositoryJob.new(repository).perform
+      end
       return true
     end
 
@@ -61,6 +62,9 @@ Scm::CreateManagedRepositoryService = Struct.new :repository do
     @rejected = I18n.t('repositories.errors.filesystem_access_failed',
                        message: e.message)
     false
+  rescue OpenProject::Scm::Exceptions::ScmError => e
+    @rejected = e.message
+    false
   end
 
   ##
@@ -68,15 +72,4 @@ Scm::CreateManagedRepositoryService = Struct.new :repository do
   def localized_rejected_reason
     @rejected ||= I18n.t('repositories.errors.not_manageable')
   end
-
-  private
-
-  ##
-  # Test if the repository exists already on filesystem.
-  def repository_exists?
-    if File.directory?(repository.root_url)
-      @rejected = I18n.t('repositories.errors.exists_on_filesystem')
-      return true
-    end
-  end
 end

app/services/scm/delete_managed_repository_service.rb

@@ -35,14 +35,17 @@ Scm::DeleteManagedRepositoryService = Struct.new :repository do
   # Registers an asynchronous job to delete the repository on disk.
   #
   def call
-    if repository.managed?
-      delete_repository
+    return false unless repository.managed?
+
+    if repository.class.manages_remote?
+      Scm::DeleteRemoteRepositoryJob.new(repository).perform
+      true
     else
-      false
+      delete_local_repository
     end
   end
 
-  def delete_repository
+  def delete_local_repository
     # Create necessary changes to repository to mark
     # it as managed by OP, but delete asynchronously.
     managed_path = repository.root_url
@@ -55,13 +58,20 @@ Scm::DeleteManagedRepositoryService = Struct.new :repository do
       # Instead, this will be refactored into a single service wrapper for
       # creating and deleting repositories, which provides transactional DB access
       # as well as filesystem access.
-      Scm::DeleteRepositoryJob.new(managed_path).perform
+      Scm::DeleteLocalRepositoryJob.new(managed_path).perform
     end
 
     true
   rescue SystemCallError => e
-    Rails.logger.error("An error occurred while accessing the repository '#{repository.root_url}'" +
-                       " on filesystem: #{e.message}")
+    @rejected = I18n.t('repositories.errors.managed_delete_local',
+                       path: repository.root_url,
+                       error_message: e.message)
     false
   end
+
+  ##
+  # Returns the error symbol
+  def localized_rejected_reason
+    @rejected ||= I18n.t('repositories.errors.managed_delete')
+  end
 end

app/workers/scm/create_local_repository_job.rb

@@ -34,10 +34,17 @@
 # We envision a repository management wrapper that covers transactional
 # creation and deletion of repositories BOTH on the database and filesystem.
 # Until then, a synchronous process is more failsafe.
-class Scm::CreateRepositoryJob
+class Scm::CreateLocalRepositoryJob
   include OpenProject::BeforeDelayedJob
 
   def initialize(repository)
+    # Cowardly refusing to override existing local repository
+    if File.directory?(repository.root_url)
+      raise OpenProject::Scm::Exceptions::ScmError.new(
+        I18n.t('repositories.errors.exists_on_filesystem')
+      )
+    end
+
     # TODO currently uses the full repository object,
     # as the Job is performed synchronously.
     # Change this to serialize the ID once its turned to process asynchronously.

app/workers/scm/create_remote_repository_job.rb

@@ -0,0 +1,42 @@
+#-- encoding: UTF-8
+#-- copyright
+# OpenProject is a project management system.
+# Copyright (C) 2012-2015 the OpenProject Foundation (OPF)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See doc/COPYRIGHT.rdoc for more details.
+#++
+
+##
+# Provides an asynchronous job to create a managed repository on a remote system
+# using a simple HTTP callback
+# Currently, this is run synchronously due to potential issues
+# with error handling.
+# We envision a repository management wrapper that covers transactional
+# creation and deletion of repositories BOTH on the database and filesystem.
+# Until then, a synchronous process is more failsafe.
+class Scm::CreateRemoteRepositoryJob < Scm::RemoteRepositoryJob
+  def perform
+    send(repository_request.merge(action: :create))
+  end
+end

app/workers/scm/delete_local_repository_job.rb

@@ -34,7 +34,7 @@
 # We envision a repository management wrapper that covers transactional
 # creation and deletion of repositories BOTH on the database and filesystem.
 # Until then, a synchronous process is more failsafe.
-class Scm::DeleteRepositoryJob
+class Scm::DeleteLocalRepositoryJob
   include OpenProject::BeforeDelayedJob
 
   def initialize(managed_path)

app/workers/scm/delete_remote_repository_job.rb

@@ -0,0 +1,41 @@
+#-- encoding: UTF-8
+#-- copyright
+# OpenProject is a project management system.
+# Copyright (C) 2012-2015 the OpenProject Foundation (OPF)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See doc/COPYRIGHT.rdoc for more details.
+#++
+
+##
+# Provides an asynchronous job to delete a managed repository on the filesystem.
+# Currently, this is run synchronously due to potential issues
+# with error handling.
+# We envision a repository management wrapper that covers transactional
+# creation and deletion of repositories BOTH on the database and filesystem.
+# Until then, a synchronous process is more failsafe.
+class Scm::DeleteRemoteRepositoryJob < Scm::RemoteRepositoryJob
+  def perform
+    send(repository_request.merge(action: :delete))
+  end
+end

app/workers/scm/remote_repository_job.rb

@@ -0,0 +1,75 @@
+#-- encoding: UTF-8
+#-- copyright
+# OpenProject is a project management system.
+# Copyright (C) 2012-2015 the OpenProject Foundation (OPF)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See doc/COPYRIGHT.rdoc for more details.
+#++
+
+##
+# Provides an asynchronous job to create a managed repository on the filesystem.
+# Currently, this is run synchronously due to potential issues
+# with error handling.
+# We envision a repository management wrapper that covers transactional
+# creation and deletion of repositories BOTH on the database and filesystem.
+# Until then, a synchronous process is more failsafe.
+class Scm::RemoteRepositoryJob
+  include OpenProject::BeforeDelayedJob
+
+  def initialize(repository)
+    # TODO currently uses the full repository object,
+    # as the Job is performed synchronously.
+    # Change this to serialize the ID once its turned to process asynchronously.
+    @repository = repository
+  end
+
+  protected
+
+  ##
+  # Submits the request to the configured managed remote as JSON.
+  def send(request)
+    uri = @repository.class.managed_remote
+    req = ::Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
+    req.body = request.to_json
+
+    ::Net::HTTP.start(uri.hostname, uri.port) do |http|
+      http.request(req)
+    end
+  end
+
+  def repository_request
+    project = @repository.project
+
+    {
+      identifier: @repository.repository_identifier,
+      vendor: @repository.vendor,
+      scm_type: @repository.scm_type,
+      project: {
+        id: project.id,
+        name: project.name,
+        identifier: project.identifier,
+      }
+    }
+  end
+end

config/locales/en.yml

@@ -1402,10 +1402,11 @@ en:
       title: "Do you really want to delete the %{repository_type} of the project %{project_name}?"
     errors:
       build_failed: "Unable to create the repository with the selected configuration. %{reason}"
+      managed_delete: "Unable to delete the managed repository."
+      managed_delete_local: "Unable to delete the local repository on filesystem at '%{path}': %{error_message}"
       empty_repository: "The repository exists, but is empty. It does not contain any revisions yet."
       exists_on_filesystem: "The repository directory already exists in the filesystem."
       filesystem_access_failed: "An error occurred while accessing the repository in the filesystem: %{message}"
-      deletion_failed: "The repository could not be deleted from disk due to lacking permissions. Please contact your administrator to resolve this issue."
       not_manageable: "This repository vendor cannot be managed by OpenProject."
       path_permission_failed: "An error occurred trying to create the following path: %{path}. Please ensure that OpenProject may write to that folder."
       unauthorized: "You're not authorized to access the repository or the credentials are invalid."

lib/open_project/scm/manageable_repository.rb

@@ -58,10 +58,24 @@ module OpenProject
         def managed_root
           scm_config[:manages]
         end
+
+        ##
+        # Returns the managed remote for this repository vendor,
+        # if any. Use +manages_remote?+ to determine whether the configuration
+        # specifies local or remote managed repositories.
+        def managed_remote
+          URI.parse(scm_config[:manages])
+        rescue URI::Error
+          nil
+        end
+
+        ##
+        # Returns whether the managed root is a remote URL to post to
+        def manages_remote?
+          managed_remote.present? && managed_remote.absolute?
+        end
       end
 
-      ##
-      #
       def manageable?
         self.class.manageable?
       end

spec/services/scm/checkout_instructions_service_spec.rb

@@ -93,7 +93,7 @@ describe Scm::CheckoutInstructionsService do
 
       it 'returns the default translated instructions' do
         expect(service.instructions)
-          .to eq(I18n.t("repositories.checkout.default_instructions.subversion"))
+          .to eq(I18n.t('repositories.checkout.default_instructions.subversion'))
       end
     end
   end

spec/services/scm/create_managed_repository_service_spec.rb

@@ -35,7 +35,7 @@ describe Scm::CreateManagedRepositoryService do
   let(:repository) { FactoryGirl.build(:repository_subversion) }
   subject(:service) { Scm::CreateManagedRepositoryService.new(repository) }
 
-  let(:config)   { {} }
+  let(:config) { {} }
 
   before do
     allow(OpenProject::Configuration).to receive(:[]).and_call_original
@@ -69,12 +69,12 @@ describe Scm::CreateManagedRepositoryService do
     end
   end
 
-  context 'with managed config' do
+  context 'with managed local config' do
     include_context 'with tmpdir'
     let(:config) {
       {
         subversion: { manages: File.join(tmpdir, 'svn') },
-        git:        { manages: File.join(tmpdir, 'git') }
+        git: { manages: File.join(tmpdir, 'git') }
       }
     }
 
@@ -86,7 +86,9 @@ describe Scm::CreateManagedRepositoryService do
     }
 
     before do
-      allow_any_instance_of(Scm::CreateRepositoryJob)
+      allow_any_instance_of(Scm::CreateLocalRepositoryJob)
+        .to receive(:repository).and_return(repository)
+      allow_any_instance_of(Scm::CreateRemoteRepositoryJob)
         .to receive(:repository).and_return(repository)
     end
 
@@ -109,7 +111,7 @@ describe Scm::CreateManagedRepositoryService do
 
     context 'with a permission error occuring in the Job' do
       before do
-        allow(Scm::CreateRepositoryJob)
+        allow(Scm::CreateLocalRepositoryJob)
           .to receive(:new).and_raise(Errno::EACCES)
       end
 
@@ -123,7 +125,7 @@ describe Scm::CreateManagedRepositoryService do
 
     context 'with an OS error occuring in the Job' do
       before do
-        allow(Scm::CreateRepositoryJob)
+        allow(Scm::CreateLocalRepositoryJob)
           .to receive(:new).and_raise(Errno::ENOENT)
       end
 
@@ -134,4 +136,41 @@ describe Scm::CreateManagedRepositoryService do
       end
     end
   end
+
+  context 'with managed remote config' do
+    let(:url) { 'http://myreposerver.example.com/api/' }
+    let(:config) {
+      {
+        subversion: { manages: url }
+      }
+    }
+
+    let(:repository) {
+      repo = Repository::Subversion.new(scm_type: :managed)
+      repo.project = project
+      repo.configure(:managed, nil)
+      repo
+    }
+
+    it 'detects the remote config' do
+      expect(repository.class.managed_remote.to_s).to eq(url)
+      expect(repository.class).to be_manages_remote
+    end
+
+    context 'with a remote callback' do
+      before do
+        stub_request(:post, url).to_return(status: 200)
+      end
+
+      it 'calls the callback' do
+        expect(Scm::CreateRemoteRepositoryJob)
+          .to receive(:new).and_call_original
+
+        expect(service.call).to be true
+        expect(WebMock)
+          .to have_requested(:post, url)
+          .with(body: hash_including(action: 'create'))
+      end
+    end
+  end
 end

spec/services/scm/delete_managed_repository_service_spec.rb

@@ -35,7 +35,7 @@ describe Scm::DeleteManagedRepositoryService do
   let(:repository) { FactoryGirl.build(:repository_subversion) }
   subject(:service) { Scm::DeleteManagedRepositoryService.new(repository) }
 
-  let(:config)   { {} }
+  let(:config) { {} }
 
   before do
     allow(OpenProject::Configuration).to receive(:[]).and_call_original
@@ -68,7 +68,7 @@ describe Scm::DeleteManagedRepositoryService do
     let(:config) {
       {
         subversion: { manages: File.join(tmpdir, 'svn') },
-        git:        { manages: File.join(tmpdir, 'git') }
+        git: { manages: File.join(tmpdir, 'git') }
       }
     }
 
@@ -82,7 +82,9 @@ describe Scm::DeleteManagedRepositoryService do
     }
 
     before do
-      allow_any_instance_of(Scm::CreateRepositoryJob)
+      allow_any_instance_of(Scm::DeleteLocalRepositoryJob)
+        .to receive(:repository).and_return(repository)
+      allow_any_instance_of(Scm::DeleteRemoteRepositoryJob)
         .to receive(:repository).and_return(repository)
     end
 
@@ -94,7 +96,7 @@ describe Scm::DeleteManagedRepositoryService do
 
     it 'does not raise an exception upon permission errors' do
       expect(File.directory?(repository.root_url)).to be true
-      expect(Scm::DeleteRepositoryJob)
+      expect(Scm::DeleteLocalRepositoryJob)
         .to receive(:new).and_raise(Errno::EACCES)
 
       expect(service.call).to be false
@@ -118,4 +120,37 @@ describe Scm::DeleteManagedRepositoryService do
       end
     end
   end
+
+  context 'with managed remote config' do
+    let(:url) { 'http://myreposerver.example.com/api/' }
+    let(:config) {
+      {
+        subversion: { manages: url }
+      }
+    }
+
+    let(:repository) {
+      repo = Repository::Subversion.new(scm_type: :managed)
+      repo.project = project
+      repo.configure(:managed, nil)
+
+      repo.save!
+      repo
+    }
+
+    before do
+      stub_request(:post, url).to_return(status: 200)
+    end
+
+    it 'calls the callback' do
+      expect(Scm::DeleteRemoteRepositoryJob)
+        .to receive(:new).and_call_original
+
+      expect(service.call).to be true
+      expect(WebMock)
+        .to have_requested(:post, url)
+        .with(body: hash_including(identifier: repository.repository_identifier,
+                                   action: 'delete'))
+    end
+  end
 end

spec/spec_helper.rb

@@ -43,6 +43,7 @@ require 'rspec/example_disabler'
 require 'capybara/rails'
 require 'capybara-screenshot/rspec'
 require 'factory_girl_rails'
+require 'webmock/rspec'
 
 Capybara.register_driver :selenium do |app|
   require 'selenium/webdriver'

spec/workers/scm/create_local_repository_job_spec.rb

@@ -29,7 +29,7 @@
 
 require 'spec_helper'
 
-describe Scm::CreateRepositoryJob do
+describe Scm::CreateLocalRepositoryJob do
   subject { described_class.new(repository) }
 
   # Allow to override configuration values to determine