Merge pull request #3631 from oliverguenther/feature/21710-reposman_apache_module

[21710] Apache Perl wrapper for repository management

app/services/scm/delete_managed_repository_service.rb

@@ -43,6 +43,9 @@ Scm::DeleteManagedRepositoryService = Struct.new :repository do
     else
       delete_local_repository
     end
+  rescue OpenProject::Scm::Exceptions::ScmError => e
+    @rejected = e.message
+    false
   end
 
   def delete_local_repository

app/workers/scm/remote_repository_job.rb

@@ -53,9 +53,27 @@ class Scm::RemoteRepositoryJob
     req = ::Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
     req.body = request.to_json
 
-    ::Net::HTTP.start(uri.hostname, uri.port) do |http|
+    response = ::Net::HTTP.start(uri.hostname, uri.port) do |http|
       http.request(req)
     end
+
+    unless response.is_a? ::Net::HTTPSuccess
+      info = try_to_parse_response(response.body)
+      raise OpenProject::Scm::Exceptions::ScmError.new(
+        I18n.t('repositories.errors.remote_call_failed',
+               code: response.code,
+               message: info['message']
+           )
+      )
+    end
+  end
+
+  def try_to_parse_response(body)
+    JSON.parse(body)
+  rescue JSON::JSONError => e
+    raise OpenProject::Scm::Exceptions::ScmError.new(
+      I18n.t('repositories.errors.remote_invalid_response')
+    )
   end
 
   def repository_request

config/locales/en.yml

@@ -1414,6 +1414,8 @@ en:
       exception_title: "Cannot access the repository: %{message}"
       disabled_or_unknown_type: "The selected type %{type} is disabled or no longer available for the SCM vendor %{vendor}."
       disabled_or_unknown_vendor: "The SCM vendor %{vendor} is disabled or no longer available."
+      remote_call_failed: "Calling the managed remote failed with message '%{message}' (Code: %{code})"
+      remote_invalid_response: "Received an invalid response from the managed remote."
     git:
       instructions:
         managed_url: "This is the URL of the managed (local) Git repository."

doc/operation_guides/manual/repository-integration.md

@@ -26,10 +26,20 @@ The following is an excerpt of the configuration and contains all required infor
 	#   Absolute path (e.g. /usr/local/bin/hg) or command name (e.g. hg.exe, bzr.exe)
 	#   On Windows, *.cmd, *.bat (e.g. hg.cmd, bzr.bat) does not work.
 	# manages:
-	#   Enable managed repositories for this vendor. This allows OpenProject to
+	#   You may either specify a local path on the filesystem or an absolute URL to call when
-	#   take control over the given path to create and delete repositories directly
+	#   repositories are to be created or deleted.
-	#   when created in the frontend.
+	#   This allows OpenProject to take control over the given path to create and delete repositories
+	#   directly when created in the frontend.
+	#
+	#   When entering a URL, OpenProject will POST to this resource when repositories are created
+	#   using the following JSON-encoded payload:
+	#     - action: The action to perform (create, delete)
+	#     - identifier: The repository identifier name
+	#     - vendor: The SCM vendor of the repository to create
+	#     - project: identifier, name and ID of the associated project
+	#
 	#   NOTE: Disabling :managed repositories using disabled_types takes precedence over this setting.
+	#
 	# disabled_types:
 	#   Disable specific repository types for this particular vendor. This allows
 	#   to restrict the available choices a project administrator has for creating repositories
@@ -51,7 +61,7 @@ The following is an excerpt of the configuration and contains all required infor
 
 With this configuration, you can create managed repositories by selecting the `managed` Git repository in the Project repository settings tab.
 
-### reposman.rb
+### Reposman.rb
 
 Part of the managed repositories functionality was previously provided with reposman.rb.
 Reposman periodically checked for new projects and automatically created a repository of a given type.
@@ -59,6 +69,55 @@ It never deleted repositories on the filesystem when their associated project wa
 
 This script has been integrated into OpenProject and extended. If you previously used reposman, please see the [upgrade guide to 5.0](./upgrade-guide.md) for further guidance on how to migrate to managed repositories.
 
+### Managing Repositories Remotely
+
+OpenProject comes with a simple webhook to call other services rather than management repositories itself.
+To enable remote managed repositories, simply pass an absolute URL to the `manages` key of a vendor in the `configuration.yml`. The following excerpt shows that configuration for Subversion, assuming your callback is `https://example.org/repos`.
+
+	scm:
+	  subversion:
+	    manages: https://example.org/repos
+	    accesstoken: <Fixed access token passed to the endpoint>
+
+Upon creating and deleting repositories in the frontend, OpenProject will POST to this endpoint a JSON object containg information on the repository.
+
+	{
+		"identifier": "seeded_project.git",
+		"vendor": "git",
+		"scm_type": "managed",
+		"project": {
+			"id": 1,
+			"name": "Seeded Project",
+			"identifier": "seeded_project"
+		},
+		"action": "create",
+		"token": <Fixed access token passed to the endpoint>
+	}
+
+Our main use-case for this feature is to reduce the complexity of permission issues around Subversion mainly in packager, for which a simple Apache wrapper script is used in `extra/Apache/OpenProjectRepoman.pm`.
+This functionality is very limited, but may be extended when other use cases arise.
+	If you're interested in setting up the integration manually outside the context of packager, the following excerpt will help you:
+	
+	PerlSwitches -I/srv/www/perl-lib -T
+	PerlLoadModule Apache::OpenProjectRepoman
+	
+	<Location /repos>
+	        SetHandler perl-script
+	
+	        # Sets the access token secret to check against
+	        AccessSecret "<Fixed access token passed to the endpoint>"
+	
+	        # Configure pairs of (vendor, path) to the wrapper
+	        PerlAddVar ScmVendorPaths "git"
+	        PerlAddVar ScmVendorPaths "/srv/repositories/git"
+	
+	        PerlAddVar ScmVendorPaths "subversion"
+	        PerlAddVar ScmVendorPaths "/srv/repositories/subversion"
+	
+	        PerlResponseHandler Apache::OpenProjectRepoman
+	</Location>
+
+
 ## Other Features
 
 OpenProject 5.0 introduces more features regarding repository management that we briefly outline in the following.
@@ -138,7 +197,7 @@ The following workarounds exist:
 
 This is a simple solution, but theoretically less secure when the server provides more than just SVN and OpenProject.
 
-#### Use filesystem ACLs
+#### Use Filesystem ACLs
 
 You can define ACLs on the managed repository root (requires compatible FS).
 You'll need the the `acl` package and define the ACL.
@@ -178,6 +237,12 @@ On many file systems, ACLS are enabled by default. On others, you might need to
 
 Note that this issue applies to mod_dav_svn only.
 
+### Use the Apache wrapper script
+
+Similar to the integration we use ourselves for the packager-based installation, you can set up Apache to manage repositories using the remote hook in OpenProject.
+
+For more information, see the section 'Managing Repositories Remotely'.
+
 ### Exemplary Apache Configuration
 
 We provide an example apache configuration. Some details are explained inline as comments.

extra/Apache/OpenProjectAuthentication.pm

@@ -1,4 +1,4 @@
-package Apache::Authn::OpenProject;
+package Apache::OpenProjectAuthentication;
 
 use strict;
 use warnings FATAL => 'all', NONFATAL => 'redefine';

extra/Apache/OpenProjectRepoman.pm

@@ -0,0 +1,182 @@
+package Apache::OpenProjectRepoman;
+
+use strict;
+use warnings FATAL => 'all', NONFATAL => 'redefine';
+
+use File::Path qw(remove_tree);
+use File::Spec ();
+
+use Apache2::Module;
+use Apache2::Module;
+use Apache2::Access;
+use Apache2::ServerRec qw();
+use Apache2::Response ();
+use Apache2::RequestRec qw();
+use Apache2::RequestUtil qw();
+use Apache2::RequestIO qw();
+use Apache2::Const -compile => qw(FORBIDDEN OK OR_AUTHCFG TAKE1 HTTP_UNPROCESSABLE_ENTITY HTTP_BAD_REQUEST OK);
+use APR::Table ();
+
+use JSON::PP;
+use Carp;
+
+
+##
+# Add AccessSecret directive to Apache, which is checked during configtest
+my @directives = (
+  {
+    name => 'AccessSecret',
+    req_override => Apache2::Const::OR_AUTHCFG,
+    args_how => Apache2::Const::TAKE1,
+    errmsg => 'Secret access token used to access the repository wrapper.',
+  }
+);
+Apache2::Module::add(__PACKAGE__, \@directives);
+
+##
+# Accepts and tests the access secret value given in the Apache configuration
+sub AccessSecret {
+  my ($self, $parms, @args) = @_;
+  $self->{token} = $args[0];
+  unless (length($self->{token}) >= 8) {
+    die "Use at least 8 characters for the repoman access token!";
+  }
+}
+
+##
+# Creates an actual repository on disk for Subversion and Git.
+sub create_repository {
+  my ($r, $vendor, $repository) = @_;
+
+  my $command = {
+    git => "git init $repository --shared --bare",
+    subversion => "svnadmin create $repository"
+  }->{$vendor};
+
+  die "No create command known for vendor '$vendor'\n" unless defined($command);
+  die "Could not create repository.\n" unless system($command) == 0;
+}
+
+##
+# Removes the repository with a given identifier on disk.
+sub delete_repository {
+  my ($r, $vendor, $repository) = @_;
+  remove_tree($repository, { safe => 1 }) if -d $repository;
+}
+
+##
+# Extract and return JSON request from the Apache request handler.
+sub parse_request {
+  my $r = shift;
+  my $len  = $r->headers_in->{'Content-Length'};
+
+  die "Request invalid.\n"  unless (defined($len) && $len > 0);
+  die "Request too large.\n"  if ($len > (2**13));
+
+  my ($buf, $content);
+  while($r->read($buf, $len)) {
+    $content .= $buf;
+  }
+
+  return decode_json($content);
+}
+
+##
+# Returns a JSON error and sets the HTTP response code to $type.
+sub make_error {
+    my ($r, $type, $msg) = @_;
+    my $response = {
+      success => JSON::PP::false,
+      message => $msg
+    };
+
+    $r->status($type) ;
+    return $response;
+  }
+
+##
+# Actual incoming request handler, that receives the JSON request
+# and determines the necessary local action from the request.
+sub _handle_request {
+    my $r = shift;
+
+  # Parse JSON request
+  my $request = parse_request($r);
+
+  # Get repository root for the current vendor
+  my %paths = $r->dir_config->get('ScmVendorPaths');
+
+  my $vendor = $request->{vendor};
+  my $repository_root = $paths{$vendor};
+
+  # Compare access token
+  my $passed_token = $request->{token};
+  my $cfg = Apache2::Module::get_config( __PACKAGE__, $r->server, $r->per_dir_config );
+
+  unless (length($passed_token) >= 8 && ($passed_token eq $cfg->{token})) {
+    return make_error($r, Apache2::Const::FORBIDDEN, 'Invalid access token');
+  }
+
+  # Abort unless repository root is configured in the Apache configuration
+  unless (defined($repository_root)) {
+    return make_error($r,
+      Apache2::Const::HTTP_UNPROCESSABLE_ENTITY,
+      "Vendor '$vendor' not configured.");
+  }
+
+  # Abort unless the repository root actually exists
+  unless (-d $repository_root) {
+    return make_error($r,
+      Apache2::Const::HTTP_UNPROCESSABLE_ENTITY,
+      "Repository path for vendor '$vendor' does not exist.");
+  }
+
+  # Determine validity of the identifier as a dir name
+  my $repository_identifier = $request->{identifier};
+  if ($repository_identifier =~ m{[\\/:*?"<>|]}) {
+    return make_error($r,
+      Apache2::Const::HTTP_UNPROCESSABLE_ENTITY,
+      "Repository identifier is an invalid filename");
+  }
+
+  # Call the necessary action on disk
+  my $target = File::Spec->catdir($repository_root, $repository_identifier);
+  my %actions = (
+    'create' => \&create_repository,
+    'delete' => \&delete_repository
+    );
+
+  my $action = $actions{$request->{action}};
+  die "Unknown action.\n"  unless defined($action);
+  $action->($r, $vendor, $target);
+
+  return {
+    success => JSON::PP::true,
+    message => "The action has completed sucessfully.",
+    repository => $target
+  };
+}
+
+##
+# Handler subroutine that is called for each request by Apache
+sub handler {
+  my $r = shift;
+
+  my $response;
+  $r->content_type('application/json');
+
+  eval {
+    $response = _handle_request($r);
+    1;
+  } or do {
+    my $err = $@;
+    chomp $err;
+    $response = make_error($r, Apache2::Const::HTTP_BAD_REQUEST, $err);
+  };
+
+
+  print encode_json($response);
+  return Apache2::Const::OK;
+}
+
+1;

spec/services/scm/create_managed_repository_service_spec.rb

@@ -172,5 +172,24 @@ describe Scm::CreateManagedRepositoryService do
           .with(body: hash_including(action: 'create'))
       end
     end
+
+    context 'with a faulty remote callback' do
+      before do
+        stub_request(:post, url)
+          .to_return(status: 400, body: { success: false, message: 'An error occurred' }.to_json)
+      end
+
+      it 'calls the callback' do
+        expect(Scm::CreateRemoteRepositoryJob)
+          .to receive(:new).and_call_original
+
+        expect(service.call).to be false
+        expect(service.localized_rejected_reason)
+          .to eq("Calling the managed remote failed with message 'An error occurred' (Code: 400)")
+        expect(WebMock)
+          .to have_requested(:post, url)
+                .with(body: hash_including(action: 'create'))
+      end
+    end
   end
 end

spec/services/scm/delete_managed_repository_service_spec.rb

@@ -134,23 +134,44 @@ describe Scm::DeleteManagedRepositoryService do
       repo.project = project
       repo.configure(:managed, nil)
 
-      repo.save!
       repo
     }
 
-    before do
+    context 'with a valid remote' do
-      stub_request(:post, url).to_return(status: 200)
+      before do
+        stub_request(:post, url).to_return(status: 200)
+      end
+
+      it 'calls the callback' do
+        expect(Scm::DeleteRemoteRepositoryJob)
+          .to receive(:new).and_call_original
+
+        expect(service.call).to be true
+        expect(WebMock)
+          .to have_requested(:post, url)
+          .with(body: hash_including(identifier: repository.repository_identifier,
+                                     action: 'delete'))
+      end
     end
 
-    it 'calls the callback' do
+    context 'with a remote callback returning an error' do
-      expect(Scm::DeleteRemoteRepositoryJob)
+      before do
-        .to receive(:new).and_call_original
+        stub_request(:post, url)
+          .to_return(status: 400, body: { success: false, message: 'An error occurred' }.to_json)
+      end
 
-      expect(service.call).to be true
+      it 'calls the callback' do
-      expect(WebMock)
+        expect(Scm::DeleteRemoteRepositoryJob)
-        .to have_requested(:post, url)
+          .to receive(:new).and_call_original
-        .with(body: hash_including(identifier: repository.repository_identifier,
+
-                                   action: 'delete'))
+        expect(service.call).to be false
+
+        expect(service.localized_rejected_reason)
+          .to eq("Calling the managed remote failed with message 'An error occurred' (Code: 400)")
+        expect(WebMock)
+          .to have_requested(:post, url)
+                .with(body: hash_including(action: 'delete'))
+      end
     end
   end
 end