- Remove configuration option for attachment search

- Remove attachment search from global search as it is currently incompatible
- Add rake task for indexing attachments that have not been indexed yet
- Make the attachments filters a EE feature.
pull/6038/head
Wieland Lindenthal 7 years ago
parent 14081ea0f1
commit 643e945b85
  1. 21
      app/models/attachment.rb
  2. 6
      app/models/queries/work_packages.rb
  3. 15
      app/models/work_package.rb
  4. 2
      app/services/authorization/enterprise_service.rb
  5. 2
      app/workers/extract_fulltext_job.rb
  6. 41
      config/configuration.yml.example
  7. 1
      config/environments/production.rb
  8. 9
      docs/operations/upgrading/manual/upgrading.md
  9. 1
      lib/open_project/configuration.rb
  10. 6
      lib/tasks/attachments.rake

@ -182,24 +182,17 @@ class Attachment < ActiveRecord::Base
end
def extract_fulltext
if OpenProject::Configuration['enable_attachment_search']
job = ExtractFulltextJob.new(id)
Delayed::Job::enqueue job
end
job = ExtractFulltextJob.new(id)
Delayed::Job::enqueue job
end
# attempts to extract the fulltext of any attachments that do not have a
# fulltext set currently. This runs inline, does not enqueue any background
# jobs.
# Extract the fulltext of any attachments where fulltext is still nil.
# This runs inline and not in a asynchronous worker.
def self.extract_fulltext
if OpenProject::Configuration['enable_fulltext_search']
Attachment.where(fulltext: nil).pluck(:id).each do |id|
job = ExtractFulltextJob.new(id)
job.perform
end
else
logger.info "fulltext search is disabled, check configuration.yml"
Attachment.where(fulltext: nil).pluck(:id).each do |id|
job = ExtractFulltextJob.new(id)
job.perform
end
end
end

@ -34,8 +34,10 @@ module Queries::WorkPackages
register.filter Query, filters_module::AssignedToFilter
register.filter Query, filters_module::AssigneeOrGroupFilter
register.filter Query, filters_module::AttachmentContentFilter
register.filter Query, filters_module::AttachmentFileNameFilter
if EnterpriseToken.allows_to?(:attachment_filters)
register.filter Query, filters_module::AttachmentContentFilter
register.filter Query, filters_module::AttachmentFileNameFilter
end
register.filter Query, filters_module::AuthorFilter
register.filter Query, filters_module::CategoryFilter
register.filter Query, filters_module::CreatedAtFilter

@ -121,16 +121,9 @@ class WorkPackage < ActiveRecord::Base
acts_as_customizable
# def self.attachments_searchable?
# OpenProject::Configuration["enable_attachment_search"] && EnterpriseToken.allows_to?(:attachment_search)
# end
acts_as_searchable columns: ['subject',
"#{table_name}.description",
"#{Journal.table_name}.notes"] +
["#{Attachment.table_name}.fulltext",
"#{Attachment.table_name}.filename",
"#{Attachment.table_name}.description"],
"#{Journal.table_name}.notes"],
include: %i(project journals).push(:attachments) ,
references: %i(projects journals).push(:attachments),
date_column: "#{quoted_table_name}.created_at",
@ -151,13 +144,13 @@ class WorkPackage < ActiveRecord::Base
acts_as_attachable after_remove: :attachments_changed, order: "#{Attachment.table_name}.filename"
after_validation :set_attachments_error_details,
if: lambda { |work_package| work_package.errors.messages.has_key? :attachments }
if: lambda { |work_package| work_package.errors.messages.has_key? :attachments }
associated_to_ask_before_destruction TimeEntry,
->(work_packages) {
->(work_packages) {
TimeEntry.on_work_packages(work_packages).count > 0
},
method(:cleanup_time_entries_before_destruction_of)
method(:cleanup_time_entries_before_destruction_of)
include WorkPackage::Journalized

@ -40,7 +40,7 @@ class Authorization::EnterpriseService
ldap_groups
custom_fields_in_projects_list
custom_actions
attachment_search).freeze
attachment_filters).freeze
def initialize(token)
self.token = token

@ -37,7 +37,7 @@ class ExtractFulltextJob < ApplicationJob
end
def perform
attachment = find_attachment(@attachment_id)
return unless attachment = find_attachment(@attachment_id)
text = TextExtractor::Resolver.new(attachment.diskfile, attachment.content_type).text if attachment.readable?

@ -348,47 +348,6 @@ default:
# after_login_default_redirect_url: '/my/page'
# after_first_login_redirect_url: '/projects/demo'
# Enable fulltext extraction and fulltext search in attachments.
# To make existing attachments fulltext searchable, run
# rake openproject:attachments:extract_fulltext
#
# Enabled by default.
#
# enable_attachment_search: true
# Text extraction helper programs.
#
# commands should write the resulting plain text to STDOUT. Use __FILE__ as
# placeholder for the file path. The values below are the defaults.
text_extractors:
# apt install poppler-utils
# pdftotext:
# - /usr/bin/pdftotext
# - -enc
# - UTF-8
# - __FILE__
# - '-'
# apt install unrtf
# unrtf:
# - /usr/bin/unrtf
# - --text
# - __FILE__
# apt install catdoc
# catdoc:
# - /usr/bin/catdoc
# - -dutf-8
# - __FILE__
# xls2csv:
# - /usr/bin/xls2csv
# - -dutf-8
# - __FILE__
# catppt:
# - /usr/bin/catppt
# - -dutf-8
# - __FILE__
# specific configuration options for production environment
# that overrides the default ones

@ -76,7 +76,6 @@ OpenProject::Application.configure do
# Set to :debug to see everything in the log.
# config.log_level = :warn
config.log_level = :debug
# Prepend all log lines with the following tags.
# config.log_tags = [ :subdomain, :uuid ]

@ -226,6 +226,15 @@ The actual upgrade commands are as follows:
[openproject@debian]# touch tmp/restart.txt
```
To make sure that all work package attachments are indexed, so that their content can be used in work package filters
run:
```bash
[openproject@debian]# RAILS_ENV="production" rake attachments:extract_fulltext
```
*Side note:* If you are using `RAILS_ENV="development"` the task `bundle exec rake assets:webpack` needs to be run. This step is not necessary for `production` because it is part of the `asset:precompile` tasks.
**NOTE** `db:seed` can also be invoked with a 'LOCALE' environment variable defined, specifying the language in which to seed. Note however, that specifying different locales for calls to `db:seed` might lead to a mixture of languages in your data. It is therefore advisable to use the same language for all calls to `db:seed`.

@ -113,7 +113,6 @@ module OpenProject
'after_login_default_redirect_url' => nil,
'after_first_login_redirect_url' => nil,
'enable_attachment_search' => true,
'main_content_language' => 'english'
}

@ -106,4 +106,10 @@ namespace :attachments do
target_attachment.mount_uploader :file, target_uploader
target_attachment.store_all! attachments
end
desc 'Extract text content from attachment that were not extracted yet.'
task extract_fulltext: [:environment] do
Attachment.extract_fulltext
end
end

Loading…
Cancel
Save