update robot.txt

/calendar does no longer exist
/activity and /activities are synonyms for the same route.
/search (global or in project) does not make sense for a robot to process.

app/views/homescreen/robots.text.erb

@@ -28,11 +28,13 @@ See COPYRIGHT and LICENSE files for more details.
 ++#%>
 User-agent: *
 <% cache @projects.cache_key(:updated_at) do %>
-  <% @projects.each do |p| -%>
-Disallow: <%= project_repository_path(p) %>
-Disallow: <%= project_work_packages_path(p) %>
-Disallow: <%= project_activity_index_path(p) %>
+  <% @projects.flat_map { |p| [p.id, p.identifier] }.each do |identifier| -%>
+Disallow: <%= project_repository_path(identifier) %>
+Disallow: <%= project_work_packages_path(identifier) %>
+Disallow: <%= project_activity_index_path(identifier) %>
+Disallow: <%= search_path(identifier) %>
   <% end -%>
 <% end %>
-Disallow: /calendar
+Disallow: <%= activities_path %>
 Disallow: /activity
+Disallow: <%= search_path %>

spec/features/homescreen/robots_spec.rb

@@ -36,11 +36,15 @@ describe 'robots.txt' do
   end
 
   it 'disallows global paths and paths from public project' do
-    expect(page).to have_content('Disallow: /calendar')
     expect(page).to have_content('Disallow: /activity')
+    expect(page).to have_content('Disallow: /activities')
+    expect(page).to have_content('Disallow: /search')
 
-    expect(page).to have_content("Disallow: /projects/#{project.identifier}/repository")
-    expect(page).to have_content("Disallow: /projects/#{project.identifier}/work_packages")
-    expect(page).to have_content("Disallow: /projects/#{project.identifier}/activity")
+    [project.identifier, project.id].each do |identifier|
+      expect(page).to have_content("Disallow: /projects/#{identifier}/repository")
+      expect(page).to have_content("Disallow: /projects/#{identifier}/work_packages")
+      expect(page).to have_content("Disallow: /projects/#{identifier}/activity")
+      expect(page).to have_content("Disallow: /projects/#{identifier}/search")
+    end
   end
 end