Implement reindex mechanism for search (#2104)

Adds a new button to repository settings to allow users to manually delete and re-create search indices. The actual re-indexing is happening in plugins that subscribe to the newly created event.

Co-authored-by: Eduard Heimbuch <eduard.heimbuch@cloudogu.com>

gradle/changelog/reindex.yaml

@@ -0,0 +1,2 @@
+- type: added
+  description: Reindex mechanism for search ([#2104](https://github.com/scm-manager/scm-manager/pull/2104))

scm-core/src/main/java/sonia/scm/search/ReindexRepositoryEvent.java

@@ -0,0 +1,40 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.search;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import sonia.scm.event.Event;
+import sonia.scm.repository.Repository;
+
+/**
+ * @since 2.39.0
+ */
+@Event
+@AllArgsConstructor
+@Getter
+public class ReindexRepositoryEvent {
+  private Repository repository;
+}

scm-ui/ui-api/src/repositories.ts

@@ -373,3 +373,24 @@ export const useRenameRepository = (repository: Repository) => {
     isRenamed: !!data
   };
 };
+
+export const useReindexRepository = () => {
+  const queryClient = useQueryClient();
+  const { mutate, isLoading, error, data } = useMutation<unknown, Error, Repository>(
+    (repository) => {
+      const link = requiredLink(repository, "reindex");
+      return apiClient.post(link);
+    },
+    {
+      onSuccess: async (_, repository) => {
+        await queryClient.invalidateQueries(repoQueryKey(repository));
+      },
+    }
+  );
+  return {
+    reindex: (repository: Repository) => mutate(repository),
+    isLoading,
+    error,
+    isRunning: !!data,
+  };
+};

scm-ui/ui-webapp/public/locales/de/repos.json

@@ -492,6 +492,12 @@
     "descriptionNotRunning": "Starten der Integritätsprüfung dieses Repositories. Dieser Vorgang kann einige Zeit in Anspruch nehmen.",
     "descriptionRunning": "Die Integritätsprüfung für dieses Repository läuft bereits und kann nicht parallel erneut gestartet werden."
   },
+  "reindex": {
+    "button": "Reindizieren",
+    "subtitle": "Suchindizes neu erstellen",
+    "description": "Löscht alle existierenden Suchindizes für dieses Repository and erstellt sie komplett neu. Dieser Vorgang kann einige Zeit in Anspruch nehmen.",
+    "started": "Die Reindizierung wurde erfolgreich gestartet. Dies ist eine asynchrone Operation und kann einige Zeit in Anspruch nehmen."
+  },
   "diff": {
     "jumpToSource": "Zur Quelldatei springen",
     "jumpToTarget": "Zur vorherigen Version der Datei springen",

scm-ui/ui-webapp/public/locales/en/repos.json

@@ -481,6 +481,12 @@
     "descriptionNotRunning": "Run the health checks for this repository. This may take a while.",
     "descriptionRunning": "Health checks for this repository are currently running and cannot be started again in parallel."
   },
+  "reindex": {
+    "button": "Reindex",
+    "subtitle": "Recreate Search Indices",
+    "description": "Deletes all existing search indices for this repository and recreates them from scratch. This may take a while.",
+    "started": "Reindexing has been started successfully. This is an asynchronous operation and may take a while."
+  },
   "archive": {
     "tooltip": "Read only. The archive cannot be changed."
   },

scm-ui/ui-webapp/src/repos/components/Reindex.tsx

@@ -0,0 +1,64 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+import { Repository } from "@scm-manager/ui-types";
+import React, { FC } from "react";
+import { useReindexRepository } from "@scm-manager/ui-api";
+import { Button, ErrorNotification, Level, Notification, Subtitle } from "@scm-manager/ui-components";
+import { useTranslation } from "react-i18next";
+
+type Props = {
+  repository: Repository;
+};
+
+const Reindex: FC<Props> = ({ repository }) => {
+  const [t] = useTranslation("repos");
+  const { reindex, error, isLoading, isRunning } = useReindexRepository();
+
+  return (
+    <>
+      <hr />
+      <ErrorNotification error={error} />
+      {isRunning ? <Notification type="success">{t("reindex.started")}</Notification> : null}
+      <Subtitle>{t("reindex.subtitle")}</Subtitle>
+      <p>{t("reindex.description")}</p>
+      <Level
+        right={
+          <Button
+            color="warning"
+            icon="sync-alt"
+            className="mt-4"
+            action={() => reindex(repository)}
+            disabled={isLoading}
+            loading={isLoading}
+          >
+            {t("reindex.button")}
+          </Button>
+        }
+      />
+    </>
+  );
+};
+
+export default Reindex;

scm-ui/ui-webapp/src/repos/containers/EditRepo.tsx

@@ -34,6 +34,7 @@ import { useUpdateRepository } from "@scm-manager/ui-api";
 import HealthCheckWarning from "./HealthCheckWarning";
 import RunHealthCheck from "./RunHealthCheck";
 import UpdateNotification from "../../components/UpdateNotification";
+import Reindex from "../components/Reindex";
 
 type Props = {
   repository: Repository;
@@ -71,6 +72,7 @@ const EditRepo: FC<Props> = ({ repository }) => {
       {(repository._links.runHealthCheck || repository.healthCheckRunning) && (
         <RunHealthCheck repository={repository} />
       )}
+      {repository._links.reindex ? <Reindex repository={repository} /> : null}
       <RepositoryDangerZone repository={repository} />
     </>
   );

scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryResource.java

@@ -30,10 +30,13 @@ import io.swagger.v3.oas.annotations.media.ExampleObject;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import sonia.scm.event.ScmEventBus;
 import sonia.scm.repository.HealthCheckService;
 import sonia.scm.repository.NamespaceAndName;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.RepositoryManager;
+import sonia.scm.repository.RepositoryPermissions;
+import sonia.scm.search.ReindexRepositoryEvent;
 import sonia.scm.web.VndMediaType;
 
 import javax.inject.Inject;
@@ -294,6 +297,26 @@ public class RepositoryResource {
     healthCheckService.fullCheck(repository);
   }
 
+  @POST
+  @Path("reindex")
+  @Operation(summary = "Manually reindex repository", description = "Asynchronously update search indices for repository", tags = "Repository")
+  @ApiResponse(responseCode = "204", description = "event submitted")
+  @ApiResponse(responseCode = "401", description = "not authenticated / invalid credentials")
+  @ApiResponse(responseCode = "403", description = "not authorized, the current user does not have owner permissions for this repository")
+  @ApiResponse(
+    responseCode = "404",
+    description = "not found, no repository with the specified namespace and name available",
+    content = @Content(
+      mediaType = VndMediaType.ERROR_TYPE,
+      schema = @Schema(implementation = ErrorDto.class)
+    ))
+  @ApiResponse(responseCode = "500", description = "internal server error")
+  public void reindex(@PathParam("namespace") String namespace, @PathParam("name") String name) {
+    Repository repository = loadBy(namespace, name).get();
+    RepositoryPermissions.custom("*", repository).check();
+    ScmEventBus.getInstance().post(new ReindexRepositoryEvent(repository));
+  }
+
   private Repository processUpdate(RepositoryDto repositoryDto, Repository existing) {
     Repository changedRepository = dtoToRepositoryMapper.map(repositoryDto, existing.getId());
     changedRepository.setPermissions(existing.getPermissions());

scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapper.java

@@ -111,6 +111,9 @@ public abstract class RepositoryToRepositoryDtoMapper extends BaseMapper<Reposit
   @ObjectFactory
   RepositoryDto createDto(Repository repository) {
     Links.Builder linksBuilder = linkingTo().self(resourceLinks.repository().self(repository.getNamespace(), repository.getName()));
+    if (RepositoryPermissions.custom("*", repository).isPermitted()) {
+      linksBuilder.single(link("reindex", resourceLinks.repository().reindex(repository.getNamespace(), repository.getName())));
+    }
     if (RepositoryPermissions.delete(repository).isPermitted()) {
       linksBuilder.single(link("delete", resourceLinks.repository().delete(repository.getNamespace(), repository.getName())));
     }

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java

@@ -439,6 +439,10 @@ class ResourceLinks {
     String runHealthCheck(String namespace, String name) {
       return repositoryLinkBuilder.method("getRepositoryResource").parameters(namespace, name).method("runHealthCheck").parameters().href();
     }
+
+    String reindex(String namespace, String name) {
+      return repositoryLinkBuilder.method("getRepositoryResource").parameters(namespace, name).method("reindex").parameters().href();
+    }
   }
 
   RepositoryCollectionLinks repositoryCollection() {

scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java

@@ -24,11 +24,13 @@
 
 package sonia.scm.api.v2.resources;
 
+import com.github.legman.Subscribe;
 import com.github.sdorra.shiro.ShiroRule;
 import com.github.sdorra.shiro.SubjectAware;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.io.Resources;
 import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import org.apache.shiro.authz.AuthorizationException;
 import org.apache.shiro.subject.SimplePrincipalCollection;
 import org.apache.shiro.subject.Subject;
 import org.jboss.resteasy.mock.MockHttpRequest;
@@ -45,6 +47,7 @@ import org.mockito.junit.MockitoJUnitRunner;
 import sonia.scm.NotFoundException;
 import sonia.scm.PageResult;
 import sonia.scm.config.ScmConfiguration;
+import sonia.scm.event.ScmEventBus;
 import sonia.scm.importexport.ExportFileExtensionResolver;
 import sonia.scm.importexport.ExportNotificationHandler;
 import sonia.scm.importexport.ExportService;
@@ -68,6 +71,7 @@ import sonia.scm.repository.api.BundleCommandBuilder;
 import sonia.scm.repository.api.Command;
 import sonia.scm.repository.api.RepositoryService;
 import sonia.scm.repository.api.RepositoryServiceFactory;
+import sonia.scm.search.ReindexRepositoryEvent;
 import sonia.scm.search.SearchEngine;
 import sonia.scm.user.User;
 import sonia.scm.web.RestDispatcher;
@@ -97,6 +101,8 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyMap;
@@ -749,6 +755,60 @@ public class RepositoryRootResourceTest extends RepositoryTestBase {
     verify(exportService).getExportInformation(repository);
   }
 
+
+  @Test
+  public void shouldDispatchReindexEvent() throws URISyntaxException {
+    ReindexTestListener listener = new ReindexTestListener();
+
+    ScmEventBus.getInstance().register(listener);
+
+    Repository repository = createRepository("space", "repo");
+
+    MockHttpRequest request = MockHttpRequest
+      .post("/" + RepositoryRootResource.REPOSITORIES_PATH_V2 + "space/repo/reindex");
+    MockHttpResponse response = new MockHttpResponse();
+
+    dispatcher.invoke(request, response);
+    assertNotNull(listener.event);
+    assertEquals(repository, listener.repository);
+  }
+
+  @Test
+  public void shouldThrowErrorWhenMissingPermissions() throws URISyntaxException {
+    Subject subject = mock(Subject.class);
+    doThrow(new AuthorizationException()).when(subject).checkPermission("repository:*:space-repo");
+    shiro.setSubject(subject);
+
+    ReindexTestListener listener = new ReindexTestListener();
+
+    ScmEventBus.getInstance().register(listener);
+
+    createRepository("space", "repo");
+
+    MockHttpRequest request = MockHttpRequest
+      .post("/" + RepositoryRootResource.REPOSITORIES_PATH_V2 + "space/repo/reindex");
+    MockHttpResponse response = new MockHttpResponse();
+
+    dispatcher.invoke(request, response);
+
+    assertEquals(403, response.getStatus());
+    assertNull(listener.event);
+  }
+
+  private static class ReindexTestListener {
+
+    private ReindexRepositoryEvent event;
+
+    private Repository repository;
+
+    @Subscribe(async = false)
+    public void onEvent(ReindexRepositoryEvent event) {
+        this.repository = event.getRepository();
+        this.event = event;
+    }
+  }
+
+
   private void mockRepositoryHandler(Set<Command> cmds) {
     RepositoryHandler repositoryHandler = mock(RepositoryHandler.class);
     RepositoryType repositoryType = mock(RepositoryType.class);

scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapperTest.java

@@ -408,6 +408,23 @@ public class RepositoryToRepositoryDtoMapperTest {
       .isEqualTo("http://example.com/base/v2/search/searchableTypes/testspace/test");
   }
 
+  @Test
+  public void shouldCreateReindexLink() {
+    Repository testRepository = createTestRepository();
+    RepositoryDto dto = mapper.map(testRepository);
+    assertThat(dto.getLinks().getLinkBy("reindex"))
+      .get()
+      .hasFieldOrPropertyWithValue("href", "http://example.com/base/v2/repositories/testspace/test/reindex");
+  }
+
+  @SubjectAware(username = "unpriv")
+  @Test
+  public void shouldNotCreateReindexLinkWithoutPermission() {
+    Repository testRepository = createTestRepository();
+    RepositoryDto dto = mapper.map(testRepository);
+    assertThat(dto.getLinks().getLinkBy("reindex")).isEmpty();
+  }
+
   private ScmProtocol mockProtocol(String type, String protocol) {
     return new MockScmProtocol(type, protocol);
   }