create history cache for file based repositories (#4283)

fixes #4280

Author: vladak

opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java

@@ -112,6 +112,7 @@
 import org.opengrok.indexer.configuration.Project;
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.history.Annotation;
+import org.opengrok.indexer.history.History;
 import org.opengrok.indexer.history.HistoryEntry;
 import org.opengrok.indexer.history.HistoryException;
 import org.opengrok.indexer.history.HistoryGuru;
@@ -633,13 +634,15 @@ public void populateDocument(Document doc, File file, String path, AbstractAnaly
     private static void populateDocumentHistory(Document doc, File file) {
         try {
             HistoryGuru histGuru = HistoryGuru.getInstance();
-            HistoryReader hr = histGuru.getHistoryReader(file);
-            if (hr != null) {
+            History history = histGuru.getHistory(file, false);
+            if (history != null) {
+                HistoryReader hr = new HistoryReader(history);
                 doc.add(new TextField(QueryBuilder.HIST, hr));
-                HistoryEntry histEntry = histGuru.getLastHistoryEntry(file, false, true);
+                HistoryEntry histEntry = history.getLastHistoryEntry();
                 if (histEntry != null) {
                     doc.add(new TextField(QueryBuilder.LASTREV, histEntry.getRevision(), Store.YES));
                 }
+                histGuru.storeHistory(file, history);
             }
         } catch (HistoryException e) {
             LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java

@@ -292,6 +292,11 @@ private static File getTagsFile(File file) {
         return new File(file.getAbsolutePath() + ".t");
     }
 
+    @Override
+    public void storeFile(History history, File file, Repository repository) throws HistoryException {
+        storeFile(history, file, repository, false);
+    }
+
     /**
      * Store {@link History} object in a file.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryCache.java

@@ -70,6 +70,16 @@ interface HistoryCache extends Cache {
      */
     void store(History history, Repository repository, @Nullable String tillRevision) throws CacheException;
 
+    /**
+     * Store the history for a file in given repository.
+     *
+     * @param history The history to store
+     * @param file file
+     * @param repository The repository whose history to store
+     * @throws HistoryException if the history cannot be stored
+     */
+    void storeFile(History history, File file, Repository repository) throws HistoryException;
+
     /**
      * Get the revision identifier for the latest cached revision in a repository.
      *

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

@@ -349,18 +349,6 @@ private void completeAnnotationWithHistory(File file, Annotation annotation, Rep
         }
     }
 
-    /**
-     * Get the appropriate history reader for given file.
-     *
-     * @param file The file to get the history reader for
-     * @throws HistoryException If an error occurs while getting the history
-     * @return A {@link HistoryReader} that may be used to read out history data for a named file
-     */
-    public HistoryReader getHistoryReader(File file) throws HistoryException {
-        History history = getHistory(file, false);
-        return history == null ? null : new HistoryReader(history);
-    }
-
     /**
      * Get the history for the specified file.
      *
@@ -906,6 +894,26 @@ public Collection<RepositoryInfo> getRepositories() {
         return repositories.values().stream().map(RepositoryInfo::new).collect(Collectors.toSet());
     }
 
+    /**
+     * Store history for a file into history cache. If the related repository does not support
+     * getting the history for directories, it will return right away without storing the history.
+     * @param file file
+     * @param history {@link History} instance
+     */
+    public void storeHistory(File file, History history) {
+        Repository repository = getRepository(file);
+        if (repository.hasHistoryForDirectories()) {
+            return;
+        }
+
+        try {
+            historyCache.storeFile(history, file, repository);
+        } catch (HistoryException e) {
+            LOGGER.log(Level.WARNING,
+                    String.format("cannot create history cache for '%s' in repository %s", file, repository), e);
+        }
+    }
+
     private void createHistoryCache(Repository repository, String sinceRevision) {
         String path = repository.getDirectoryName();
         String type = repository.getClass().getSimpleName();
@@ -1027,13 +1035,23 @@ public void createHistoryCache(Collection<String> repositories) {
     /**
      * Clear entry for single file from history cache.
      * @param path path to the file relative to the source root
+     * @param removeHistory whether to remove history cache entry for the path
      */
-    public void clearHistoryCacheFile(String path) {
+    public void clearHistoryCacheFile(String path, boolean removeHistory) {
         if (!useHistoryCache()) {
             return;
         }
 
-        historyCache.clearFile(path);
+        Repository repository = getRepository(new File(env.getSourceRootFile(), path));
+        if (repository == null) {
+            return;
+        }
+
+        // Repositories that do not support getting history for directories do not undergo
+        // incremental history cache generation, so for these the removeHistory parameter is not honored.
+        if (!repository.hasHistoryForDirectories() || removeHistory) {
+            historyCache.clearFile(path);
+        }
     }
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryReader.java

@@ -18,14 +18,16 @@
  */
 
 /*
- * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opengrok.indexer.history;
 
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.List;
+
+import org.jetbrains.annotations.NotNull;
 import org.opengrok.indexer.util.IOUtils;
 
 /**
@@ -36,12 +38,12 @@ public class HistoryReader extends Reader {
     private final List<HistoryEntry> entries;
     private Reader input;
 
-    HistoryReader(History history) {
+    public HistoryReader(History history) {
         entries = history.getHistoryEntries();
     }
 
     @Override
-    public int read(char[] cbuf, int off, int len) throws IOException {
+    public int read(char @NotNull [] cbuf, int off, int len) throws IOException {
         if (input == null) {
             input = createInternalReader();
         }

opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.index;
@@ -1044,8 +1044,8 @@ private void removeXrefFile(String path) {
         completer.add(pending);
     }
 
-    private void removeHistoryFile(String path) {
-        HistoryGuru.getInstance().clearHistoryCacheFile(path);
+    private void removeHistoryCacheFile(String path, boolean removeHistory) {
+        HistoryGuru.getInstance().clearHistoryCacheFile(path, removeHistory);
     }
 
     private void removeAnnotationFile(String path) {
@@ -1056,7 +1056,7 @@ private void removeAnnotationFile(String path) {
      * Remove a stale file from the index database and potentially also from history cache,
      * and queue the removal of the associated xref file.
      *
-     * @param removeHistory if false, do not remove history cache and annotation cache for this file
+     * @param removeHistory if false, do not remove history cache for this file
      * @throws java.io.IOException if an error occurs
      */
     private void removeFile(boolean removeHistory) throws IOException {
@@ -1070,9 +1070,7 @@ private void removeFile(boolean removeHistory) throws IOException {
 
         removeXrefFile(path);
 
-        if (removeHistory) {
-            removeHistoryFile(path);
-        }
+        removeHistoryCacheFile(path, removeHistory);
 
         /*
          * Even when the history should not be removed (incremental reindex), annotation should,

opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java

@@ -25,6 +25,7 @@
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -70,8 +71,10 @@
 import org.opengrok.indexer.history.RepositoryFactory;
 import org.opengrok.indexer.history.RepositoryInfo;
 import org.opengrok.indexer.history.RepositoryWithHistoryTraversal;
+import org.opengrok.indexer.history.SCCSRepository;
 import org.opengrok.indexer.search.QueryBuilder;
 import org.opengrok.indexer.search.SearchEngine;
+import org.opengrok.indexer.util.Executor;
 import org.opengrok.indexer.util.ForbiddenSymlinkException;
 import org.opengrok.indexer.util.IOUtils;
 import org.opengrok.indexer.util.TandemPath;
@@ -91,6 +94,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.opengrok.indexer.condition.RepositoryInstalled.Type.CVS;
+import static org.opengrok.indexer.condition.RepositoryInstalled.Type.SCCS;
 
 /**
  * Unit tests for the {@code IndexDatabase} class.
@@ -222,8 +226,14 @@ private void checkDataExistence(String fileName, boolean shouldExist) {
                 } else {
                     assertFalse(historyGuru.hasAnnotationCacheForFile(file));
                 }
+            } else if (dirName.equals("historycache")) {
+                if (shouldExist) {
+                    assertTrue(historyGuru.hasHistoryCacheForFile(file));
+                } else {
+                    assertFalse(historyGuru.hasHistoryCacheForFile(file));
+                }
             } else {
-                cacheFile = TandemPath.join(fileName, dirName.equals(IndexDatabase.XREF_DIR) ? ".gz" : "");
+                cacheFile = TandemPath.join(fileName, ".gz");
                 File dataFile = new File(dataDir, cacheFile);
 
                 if (shouldExist) {
@@ -973,4 +983,59 @@ void testAnnotationCacheProjectTunable(boolean useAnnotationCache, boolean isHis
         env.setDataRoot(dataRootOrig);
         IOUtils.removeRecursive(dataRoot);
     }
+
+    private static void runSccsCommand(File reposRoot, String... args) {
+        List<String> cmdargs = new ArrayList<>();
+        SCCSRepository repo = new SCCSRepository();
+        cmdargs.add(repo.getRepoCommand());
+        Collections.addAll(cmdargs, args);
+        Executor exec = new Executor(cmdargs, reposRoot);
+        int exitCode = exec.exec();
+        assertEquals(0, exitCode, "command '" + cmdargs.toString() + "'failed."
+                + "\nexit code: " + exitCode
+                + "\nstdout:\n" + exec.getOutputString()
+                + "\nstderr:\n" + exec.getErrorString());
+    }
+
+    @Test
+    @EnabledForRepository(SCCS)
+    void testHistoryCacheForFileBasedRepository() throws Exception {
+        String projectName = "teamware";
+        Project project = env.getProjects().get(projectName);
+        assertNotNull(project);
+        IndexDatabase idb = new IndexDatabase(project);
+        assertNotNull(idb);
+
+        String fileName = "header.h";
+        File repoRoot = new File(repository.getSourceRoot(), projectName);
+        File file = new File(repoRoot, fileName);
+        assertTrue(file.exists());
+
+        // Check that the history cache entry for the file exists.
+        checkDataExistence(projectName + File.separator + fileName, true);
+        HistoryGuru historyGuru = HistoryGuru.getInstance();
+        HistoryEntry historyEntry = historyGuru.getLastHistoryEntry(file, false, false);
+        assertNotNull(historyEntry);
+        assertEquals("1.1", historyEntry.getRevision());
+
+        // Update and change the file.
+        runSccsCommand(repoRoot, "clean");
+        runSccsCommand(repoRoot, "get", "-e", fileName);
+        try (FileWriter fileWriter = new FileWriter(file, true)) {
+            fileWriter.write("#define FOO 42");
+        }
+        runSccsCommand(repoRoot, "delget", "-yfoo", fileName);
+        idb.update();
+
+        // Recheck the history cache entry.
+        assertTrue(historyGuru.hasHistoryCacheForFile(file));
+        historyEntry = historyGuru.getLastHistoryEntry(file, false, false);
+        assertNotNull(historyEntry);
+        assertEquals("1.2", historyEntry.getRevision());
+
+        // The SCCS "clean" operation above wiped all the other files, so their history cache entries should be gone.
+        File otherFile = new File(repoRoot, "main.c");
+        assertFalse(otherFile.exists());
+        assertFalse(historyGuru.hasHistoryCacheForFile(otherFile));
+    }
 }