directory listing API (#4308)

fixes #3047
fixes #3738

Author: vladak

apiary.apib

@@ -197,6 +197,41 @@ The `Content-type` header of the reply will be set accordingly.
             }
 
 
+
+## Directory listing [/list{?path}]
+
+### get directory entries [GET]
+
++ Parameters
+  + path (string) - path of file/directory to get listing for, relative to source root, starting with /
+
++ Response 200 (application/json)
+  + Body
+
+            [
+              {
+                "path": "/lucene/.github",
+                "numLines": 178,
+                "loc": 125,
+                "date": 1673456294670,
+                "description": null,
+                "pathDescription": "",
+                "isDirectory": true,
+                "size": null
+              },
+              {
+                "path": "/lucene/.asf.yaml",
+                "numLines": 25,
+                "loc": 21,
+                "date": 1673456294670,
+                "description": null,
+                "pathDescription": "",
+                "isDirectory": false,
+                "size": 574
+              }
+            ]
+
+
 ## Include files reload [/system/includes/reload]
 
 ### reloads all include files for web application [PUT]

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

@@ -712,13 +712,14 @@ public boolean hasAnnotationCacheForFile(File file) {
     }
 
     /**
-     * Get the last modified times and descriptions for all files and subdirectories in the specified directory.
+     * Get the last modified times and descriptions for all files and subdirectories in the specified directory
+     * and set it into the entries provided.
      * @param directory the directory whose files to check
      * @param entries list of {@link DirectoryEntry} instances
      * @return whether to fall back to file system based time stamps if the date is {@code null}
      * @throws org.opengrok.indexer.history.CacheException if history cannot be retrieved
      */
-    public boolean getLastHistoryEntries(File directory, List<DirectoryEntry> entries) throws CacheException {
+    public boolean fillLastHistoryEntries(File directory, List<DirectoryEntry> entries) throws CacheException {
 
         if (!env.isUseHistoryCacheForDirectoryListing()) {
             LOGGER.log(Level.FINEST, "using history cache to retrieve last modified times for ''{0}}'' is disabled",

opengrok-indexer/src/main/java/org/opengrok/indexer/search/DirectoryEntry.java

@@ -35,10 +35,12 @@
 public class DirectoryEntry {
 
     private final File file;
-    private final NullableNumLinesLOC extra;
+    private NullableNumLinesLOC extra;
 
     private String description;
 
+    private String pathDescription;
+
     private Date date;
 
     /**
@@ -77,6 +79,10 @@ public NullableNumLinesLOC getExtra() {
         return extra;
     }
 
+    public void setExtra(NullableNumLinesLOC extra) {
+        this.extra = extra;
+    }
+
     public String getDescription() {
         return description;
     }
@@ -92,4 +98,12 @@ public Date getDate() {
     public void setDate(Date date) {
         this.date = date;
     }
+
+    public String getPathDescription() {
+        return pathDescription;
+    }
+
+    public void setPathDescription(String pathDescription) {
+        this.pathDescription = pathDescription;
+    }
 }

opengrok-indexer/src/main/java/org/opengrok/indexer/search/DirectoryExtraReader.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.search;
@@ -59,8 +59,7 @@ public class DirectoryExtraReader {
      * @return a list of results, limited to 2000 values
      * @throws IOException if an error occurs searching the index
      */
-    public List<NullableNumLinesLOC> search(IndexSearcher searcher, String path)
-            throws IOException {
+    public List<NullableNumLinesLOC> search(IndexSearcher searcher, String path) throws IOException {
         if (searcher == null) {
             throw new IllegalArgumentException("`searcher' is null");
         }
@@ -87,9 +86,7 @@ public List<NullableNumLinesLOC> search(IndexSearcher searcher, String path)
                 "search.latency", new String[]{"category", "extra",
                         "outcome", hits.scoreDocs.length > 0 ? "success" : "empty"});
 
-        List<NullableNumLinesLOC> results = processHits(searcher, hits);
-
-        return results;
+        return processHits(searcher, hits);
     }
 
     private List<NullableNumLinesLOC> processHits(IndexSearcher searcher, TopDocs hits)

opengrok-indexer/src/main/java/org/opengrok/indexer/util/FileExtraZipper.java

@@ -18,16 +18,15 @@
  */
 
 /*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2017, 2020, Chris Fraire <[email protected]>.
  */
 package org.opengrok.indexer.util;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import org.opengrok.indexer.analysis.NullableNumLinesLOC;
 import org.opengrok.indexer.search.DirectoryEntry;
@@ -41,29 +40,23 @@ public class FileExtraZipper {
 
     /**
      * Merge the specified lists by looking up a possible entry in
-     * {@code extras} for every element in {@code files}.
-     * @param dir the files' directory
-     * @param files the file names
-     * @param extras some OpenGrok-analyzed extra metadata
-     * @return a list of the same size as {@code files}
+     * {@code extras} for every element in {@code entries}.
+     *
+     * @param entries list of {@link DirectoryEntry} instances
+     * @param extras  some OpenGrok-analyzed extra metadata
      */
-    public List<DirectoryEntry> zip(File dir, List<String> files, List<NullableNumLinesLOC> extras) {
+    public void zip(List<DirectoryEntry> entries, List<NullableNumLinesLOC> extras) {
 
         if (extras == null) {
-            return files.stream().map(f -> new DirectoryEntry(new File(dir, f))).collect(Collectors.toList());
+            return;
         }
 
         Map<String, NullableNumLinesLOC> byName = indexExtraByName(extras);
 
-        List<DirectoryEntry> result = new ArrayList<>(files.size());
-        for (String file : files) {
-            File fileobj = new File(dir, file);
-            NullableNumLinesLOC extra = findExtra(byName, fileobj);
-            DirectoryEntry entry = new DirectoryEntry(fileobj, extra);
-            result.add(entry);
+        for (DirectoryEntry entry : entries) {
+            NullableNumLinesLOC extra = findExtra(byName, entry.getFile());
+            entry.setExtra(extra);
         }
-
-        return result;
     }
 
     private NullableNumLinesLOC findExtra(Map<String, NullableNumLinesLOC> byName, File fileobj) {

opengrok-indexer/src/test/java/org/opengrok/indexer/history/HistoryGuruTest.java

@@ -452,7 +452,7 @@ void testGetLastHistoryEntriesWrtMergeCommits(boolean isMergeCommitsEnabled) thr
         }
         boolean useHistoryCacheForDirectoryListingOrig = env.isUseHistoryCacheForDirectoryListing();
         env.setUseHistoryCacheForDirectoryListing(true);
-        boolean fallback = instance.getLastHistoryEntries(repositoryRoot, directoryEntries);
+        boolean fallback = instance.fillLastHistoryEntries(repositoryRoot, directoryEntries);
         if (isMergeCommitsEnabled) {
             assertFalse(fallback);
         } else {

opengrok-indexer/src/test/java/org/opengrok/indexer/web/DummyHttpServletRequest.java

@@ -182,8 +182,17 @@ public String getPathTranslated() {
         throw new UnsupportedOperationException("Not supported yet.");
     }
 
+    private String contextPath;
+
+    public void setContextPath(String contextPath) {
+        this.contextPath = contextPath;
+    }
+
     @Override
     public String getContextPath() {
+        if (contextPath != null) {
+            return contextPath;
+        }
         throw new UnsupportedOperationException("Not supported yet.");
     }
 
@@ -224,9 +233,18 @@ public StringBuffer getRequestURL() {
 
     @Override
     public String getServletPath() {
+        if (servletPath != null) {
+            return servletPath;
+        }
         throw new UnsupportedOperationException("Not supported yet.");
     }
 
+    private String servletPath;
+
+    public void setServletPath(String path) {
+        this.servletPath = path;
+    }
+
     @Override
     public HttpSession getSession(boolean bln) {
         if (bln) {