[JENKINS-64383] combined refrepo became our bottleneck, support a fanout location too

src/main/java/hudson/plugins/git/GitAPI.java

@@ -156,6 +156,11 @@ public GitClient subGit(String subdir) {
         return Git.USE_CLI ? super.subGit(subdir) :  jgit.subGit(subdir);
     }
 
+    /** {@inheritDoc} */
+    public GitClient newGit(String somedir) {
+        return Git.USE_CLI ? super.newGit(somedir) :  jgit.newGit(somedir);
+    }
+
     /** {@inheritDoc} */
     public void setRemoteUrl(String name, String url) throws GitException, InterruptedException {
         if (Git.USE_CLI) super.setRemoteUrl(name, url); else  jgit.setRemoteUrl(name, url);

src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java

@@ -345,6 +345,12 @@ public GitClient subGit(String subdir) {
         return new CliGitAPIImpl(gitExe, new File(workspace, subdir), listener, environment);
     }
 
+    /** {@inheritDoc} */
+    @Override
+    public GitClient newGit(String somedir) {
+        return new CliGitAPIImpl(gitExe, new File(somedir), listener, environment);
+    }
+
     /**
      * Initialize an empty repository for further git operations.
      *
@@ -812,31 +818,51 @@ public void execute() throws GitException, InterruptedException {
                 }
 
                 if (reference != null && !reference.isEmpty()) {
-                    File referencePath = new File(reference);
-                    if (!referencePath.exists())
-                        listener.getLogger().println("[WARNING] Reference path does not exist: " + reference);
-                    else if (!referencePath.isDirectory())
-                        listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference);
-                    else {
-                        // reference path can either be a normal or a base repository
-                        File objectsPath = new File(referencePath, ".git/objects");
-                        if (!objectsPath.isDirectory()) {
-                            // reference path is bare repo
-                            objectsPath = new File(referencePath, "objects");
+                    if (isParameterizedReferenceRepository(reference)) {
+                        // LegacyCompatibleGitAPIImpl.java has a logging trace, but not into build console via listener
+                        listener.getLogger().println("[INFO] The git reference repository path '" +
+                            reference + "' is parameterized, it may take a few git queries logged " +
+                            "below to resolve it into a particular directory name");
+                    }
+                    File referencePath = findParameterizedReferenceRepository(reference, url);
+                    if (referencePath == null) {
+                        listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + reference);
+                    } else {
+                        if (!referencePath.getPath().equals(reference)) {
+                            // Note: both these logs are needed, they are used in selftest
+                            String msg = "Parameterized reference path ";
+                            msg += "'" + reference + "'";
+                            msg += " replaced with: ";
+                            msg += "'" + referencePath.getPath() + "'";
+                            if (referencePath.exists()) {
+                                listener.getLogger().println("[WARNING] " + msg);
+                            } else {
+                                listener.getLogger().println("[WARNING] " + msg + " does not exist");
+                            }
+                            reference = referencePath.getPath();
                         }
-                        if (!objectsPath.isDirectory())
-                            listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath);
+
+                        if (!referencePath.exists())
+                            listener.getLogger().println("[WARNING] Reference path does not exist: " + reference);
+                        else if (!referencePath.isDirectory())
+                            listener.getLogger().println("[WARNING] Reference path is not a directory: " + reference);
                         else {
-                            File alternates = new File(workspace, ".git/objects/info/alternates");
-                            try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) {
-                                String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/');
-                                listener.getLogger().println("Using reference repository: " + reference);
-                                // git implementations on windows also use
-                                w.print(absoluteReference);
-                            } catch (UnsupportedEncodingException ex) {
-                                listener.error("Default character set is an unsupported encoding");
-                            } catch (FileNotFoundException e) {
-                                listener.error("Failed to setup reference");
+                            File objectsPath = getObjectsFile(referencePath);
+                            if (objectsPath == null || !objectsPath.isDirectory())
+                                listener.getLogger().println("[WARNING] Reference path does not contain an objects directory (not a git repo?): " + objectsPath);
+                            else {
+                                // Go behind git's back to write a meta file in new workspace
+                                File alternates = new File(workspace, ".git/objects/info/alternates");
+                                try (PrintWriter w = new PrintWriter(alternates, Charset.defaultCharset().toString())) {
+                                    String absoluteReference = objectsPath.getAbsolutePath().replace('\\', '/');
+                                    listener.getLogger().println("Using reference repository: " + reference);
+                                    // git implementations on windows also use
+                                    w.print(absoluteReference);
+                                } catch (UnsupportedEncodingException ex) {
+                                    listener.error("Default character set is an unsupported encoding");
+                                } catch (FileNotFoundException e) {
+                                    listener.error("Failed to setup reference");
+                                }
                             }
                         }
                     }
@@ -1428,13 +1454,15 @@ public void execute() throws GitException, InterruptedException {
                     }
                 }
                 if ((ref != null) && !ref.isEmpty()) {
-                    File referencePath = new File(ref);
-                    if (!referencePath.exists())
-                        listener.getLogger().println("[WARNING] Reference path does not exist: " + ref);
-                    else if (!referencePath.isDirectory())
-                        listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref);
-                    else
-                        args.add("--reference", ref);
+                    if (!isParameterizedReferenceRepository(ref)) {
+                        File referencePath = new File(ref);
+                        if (!referencePath.exists())
+                            listener.getLogger().println("[WARNING] Reference path does not exist: " + ref);
+                        else if (!referencePath.isDirectory())
+                            listener.getLogger().println("[WARNING] Reference path is not a directory: " + ref);
+                        else
+                            args.add("--reference", ref);
+                    } // else handled below in per-module loop
                 }
                 if (shallow) {
                     if (depth == null) {
@@ -1484,9 +1512,34 @@ else if (!referencePath.isDirectory())
                         listener.error("Invalid repository for " + sModuleName);
                         throw new GitException("Invalid repository for " + sModuleName);
                     }
+                    String strURIish = null;
+                    if (urIish != null) {
+                        strURIish = urIish.toPrivateString();
+                    }
+
+                    if (isParameterizedReferenceRepository(ref)) {
+                        File referencePath = findParameterizedReferenceRepository(ref, strURIish);
+                        if (referencePath == null) {
+                            listener.getLogger().println("[ERROR] Could not make File object from reference path, skipping its use: " + ref);
+                        } else {
+                            String expRef = null;
+                            if (referencePath.getPath().equals(ref)) {
+                                expRef = ref;
+                            } else {
+                                expRef = referencePath.getPath();
+                                expRef += " (expanded from " + ref + ")";
+                            }
+                            if (!referencePath.exists())
+                                listener.getLogger().println("[WARNING] Reference path does not exist: " + expRef);
+                            else if (!referencePath.isDirectory())
+                                listener.getLogger().println("[WARNING] Reference path is not a directory: " + expRef);
+                            else
+                                args.add("--reference", referencePath.getPath());
+                        }
+                    }
 
                     // Find credentials for this URL
-                    StandardCredentials cred = credentials.get(urIish.toPrivateString());
+                    StandardCredentials cred = credentials.get(strURIish);
                     if (parentCredentials) {
                         String parentUrl = getRemoteUrl(getDefaultRemote());
                         URIish parentUri = null;
@@ -1594,6 +1647,60 @@ public void setSubmoduleUrl(String name, String url) throws GitException, Interr
         return StringUtils.trim(firstLine(result));
     }
 
+    /** {@inheritDoc} */
+    @Override
+    public @CheckForNull Map<String, String> getRemoteUrls() throws GitException, InterruptedException {
+        String result = launchCommand( "config", "--local", "--list" );
+        Map<String, String> uriNames = new HashMap<>();
+        for (String line : result.split("\\R+")) {
+            line = StringUtils.trim(line);
+            if (!line.startsWith("remote.") || !line.contains(".url=")) {
+                continue;
+            }
+
+            String remoteName = StringUtils.substringBetween(line, "remote.", ".url=");
+            String remoteUri = StringUtils.substringAfter(line, ".url=");
+
+            // If uri String values end up identical, Map only stores one entry
+            uriNames.put(remoteUri, remoteName);
+
+            try {
+                URI u = new URI(remoteUri);
+                uriNames.put(u.toASCIIString(), remoteName);
+                URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment());
+                uriNames.put(uSafe.toString(), remoteName);
+                uriNames.put(uSafe.toASCIIString(), remoteName);
+            } catch (URISyntaxException ue) {} // ignore, move along
+        }
+        return uriNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public @CheckForNull Map<String, String> getRemotePushUrls() throws GitException, InterruptedException {
+        String result = launchCommand( "config", "--local", "--list" );
+        Map<String, String> uriNames = new HashMap<>();
+        for (String line : result.split("\\R+")) {
+            line = StringUtils.trim(line);
+            if (!line.startsWith("remote.") || !line.contains(".pushurl=")) {
+                continue;
+            }
+
+            String remoteName = StringUtils.substringBetween(line, "remote.", ".pushurl=");
+            String remoteUri = StringUtils.substringAfter(line, ".pushurl=");
+            uriNames.put(remoteUri, remoteName);
+
+            try {
+                URI u = new URI(remoteUri);
+                uriNames.put(u.toASCIIString(), remoteName);
+                URI uSafe = new URI(u.getScheme(), u.getHost(), u.getPath(), u.getFragment());
+                uriNames.put(uSafe.toString(), remoteName);
+                uriNames.put(uSafe.toASCIIString(), remoteName);
+            } catch (URISyntaxException ue) {} // ignore, move along
+        }
+        return uriNames;
+    }
+
     /** {@inheritDoc} */
     @Override
     public void setRemoteUrl(String name, String url) throws GitException, InterruptedException {
@@ -2693,7 +2800,12 @@ private String launchCommandIn(ArgumentListBuilder args, File workDir, EnvVars e
             }
 
             if (status != 0) {
-                throw new GitException("Command \"" + command + "\" returned status code " + status + ":\nstdout: " + stdout + "\nstderr: "+ stderr);
+                if (workDir == null)
+                    workDir = java.nio.file.Paths.get(".").toAbsolutePath().normalize().toFile();
+                throw new GitException("Command \"" + command +
+                    "\" executed in workdir \"" + workDir.toString() +
+                    "\" returned status code " + status +
+                    ":\nstdout: " + stdout + "\nstderr: " + stderr);
             }
 
             return stdout;

src/main/java/org/jenkinsci/plugins/gitclient/GitClient.java

@@ -223,6 +223,28 @@ public interface GitClient {
      */
     String getRemoteUrl(String name) throws GitException, InterruptedException;
 
+    /**
+     * getRemoteUrls.
+     *
+     * @return a Map where String keys represent URIs (with and
+     *         without passwords, if any; ASCII or not, if
+     *         applicable) for all remotes configured in this
+     *         repository/workspace, and values represent names.
+     *         There may be several URIs corresponding to same name.
+     */
+    public Map<String, String> getRemoteUrls() throws GitException, InterruptedException;
+
+    /**
+     * getRemotePushUrls.
+     *
+     * @return a Map where String keys represent push-only URIs
+     *         (with and without passwords, if any; ASCII or not,
+     *         if applicable) for all remotes configured in this
+     *         repository/workspace, and values represent names.
+     *         There may be several URIs corresponding to same name.
+     */
+    public Map<String, String> getRemotePushUrls() throws GitException, InterruptedException;
+
     /**
      * For a given repository, set a remote's URL
      *
@@ -703,6 +725,17 @@ public interface GitClient {
     List<ObjectId> revList(String ref) throws GitException, InterruptedException;
 
 
+    // --- new instance of same applied class
+
+    /**
+     * newGit.
+     *
+     * @return a IGitAPI implementation to manage another git repository
+     *   with same general settings and implementation as the current one.
+     * @param somedir a {@link java.lang.String} object.
+     */
+    GitClient newGit(String somedir);
+
     // --- submodules
 
     /**