diff --git a/src/main/conf/configuration.xml b/src/main/conf/configuration.xml
index e471ed9..eb5470d 100644
--- a/src/main/conf/configuration.xml
+++ b/src/main/conf/configuration.xml
@@ -24,6 +24,9 @@
+
+ true
+
@@ -51,6 +54,7 @@
git://github.com/edkennard/Integration.Git.Test2.git
+ true
@@ -64,6 +68,7 @@
git://github.com/edkennard/Integration.Git.Test3.git
+ false
diff --git a/src/main/java/com/versionone/git/GitConnector.java b/src/main/java/com/versionone/git/GitConnector.java
index d8b5277..7275c02 100644
--- a/src/main/java/com/versionone/git/GitConnector.java
+++ b/src/main/java/com/versionone/git/GitConnector.java
@@ -16,8 +16,10 @@
import org.eclipse.jgit.storage.file.FileRepository;
import org.eclipse.jgit.transport.*;
+import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.text.DecimalFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -50,7 +52,7 @@ public GitConnector(GitConnection gitConnection, String repositoryId, String loc
}
public void initRepository() throws GitException {
- LOG.debug("Initalizing repository...");
+ LOG.debug("Initalizing repository...");
try {
cloneRepository();
@@ -82,7 +84,10 @@ public boolean shouldAdd(ChangeSetInfo changeSet) {
}
};
- traverseChanges(builder);
+ Iterable changeSets = getChangeSetsFromCommits();
+
+ for (ChangeSetInfo changeSet : changeSets)
+ builder.add(changeSet);
return builder.build();
} catch(NotSupportedException ex) {
@@ -94,68 +99,58 @@ public boolean shouldAdd(ChangeSetInfo changeSet) {
}
}
- private void traverseChanges(ChangeSetListBuilder builder) throws GitException {
-
- Iterable commits = getCommits();
-
- for (RevCommit commit : commits) {
-
- // jGit returns data in seconds
- long millisecond = commit.getCommitTime() * 1000l;
- ChangeSetInfo info = new ChangeSetInfo(
- gitConnection,
- commit.getAuthorIdent().getName(),
- commit.getFullMessage().trim(),
- commit.getId().getName(),
- new Date(millisecond));
-
- if(gitConnection.getUseBranchName()) {
- List branches = getBranchNames(commit);
- for(String branch : branches) {
- fillReferences(branch, info.getReferences());
- }
- } else {
- fillReferences(info.getMessage(), info.getReferences());
- }
-
- builder.add(info);
- }
- }
-
- private Iterable getCommits() throws GitException {
- ArrayList commits = new ArrayList();
+ private Iterable getChangeSetsFromCommits() throws GitException {
+ Map changeSetMap = new HashMap();
try {
Git git = new Git(local);
- Map refs;
+ Map refs = new HashMap();
- // Either filter by just the watched branch if one is specified, or get all branch refs
+ // Either filter by just the watched branch if one is specified, or get all branch refs while respecting filter setting
if (gitConnection.getWatchedBranch() != null && !gitConnection.getWatchedBranch().trim().isEmpty()) {
String branchName = Constants.R_REMOTES + "/" + Constants.DEFAULT_REMOTE_NAME + "/" + gitConnection.getWatchedBranch();
- refs = new HashMap();
refs.put(branchName, local.getRef(branchName));
- }
- else if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilter().trim().isEmpty()) {
- refs = local.getAllRefs();
- for (String ref : refs.keySet()) {
- if (ref.contains(gitConnection.getBranchFilter())) {
- LOG.debug("Remove branch " + ref + " from the watching list since the GitConnection is configured to filter " + gitConnection.getBranchFilter());
- refs.remove(ref);
- }
- }
+ LOG.info(String.format("Checking branch '%s'...", branchName));
}
- else
- refs = local.getAllRefs();
+ else {
+ Map allRefs = local.getAllRefs();
- // Iterate through each branch checking for any new commits since the last one processed
- for (String ref : refs.keySet()) {
+ LOG.debug(String.format("Filtering %s reference%s...", allRefs.size(), refs.size() != 1 ? "s" : ""));
+
+ // Iterate through refs, filtering out tags or ones named in the branch filter
+ for (Map.Entry refEntry : allRefs.entrySet()) {
+
+ String refKey = refEntry.getKey();
- try {
// Skip anything other than branches (e.g. tags) since they're not commit objects and
// will throw an IncorrectObjectTypeException when setting the log command range
- if (!ref.contains("refs/remotes/origin"))
+ if (!refKey.contains("refs/remotes/origin")) {
+ LOG.debug(String.format("Ignoring %s, not a branch", refKey));
continue;
+ }
+
+ // Skip any refs matching the connection's branch filter
+ if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilter().trim().isEmpty() && refKey.contains(gitConnection.getBranchFilter())) {
+ LOG.debug(String.format("Ignoring %s, matched the connection's branch filter %s", refKey, gitConnection.getBranchFilter()));
+ continue;
+ }
+
+ // Add ref to list for processing
+ LOG.debug(String.format("Adding branch %s to list for checking", refKey));
+ refs.put(refKey, refEntry.getValue());
+ }
+ allRefs.clear();
+ LOG.debug(String.format("Checking %s branch%s...", refs.size(), refs.size() != 1 ? "es" : ""));
+ }
+
+ int refCounter = 0;
+
+ // Iterate through each branch checking for any new commits since the last one processed
+ for (String ref : refs.keySet()) {
+
+ refCounter++;
+ try {
// For each branch traversal use a new log object, since they're intended to be called only once
LogCommand logCommand = git.log();
@@ -172,18 +167,32 @@ else if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilt
if (persistedHash != null) {
AnyObjectId persistedHeadId = local.resolve(persistedHash);
- LOG.debug("Checking branch " + ref + " for new commits since the last one processed (" + persistedHash + ")...");
- //here we get lock for directory
+ LOG.debug(String.format("Checking branch %s (%s of %s) for new commits since the last one processed (%s)...",
+ ref, refCounter, refs.size(), persistedHash));
+
+ // Here we get lock for directory
logCommand.addRange(persistedHeadId, headId);
} else {
logCommand.add(headId);
- LOG.debug("Last commit processed on branch " + ref + " was not found so processing commits from the beginning.");
+ LOG.debug(String.format("Last commit processed on branch %s (%s of %s) was not found so processing commits from the beginning...",
+ ref, refCounter, refs.size()));
}
- if(!headHash.equals(persistedHash)) {
+ if (!headHash.equals(persistedHash)) {
+ int newCommitCounter = 0;
+
+ // Search for new commits then immediately convert them into much
+ // less memory-intensive ChangeSetInfo objects containing only the info we need
for (RevCommit commit : logCommand.call()) {
- if (!commits.contains(commit))
- commits.add(commit);
+ if (!changeSetMap.containsKey(commit.getId().getName())) {
+ newCommitCounter++;
+ ChangeSetInfo changeSet = getChangeSetFromCommit(commit);
+ changeSetMap.put(changeSet.getRevision(), changeSet);
+ }
+ }
+ if (newCommitCounter > 0) {
+ DecimalFormat df = new DecimalFormat("#,###");
+ LOG.debug(String.format("%s new commit(s) found (%s in total)", df.format(newCommitCounter), df.format(changeSetMap.size())));
}
storage.persistLastCommit(headHash, repositoryId, ref);
} else {
@@ -200,12 +209,39 @@ else if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilt
throw new GitException(ex);
}
+ // Convert map to list then immediately clear map
+ ArrayList changeSetList = new ArrayList(changeSetMap.values());
+ changeSetMap.clear();
+
// Sort commits by commit time which is needed when they've been taken
// from multiple branches since they won't be listed chronologically
- Comparator comparator = new GitCommitComparator();
- Collections.sort(commits, comparator);
+ Comparator comparator = new ChangeSetComparator();
+ Collections.sort(changeSetList, comparator);
- return commits;
+ return changeSetList;
+ }
+
+ private ChangeSetInfo getChangeSetFromCommit(RevCommit commit) {
+
+ // jGit returns data in seconds
+ long millisecond = commit.getCommitTime() * 1000l;
+ ChangeSetInfo info = new ChangeSetInfo(
+ gitConnection,
+ commit.getAuthorIdent().getName(),
+ commit.getFullMessage().trim(),
+ commit.getId().getName(),
+ new Date(millisecond));
+
+ if (gitConnection.getUseBranchName()) {
+ List branches = getBranchNames(commit);
+ for (String branch : branches) {
+ fillReferences(branch, info.getReferences());
+ }
+ } else {
+ fillReferences(info.getMessage(), info.getReferences());
+ }
+
+ return info;
}
private void fillReferences(String message, List references) {
@@ -246,12 +282,27 @@ private List getBranchNames(RevCommit commit) {
}
private void cloneRepository() throws IOException, URISyntaxException {
- LOG.debug("Cloning repository...");
+
+ File directory = new File(localDirectory);
local = new FileRepository(localDirectory);
- local.create();
URIish uri = new URIish(gitConnection.getRepositoryPath());
+ // Unless this repo is configured to always clone on startup,
+ // only re-create it if it doesn't already exist. This avoids re-cloning every
+ // restart of the integration, which for larger repos can be unnecessarily time consuming.
+ if (gitConnection.getAlwaysCloneOnStartup() || !directory.exists() || !local.getObjectDatabase().exists()) {
+
+ if (directory.exists() && !Utilities.deleteDirectory(directory))
+ LOG.warn(localDirectory + " couldn't be deleted, cloning may fail");
+
+ LOG.info("Cloning repository...");
+ local.create();
+ }
+ else {
+ LOG.info("Not re-cloning repository as it's already present. If you always want to re-clone, switch the AlwaysCloneOnStartup configuration on for this repo");
+ }
+
remoteConfig = new RemoteConfig(local.getConfig(), remoteName);
remoteConfig.addURI(uri);
@@ -295,12 +346,12 @@ public void endTask() {}
}
/** Compares two commits and sorts them by commit time in ascending order */
- private class GitCommitComparator implements Comparator {
+ private class ChangeSetComparator implements Comparator {
public int compare(Object object1, Object object2) {
- RevCommit commit1 = (RevCommit)object1;
- RevCommit commit2 = (RevCommit)object2;
+ ChangeSetInfo change1 = (ChangeSetInfo)object1;
+ ChangeSetInfo change2 = (ChangeSetInfo)object2;
- return commit1.getCommitTime() - commit2.getCommitTime();
+ return change1.getChangeDate().compareTo(change2.getChangeDate());
}
}
}
\ No newline at end of file
diff --git a/src/main/java/com/versionone/git/GitPollTask.java b/src/main/java/com/versionone/git/GitPollTask.java
index 0f7faae..9fdca36 100644
--- a/src/main/java/com/versionone/git/GitPollTask.java
+++ b/src/main/java/com/versionone/git/GitPollTask.java
@@ -26,7 +26,7 @@ public class GitPollTask extends TimerTask {
v1Connector.connect(configuration.getVersionOneConnection());
changeSetWriter = new ChangeSetWriter(configuration.getChangeSet(), v1Connector);
- cleanupLocalDirectory();
+ createLocalDirectory();
serviceInitialize();
}
@@ -97,18 +97,19 @@ private boolean initializeGitService(GitService service) {
return true;
}
- private void cleanupLocalDirectory() {
- LOG.debug(String.format("Resetting local directory %s...", configuration.getLocalDirectory()));
+ private void createLocalDirectory() {
- if (!Utilities.deleteDirectory(new File(configuration.getLocalDirectory()))) {
- LOG.warn(configuration.getLocalDirectory() + " couldn't be reset, possibly due to this being the first time the service has been run");
- }
+ File directory = new File(configuration.getLocalDirectory());
+
+ if (!directory.exists()) {
+ LOG.info(String.format("Creating local directory %s...", configuration.getLocalDirectory()));
- boolean result = new File(configuration.getLocalDirectory()).mkdir();
+ boolean result = directory.mkdir();
- if (!result) {
- LOG.fatal(configuration.getLocalDirectory() + " couldn't be created");
- System.exit(-1);
+ if (!result) {
+ LOG.fatal(configuration.getLocalDirectory() + " couldn't be created");
+ System.exit(-1);
+ }
}
}
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/versionone/git/configuration/GitConnection.java b/src/main/java/com/versionone/git/configuration/GitConnection.java
index c5260b5..35db9a3 100644
--- a/src/main/java/com/versionone/git/configuration/GitConnection.java
+++ b/src/main/java/com/versionone/git/configuration/GitConnection.java
@@ -5,6 +5,8 @@
public class GitConnection {
@XmlElement(name = "Path")
private String repositoryPath;
+ @XmlElement(name = "AlwaysCloneOnStartup")
+ private Boolean alwaysCloneOnStartup = false;
@XmlElement(name = "WatchedBranchName")
private String watchedBranch;
@XmlElement(name = "BranchFilter")
@@ -23,6 +25,10 @@ public String getRepositoryPath() {
return repositoryPath;
}
+ public Boolean getAlwaysCloneOnStartup() {
+ return alwaysCloneOnStartup;
+ }
+
public String getPassword() {
return password;
}
@@ -57,6 +63,7 @@ public boolean equals(Object o) {
if (passphrase != null ? !passphrase.equals(that.passphrase) : that.passphrase != null) return false;
if (password != null ? !password.equals(that.password) : that.password != null) return false;
if (!repositoryPath.equals(that.repositoryPath)) return false;
+ if (!alwaysCloneOnStartup.equals(that.alwaysCloneOnStartup)) return false;
if (!useBranchName.equals(that.useBranchName)) return false;
if (watchedBranch != null ? !watchedBranch.equals(that.watchedBranch) : that.watchedBranch != null) return false;
if (link != null ? !link.equals(that.link) : that.link != null) return false;
@@ -66,6 +73,7 @@ public boolean equals(Object o) {
@Override
public int hashCode() {
int result = repositoryPath.hashCode();
+ result = 31 * result + alwaysCloneOnStartup.hashCode();
result = 31 * result + (watchedBranch != null ? watchedBranch.hashCode() : 0);
result = 31 * result + (password != null ? password.hashCode() : 0);
result = 31 * result + (passphrase != null ? passphrase.hashCode() : 0);
diff --git a/src/test/java/com/versionone/git/ConfigurationTester.java b/src/test/java/com/versionone/git/ConfigurationTester.java
index 59f15f5..d7b1e0c 100644
--- a/src/test/java/com/versionone/git/ConfigurationTester.java
+++ b/src/test/java/com/versionone/git/ConfigurationTester.java
@@ -30,6 +30,7 @@ public void config() throws IOException {
Assert.assertEquals("Proxy user name is incorrect.", "proxyUser", proxy.getUserName());
Assert.assertEquals("Proxy password is incorrect.", "proxyUserPass", proxy.getPassword());
Assert.assertEquals("Git repository path is incorrect.", "https://github.com/account/repo.git", git.getRepositoryPath());
+ Assert.assertEquals("Incorrect setting for always cloning on startup.", true, git.getAlwaysCloneOnStartup());
Assert.assertEquals("Git password is incorrect.", "password", git.getPassword());
Assert.assertEquals("Git passphrase is incorrect.", "passphrase", git.getPassphrase());
Assert.assertEquals("Git branch name is incorrect.", "master", git.getWatchedBranch());
diff --git a/src/test/resources/test_configuration.xml b/src/test/resources/test_configuration.xml
index 118921f..8caa008 100644
--- a/src/test/resources/test_configuration.xml
+++ b/src/test/resources/test_configuration.xml
@@ -23,6 +23,7 @@
https://github.com/account/repo.git
password
passphrase
+ true
master
false
personal
@@ -38,6 +39,7 @@
https://github.com/account/repo.git
password
passphrase
+ true
master
false
personal