Skip to content

Optimisations #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edkennard wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Optimisations #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
83aa448
Added AlwaysCloneOnStartup configuration for GitConnections, to optio…
edkennard Nov 9, 2014
2e5538d
Improved performance and reduced memory usage when scanning for new c…
edkennard Nov 9, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/main/conf/configuration.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
<Password></Password>
<SshPassphrase></SshPassphrase>

<!-- Set AlwaysCloneOnStartup to true to always delete then re-clone a repo as the service starts -->
<AlwaysCloneOnStartup>true</AlwaysCloneOnStartup>

<!-- Specify a single branch to watch, or leave blank to watch all branches -->
<WatchedBranchName></WatchedBranchName>

Expand Down Expand Up @@ -51,6 +54,7 @@
</GitConnection>
<GitConnection>
<Path>git://github.com/edkennard/Integration.Git.Test2.git</Path>
<AlwaysCloneOnStartup>true</AlwaysCloneOnStartup>
<Password></Password>
<SshPassphrase></SshPassphrase>
<WatchedBranchName></WatchedBranchName>
Expand All @@ -64,6 +68,7 @@
</GitConnection>
<GitConnection>
<Path>git://github.com/edkennard/Integration.Git.Test3.git</Path>
<AlwaysCloneOnStartup>false</AlwaysCloneOnStartup>
<Password></Password>
<SshPassphrase></SshPassphrase>
<WatchedBranchName></WatchedBranchName>
Expand Down
181 changes: 116 additions & 65 deletions src/main/java/com/versionone/git/GitConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@
import org.eclipse.jgit.storage.file.FileRepository;
import org.eclipse.jgit.transport.*;

import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.DecimalFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -50,7 +52,7 @@ public GitConnector(GitConnection gitConnection, String repositoryId, String loc
}

public void initRepository() throws GitException {
LOG.debug("Initalizing repository...");
LOG.debug("Initalizing repository...");

try {
cloneRepository();
Expand Down Expand Up @@ -82,7 +84,10 @@ public boolean shouldAdd(ChangeSetInfo changeSet) {
}
};

traverseChanges(builder);
Iterable<ChangeSetInfo> changeSets = getChangeSetsFromCommits();

for (ChangeSetInfo changeSet : changeSets)
builder.add(changeSet);

return builder.build();
} catch(NotSupportedException ex) {
Expand All @@ -94,68 +99,58 @@ public boolean shouldAdd(ChangeSetInfo changeSet) {
}
}

private void traverseChanges(ChangeSetListBuilder builder) throws GitException {

Iterable<RevCommit> commits = getCommits();

for (RevCommit commit : commits) {

// jGit returns data in seconds
long millisecond = commit.getCommitTime() * 1000l;
ChangeSetInfo info = new ChangeSetInfo(
gitConnection,
commit.getAuthorIdent().getName(),
commit.getFullMessage().trim(),
commit.getId().getName(),
new Date(millisecond));

if(gitConnection.getUseBranchName()) {
List<String> branches = getBranchNames(commit);
for(String branch : branches) {
fillReferences(branch, info.getReferences());
}
} else {
fillReferences(info.getMessage(), info.getReferences());
}

builder.add(info);
}
}

private Iterable<RevCommit> getCommits() throws GitException {
ArrayList<RevCommit> commits = new ArrayList<RevCommit>();
private Iterable<ChangeSetInfo> getChangeSetsFromCommits() throws GitException {
Map<String, ChangeSetInfo> changeSetMap = new HashMap<String, ChangeSetInfo>();

try {
Git git = new Git(local);
Map<String, Ref> refs;
Map<String, Ref> refs = new HashMap();

// Either filter by just the watched branch if one is specified, or get all branch refs
// Either filter by just the watched branch if one is specified, or get all branch refs while respecting filter setting
if (gitConnection.getWatchedBranch() != null && !gitConnection.getWatchedBranch().trim().isEmpty()) {
String branchName = Constants.R_REMOTES + "/" + Constants.DEFAULT_REMOTE_NAME + "/" + gitConnection.getWatchedBranch();
refs = new HashMap();
refs.put(branchName, local.getRef(branchName));
}
else if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilter().trim().isEmpty()) {
refs = local.getAllRefs();
for (String ref : refs.keySet()) {
if (ref.contains(gitConnection.getBranchFilter())) {
LOG.debug("Remove branch " + ref + " from the watching list since the GitConnection is configured to filter " + gitConnection.getBranchFilter());
refs.remove(ref);
}
}
LOG.info(String.format("Checking branch '%s'...", branchName));
}
else
refs = local.getAllRefs();
else {
Map<String, Ref> allRefs = local.getAllRefs();

// Iterate through each branch checking for any new commits since the last one processed
for (String ref : refs.keySet()) {
LOG.debug(String.format("Filtering %s reference%s...", allRefs.size(), refs.size() != 1 ? "s" : ""));

// Iterate through refs, filtering out tags or ones named in the branch filter
for (Map.Entry<String, Ref> refEntry : allRefs.entrySet()) {

String refKey = refEntry.getKey();

try {
// Skip anything other than branches (e.g. tags) since they're not commit objects and
// will throw an IncorrectObjectTypeException when setting the log command range
if (!ref.contains("refs/remotes/origin"))
if (!refKey.contains("refs/remotes/origin")) {
LOG.debug(String.format("Ignoring %s, not a branch", refKey));
continue;
}

// Skip any refs matching the connection's branch filter
if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilter().trim().isEmpty() && refKey.contains(gitConnection.getBranchFilter())) {
LOG.debug(String.format("Ignoring %s, matched the connection's branch filter %s", refKey, gitConnection.getBranchFilter()));
continue;
}

// Add ref to list for processing
LOG.debug(String.format("Adding branch %s to list for checking", refKey));
refs.put(refKey, refEntry.getValue());
}
allRefs.clear();
LOG.debug(String.format("Checking %s branch%s...", refs.size(), refs.size() != 1 ? "es" : ""));
}

int refCounter = 0;

// Iterate through each branch checking for any new commits since the last one processed
for (String ref : refs.keySet()) {

refCounter++;

try {
// For each branch traversal use a new log object, since they're intended to be called only once
LogCommand logCommand = git.log();

Expand All @@ -172,18 +167,32 @@ else if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilt

if (persistedHash != null) {
AnyObjectId persistedHeadId = local.resolve(persistedHash);
LOG.debug("Checking branch " + ref + " for new commits since the last one processed (" + persistedHash + ")...");
//here we get lock for directory
LOG.debug(String.format("Checking branch %s (%s of %s) for new commits since the last one processed (%s)...",
ref, refCounter, refs.size(), persistedHash));

// Here we get lock for directory
logCommand.addRange(persistedHeadId, headId);
} else {
logCommand.add(headId);
LOG.debug("Last commit processed on branch " + ref + " was not found so processing commits from the beginning.");
LOG.debug(String.format("Last commit processed on branch %s (%s of %s) was not found so processing commits from the beginning...",
ref, refCounter, refs.size()));
}

if(!headHash.equals(persistedHash)) {
if (!headHash.equals(persistedHash)) {
int newCommitCounter = 0;

// Search for new commits then immediately convert them into much
// less memory-intensive ChangeSetInfo objects containing only the info we need
for (RevCommit commit : logCommand.call()) {
if (!commits.contains(commit))
commits.add(commit);
if (!changeSetMap.containsKey(commit.getId().getName())) {
newCommitCounter++;
ChangeSetInfo changeSet = getChangeSetFromCommit(commit);
changeSetMap.put(changeSet.getRevision(), changeSet);
}
}
if (newCommitCounter > 0) {
DecimalFormat df = new DecimalFormat("#,###");
LOG.debug(String.format("%s new commit(s) found (%s in total)", df.format(newCommitCounter), df.format(changeSetMap.size())));
}
storage.persistLastCommit(headHash, repositoryId, ref);
} else {
Expand All @@ -200,12 +209,39 @@ else if (gitConnection.getBranchFilter() != null && !gitConnection.getBranchFilt
throw new GitException(ex);
}

// Convert map to list then immediately clear map
ArrayList<ChangeSetInfo> changeSetList = new ArrayList<ChangeSetInfo>(changeSetMap.values());
changeSetMap.clear();

// Sort commits by commit time which is needed when they've been taken
// from multiple branches since they won't be listed chronologically
Comparator comparator = new GitCommitComparator();
Collections.sort(commits, comparator);
Comparator comparator = new ChangeSetComparator();
Collections.sort(changeSetList, comparator);

return commits;
return changeSetList;
}

private ChangeSetInfo getChangeSetFromCommit(RevCommit commit) {

// jGit returns data in seconds
long millisecond = commit.getCommitTime() * 1000l;
ChangeSetInfo info = new ChangeSetInfo(
gitConnection,
commit.getAuthorIdent().getName(),
commit.getFullMessage().trim(),
commit.getId().getName(),
new Date(millisecond));

if (gitConnection.getUseBranchName()) {
List<String> branches = getBranchNames(commit);
for (String branch : branches) {
fillReferences(branch, info.getReferences());
}
} else {
fillReferences(info.getMessage(), info.getReferences());
}

return info;
}

private void fillReferences(String message, List<String> references) {
Expand Down Expand Up @@ -246,12 +282,27 @@ private List<String> getBranchNames(RevCommit commit) {
}

private void cloneRepository() throws IOException, URISyntaxException {
LOG.debug("Cloning repository...");

File directory = new File(localDirectory);
local = new FileRepository(localDirectory);
local.create();

URIish uri = new URIish(gitConnection.getRepositoryPath());

// Unless this repo is configured to always clone on startup,
// only re-create it if it doesn't already exist. This avoids re-cloning every
// restart of the integration, which for larger repos can be unnecessarily time consuming.
if (gitConnection.getAlwaysCloneOnStartup() || !directory.exists() || !local.getObjectDatabase().exists()) {

if (directory.exists() && !Utilities.deleteDirectory(directory))
LOG.warn(localDirectory + " couldn't be deleted, cloning may fail");

LOG.info("Cloning repository...");
local.create();
}
else {
LOG.info("Not re-cloning repository as it's already present. If you always want to re-clone, switch the AlwaysCloneOnStartup configuration on for this repo");
}

remoteConfig = new RemoteConfig(local.getConfig(), remoteName);
remoteConfig.addURI(uri);

Expand Down Expand Up @@ -295,12 +346,12 @@ public void endTask() {}
}

/** Compares two commits and sorts them by commit time in ascending order */
private class GitCommitComparator implements Comparator {
private class ChangeSetComparator implements Comparator {
public int compare(Object object1, Object object2) {
RevCommit commit1 = (RevCommit)object1;
RevCommit commit2 = (RevCommit)object2;
ChangeSetInfo change1 = (ChangeSetInfo)object1;
ChangeSetInfo change2 = (ChangeSetInfo)object2;

return commit1.getCommitTime() - commit2.getCommitTime();
return change1.getChangeDate().compareTo(change2.getChangeDate());
}
}
}
23 changes: 12 additions & 11 deletions src/main/java/com/versionone/git/GitPollTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class GitPollTask extends TimerTask {
v1Connector.connect(configuration.getVersionOneConnection());

changeSetWriter = new ChangeSetWriter(configuration.getChangeSet(), v1Connector);
cleanupLocalDirectory();
createLocalDirectory();
serviceInitialize();
}

Expand Down Expand Up @@ -97,18 +97,19 @@ private boolean initializeGitService(GitService service) {
return true;
}

private void cleanupLocalDirectory() {
LOG.debug(String.format("Resetting local directory %s...", configuration.getLocalDirectory()));
private void createLocalDirectory() {

if (!Utilities.deleteDirectory(new File(configuration.getLocalDirectory()))) {
LOG.warn(configuration.getLocalDirectory() + " couldn't be reset, possibly due to this being the first time the service has been run");
}
File directory = new File(configuration.getLocalDirectory());

if (!directory.exists()) {
LOG.info(String.format("Creating local directory %s...", configuration.getLocalDirectory()));

boolean result = new File(configuration.getLocalDirectory()).mkdir();
boolean result = directory.mkdir();

if (!result) {
LOG.fatal(configuration.getLocalDirectory() + " couldn't be created");
System.exit(-1);
if (!result) {
LOG.fatal(configuration.getLocalDirectory() + " couldn't be created");
System.exit(-1);
}
}
}
}
}
8 changes: 8 additions & 0 deletions src/main/java/com/versionone/git/configuration/GitConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
public class GitConnection {
@XmlElement(name = "Path")
private String repositoryPath;
@XmlElement(name = "AlwaysCloneOnStartup")
private Boolean alwaysCloneOnStartup = false;
@XmlElement(name = "WatchedBranchName")
private String watchedBranch;
@XmlElement(name = "BranchFilter")
Expand All @@ -23,6 +25,10 @@ public String getRepositoryPath() {
return repositoryPath;
}

public Boolean getAlwaysCloneOnStartup() {
return alwaysCloneOnStartup;
}

public String getPassword() {
return password;
}
Expand Down Expand Up @@ -57,6 +63,7 @@ public boolean equals(Object o) {
if (passphrase != null ? !passphrase.equals(that.passphrase) : that.passphrase != null) return false;
if (password != null ? !password.equals(that.password) : that.password != null) return false;
if (!repositoryPath.equals(that.repositoryPath)) return false;
if (!alwaysCloneOnStartup.equals(that.alwaysCloneOnStartup)) return false;
if (!useBranchName.equals(that.useBranchName)) return false;
if (watchedBranch != null ? !watchedBranch.equals(that.watchedBranch) : that.watchedBranch != null) return false;
if (link != null ? !link.equals(that.link) : that.link != null) return false;
Expand All @@ -66,6 +73,7 @@ public boolean equals(Object o) {
@Override
public int hashCode() {
int result = repositoryPath.hashCode();
result = 31 * result + alwaysCloneOnStartup.hashCode();
result = 31 * result + (watchedBranch != null ? watchedBranch.hashCode() : 0);
result = 31 * result + (password != null ? password.hashCode() : 0);
result = 31 * result + (passphrase != null ? passphrase.hashCode() : 0);
Expand Down
1 change: 1 addition & 0 deletions src/test/java/com/versionone/git/ConfigurationTester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public void config() throws IOException {
Assert.assertEquals("Proxy user name is incorrect.", "proxyUser", proxy.getUserName());
Assert.assertEquals("Proxy password is incorrect.", "proxyUserPass", proxy.getPassword());
Assert.assertEquals("Git repository path is incorrect.", "https://github.com/account/repo.git", git.getRepositoryPath());
Assert.assertEquals("Incorrect setting for always cloning on startup.", true, git.getAlwaysCloneOnStartup());
Assert.assertEquals("Git password is incorrect.", "password", git.getPassword());
Assert.assertEquals("Git passphrase is incorrect.", "passphrase", git.getPassphrase());
Assert.assertEquals("Git branch name is incorrect.", "master", git.getWatchedBranch());
Expand Down
2 changes: 2 additions & 0 deletions src/test/resources/test_configuration.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
<Path>https://github.com/account/repo.git</Path>
<Password>password</Password>
<SshPassphrase>passphrase</SshPassphrase>
<AlwaysCloneOnStartup>true</AlwaysCloneOnStartup>
<WatchedBranchName>master</WatchedBranchName>
<UseBranchName>false</UseBranchName>
<BranchFilter>personal</BranchFilter>
Expand All @@ -38,6 +39,7 @@
<Path>https://github.com/account/repo.git</Path>
<Password>password</Password>
<SshPassphrase>passphrase</SshPassphrase>
<AlwaysCloneOnStartup>true</AlwaysCloneOnStartup>
<WatchedBranchName>master</WatchedBranchName>
<UseBranchName>false</UseBranchName>
<BranchFilter>personal</BranchFilter>
Expand Down