com.centurylink.mdw.dataaccess.file.VersionControlGit Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of mdw-common Show documentation
Show all versions of mdw-common Show documentation
MDW is a workflow framework specializing in microservice orchestration
/*
* Copyright (C) 2017 CenturyLink, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.centurylink.mdw.dataaccess.file;
import com.centurylink.mdw.cli.Delete;
import com.centurylink.mdw.dataaccess.AssetRevision;
import com.centurylink.mdw.dataaccess.VersionControl;
import com.centurylink.mdw.dataaccess.file.GitDiffs.DiffType;
import com.centurylink.mdw.model.asset.CommitInfo;
import com.centurylink.mdw.util.file.VersionProperties;
import com.google.common.io.Files;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.CreateBranchCommand.SetupUpstreamMode;
import org.eclipse.jgit.api.ResetCommand.ResetType;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.TransportException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.dircache.DirCacheIterator;
import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.*;
import org.eclipse.jgit.pgm.Main;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.EmptyTreeIterator;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
import java.io.*;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.*;
public class VersionControlGit implements VersionControl {
public static final String VERSIONS_FILE = ".mdw/versions";
public static final char NEWLINE_CHAR = 0x0a;
/**
* should be sufficient, but we could change to 8 or more
* (com.centurylink.mdw.abbreviated.id.length system property)
*/
public static int ABBREVIATED_ID_LENGTH = 7;
private String repositoryUrl;
protected String getRepositoryUrl() { return repositoryUrl; }
private File localDir;
private Repository localRepo;
private Git git;
private Map file2id;
private Map id2file;
private Map pkg2versions;
private CredentialsProvider credentialsProvider;
public CredentialsProvider getCredentialsProvider() { return this.credentialsProvider; };
public void setCredentialsProvider(CredentialsProvider provider) { this.credentialsProvider = provider; }
public VersionControlGit() {
}
public void connect(String repositoryUrl, String user, String password, File localDir) throws IOException {
if (repositoryUrl != null && !repositoryUrl.isEmpty()) {
if (user == null) {
this.repositoryUrl = repositoryUrl;
}
else {
int slashSlash = repositoryUrl.indexOf("//");
this.repositoryUrl = repositoryUrl.substring(0, slashSlash + 2) + user + ":" + password + "@" + repositoryUrl.substring(slashSlash + 2);
}
}
this.localDir = localDir;
localRepo = new FileRepository(new File(localDir + "/.git"));
git = new Git(localRepo);
if (credentialsProvider == null) {
if (user != null && password != null)
credentialsProvider = new UsernamePasswordCredentialsProvider(user, password);
}
file2id = new HashMap();
id2file = new HashMap();
pkg2versions = new HashMap<>();
String idLengthProp = System.getProperty("com.centurylink.mdw.abbreviated.id.length");
if (idLengthProp != null)
ABBREVIATED_ID_LENGTH = Integer.parseInt(idLengthProp);
}
public synchronized void reconnect() throws IOException {
Repository newLocalRepo = new FileRepository(new File(localDir + "/.git"));
git = new Git(newLocalRepo);
localRepo.close();
localRepo = newLocalRepo;
}
public String toString() {
return localDir + "->" + repositoryUrl;
}
/**
* Cannot use git object hash since identical files would return duplicate
* ids. Hash using git algorithm based on the relative file path.
*/
public long getId(File file) throws IOException {
Long id = file2id.get(file);
if (id == null) {
id = gitHash(file);
file2id.put(file, id);
id2file.put(id, file);
}
return id;
}
/**
* This produces the same hash for a given object that Git 'hash-object' creates
*/
public String getGitId(File input) throws IOException {
String hash = "";
if (input.isFile()) {
FileInputStream fis = null;
try {
int fileSize = (int)input.length();
fis = new FileInputStream(input);
byte[] fileBytes = new byte[fileSize];
fis.read(fileBytes);
//hash = localRepo.newObjectInserter().idFor(Constants.OBJ_BLOB, fileBytes).getName(); // This is slower than below code (even if reusing ObjectInserter instance)
String blob = "blob " + fileSize + "\0" + new String(fileBytes);
MessageDigest md = MessageDigest.getInstance("SHA-1");
byte[] bytes = md.digest(blob.getBytes());
hash = byteArrayToHexString(bytes);
}
catch (Throwable ex) {
throw new IOException(ex.getMessage(), ex);
}
finally {
if (fis != null)
fis.close();
}
}
return hash;
}
protected long getLongId(ObjectId objectId) {
String h = objectId.abbreviate(ABBREVIATED_ID_LENGTH).name();
return Long.parseLong(h, 16);
}
public File getFile(long id) {
return id2file.get(id);
}
public void clearId(File file) {
Long id = file2id.remove(file);
if (id != null)
id2file.remove(id);
}
public void clear() {
file2id.clear();
id2file.clear();
pkg2versions.clear();
}
public void deleteRev(File file) throws IOException {
VersionProperties verProps = getVersionProps(file.getParentFile());
String val = verProps.getProperty(file.getName());
if (val != null) {
verProps.remove(file.getName());
verProps.save();
}
}
public AssetRevision getRevision(File file) throws IOException {
return getRevisionInVersionsFile(file);
}
public void setRevision(File file, AssetRevision rev) throws IOException {
setRevisionInVersionsFile(file, rev);
}
public AssetRevision getRevisionInVersionsFile(File file) throws IOException {
Properties verProps = getVersionProps(file.getParentFile());
if (verProps == null)
return null;
String propVal = verProps.getProperty(file.getName());
if (propVal == null) {
return null;
}
else {
return parseAssetRevision(propVal.trim());
}
}
public void setRevisionInVersionsFile(File file, AssetRevision rev) throws IOException {
VersionProperties verProps = getVersionProps(file.getParentFile());
if (verProps == null) {
// presumably newly-created package
File verFile = new File(file.getParentFile() + "/" + VERSIONS_FILE);
Files.write("".getBytes(), verFile);
verProps = new VersionProperties(verFile);
pkg2versions.put(file.getParentFile(), verProps);
}
verProps.setProperty(file.getName(), String.valueOf(rev.getVersion()));
verProps.save();
}
public static AssetRevision parseAssetRevision(String propertyValue) {
AssetRevision assetRevision = new AssetRevision();
int firstSpace = propertyValue.indexOf(' ');
if (firstSpace > 0) {
// includes comment
assetRevision.setVersion(Integer.parseInt(propertyValue.substring(0, firstSpace)));
assetRevision.setComment(propertyValue.substring(firstSpace + 1).replace(NEWLINE_CHAR, '\n'));
}
else {
assetRevision.setVersion(Integer.parseInt(propertyValue));
}
return assetRevision;
}
private VersionProperties getVersionProps(File pkgDir) throws IOException {
VersionProperties props = pkg2versions.get(pkgDir);
if (props == null) {
File file = new File(pkgDir + "/" + VERSIONS_FILE);
if (file.exists()) {
props = new VersionProperties(file);
pkg2versions.put(pkgDir, props);
}
else if (file.getAbsolutePath().indexOf(localDir.getAbsolutePath()) < 0) {
return getVersionProps(new File(localDir.getAbsolutePath() + "/" + pkgDir.getPath()));
}
}
return props;
}
/**
* Use git hashing algorithm.
* http://stackoverflow.com/questions/7225313/how-does-git-compute-file-hashes
*/
public Long gitHash(File input) throws IOException {
String path = getLogicalPath(input);
String blob = "blob " + path.length() + "\0" + path;
MessageDigest md = null;
try {
md = MessageDigest.getInstance("SHA-1");
byte[] bytes = md.digest(blob.getBytes());
String h = byteArrayToHexString(bytes).substring(0, 7);
return Long.parseLong(h, 16);
}
catch (NoSuchAlgorithmException ex) {
throw new IOException(ex.getMessage(), ex);
}
}
/**
* Seems slightly slower than gitHash().
*/
protected ObjectId gitHashJgit(File input) throws IOException {
String path = getLogicalPath(input);
String blob = "blob " + path.length() + "\0" + path;
MessageDigest md = null;
try {
md = MessageDigest.getInstance("SHA-1");
byte[] bytes = md.digest(blob.getBytes());
return ObjectId.fromRaw(bytes);
}
catch (NoSuchAlgorithmException ex) {
throw new IOException(ex.getMessage(), ex);
}
}
private String getLogicalPath(File file) {
return file.getPath().replace('\\', '/');
}
public String getCommit() throws IOException {
ObjectId head = localRepo.resolve(Constants.HEAD);
if (head != null)
return head.getName();
else
return null;
}
public String getCommitForTag(String tag) throws Exception {
fetch();
List tagRefs = git.tagList().call();
for (Ref tagRef : tagRefs) {
if (tagRef.getName().equals("refs/tags/" + tag))
return tagRef.getObjectId().name();
}
return null;
}
/**
* Get remote HEAD commit.
*/
public String getRemoteCommit(String branch) throws Exception {
fetch();
ObjectId commit = localRepo.resolve("origin/" + branch);
if (commit != null)
return commit.getName();
else
return null;
}
public long getCommitTime(String commitId) throws Exception {
try (RevWalk revWalk = new RevWalk(localRepo)) {
return revWalk.parseCommit(ObjectId.fromString(commitId)).getCommitterIdent().getWhen().getTime();
}
}
public String getBranch() throws IOException {
return localRepo.getBranch();
}
/**
* Does not do anything if already on target branch.
*/
public void checkout(String branch) throws Exception {
if (!branch.equals(getBranch())) {
createBranchIfNeeded(branch);
git.checkout().setName(branch).setStartPoint("origin/" + branch)
.setUpstreamMode(CreateBranchCommand.SetupUpstreamMode.TRACK).call();
// for some reason jgit needs this when branch is switched
git.checkout().setName(branch).call();
}
}
public void checkoutTag(String tag) throws Exception {
if (localRepo.getTags().get(tag) != null)
git.checkout().setName(localRepo.getTags().get(tag).getName()).call();
}
/**
* Actually a workaround since JGit does not support sparse checkout:
* https://bugs.eclipse.org/bugs/show_bug.cgi?id=383772.
* Performs a HARD reset and FORCED checkout then deletes non-path items.
* Only to be used on server (not Designer).
*/
public void sparseCheckout(String branch, String path) throws Exception {
fetch(); // in case the branch is not known locally
hardReset();
checkout(branch);
pull(branch); // pull before delete or next pull may add non-path items back
// delete non-path items
List preserveList = new ArrayList();
preserveList.add(new File(localDir + "/.git"));
preserveList.add(new File(localDir + "/" + path));
new Delete(localDir, true).run();
}
public void hardCheckout(String branch) throws Exception {
hardCheckout(branch, false);
}
public void hardTagCheckout(String tag) throws Exception {
hardTagCheckout(tag, false);
}
/**
* Performs a HARD reset and FORCED checkout.
* Only to be used on server (not Designer).
* FIXME: path is ignored
*/
public void hardCheckout(String branch, Boolean hard) throws Exception {
fetch(); // in case the branch is not known locally
if (hard)
hardReset();
checkout(branch);
pull(branch); // pull before delete or next pull may add non-path items back
}
public void hardTagCheckout(String tag, Boolean hard) throws Exception {
fetch(); // in case the tag is not known locally
if (hard)
hardReset();
checkoutTag(tag);
}
public void hardReset() throws Exception {
git.reset().setMode(ResetType.HARD).call();
git.clean().call();
}
/**
* Create a local branch for remote tracking if it doesn't exist already.
*/
protected void createBranchIfNeeded(String branch) throws Exception {
fetch(); // in case the branch is not known locally
if (localRepo.findRef(branch) == null) {
git.branchCreate()
.setName(branch)
.setUpstreamMode(SetupUpstreamMode.TRACK)
.setStartPoint("origin/" + branch)
.call();
}
}
public void commit(String path, String msg) throws Exception {
git.commit().setOnly(path).setMessage(msg).call();
}
public void commit(List paths, String msg) throws Exception {
CommitCommand commit = git.commit().setMessage(msg);
for (String path : paths)
commit.setOnly(path);
commit.call();
}
public void push() throws Exception {
PushCommand push = git.push();
if (credentialsProvider != null)
push.setCredentialsProvider(credentialsProvider);
push.call();
}
public void fetch() throws Exception {
FetchCommand fetchCommand = git.fetch();
if (credentialsProvider != null)
fetchCommand.setCredentialsProvider(credentialsProvider);
try {
fetchCommand.call();
}
catch (JGitInternalException | TransportException ex) {
// LocalRepo object might be out of sync with actual local repo, so recreate objects for next time
reconnect();
throw ex;
}
}
public void cloneRepo() throws Exception {
cloneRepo(null);
}
public void cloneRepo(String branch) throws Exception {
CloneCommand cloneCommand = Git.cloneRepository().setURI(repositoryUrl).setDirectory(localRepo.getDirectory().getParentFile());
if (branch != null)
cloneCommand.setBranch(branch);
if (credentialsProvider != null)
cloneCommand.setCredentialsProvider(credentialsProvider);
cloneCommand.call();
}
/**
* Different from cloneRepo in that will clone only a single branch.
* For quicker clone of specific branch/tag.
*/
public void cloneBranch(String branch) throws Exception {
CloneCommand cloneCommand = Git.cloneRepository()
.setURI(repositoryUrl)
.setDirectory(localRepo.getDirectory().getParentFile())
.setBranchesToClone(Arrays.asList(branch))
.setCloneAllBranches(false)
.setBranch(branch);
if (credentialsProvider != null)
cloneCommand.setCredentialsProvider(credentialsProvider);
Git git = cloneCommand.call();
git.getRepository().close();
git.close();
}
public Status getStatus() throws Exception {
return getStatus(null);
}
public Status getStatus(String path) throws Exception {
fetch();
StatusCommand sc = git.status();
if (path != null)
sc.addPath(path);
return sc.call();
}
public void add(String filePattern) throws GitAPIException {
git.add().addFilepattern(filePattern).call();
}
public void add(List paths) throws Exception {
AddCommand add = git.add();
for (String path : paths)
add.addFilepattern(path);
add.call();
}
public void cloneNoCheckout() throws Exception {
cloneNoCheckout(false);
}
/**
* In lieu of sparse checkout since it's not yet supported in JGit:
* https://bugs.eclipse.org/bugs/show_bug.cgi?id=383772
*/
public void cloneNoCheckout(boolean withProgress) throws Exception {
CloneCommand clone = Git.cloneRepository().setURI(repositoryUrl).setDirectory(localRepo.getDirectory().getParentFile()).setNoCheckout(true);
// https://bugs.eclipse.org/bugs/show_bug.cgi?id=442029
if (credentialsProvider != null)
clone.setCredentialsProvider(credentialsProvider);
if (withProgress)
clone.setProgressMonitor(new TextProgressMonitor(new PrintWriter(System.out)));
clone.call();
}
public boolean localRepoExists() {
return localRepo.getDirectory() != null && localRepo.getDirectory().exists();
}
public boolean isTracked(String path) throws IOException {
ObjectId objectId = localRepo.resolve(Constants.HEAD);
RevTree tree;
RevWalk walk = null;
if (objectId != null) {
walk = new RevWalk(localRepo);
tree = walk.parseTree(objectId);
}
else {
tree = null;
}
try (TreeWalk treeWalk = new TreeWalk(localRepo)) {
treeWalk.setRecursive(true);
if (tree != null)
treeWalk.addTree(tree);
else
treeWalk.addTree(new EmptyTreeIterator());
treeWalk.addTree(new DirCacheIterator(localRepo.readDirCache()));
treeWalk.setFilter(PathFilterGroup.createFromStrings(Collections.singleton(path)));
return treeWalk.next();
}
finally {
if (walk != null)
walk.close();
}
}
public void pull(String branch) throws Exception {
PullCommand pull = git.pull().setRemote("origin").setRemoteBranchName(branch);
if (credentialsProvider != null)
pull.setCredentialsProvider(credentialsProvider);
pull.call();
}
public static String byteArrayToHexString(byte[] b) {
String result = "";
for (int i = 0; i < b.length; i++) {
result += Integer.toString((b[i] & 0xff) + 0x100, 16).substring(1);
}
return result;
}
/**
* @deprecated use {@link #getRelativePath(Path)}
* this has issues with relative git local paths like ../
*/
@Deprecated
public String getRelativePath(File file) {
String localPath = localDir.getAbsolutePath();
if (localPath.endsWith("\\.") || localPath.endsWith("/."))
localPath = localPath.substring(0, localPath.length() - 2);
return file.getAbsolutePath().substring(localPath.length() + 1).replace('\\', '/');
}
public String getRelativePath(Path path) {
return localDir.toPath().normalize().relativize(path.normalize()).toString().replace('\\', '/');
}
public GitDiffs getDiffsForTag(String tag, String path) throws Exception {
fetch();
ObjectId obj = localRepo.resolve("refs/tags/" + tag + "^{tree}");
if (obj == null)
throw new IOException("Unable to determine Git Diffs: path " + path + " not found on tag " + tag);
return getDiffs(obj, path);
}
public GitDiffs getDiffs(String branch, String path) throws Exception {
fetch();
ObjectId obj = localRepo.resolve("origin/" + branch + "^{tree}");
if (obj == null)
throw new IOException("Unable to determine Git Diffs: path " + path + " not found on branch " + branch);
return getDiffs(obj, path);
}
private GitDiffs getDiffs(ObjectId objectId, String path) throws Exception {
GitDiffs diffs = new GitDiffs();
CanonicalTreeParser newTreeIter = new CanonicalTreeParser();
newTreeIter.reset(localRepo.newObjectReader(), objectId);
DiffCommand dc = git.diff().setNewTree(newTreeIter);
if (path != null)
dc.setPathFilter(PathFilter.create(path));
dc.setShowNameAndStatusOnly(true);
for (DiffEntry diff : dc.call()) {
if (diff.getChangeType() == ChangeType.ADD || diff.getChangeType() == ChangeType.COPY) {
diffs.add(DiffType.MISSING, diff.getNewPath());
}
else if (diff.getChangeType() == ChangeType.MODIFY) {
diffs.add(DiffType.DIFFERENT, diff.getNewPath());
}
else if (diff.getChangeType() == ChangeType.DELETE) {
diffs.add(DiffType.EXTRA, diff.getOldPath());
}
else if (diff.getChangeType() == ChangeType.RENAME) {
diffs.add(DiffType.MISSING, diff.getNewPath());
diffs.add(DiffType.EXTRA, diff.getOldPath());
}
}
// we're purposely omitting folders
Status status = git.status().addPath(path).call();
for (String untracked : status.getUntracked()) {
if (!untracked.startsWith(path + "/Archive/"))
diffs.add(DiffType.EXTRA, untracked);
}
for (String added : status.getAdded()) {
diffs.add(DiffType.EXTRA, added);
}
for (String missing : status.getMissing()) {
diffs.add(DiffType.MISSING, missing);
}
for (String removed : status.getRemoved()) {
diffs.add(DiffType.MISSING, removed);
}
for (String changed : status.getChanged()) {
diffs.add(DiffType.DIFFERENT, changed);
}
for (String modified : status.getModified()) {
diffs.add(DiffType.DIFFERENT, modified);
}
for (String conflict : status.getConflicting()) {
diffs.add(DiffType.DIFFERENT, conflict);
}
return diffs;
}
/**
* Does not fetch.
*/
public CommitInfo getCommitInfo(String path) throws Exception {
Iterator revCommits = git.log().addPath(path).setMaxCount(1).call().iterator();
if (revCommits.hasNext()) {
RevCommit revCommit = revCommits.next();
CommitInfo commitInfo = new CommitInfo(revCommit.getId().name());
PersonIdent committerIdent = revCommit.getCommitterIdent();
commitInfo.setCommitter(committerIdent.getName());
commitInfo.setEmail(committerIdent.getEmailAddress());
if ((commitInfo.getCommitter() == null || commitInfo.getCommitter().isEmpty()) && commitInfo.getEmail() != null)
commitInfo.setCommitter(commitInfo.getEmail());
commitInfo.setDate(committerIdent.getWhen());
commitInfo.setMessage(revCommit.getShortMessage());
return commitInfo;
}
return null;
}
public ObjectStream getRemoteContentStream(String branch, String path) throws Exception {
ObjectId id = localRepo.resolve("refs/remotes/origin/" + branch);
try (ObjectReader reader = localRepo.newObjectReader();
RevWalk walk = new RevWalk(reader)) {
RevCommit commit = walk.parseCommit(id);
RevTree tree = commit.getTree();
TreeWalk treewalk = TreeWalk.forPath(reader, path, tree);
if (treewalk != null) {
return reader.open(treewalk.getObjectId(0)).openStream();
}
else {
return null;
}
}
}
public String getRemoteContentString(String branch, String path) throws Exception {
InputStream in = null;
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
in = getRemoteContentStream(branch, path);
if (in == null)
return null;
int read = 0;
byte[] bytes = new byte[1024];
while ((read = in.read(bytes)) != -1)
out.write(bytes, 0, read);
}
finally {
if (in != null)
in.close();
}
return out.toString();
}
public boolean exists() {
return new File(localDir + "/.git").isDirectory();
}
/**
* Execute an arbitrary git command.
*/
public void git(String... args) throws Exception {
Main.main(args);
}
public byte[] readFromCommit(String commitId, String path) throws Exception {
try (RevWalk revWalk = new RevWalk(localRepo)) {
RevCommit commit = revWalk.parseCommit(ObjectId.fromString(commitId));
// use commit's tree find the path
RevTree tree = commit.getTree();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (TreeWalk treeWalk = new TreeWalk(localRepo)) {
treeWalk.addTree(tree);
treeWalk.setRecursive(true);
treeWalk.setFilter(PathFilter.create(path));
if (!treeWalk.next()) {
return null;
}
ObjectId objectId = treeWalk.getObjectId(0);
ObjectLoader loader = localRepo.open(objectId);
loader.copyTo(baos);
}
revWalk.dispose();
return baos.toByteArray();
}
}
public byte[] readFromHead(String filePath) throws Exception {
try {
return readFromCommit(ObjectId.toString(localRepo.resolve(Constants.HEAD)), filePath);
}
catch (Exception e) { // MDW Studio throws MissingObjectExceptions after a pull, on Windows at least
reconnect();
return readFromCommit(ObjectId.toString(localRepo.resolve(Constants.HEAD)), filePath);
}
}
}