io.resys.hdes.client.spi.git.GitFiles Maven / Gradle / Ivy

package io.resys.hdes.client.spi.git;

/*-
 * #%L
 * hdes-client-api
 * %%
 * Copyright (C) 2020 - 2021 Copyright 2020 ReSys OÜ
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import io.resys.hdes.client.api.HdesStore.CreateStoreEntity;
import io.resys.hdes.client.api.ast.AstBody.AstBodyType;
import io.resys.hdes.client.api.ast.AstCommand;
import io.resys.hdes.client.api.ast.AstCommand.AstCommandValue;
import io.resys.hdes.client.spi.GitConfig;
import io.resys.hdes.client.spi.GitConfig.GitEntry;
import io.resys.hdes.client.spi.GitConfig.GitFile;
import io.resys.hdes.client.spi.GitConfig.GitFileReload;
import io.resys.hdes.client.spi.ImmutableGitEntry;
import io.resys.hdes.client.spi.ImmutableGitFile;
import io.resys.hdes.client.spi.ImmutableGitFileReload;
import io.resys.hdes.client.spi.staticresources.Sha2;
import io.resys.hdes.client.spi.util.HdesAssert;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.ResetCommand.ResetType;
import org.eclipse.jgit.api.errors.CheckoutConflictException;
import org.eclipse.jgit.api.errors.EmptyCommitException;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.DiffEntry;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevSort;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
import org.eclipse.jgit.treewalk.filter.TreeFilter;
import org.ehcache.Cache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;

public class GitFiles {
  private static final Logger LOGGER = LoggerFactory.getLogger(GitFiles.class);
  private static final String SEPARATOR = "/";
  private final GitConfig conn;
  
  public GitFiles(GitConfig connection) {
    super();
    this.conn = connection;
  }
  
  public List push(List gitFiles) {
    final var git = conn.getClient();
    final var repo = git.getRepository();    
    final var callback = conn.getCallback();
    final var creds = conn.getCreds().get();
    try {
      final var start = repo.resolve(Constants.HEAD);
      
      // pull
      git.pull().setTransportConfigCallback(callback).call().getFetchResult();
      
      // add new files
      var add = git.add();
      for(final var gitFile : gitFiles) {
        add = add.addFilepattern(gitFile.getTreeValue());
      }
      add.call();
      
      // commit changes
      git.commit()
      .setAll(true)
      .setAllowEmpty(false)
      .setMessage("Batch import")
      .setAuthor(creds.getUser(), creds.getEmail())
      .setCommitter(creds.getUser(), creds.getEmail())
      .call();

      // push
      git.push().setTransportConfigCallback(callback).call();
      final var end = repo.resolve(Constants.HEAD);
      
      LOGGER.debug("Pushing changes...");
      return diff(start, end);
      
    } catch(CheckoutConflictException e) {
      LOGGER.error("Conflict, resetting... " +  e.getMessage(), e);
      try {
        git.reset().setMode(ResetType.HARD).call();
        git.pull().setTransportConfigCallback(callback).call();
      } catch(Exception ex) {
        LOGGER.error(e.getMessage(), e);
        throw new RuntimeException(e.getMessage(), e);
      }
      throw new RuntimeException(e.getMessage(), e);
    } catch(EmptyCommitException e) {
      LOGGER.debug("nothing to commit");
    } catch(Exception e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e.getMessage(), e);
    }
    
    return Collections.emptyList();
  }
  
  public List push(GitFile gitFile) {
    final var git = conn.getClient();
    final var repo = git.getRepository();    
    final var callback = conn.getCallback();
    final var creds = conn.getCreds().get();
    try {
      final var start = repo.resolve(Constants.HEAD);
      
      // pull
      git.pull().setTransportConfigCallback(callback).call().getFetchResult();
      
      // add new files
      git.add().addFilepattern(gitFile.getTreeValue()).call();

      // commit changes
      git.commit()
      .setAll(true)
      .setAllowEmpty(false)
      .setMessage("Changes to: " + gitFile.getBodyType() + " file: " + gitFile.getTreeValue())
      .setAuthor(creds.getUser(), creds.getEmail())
      .setCommitter(creds.getUser(), creds.getEmail())
      .call();

      // push
      git.push().setTransportConfigCallback(callback).call();
      final var end = repo.resolve(Constants.HEAD);
      
      LOGGER.debug("Pushing changes...");
      return diff(start, end);
      
    } catch(CheckoutConflictException e) {
      LOGGER.error("Conflict, resetting... " +  e.getMessage(), e);
      try {
        git.reset().setMode(ResetType.HARD).call();
        git.pull().setTransportConfigCallback(callback).call();
      } catch(Exception ex) {
        LOGGER.error(e.getMessage(), e);
        throw new RuntimeException(e.getMessage(), e);
      }
      throw new RuntimeException(e.getMessage(), e);
    } catch(EmptyCommitException e) {
      LOGGER.debug("nothing to commit");
    } catch(Exception e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e.getMessage(), e);
    }
    
    return Collections.emptyList();
  }
  
  public GitEntry readEntry(GitFile entry) {
    final var git = conn.getClient();
    final var repo = git.getRepository();
    
    try {
      final var start = repo.resolve(Constants.HEAD);
      return readEntry(entry, start);
    } catch (IOException e) {
      throw new RuntimeException("Failed to load timestamps for: " + entry.getTreeValue() + "!" + e.getMessage(), e);
    }
  }
  
  public GitEntry readEntry(GitFile entry, ObjectId start) {

    final var git = conn.getClient();
    final var repo = git.getRepository();
    try(final var revWalk = new RevWalk(repo)) {
      
      final TreeFilter treeFilter = AndTreeFilter.create(
          PathFilterGroup.createFromStrings(entry.getTreeValue()), 
          TreeFilter.ANY_DIFF);

      String revision = start.getName();
      Timestamp created = Timestamp.valueOf(LocalDateTime.MIN);
      Timestamp modified = Timestamp.valueOf(LocalDateTime.MIN);
      try {
        final var commit = revWalk.parseCommit(start);
        
        revWalk.reset();
        revWalk.markStart(commit);
        revWalk.setTreeFilter(treeFilter);
        revWalk.sort(RevSort.COMMIT_TIME_DESC);
        final var modTree = revWalk.next();
        modified = (modTree != null ? new Timestamp(modTree.getCommitTime() * 1000L) : new Timestamp(System.currentTimeMillis()));
        revision = modTree != null ? modTree.getName() : start.getName();
        
        revWalk.reset();
        revWalk.markStart(commit);
        revWalk.setTreeFilter(treeFilter);
        revWalk.sort(RevSort.COMMIT_TIME_DESC);
        revWalk.sort(RevSort.REVERSE, true);
        created = new Timestamp(System.currentTimeMillis());

      } catch(Exception e) {
        LOGGER.error(
            "Failed to create asset from file: '" + entry.getTreeValue() + "'" + System.lineSeparator() +
            "because of: " + e.getMessage() + System.lineSeparator() +
            "with content: " + entry.getBlobValue() 
            , e);
      }
      
      
      final var commands = conn.getSerializer().read(entry.getBlobValue());      
      final var result = ImmutableGitEntry.builder()
          .id(entry.getId())
          .revision(revision)
          .bodyType(entry.getBodyType())
          .treeValue(entry.getTreeValue())
          .blobValue(entry.getBlobValue())
          .created(created)
          .modified(modified)
          .blobHash(Sha2.blob(entry.getBlobValue()))
          .commands(commands)
          .build();

      if(LOGGER.isDebugEnabled()) {
        final var msg = new StringBuilder()
            .append("Loading path: ").append(result.getTreeValue()).append(System.lineSeparator())
            .append("  - blob murmur3_128: ").append(result.getBlobHash()).append(System.lineSeparator())
            .append("  - body type: ").append(result.getBodyType()).append(System.lineSeparator())
            .append("  - created: ").append(result.getCreated()).append(System.lineSeparator())
            .append("  - modified: ").append(result.getModified()).append(System.lineSeparator())
            .append("  - revision: ").append(result.getRevision()).append(System.lineSeparator());
        LOGGER.debug(msg.toString());
      }
      
      return result;
      
    } catch (Exception e) {
      throw new RuntimeException("Failed to load timestamps for: " + entry.getTreeValue() + "!" + System.lineSeparator() + e.getMessage(), e);
    }
  }
  
  public Map.Entry> tag(CreateStoreEntity entity) {
    final var git = conn.getClient();
    final var callback = conn.getCallback();
    final var creds = conn.getCreds().get();
    final var repo = git.getRepository();
    
    try {
      final var start = repo.resolve(Constants.HEAD);
      git.pull().setTransportConfigCallback(callback).call().getFetchResult();
      final var name = entity.getBody().stream().filter(e -> e.getType() == AstCommandValue.SET_TAG_NAME).findFirst().get().getValue();
      
      Ref ref = git.tag().setName(name).setAnnotated(true)
          .setMessage("tag created")
          .setTagger(new PersonIdent(creds.getUser(), creds.getEmail()))
          .call();
      git.push().setPushTags().setTransportConfigCallback(callback).call();
      final var end = repo.resolve(Constants.HEAD);
      
      final var result = new ArrayList<>(diff(start, end));
      result.add(ImmutableGitFileReload.builder().treeValue(ref.getName()).id(name).bodyType(AstBodyType.TAG).build());
      return Map.entry(ref.getObjectId().getName(), result);
    } catch(Exception e) {
      LOGGER.error(e.getMessage(), e);
      throw new RuntimeException(e.getMessage(), e);
    }
  }
  
  private List diff(ObjectId oldHead, ObjectId newHead) {
    final var git = conn.getClient();
    final var repo = git.getRepository();
    
    try (final var reader = repo.newObjectReader()) {
      final var result = new ArrayList();
      
      final var oldTreeIter = new CanonicalTreeParser();
      oldTreeIter.reset(reader, repo.parseCommit(oldHead).getTree());
      
      final var newTreeIter = new CanonicalTreeParser();
      newTreeIter.reset(reader, repo.parseCommit(newHead).getTree());

      // finally get the list of changed files
      final List diffs = git.diff().setNewTree(newTreeIter).setOldTree(oldTreeIter).call();
      for (final DiffEntry entry : diffs) {
        // example: src/main/resources/assets/flow/2d8958a2-44eb-4020-9c17-9bb83daa7434.json

        final var oldId = getId(entry.getOldPath());
        final var id = getId(entry.getNewPath());
        
        if(id.isEmpty()) {
          final var bodyType = getBodyType(entry.getOldPath());
          result.add(ImmutableGitFileReload.builder()
              .id(oldId.get())
              .treeValue(entry.getOldPath())
              .bodyType(bodyType)
              .build());  
        } else {
          final var content = getContent(entry.getNewPath());
          final var bodyType = getBodyType(entry.getNewPath());
          final var treeValue = entry.getNewPath();
          final var gitFile = ImmutableGitFile.builder()
            .id(id.get())
            .treeValue(treeValue)
            .blobValue(content)
            .bodyType(bodyType)
            .blobHash(Sha2.blob(content))
            .build();
          result.add(ImmutableGitFileReload.builder()
              .id(id.get())
              .treeValue(treeValue)
              .file(gitFile)
              .bodyType(bodyType)
              .build());
          
        }        
      }
      return result;
    } catch(Exception e) {
      throw new RuntimeException(e.getMessage(), e);
    }    
  }
  
  private Optional getId(String path) {
    if(path.indexOf(".json") < 0) {
      return Optional.empty();
    }

    final var paths = path.split(SEPARATOR);
    final var fileName = paths[paths.length -1];
    final var id = fileName.substring(0, fileName.indexOf("."));
    
    return Optional.of(id);
  }
  
  private AstBodyType getBodyType(String path) {
    if(path.contains(SEPARATOR + "dt" + SEPARATOR)) {
      return AstBodyType.DT;
    } else if(path.contains(SEPARATOR + "flow" + SEPARATOR)) {
      return AstBodyType.FLOW;
    } else if(path.contains(SEPARATOR + "flowtask" + SEPARATOR)) {
      return AstBodyType.FLOW_TASK;
    } else if(path.contains(SEPARATOR + "tag" + SEPARATOR)) {
      return AstBodyType.TAG;
    } else if(path.contains(SEPARATOR + "branch" + SEPARATOR)) {
      return AstBodyType.BRANCH;
    }
    
    throw new RuntimeException("Failed to load asset body type: " + path + "!");
  }
  
  private String getContent(String path) {
    try {
      return IOUtils.toString(new FileInputStream(conn.getAbsolutePath() + SEPARATOR + path), StandardCharsets.UTF_8);
    } catch (IOException e) {
      throw new RuntimeException("Failed to load asset content from: " + path + "!" + e.getMessage(), e);
    }
  }
  
  
  public GitFile create(AstBodyType bodyType, List body) throws IOException {
    final var location = conn.getLocation();
    final var id = UUID.randomUUID().toString();
    final var resourceName = location.getAbsolutePath(bodyType, id);
    final var assetName = resourceName.startsWith("file:") ? resourceName.substring(5) : resourceName;
    final var outputFile = new File(assetName);
    
    if(outputFile.exists()) {
      throw new RuntimeException("Can't create asset: '" + assetName + "' because it's already created!");
    } else {
      outputFile.getParentFile().mkdirs();
      boolean created = outputFile.createNewFile();
      HdesAssert.isTrue(created, () -> "Failed to create new file: " + assetName);
    }
    
    if(LOGGER.isDebugEnabled()) {
      LOGGER.debug("Created new file: " + outputFile.getCanonicalPath());
    }
    
    final var blob = conn.getSerializer().write(body);
    final var fileOutputStream = new FileOutputStream(outputFile);
    try {
      IOUtils.copy(new ByteArrayInputStream(blob.getBytes(StandardCharsets.UTF_8)), fileOutputStream);
    } catch(Exception e) {
      throw new RuntimeException(e.getMessage(), e);
    } finally {
      fileOutputStream.close();          
    }
    
    return ImmutableGitFile.builder()
        .id(id)
        .treeValue(location.resolveTreeValue(conn.getAssetsPath(), bodyType, id))
        .blobValue(blob)
        .blobHash(Sha2.blob(blob))
        .bodyType(bodyType)
        .build();
  }
  
  public GitFile update(String id, AstBodyType bodyType, List body) throws IOException {
    final var location = conn.getLocation();
    final var resourceName = location.getAbsolutePath(bodyType, id);
    final var assetName = resourceName.startsWith("file:") ? resourceName.substring(5) : resourceName;
    final var outputFile = new File(assetName);
    
    if(!outputFile.exists()) {
      throw new RuntimeException("Can't update asset: '" + assetName + "' because it does not exist!"); 
    }
     
    final var blob = conn.getSerializer().write(body);
    final var fileOutputStream = new FileOutputStream(outputFile);
    try {
      IOUtils.copy(new ByteArrayInputStream(blob.getBytes(StandardCharsets.UTF_8)), fileOutputStream);
    } catch(Exception e) {
      throw new RuntimeException(e.getMessage(), e);
    } finally {
      fileOutputStream.close();          
    }
    
    return ImmutableGitFile.builder()
        .id(id)
        .treeValue(location.resolveTreeValue(conn.getAssetsPath(), bodyType, id))
        .blobValue(blob)
        .bodyType(bodyType)
        .blobHash(Sha2.blob(blob))
        .build();
  }

  private String getCommitMessage(List gitFiles) {
    final var branchFile = gitFiles.stream().filter(f -> f.getBodyType().equals(AstBodyType.BRANCH)).findFirst();
    if (gitFiles.size() == 1 && branchFile.isEmpty()) {
      return "Delete: " + gitFiles.get(0).getBodyType() + " file: " + gitFiles.get(0).getTreeValue();
    } else {
      final var branchName = branchFile.get().getCommands().stream().
          filter(c -> c.getType().equals(AstCommand.AstCommandValue.SET_BRANCH_NAME))
          .collect(Collectors.toList())
          .get(0).getValue();
      return "Delete: BRANCH " + branchName + " and its assets";
    }
  }

  public List delete(List gitFiles) throws IOException {
    final Cache cache = conn.getCacheManager().getCache(conn.getCacheName(), String.class, GitEntry.class);
    final var git = conn.getClient();
    final var callback = conn.getCallback();
    final var creds = conn.getCreds().get();
    final var repo = git.getRepository();
    final var start = repo.resolve(Constants.HEAD);
    final var result = new ArrayList();

    try {
      for (final var gitFile : gitFiles) {
        if (gitFile.getBodyType() == AstBodyType.TAG) {
          final String filter = "refs/tags/" + gitFile.getCommands().stream()
              .filter(c -> c.getType() == AstCommandValue.SET_TAG_NAME)
              .findFirst().get().getValue();
          final Optional target = git.tagList().call().stream()
              .filter(ref -> ref.getName().equals(filter))
              .findFirst();

          if (target.isPresent()) {

            git.tagDelete().setTags(target.get().getName()).call();

            //delete branch 'branchToDelete' on remote 'origin'
            RefSpec refSpec = new RefSpec()
                .setSource(null)
                .setDestination(target.get().getName());
            git.push().setRefSpecs(refSpec).setRemote("origin").setTransportConfigCallback(conn.getCallback()).call();
            result.add(
                ImmutableGitFileReload.builder()
                    .id(gitFile.getId())
                    .bodyType(AstBodyType.TAG)
                    .treeValue(gitFile.getTreeValue()).build());
          } else {
            final String msg = "Can't find tag: " + filter + "!";
            LOGGER.error(msg);
          }
        }

        // pull
        git.pull().setTransportConfigCallback(callback).call().getFetchResult();

        final var treeValue = gitFile.getTreeValue().startsWith("/") ? gitFile.getTreeValue() : "/" + gitFile.getTreeValue();
        final var absolutePath = conn.getAbsolutePath().startsWith("/") ? conn.getAbsolutePath() : "/" + conn.getAbsolutePath();
        final var fullPath = absolutePath + treeValue;

        LOGGER.debug("Removing assets from git: " + fullPath + "");
        final var file = new File(URI.create("file:" + fullPath));

        boolean deleted = file.delete();
        if (!deleted) {
          throw new RuntimeException("Cant delete assets from git: " + fullPath + "");
        }

        // add new files
        git.add().addFilepattern(gitFile.getTreeValue()).call();
      }

      // commit changes
      git.commit()
          .setAll(true)
          .setAllowEmpty(false)
          .setMessage(getCommitMessage(gitFiles))
          .setAuthor(creds.getUser(), creds.getEmail())
          .setCommitter(creds.getUser(), creds.getEmail())
          .call();

      // push
      git.push().setTransportConfigCallback(callback).call();

    } catch (Exception e) {
      LOGGER.error("Failed to delete assets: '" + gitFiles.stream().map(GitEntry::getId).reduce((a, b) -> a + "," + b).get() + "'!" + System.lineSeparator() + e.getMessage(), e);
      try {
        git.reset().setMode(ResetType.HARD).call();
      } catch (GitAPIException e1) {
        throw new RuntimeException(e.getMessage(), e);
      }
    }

    final var end = repo.resolve(Constants.HEAD);
    result.addAll(diff(start, end));

    return result;
  }
  
  public static Builder builder() {
    return new Builder();
  }
  
  public static class Builder {
    private GitConfig conn;
    
    public Builder git(GitConfig conn) {
      this.conn = conn;
      return this;
    }
    
    public GitFiles build() {
      HdesAssert.notNull(conn, () -> "git connection must be defined!");
      return new GitFiles(conn);
    }
  }
}