org.jfrog.build.util.DependenciesDownloaderHelper Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of build-info-client Show documentation
There is a newer version: 2.41.24
package org.jfrog.build.util;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.jfrog.build.api.Dependency;
import org.jfrog.build.api.builder.DependencyBuilder;
import org.jfrog.build.api.dependency.DownloadableArtifact;
import org.jfrog.build.api.dependency.pattern.PatternType;
import org.jfrog.build.api.util.Log;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Helper class for downloading dependencies
 *
 * @author Shay Yaakov
 */
public class DependenciesDownloaderHelper {

    private DependenciesDownloader downloader;
    private Log log;

    public DependenciesDownloaderHelper(DependenciesDownloader downloader, Log log) {
        this.downloader = downloader;
        this.log = log;
    }

    public List downloadDependencies(Set downloadableArtifacts) throws IOException {
        List dependencies = Lists.newArrayList();
        Set downloadedArtifacts = Sets.newHashSet();
        for (DownloadableArtifact downloadableArtifact : downloadableArtifacts) {
            Dependency dependency = downloadArtifact(downloadableArtifact);
            if (dependency != null) {
                dependencies.add(dependency);
                downloadedArtifacts.add(downloadableArtifact);
            }
        }

        removeUnusedArtifactsFromLocal(downloadedArtifacts);
        return dependencies;
    }

    private void removeUnusedArtifactsFromLocal(Set downloadableArtifacts) throws IOException {
        Set forDeletionFiles = Sets.newHashSet();
        Set allResolvesFiles = Sets.newHashSet();
        for (DownloadableArtifact downloadableArtifact : downloadableArtifacts) {
            String fileDestination = downloader.getTargetDir(downloadableArtifact.getTargetDirPath(),
                    downloadableArtifact.getRelativeDirPath());
            allResolvesFiles.add(fileDestination);
            if (PatternType.DELETE.equals(downloadableArtifact.getPatternType())) {
                forDeletionFiles.add(fileDestination);
            }
        }

        downloader.removeUnusedArtifactsFromLocal(allResolvesFiles, forDeletionFiles);
    }


    private Dependency downloadArtifact(DownloadableArtifact downloadableArtifact) throws IOException {
        Dependency dependencyResult = null;
        String filePath = downloadableArtifact.getFilePath();
        String matrixParams = downloadableArtifact.getMatrixParameters();
        final String uri = downloadableArtifact.getRepoUrl() + '/' + filePath;
        final String uriWithParams = (StringUtils.isBlank(matrixParams) ? uri : uri + ';' + matrixParams);
        String fileDestination = downloader.getTargetDir(downloadableArtifact.getTargetDirPath(),
                downloadableArtifact.getRelativeDirPath());

        try {
            dependencyResult = getDependencyLocally(uriWithParams, fileDestination);
            if (dependencyResult == null) {
                log.info("Downloading '" + uriWithParams + "' ...");
                HttpResponse httpResponse = downloader.getClient().downloadArtifact(uriWithParams);
                InputStream inputStream = httpResponse.getEntity().getContent();
                Map checksumsMap = downloader.saveDownloadedFile(inputStream, fileDestination);

                // If the checksums map is null then something went wrong and we should fail the build
                if (checksumsMap == null) {
                    throw new IOException("Received null checksums map");
                }

                String md5 = validateMd5Checksum(httpResponse, checksumsMap.get("md5"));
                String sha1 = validateSha1Checksum(httpResponse, checksumsMap.get("sha1"));

                log.info("Successfully downloaded '" + uriWithParams + "' to '" + fileDestination + "'");
                dependencyResult = new DependencyBuilder().id(filePath).md5(md5).sha1(sha1).build();
            }
        } catch (FileNotFoundException e) {
            if (StringUtils.isNotBlank(matrixParams)) {
                String skippedMessage = "Skipping download of '" + uriWithParams + "' due to matrix params mismatch.";
                log.debug(skippedMessage);
            } else {
                String warningMessage = "Error occurred while resolving published dependency: " + e.getMessage();
                log.warn(warningMessage);
            }
        }

        return dependencyResult;
    }

    /**
     * Perform HEAD request to get the artifact checksums and check if the local file is the same.
     *
     * @param uriWithParams The uri for the artifact to perform HEAD request to
     * @param filePath      The locally file path
     */
    private Dependency getDependencyLocally(String uriWithParams, String filePath) throws IOException {
        Dependency dependencyResult = null;
        HttpResponse artifactChecksums = downloader.getClient().getArtifactChecksums(uriWithParams);
        String md5 = getMD5ChecksumFromResponse(artifactChecksums);
        String sha1 = getSHA1ChecksumFromResponse(artifactChecksums);

        if (downloader.isFileExistsLocally(filePath, md5, sha1)) {
            log.debug("File '" + filePath + "' already exists locally, skipping remote download.");
            dependencyResult = new DependencyBuilder().id(filePath).md5(md5).sha1(sha1).build();
        }

        return dependencyResult;
    }

    private String validateMd5Checksum(HttpResponse httpResponse, String calculatedMd5) throws IOException {
        String md5ChecksumFromResponse = getMD5ChecksumFromResponse(httpResponse);
        if (!StringUtils.equals(getMD5ChecksumFromResponse(httpResponse), calculatedMd5)) {
            String errorMessage = "Calculated MD5 checksum is different from original, "
                    + "Original: '" + md5ChecksumFromResponse + "' Calculated: '" + calculatedMd5 + "'";
            throw new IOException(errorMessage);
        }
        return md5ChecksumFromResponse == null ? "" : md5ChecksumFromResponse;
    }

    private String validateSha1Checksum(HttpResponse httpResponse, String calculatedSha1) throws IOException {
        String sha1ChecksumFromResponse = getSHA1ChecksumFromResponse(httpResponse);
        if (!StringUtils.equals(sha1ChecksumFromResponse, calculatedSha1)) {
            String errorMessage = "Calculated SHA-1 checksum is different from original, "
                    + "Original: '" + sha1ChecksumFromResponse + "' Calculated: '" + calculatedSha1 + "'";
            throw new IOException(errorMessage);
        }
        return sha1ChecksumFromResponse == null ? "" : sha1ChecksumFromResponse;
    }

    private String getSHA1ChecksumFromResponse(HttpResponse artifactChecksums) {
        String sha1 = null;
        Header sha1Header = artifactChecksums.getFirstHeader("X-Checksum-Sha1");
        if (sha1Header != null) {
            sha1 = sha1Header.getValue();
        }
        return sha1;
    }

    private String getMD5ChecksumFromResponse(HttpResponse artifactChecksums) {
        String md5 = null;
        Header md5Header = artifactChecksums.getFirstHeader("X-Checksum-Md5");
        if (md5Header != null) {
            md5 = md5Header.getValue();
        }
        return md5;
    }
}