org.metaeffekt.artifact.resolver.pypi.PyPICliAdapter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of ae-artifact-resolver Show documentation
There is a newer version: 0.134.0
/*
 * Copyright 2021-2024 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.metaeffekt.artifact.resolver.pypi;

import org.metaeffekt.artifact.resolver.ResolverResult;
import org.metaeffekt.artifact.resolver.generic.AbstractCliAdapter;
import org.metaeffekt.artifact.resolver.generic.FileLocation;
import org.metaeffekt.artifact.resolver.model.DownloadLocation;
import org.metaeffekt.core.util.ExecUtils;
import org.metaeffekt.core.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class PyPICliAdapter extends AbstractCliAdapter {

    private static final Logger LOG = LoggerFactory.getLogger(PyPICliAdapter.class);

    public PyPICliAdapter(DownloadLocation downloadLocation) {
        super(downloadLocation);
    }

    public ResolverResult resolveSourceArtifact(PyPIArtifactReference artifactReference) {

        final List commandParts = new ArrayList<>();
        commandParts.add("python3");
        commandParts.add("-m");
        commandParts.add("pip");
        commandParts.add("download");
        commandParts.add("--no-deps");
        commandParts.add("--no-dependencies");
        commandParts.add("--no-binary");
        commandParts.add(":all:");
        commandParts.add(artifactReference.derivePipRequirementSpec());

        final String modifier = "source";

        return resolveArtifactUsingCommand(artifactReference, commandParts, modifier);
    }

    public ResolverResult resolveBinaryArtifact(PyPIArtifactReference artifactReference) {
        final List commandParts = new ArrayList<>();
        commandParts.add("python3");
        commandParts.add("-m");
        commandParts.add("pip");
        commandParts.add("download");
        commandParts.add("--no-deps");
        commandParts.add("--only-binary");
        commandParts.add(":all:");
        commandParts.add(artifactReference.derivePipRequirementSpec());


        final String partModifier = "binary";

        return resolveArtifactUsingCommand(artifactReference, commandParts, partModifier);
    }

    private ResolverResult resolveArtifactUsingCommand(PyPIArtifactReference artifactReference, List commandParts, String partModifier) {
        final FileLocation fileLocation = artifactReference.deriveFileLocation();
        final File targetFolder = getDownloadLocation().deriveDownloadFolder(fileLocation.getEcosystem(), fileLocation.getFilename());

        final String artifactQualifier = artifactReference.derivePipRequirementSpec();

        // PROBLEM: we do not know the filename of what is being downloaded. Therefore, we always use marker files
        // to indicate a file was successfully downloaded; or is not available.
        final String markerFileName = artifactReference.deriveMarkerFileName(partModifier);

        final File markerBaseDir = new File(targetFolder, "markers");
        final File markerFile = new File(markerBaseDir, markerFileName);

        boolean reattempt = false;
        if (markerFile.exists() && markerFile.length() > 0) {
            reattempt = true;
            try {
                // the resulting file path is included in the marker
                final String downloadedFilePath = FileUtils.readFileToString(markerFile, FileUtils.ENCODING_UTF_8);
                final File downloadedFile = new File(downloadedFilePath);
                if (downloadedFile.exists()) {
                    reattempt = false;
                    LOG.info("Skipping download for ref [{}]: file [{}] exists.", artifactQualifier, downloadedFile);
                    return resolve(downloadedFile);
                }
            } catch (IOException e) {
                LOG.info("Detected incomplete download. Reattempting [{}].", artifactQualifier);
            }
        }

        // do not reattempt download as long as a recent (less than a day old) marker file exists
        if (!reattempt) {
            if (hasRecentFailedDownloadAttempt(markerFile)) {
                LOG.info("Skipping download [{}]. Recent attempt failed.", artifactQualifier);
                return null;
            }
        }

        final File downloadFolder = new File(markerBaseDir, "tmp-" + partModifier);

        String errorMessage = null;

        if (!downloadFolder.exists()) {
            try {
                FileUtils.forceMkdir(downloadFolder);
            } catch (IOException e) {
                errorMessage = "Cannot create intermediate download folder: " + downloadFolder;
            }
        }

        if (downloadFolder.exists()) {
            try {
                ExecUtils.ExecParam execParam = new ExecUtils.ExecParam((commandParts));
                execParam.retainErrorOutputs();
                execParam.setWorkingDir(downloadFolder);

                ExecUtils.executeAndThrowIOExceptionOnFailure(execParam);
                final File downloadedFile = FileUtils.findSingleFile(downloadFolder, "*.*");

                if (downloadedFile != null) {
                    final File targetFile = new File(targetFolder, downloadedFile.getName());

                    // delete file in final location if it already exists
                    if (targetFile.exists()) {
                        FileUtils.deleteQuietly(targetFile);
                    }

                    // move to final location
                    FileUtils.moveFile(downloadedFile, targetFile);

                    // create marker with path as content
                    FileUtils.write(markerFile, targetFile.getCanonicalFile().getAbsolutePath(), FileUtils.ENCODING_UTF_8);

                    // return the target file
                    return resolve(targetFile);
                } else {
                    errorMessage = String.format("Downloaded file for %s not detected in folder %s", artifactQualifier, downloadFolder);
                }
            } catch (IOException e) {
                errorMessage = e.getMessage();
            } finally {
                FileUtils.deleteDirectoryQuietly(downloadFolder);
            }

        }

        // report error to the console
        LOG.error("Cannot access PyPI package {}. {}", artifactReference.derivePipRequirementSpec(), errorMessage);

        // touch marker file to prevent further attempts (until expiry)
        try {
            FileUtils.touch(markerFile);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        return null;
    }

}