All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.eclipse.steady.python.pip.PipWrapper Maven / Gradle / Ivy

/**
 * This file is part of Eclipse Steady.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 * SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors
 */
package org.eclipse.steady.python.pip;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.logging.log4j.Logger;
import org.eclipse.steady.python.ProcessWrapper;
import org.eclipse.steady.python.ProcessWrapperException;
import org.eclipse.steady.python.utils.PythonConfiguration;
import org.eclipse.steady.shared.json.JacksonUtil;
import org.eclipse.steady.shared.util.FileUtil;
import org.eclipse.steady.shared.util.StopWatch;
import org.eclipse.steady.shared.util.StringList;
import org.eclipse.steady.shared.util.ThreadUtil;
import org.eclipse.steady.shared.util.VulasConfiguration;
import org.eclipse.steady.shared.util.StringList.CaseSensitivity;
import org.eclipse.steady.shared.util.StringList.ComparisonMode;

/**
 * 

PipWrapper class.

*/ public class PipWrapper { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(); static final Pattern DOWNLOAD_PATTERN = Pattern.compile("^\\s*Downloading\\s*(http\\S*).*$"); static final Pattern SAVED_PATTERN_1 = Pattern.compile("^\\s*File was already downloaded\\s*(.*)$"); static final Pattern SAVED_PATTERN_2 = Pattern.compile("^\\s*Saved\\s*(.*)$"); static final String PACK_PATTERN_REGEX = ".*-.*"; private Path pathToPip = null; private Path logDir = null; private StringList ignorePacks = new StringList(); /** * Assumes that the pip executable is part of the PATH environment variable. * * @throws org.eclipse.steady.python.ProcessWrapperException if any. */ public PipWrapper() throws ProcessWrapperException { this(Paths.get("pip"), null); } /** * Creates a new wrapper for the pip executable at the given path. * * @param _path_to_pip a {@link java.nio.file.Path} object. * @param _log_dir a {@link java.nio.file.Path} object. * @throws org.eclipse.steady.python.ProcessWrapperException if any. */ public PipWrapper(Path _path_to_pip, Path _log_dir) throws ProcessWrapperException { this.pathToPip = _path_to_pip; if (_log_dir != null) this.logDir = _log_dir; else { try { this.logDir = FileUtil.createTmpDir("vulas-pip-"); log.info("Created tmp directory [" + this.logDir + "]"); } catch (IOException e) { throw new ProcessWrapperException("Cannot create tmp directory: " + e.getMessage()); } } // Not to be added as dependencies final String[] ignore_packs = VulasConfiguration.getGlobal() .getStringArray(PythonConfiguration.PY_BOM_IGNORE_PACKS, new String[] {}); this.ignorePacks.addAll(ignore_packs, true); } private boolean ignorePackage(String _p) { return this.ignorePacks.contains(_p, ComparisonMode.EQUALS, CaseSensitivity.CASE_SENSITIVE); } /** *

isAvailable.

* * @return a boolean. */ public boolean isAvailable() { boolean exists = false; try { final Process p = new ProcessBuilder(this.pathToPip.toString()).start(); final int exit_code = p.waitFor(); exists = exit_code == 0; } catch (IOException ioe) { log.error("Error calling [pip]: " + ioe.getMessage(), ioe); } catch (InterruptedException ie) { log.error("Error calling [pip]: " + ie.getMessage(), ie); } return exists; } /** * Calls pip install on the given project {@link Path} and the returns a list of all installed packages (including the dependencies). * * @param _project a {@link java.nio.file.Path} object. * @return a {@link java.util.Set} object. */ public Set installPackages(Path _project) { Set packages = null; try { // Make download dir final Path download_dir = Paths.get(logDir.toString(), "pip-download"); FileUtil.createDirectory(download_dir); // Download all deps ProcessWrapper pw = new ProcessWrapper(); pw.setCommand( this.pathToPip, "download", "-d", download_dir.toString(), "--no-cache-dir", _project.toString()); pw.setPath(logDir); Thread t = new Thread(pw); t.start(); t.join(); final Path download_info = pw.getOutFile(); // Install all deps pw = new ProcessWrapper(); pw.setCommand(this.pathToPip, "install", _project.toString()); pw.setPath(logDir); t = new Thread(pw); t.start(); t.join(); // Get all deps packages = this.getListPackages(); // Enrich with download info this.searchDownloadInfo(packages, FileUtil.readFile(download_info)); } catch (ProcessWrapperException e) { log.error("Error calling installing packages: " + e.getMessage(), e); } catch (IOException e) { log.error("Error calling installing packages: " + e.getMessage(), e); } catch (InterruptedException e) { log.error("Error calling installing packages: " + e.getMessage(), e); } return packages; } /** * Calls pip freeze and pip show <package> in order to create and return all {@link PipInstalledPackage} of the Python environment. * * @return a {@link java.util.Set} object. * @throws org.eclipse.steady.python.ProcessWrapperException if any. * @throws java.io.IOException if any. * @throws java.lang.InterruptedException if any. */ public Set getFreezePackages() throws ProcessWrapperException, IOException, InterruptedException { final StopWatch sw = new StopWatch("pip freeze").start(); Set packages = null; ProcessWrapper pw = new ProcessWrapper().setCommand(this.pathToPip, "freeze").setPath(logDir); final Thread t = new Thread(pw, "pip"); t.start(); t.join(); if (pw.terminatedWithSuccess()) { // Create packages packages = this.parsePipFreezeOutput(pw.getOutFile()); // Log log.info("Found [" + packages.size() + "] pip packages:"); for (PipInstalledPackage pack : packages) log.info(" " + pack); // Call pip show and pip download in separate threads final ExecutorService pool = Executors.newFixedThreadPool(ThreadUtil.getNoThreads(2)); // newSingleThreadExecutor(); final Set> futures = new HashSet>(); for (PipInstalledPackage pack : packages) { futures.add(pool.submit(new PipShow(pack))); futures.add(pool.submit(new PipDownload(pack))); } pool.shutdown(); try { while (!pool.awaitTermination(10, TimeUnit.SECONDS)) { int done = 0; for (Future f : futures) { if (f.isDone()) done++; } sw.lap("[" + done + "/" + futures.size() + "] pip jobs are done"); } } catch (InterruptedException e) { PipWrapper.log.error("Interrupt exception"); } } sw.stop(); return packages; } /** * Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package. * @param _file * @return * @throws IOException */ private Set parsePipFreezeOutput(Path _file) throws IOException { final Set packs = new TreeSet(); // The pattern to search for final Pattern pattern = Pattern.compile("^(.*)==(.*)$"); // Read line by line final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile())); String line; while ((line = reader.readLine()) != null) { final Matcher m = pattern.matcher(line); if (m.matches()) { if (this.ignorePackage(m.group(1))) { log.warn("Package [" + m.group(1) + "] not added as installed package"); } else { packs.add(new PipInstalledPackage(m.group(1), m.group(2))); } } } reader.close(); return packs; } /** * Calls pip list and pip show <package> in order to create and return all {@link PipInstalledPackage}s of the Python environment. * * @return a {@link java.util.Set} object. * @throws org.eclipse.steady.python.ProcessWrapperException if any. * @throws java.io.IOException if any. * @throws java.lang.InterruptedException if any. */ public Set getListPackages() throws ProcessWrapperException, IOException, InterruptedException { final StopWatch sw = new StopWatch("pip list").start(); // Try legacy format ProcessWrapper pw = new ProcessWrapper() .setCommand(this.pathToPip, "list", "--format", "legacy") .setPath(logDir); Thread t = new Thread(pw, "pip"); t.start(); t.join(); Set packages = null; if (pw.terminatedWithSuccess()) { packages = this.parsePipListOutput(pw.getOutFile()); } // Try JSON format else { log.info("Legacy format did not work, trying JSON format..."); pw = new ProcessWrapper() .setCommand(this.pathToPip, "list", "--format", "json") .setPath(logDir); t = new Thread(pw, "pip"); t.start(); t.join(); if (pw.terminatedWithSuccess()) packages = this.deserializePipListOutput(pw.getOutFile()); } // Collect package details if (packages != null) { log.info("Found [" + packages.size() + "] pip packages:"); for (PipInstalledPackage pack : packages) log.info(" " + pack); // Call pip show in separate threads final ExecutorService pool = Executors.newFixedThreadPool(ThreadUtil.getNoThreads(2)); // newSingleThreadExecutor(); final Set> futures = new HashSet>(); for (PipInstalledPackage pack : packages) { futures.add(pool.submit(new PipShow(pack))); } pool.shutdown(); try { while (!pool.awaitTermination(10, TimeUnit.SECONDS)) { int done = 0; for (Future f : futures) { if (f.isDone()) done++; } sw.lap("[" + done + "/" + futures.size() + "] pip jobs are done"); } } catch (InterruptedException e) { PipWrapper.log.error("Interrupt exception"); } } else { log.warn("No pip packages found with pip list"); } sw.stop(); return packages; } /** * Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package. * @param _file * @return * @throws IOException */ private Set parsePipListOutput(Path _file) throws IOException { final Set packs = new TreeSet(); // The pattern to search for // HP, 17.04.2018: There seem to be versions of PIP with and without using brackets final Pattern pattern_brackets = Pattern.compile("^(.*)\\s+\\((.*)\\)$"); // final Pattern pattern = Pattern.compile("^(.*)\\s+(.*)$"); // Read line by line final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile())); String line; while ((line = reader.readLine()) != null) { final Matcher mb = pattern_brackets.matcher(line); if (mb.matches()) { if (this.ignorePackage(mb.group(1))) { log.warn("Package [" + mb.group(1) + "] not added as installed package"); } else { packs.add(new PipInstalledPackage(mb.group(1), mb.group(2))); } } } reader.close(); return packs; } /** * Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package. * @param _file * @return * @throws IOException */ private Set deserializePipListOutput(Path _file) throws IOException { // Deserialize final String json = FileUtil.readFile(_file); final PipPackageJson[] packs = (PipPackageJson[]) JacksonUtil.asObject(json, PipPackageJson[].class); // Create set final Set set = new HashSet(); for (PipPackageJson p : packs) { if (this.ignorePackage(p.getName())) { log.warn("Package [" + p.getName() + "] not added as installed package"); } else { set.add(new PipInstalledPackage(p.getName(), p.getVersion())); } } return set; } /** * Calls pip show in order to set the properties of the given {@link PipInstalledPackage}. * @param _p * @param _pool TODO */ final class PipShow implements Callable { private PipInstalledPackage pack = null; PipShow(PipInstalledPackage _pack) { this.pack = _pack; } public PipInstalledPackage call() throws ProcessWrapperException, IOException { final ProcessWrapper pw = new ProcessWrapper(this.pack.getName()); pw.setCommand(pathToPip, "show", this.pack.getName()).setPath(logDir); pw.run(); if (pw.terminatedWithSuccess()) { this.pack.addProperties(parsePipShowOutput(pw.getOutFile())); log.info("Added properties to " + this.pack); } else { log.info("Properties for " + this.pack + " are missing"); } return this.pack; } } private Map parsePipShowOutput(Path _file) throws IOException { final Map props = new HashMap(); // The pattern to search for final Pattern pattern = Pattern.compile("^([^:]*):(.*)$"); // Read line by line final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile())); String line; while ((line = reader.readLine()) != null) { final Matcher m = pattern.matcher(line); if (m.matches()) props.put(m.group(1).trim(), m.group(2).trim()); } reader.close(); return props; } /** * Calls pip download in order to add download path and url of the given {@link PipInstalledPackage}. * @param _p */ final class PipDownload implements Callable { private PipInstalledPackage pack = null; PipDownload(PipInstalledPackage _pack) { this.pack = _pack; } public PipInstalledPackage call() throws ProcessWrapperException, IOException { // Make download dir // final Path download_dir = Paths.get(logDir.toString(), "pip-download"); final Path download_dir = FileUtil.createTmpDir(this.pack.getName() + "-"); // Download all deps ProcessWrapper pw = new ProcessWrapper(); pw.setCommand( pathToPip, "download", "-d", download_dir.toString(), "--no-cache-dir", this.pack.getName() + "==" + this.pack.getVersion()); pw.setPath(logDir); pw.run(); final Path download_info = pw.getOutFile(); // Enrich with download info searchDownloadInfo(this.pack, FileUtil.readFile(download_info)); return this.pack; } } /** * Searches the provided output of 'pip download' for download URL and download path of the given packages. * @param _packs * @param _out standard out of pip download */ void searchDownloadInfo(Set _packs, String _out) throws IOException { if (_packs != null) for (PipInstalledPackage pack : _packs) this.searchDownloadInfo(pack, _out); } /** * Searches the provided output of 'pip download' for download URL and download path of the given package. * @param _p * @param _out standard out of pip download */ private void searchDownloadInfo(PipInstalledPackage _p, String _out) throws IOException { // Find all download URLs and files in pip output final Set urls = new HashSet(); final Set files = new HashSet(); final BufferedReader r = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(_out.getBytes()))); String line = null; while ((line = r.readLine()) != null) { final Matcher d = PipWrapper.DOWNLOAD_PATTERN.matcher(line); final Matcher s1 = PipWrapper.SAVED_PATTERN_1.matcher(line); final Matcher s2 = PipWrapper.SAVED_PATTERN_2.matcher(line); if (d.matches()) urls.add(d.group(1)); else if (s1.matches()) files.add(s1.group(1)); else if (s2.matches()) files.add(s2.group(1)); } // Regex to search for the file name String regex = PACK_PATTERN_REGEX.replace("", _p.getName().toLowerCase()); regex = regex.replace("", _p.getVersion().toLowerCase()); regex = regex.replace( "-", "[_-]{1,1}"); // For some reason, the file names of some packages use _ instead of - final Pattern p = Pattern.compile(regex); for (String url : urls) { final Matcher m = p.matcher(url.toLowerCase()); if (m.matches()) { _p.setDownloadUrl(url); break; } } for (String file : files) { final Matcher m = p.matcher(file.toLowerCase()); if (m.matches()) { _p.setDownloadPath(Paths.get(file)); break; } } // Warn if no info could be found if (_p.getDownloadPath() == null && _p.getDownloadUrl() == null) log.warn("Download path and URL for " + _p + " missing"); else if (_p.getDownloadPath() == null) log.warn("Download path for " + _p + " missing"); else if (_p.getDownloadUrl() == null) log.warn("Download URL for " + _p + " missing"); else log.info("Found download info for " + _p); } /** * Helper class for deserializing the output of pip list --format json. */ static class PipPackageJson { String name; String version; String installer; String location; public String getName() { return name; } public void setName(String name) { this.name = name; } public String getVersion() { return version; } public void setVersion(String version) { this.version = version; } public String getInstaller() { return installer; } public void setInstaller(String installer) { this.installer = installer; } public String getLocation() { return location; } public void setLocation(String location) { this.location = location; } } }




© 2015 - 2025 Weber Informatics LLC | Privacy Policy