org.eclipse.steady.python.pip.PipWrapper Maven / Gradle / Ivy
/**
* This file is part of Eclipse Steady.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* SPDX-FileCopyrightText: Copyright (c) 2018-2020 SAP SE or an SAP affiliate company and Eclipse Steady contributors
*/
package org.eclipse.steady.python.pip;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.logging.log4j.Logger;
import org.eclipse.steady.python.ProcessWrapper;
import org.eclipse.steady.python.ProcessWrapperException;
import org.eclipse.steady.python.utils.PythonConfiguration;
import org.eclipse.steady.shared.json.JacksonUtil;
import org.eclipse.steady.shared.util.FileUtil;
import org.eclipse.steady.shared.util.StopWatch;
import org.eclipse.steady.shared.util.StringList;
import org.eclipse.steady.shared.util.ThreadUtil;
import org.eclipse.steady.shared.util.VulasConfiguration;
import org.eclipse.steady.shared.util.StringList.CaseSensitivity;
import org.eclipse.steady.shared.util.StringList.ComparisonMode;
/**
* PipWrapper class.
*/
public class PipWrapper {
private static final Logger log = org.apache.logging.log4j.LogManager.getLogger();
static final Pattern DOWNLOAD_PATTERN = Pattern.compile("^\\s*Downloading\\s*(http\\S*).*$");
static final Pattern SAVED_PATTERN_1 =
Pattern.compile("^\\s*File was already downloaded\\s*(.*)$");
static final Pattern SAVED_PATTERN_2 = Pattern.compile("^\\s*Saved\\s*(.*)$");
static final String PACK_PATTERN_REGEX = ".*-.*";
private Path pathToPip = null;
private Path logDir = null;
private StringList ignorePacks = new StringList();
/**
* Assumes that the pip executable is part of the PATH environment variable.
*
* @throws org.eclipse.steady.python.ProcessWrapperException if any.
*/
public PipWrapper() throws ProcessWrapperException {
this(Paths.get("pip"), null);
}
/**
* Creates a new wrapper for the pip executable at the given path.
*
* @param _path_to_pip a {@link java.nio.file.Path} object.
* @param _log_dir a {@link java.nio.file.Path} object.
* @throws org.eclipse.steady.python.ProcessWrapperException if any.
*/
public PipWrapper(Path _path_to_pip, Path _log_dir) throws ProcessWrapperException {
this.pathToPip = _path_to_pip;
if (_log_dir != null) this.logDir = _log_dir;
else {
try {
this.logDir = FileUtil.createTmpDir("vulas-pip-");
log.info("Created tmp directory [" + this.logDir + "]");
} catch (IOException e) {
throw new ProcessWrapperException("Cannot create tmp directory: " + e.getMessage());
}
}
// Not to be added as dependencies
final String[] ignore_packs =
VulasConfiguration.getGlobal()
.getStringArray(PythonConfiguration.PY_BOM_IGNORE_PACKS, new String[] {});
this.ignorePacks.addAll(ignore_packs, true);
}
private boolean ignorePackage(String _p) {
return this.ignorePacks.contains(_p, ComparisonMode.EQUALS, CaseSensitivity.CASE_SENSITIVE);
}
/**
* isAvailable.
*
* @return a boolean.
*/
public boolean isAvailable() {
boolean exists = false;
try {
final Process p = new ProcessBuilder(this.pathToPip.toString()).start();
final int exit_code = p.waitFor();
exists = exit_code == 0;
} catch (IOException ioe) {
log.error("Error calling [pip]: " + ioe.getMessage(), ioe);
} catch (InterruptedException ie) {
log.error("Error calling [pip]: " + ie.getMessage(), ie);
}
return exists;
}
/**
* Calls pip install on the given project {@link Path} and the returns a list of all installed packages (including the dependencies).
*
* @param _project a {@link java.nio.file.Path} object.
* @return a {@link java.util.Set} object.
*/
public Set installPackages(Path _project) {
Set packages = null;
try {
// Make download dir
final Path download_dir = Paths.get(logDir.toString(), "pip-download");
FileUtil.createDirectory(download_dir);
// Download all deps
ProcessWrapper pw = new ProcessWrapper();
pw.setCommand(
this.pathToPip,
"download",
"-d",
download_dir.toString(),
"--no-cache-dir",
_project.toString());
pw.setPath(logDir);
Thread t = new Thread(pw);
t.start();
t.join();
final Path download_info = pw.getOutFile();
// Install all deps
pw = new ProcessWrapper();
pw.setCommand(this.pathToPip, "install", _project.toString());
pw.setPath(logDir);
t = new Thread(pw);
t.start();
t.join();
// Get all deps
packages = this.getListPackages();
// Enrich with download info
this.searchDownloadInfo(packages, FileUtil.readFile(download_info));
} catch (ProcessWrapperException e) {
log.error("Error calling installing packages: " + e.getMessage(), e);
} catch (IOException e) {
log.error("Error calling installing packages: " + e.getMessage(), e);
} catch (InterruptedException e) {
log.error("Error calling installing packages: " + e.getMessage(), e);
}
return packages;
}
/**
* Calls pip freeze and pip show <package> in order to create and return all {@link PipInstalledPackage} of the Python environment.
*
* @return a {@link java.util.Set} object.
* @throws org.eclipse.steady.python.ProcessWrapperException if any.
* @throws java.io.IOException if any.
* @throws java.lang.InterruptedException if any.
*/
public Set getFreezePackages()
throws ProcessWrapperException, IOException, InterruptedException {
final StopWatch sw = new StopWatch("pip freeze").start();
Set packages = null;
ProcessWrapper pw = new ProcessWrapper().setCommand(this.pathToPip, "freeze").setPath(logDir);
final Thread t = new Thread(pw, "pip");
t.start();
t.join();
if (pw.terminatedWithSuccess()) {
// Create packages
packages = this.parsePipFreezeOutput(pw.getOutFile());
// Log
log.info("Found [" + packages.size() + "] pip packages:");
for (PipInstalledPackage pack : packages) log.info(" " + pack);
// Call pip show and pip download in separate threads
final ExecutorService pool =
Executors.newFixedThreadPool(ThreadUtil.getNoThreads(2)); // newSingleThreadExecutor();
final Set> futures = new HashSet>();
for (PipInstalledPackage pack : packages) {
futures.add(pool.submit(new PipShow(pack)));
futures.add(pool.submit(new PipDownload(pack)));
}
pool.shutdown();
try {
while (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
int done = 0;
for (Future f : futures) {
if (f.isDone()) done++;
}
sw.lap("[" + done + "/" + futures.size() + "] pip jobs are done");
}
} catch (InterruptedException e) {
PipWrapper.log.error("Interrupt exception");
}
}
sw.stop();
return packages;
}
/**
* Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package.
* @param _file
* @return
* @throws IOException
*/
private Set parsePipFreezeOutput(Path _file) throws IOException {
final Set packs = new TreeSet();
// The pattern to search for
final Pattern pattern = Pattern.compile("^(.*)==(.*)$");
// Read line by line
final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile()));
String line;
while ((line = reader.readLine()) != null) {
final Matcher m = pattern.matcher(line);
if (m.matches()) {
if (this.ignorePackage(m.group(1))) {
log.warn("Package [" + m.group(1) + "] not added as installed package");
} else {
packs.add(new PipInstalledPackage(m.group(1), m.group(2)));
}
}
}
reader.close();
return packs;
}
/**
* Calls pip list and pip show <package> in order to create and return all {@link PipInstalledPackage}s of the Python environment.
*
* @return a {@link java.util.Set} object.
* @throws org.eclipse.steady.python.ProcessWrapperException if any.
* @throws java.io.IOException if any.
* @throws java.lang.InterruptedException if any.
*/
public Set getListPackages()
throws ProcessWrapperException, IOException, InterruptedException {
final StopWatch sw = new StopWatch("pip list").start();
// Try legacy format
ProcessWrapper pw =
new ProcessWrapper()
.setCommand(this.pathToPip, "list", "--format", "legacy")
.setPath(logDir);
Thread t = new Thread(pw, "pip");
t.start();
t.join();
Set packages = null;
if (pw.terminatedWithSuccess()) {
packages = this.parsePipListOutput(pw.getOutFile());
}
// Try JSON format
else {
log.info("Legacy format did not work, trying JSON format...");
pw =
new ProcessWrapper()
.setCommand(this.pathToPip, "list", "--format", "json")
.setPath(logDir);
t = new Thread(pw, "pip");
t.start();
t.join();
if (pw.terminatedWithSuccess()) packages = this.deserializePipListOutput(pw.getOutFile());
}
// Collect package details
if (packages != null) {
log.info("Found [" + packages.size() + "] pip packages:");
for (PipInstalledPackage pack : packages) log.info(" " + pack);
// Call pip show in separate threads
final ExecutorService pool =
Executors.newFixedThreadPool(ThreadUtil.getNoThreads(2)); // newSingleThreadExecutor();
final Set> futures = new HashSet>();
for (PipInstalledPackage pack : packages) {
futures.add(pool.submit(new PipShow(pack)));
}
pool.shutdown();
try {
while (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
int done = 0;
for (Future f : futures) {
if (f.isDone()) done++;
}
sw.lap("[" + done + "/" + futures.size() + "] pip jobs are done");
}
} catch (InterruptedException e) {
PipWrapper.log.error("Interrupt exception");
}
} else {
log.warn("No pip packages found with pip list");
}
sw.stop();
return packages;
}
/**
* Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package.
* @param _file
* @return
* @throws IOException
*/
private Set parsePipListOutput(Path _file) throws IOException {
final Set packs = new TreeSet();
// The pattern to search for
// HP, 17.04.2018: There seem to be versions of PIP with and without using brackets
final Pattern pattern_brackets = Pattern.compile("^(.*)\\s+\\((.*)\\)$");
// final Pattern pattern = Pattern.compile("^(.*)\\s+(.*)$");
// Read line by line
final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile()));
String line;
while ((line = reader.readLine()) != null) {
final Matcher mb = pattern_brackets.matcher(line);
if (mb.matches()) {
if (this.ignorePackage(mb.group(1))) {
log.warn("Package [" + mb.group(1) + "] not added as installed package");
} else {
packs.add(new PipInstalledPackage(mb.group(1), mb.group(2)));
}
}
}
reader.close();
return packs;
}
/**
* Parses the output of pip list, and instantiates {@link PipInstalledPackage} for every installed pip package.
* @param _file
* @return
* @throws IOException
*/
private Set deserializePipListOutput(Path _file) throws IOException {
// Deserialize
final String json = FileUtil.readFile(_file);
final PipPackageJson[] packs =
(PipPackageJson[]) JacksonUtil.asObject(json, PipPackageJson[].class);
// Create set
final Set set = new HashSet();
for (PipPackageJson p : packs) {
if (this.ignorePackage(p.getName())) {
log.warn("Package [" + p.getName() + "] not added as installed package");
} else {
set.add(new PipInstalledPackage(p.getName(), p.getVersion()));
}
}
return set;
}
/**
* Calls pip show in order to set the properties of the given {@link PipInstalledPackage}.
* @param _p
* @param _pool TODO
*/
final class PipShow implements Callable {
private PipInstalledPackage pack = null;
PipShow(PipInstalledPackage _pack) {
this.pack = _pack;
}
public PipInstalledPackage call() throws ProcessWrapperException, IOException {
final ProcessWrapper pw = new ProcessWrapper(this.pack.getName());
pw.setCommand(pathToPip, "show", this.pack.getName()).setPath(logDir);
pw.run();
if (pw.terminatedWithSuccess()) {
this.pack.addProperties(parsePipShowOutput(pw.getOutFile()));
log.info("Added properties to " + this.pack);
} else {
log.info("Properties for " + this.pack + " are missing");
}
return this.pack;
}
}
private Map parsePipShowOutput(Path _file) throws IOException {
final Map props = new HashMap();
// The pattern to search for
final Pattern pattern = Pattern.compile("^([^:]*):(.*)$");
// Read line by line
final BufferedReader reader = new BufferedReader(new FileReader(_file.toFile()));
String line;
while ((line = reader.readLine()) != null) {
final Matcher m = pattern.matcher(line);
if (m.matches()) props.put(m.group(1).trim(), m.group(2).trim());
}
reader.close();
return props;
}
/**
* Calls pip download in order to add download path and url of the given {@link PipInstalledPackage}.
* @param _p
*/
final class PipDownload implements Callable {
private PipInstalledPackage pack = null;
PipDownload(PipInstalledPackage _pack) {
this.pack = _pack;
}
public PipInstalledPackage call() throws ProcessWrapperException, IOException {
// Make download dir
// final Path download_dir = Paths.get(logDir.toString(), "pip-download");
final Path download_dir = FileUtil.createTmpDir(this.pack.getName() + "-");
// Download all deps
ProcessWrapper pw = new ProcessWrapper();
pw.setCommand(
pathToPip,
"download",
"-d",
download_dir.toString(),
"--no-cache-dir",
this.pack.getName() + "==" + this.pack.getVersion());
pw.setPath(logDir);
pw.run();
final Path download_info = pw.getOutFile();
// Enrich with download info
searchDownloadInfo(this.pack, FileUtil.readFile(download_info));
return this.pack;
}
}
/**
* Searches the provided output of 'pip download' for download URL and download path of the given packages.
* @param _packs
* @param _out standard out of pip download
*/
void searchDownloadInfo(Set _packs, String _out) throws IOException {
if (_packs != null) for (PipInstalledPackage pack : _packs) this.searchDownloadInfo(pack, _out);
}
/**
* Searches the provided output of 'pip download' for download URL and download path of the given package.
* @param _p
* @param _out standard out of pip download
*/
private void searchDownloadInfo(PipInstalledPackage _p, String _out) throws IOException {
// Find all download URLs and files in pip output
final Set urls = new HashSet();
final Set files = new HashSet();
final BufferedReader r =
new BufferedReader(new InputStreamReader(new ByteArrayInputStream(_out.getBytes())));
String line = null;
while ((line = r.readLine()) != null) {
final Matcher d = PipWrapper.DOWNLOAD_PATTERN.matcher(line);
final Matcher s1 = PipWrapper.SAVED_PATTERN_1.matcher(line);
final Matcher s2 = PipWrapper.SAVED_PATTERN_2.matcher(line);
if (d.matches()) urls.add(d.group(1));
else if (s1.matches()) files.add(s1.group(1));
else if (s2.matches()) files.add(s2.group(1));
}
// Regex to search for the file name
String regex = PACK_PATTERN_REGEX.replace("", _p.getName().toLowerCase());
regex = regex.replace("", _p.getVersion().toLowerCase());
regex =
regex.replace(
"-",
"[_-]{1,1}"); // For some reason, the file names of some packages use _ instead of -
final Pattern p = Pattern.compile(regex);
for (String url : urls) {
final Matcher m = p.matcher(url.toLowerCase());
if (m.matches()) {
_p.setDownloadUrl(url);
break;
}
}
for (String file : files) {
final Matcher m = p.matcher(file.toLowerCase());
if (m.matches()) {
_p.setDownloadPath(Paths.get(file));
break;
}
}
// Warn if no info could be found
if (_p.getDownloadPath() == null && _p.getDownloadUrl() == null)
log.warn("Download path and URL for " + _p + " missing");
else if (_p.getDownloadPath() == null) log.warn("Download path for " + _p + " missing");
else if (_p.getDownloadUrl() == null) log.warn("Download URL for " + _p + " missing");
else log.info("Found download info for " + _p);
}
/**
* Helper class for deserializing the output of pip list --format json.
*/
static class PipPackageJson {
String name;
String version;
String installer;
String location;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public String getInstaller() {
return installer;
}
public void setInstaller(String installer) {
this.installer = installer;
}
public String getLocation() {
return location;
}
public void setLocation(String location) {
this.location = location;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy