com.salesforce.dockerfileimageupdate.subcommands.impl.All Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of dockerfile-image-update Show documentation
Show all versions of dockerfile-image-update Show documentation
This tool provides a mechanism to make security updates to docker images at scale.
The tool searches github for declared docker images and sends pull requests to projects that are not using
the desired version of the requested docker image.
/*
* Copyright (c) 2018, salesforce.com, inc.
* All rights reserved.
* Licensed under the BSD 3-Clause license.
* For full license text, see LICENSE.txt file in the repo root or
* https://opensource.org/licenses/BSD-3-Clause
*/
package com.salesforce.dockerfileimageupdate.subcommands.impl;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
import com.salesforce.dockerfileimageupdate.SubCommand;
import com.salesforce.dockerfileimageupdate.model.PullRequestInfo;
import com.salesforce.dockerfileimageupdate.repository.GitHub;
import com.salesforce.dockerfileimageupdate.subcommands.ExecutableWithNamespace;
import com.salesforce.dockerfileimageupdate.utils.Constants;
import com.salesforce.dockerfileimageupdate.utils.DockerfileGitHubUtil;
import com.salesforce.dockerfileimageupdate.utils.ResultsProcessor;
import net.sourceforge.argparse4j.inf.Namespace;
import org.kohsuke.github.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Paths;
import java.util.*;
@SubCommand(help="updates all repositories' Dockerfiles",
requiredParams = {Constants.STORE})
public class All implements ExecutableWithNamespace {
private static final Logger log = LoggerFactory.getLogger(All.class);
private DockerfileGitHubUtil dockerfileGitHubUtil;
@Override
public void execute(final Namespace ns, final DockerfileGitHubUtil dockerfileGitHubUtil) throws IOException, InterruptedException {
loadDockerfileGithubUtil(dockerfileGitHubUtil);
Map imageToTagMap = new HashMap<>();
Multimap imagesFoundInParentRepo = ArrayListMultimap.create();
Multimap pathToDockerfilesInParentRepo = ArrayListMultimap.create();
Set> imageToTagStore = parseStoreToImagesMap(ns.get(Constants.STORE));
for (Map.Entry imageToTag : imageToTagStore) {
String image = imageToTag.getKey();
log.info("Repositories with image {} being forked.", image);
imageToTagMap.put(image, imageToTag.getValue().getAsString());
PagedSearchIterable contentsWithImage =
this.dockerfileGitHubUtil.findFilesWithImage(image, ns.get(Constants.GIT_ORG));
forkRepositoriesFound(pathToDockerfilesInParentRepo,
imagesFoundInParentRepo, contentsWithImage, image);
}
GHMyself currentUser = this.dockerfileGitHubUtil.getMyself();
if (currentUser == null) {
throw new IOException("Could not retrieve authenticated user.");
}
log.info("Retrieving all the forks...");
List listOfCurrUserRepos =
dockerfileGitHubUtil.getGHRepositories(pathToDockerfilesInParentRepo, currentUser);
List exceptions = new ArrayList<>();
List skippedRepos = new ArrayList<>();
for (GHRepository currUserRepo : listOfCurrUserRepos) {
try {
changeDockerfiles(ns, pathToDockerfilesInParentRepo, imagesFoundInParentRepo, imageToTagMap, currUserRepo,
skippedRepos);
} catch (IOException e) {
log.error(String.format("Error changing Dockerfile for %s", currUserRepo.getName()), e);
exceptions.add(e);
}
}
ResultsProcessor.processResults(skippedRepos, exceptions, log);
}
protected void loadDockerfileGithubUtil(DockerfileGitHubUtil _dockerfileGitHubUtil) {
dockerfileGitHubUtil = _dockerfileGitHubUtil;
}
protected void forkRepositoriesFound(Multimap pathToDockerfilesInParentRepo,
Multimap imagesFoundInParentRepo,
PagedSearchIterable contentsWithImage,
String image) throws IOException {
log.info("Forking {} repositories...", contentsWithImage.getTotalCount());
List parentReposForked = new ArrayList<>();
GHRepository parent;
String parentRepoName = null;
for (GHContent c : contentsWithImage) {
/* Kohsuke's GitHub API library, when retrieving the forked repository, looks at the name of the parent to
* retrieve. The issue with that is: GitHub, when forking two or more repositories with the same name,
* automatically fixes the names to be unique (by appending "-#" to the end). Because of this edge case, we
* cannot save the forks and iterate over the repositories; else, we end up missing/not updating the
* repositories that were automatically fixed by GitHub. Instead, we save the names of the parent repos
* in the map above, find the list of repositories under the authorized user, and iterate through that list.
*/
parent = c.getOwner();
parentRepoName = parent.getFullName();
if (parent.isFork()) {
log.warn("Skipping {} because it's a fork already. Sending a PR to a fork is unsupported at the moment.",
parentRepoName);
} else {
// fork the parent if not already forked
if (!parentReposForked.contains(parentRepoName)) {
// TODO: Need to close PR!
GHRepository fork = dockerfileGitHubUtil.getOrCreateFork(parent);
GHPullRequest pr = getPullRequestWithPullReqIdentifier(parent);
// Only reason we close the existing PR, delete fork and re-fork, is because there is no way to
// determine if the existing fork is compatible with it's parent.
if (pr != null) {
// close the pull-request since the fork is out of date
log.info("closing existing pr: {}", pr.getUrl());
try {
pr.close();
} catch (IOException e) {
log.info("Issues closing the pull request '{}'. Moving ahead...", pr.getUrl());
}
}
if (fork == null) {
log.info("Could not fork {}", parentRepoName);
} else {
// Add repos to pathToDockerfilesInParentRepo and imagesFoundInParentRepo only if we forked it successfully.
pathToDockerfilesInParentRepo.put(parentRepoName, c.getPath());
imagesFoundInParentRepo.put(parentRepoName, image);
parentReposForked.add(parentRepoName);
}
}
}
}
log.info("Path to Dockerfiles in repo '{}': {}", parentRepoName, pathToDockerfilesInParentRepo);
log.info("All images found in repo '{}': {}", parentRepoName, imagesFoundInParentRepo);
}
protected Set> parseStoreToImagesMap(String storeName)
throws IOException, InterruptedException {
GHMyself myself = dockerfileGitHubUtil.getMyself();
String login = myself.getLogin();
GHRepository store = dockerfileGitHubUtil.getRepo(Paths.get(login, storeName).toString());
GHContent storeContent = dockerfileGitHubUtil.tryRetrievingContent(store, Constants.STORE_JSON_FILE,
store.getDefaultBranch());
if (storeContent == null) {
return Collections.emptySet();
}
JsonElement json;
try (InputStream stream = storeContent.read(); InputStreamReader streamR = new InputStreamReader(stream)) {
try {
json = JsonParser.parseReader(streamR);
} catch (JsonParseException e) {
log.warn("Not a JSON format store.");
return Collections.emptySet();
}
}
JsonElement imagesJson = json.getAsJsonObject().get("images");
return imagesJson.getAsJsonObject().entrySet();
}
protected void changeDockerfiles(Namespace ns,
Multimap pathToDockerfilesInParentRepo,
Multimap imagesFoundInParentRepo,
Map imageToTagMap,
GHRepository currUserRepo,
List skippedRepos) throws IOException, InterruptedException {
/* The Github API does not provide the parent if retrieved through a list. If we want to access its parent,
* we need to retrieve it once again.
*/
GHRepository forkedRepo;
if (currUserRepo.isFork()) {
try {
forkedRepo = dockerfileGitHubUtil.getRepo(currUserRepo.getFullName());
} catch (FileNotFoundException e) {
/* The edge case here: If a different command calls getGHRepositories, and then this command calls
* it again within 60 seconds, it will still have the same list of repositories (because of caching).
* However, between the previous and current call, if some of those repositories are deleted, the call
* above may cause a FileNotFoundException. This clause prevents that exception from stopping our call;
* we do not need to stop because getGHRepositories checks that we have all the repositories we need.
*
* The integration test calls the testParent -> testAllCommand -> testIdempotency, and the
* testIdempotency was failing because of this edge condition.
*/
log.warn("This repository does not exist. The list of repositories must be outdated, but the list" +
"contains the repositories we need, so we ignore this error.");
return;
}
} else {
return;
}
GHRepository parent = forkedRepo.getParent();
if (GitHub.shouldNotProcessDockerfilesInRepo(pathToDockerfilesInParentRepo, parent)) return;
log.info("Fixing Dockerfiles in {}...", forkedRepo.getFullName());
String parentName = parent.getFullName();
String branch = (ns.get(Constants.GIT_BRANCH) == null) ? forkedRepo.getDefaultBranch() : ns.get(Constants.GIT_BRANCH);
String pathToDockerfile;
String image;
String tag;
GHContent content;
boolean isContentModified = false;
boolean isRepoSkipped = true;
Iterator pathToDockerfileInParentRepoIterator = pathToDockerfilesInParentRepo.get(parentName).iterator();
Iterator imagesFoundInParentRepoIterator = imagesFoundInParentRepo.get(parentName).iterator();
while (pathToDockerfileInParentRepoIterator.hasNext()) {
pathToDockerfile = pathToDockerfileInParentRepoIterator.next();
image = imagesFoundInParentRepoIterator.next();
tag = imageToTagMap.get(image);
log.info("pathToDockerfile: {} , image: {}, tag: {}", pathToDockerfile, image, tag);
content = dockerfileGitHubUtil.tryRetrievingContent(forkedRepo, pathToDockerfile, branch);
if (content == null) {
log.info("No Dockerfile found at path: '{}'", pathToDockerfile);
} else {
dockerfileGitHubUtil.modifyOnGithub(content, branch, image, tag,
ns.get(Constants.GIT_ADDITIONAL_COMMIT_MESSAGE));
isContentModified = true;
isRepoSkipped = false;
}
}
if (isRepoSkipped) {
log.info("Skipping repo '{}' because contents of it's fork could not be retrieved. Moving ahead...",
parentName);
skippedRepos.add(forkedRepo.getFullName());
}
if (isContentModified) {
// Passing null for image/tag to temporarily maintain old behavior of using constant.
PullRequestInfo pullRequestInfo =
new PullRequestInfo(ns.get(Constants.GIT_PR_TITLE),null, null);
dockerfileGitHubUtil.createPullReq(parent, branch, forkedRepo, pullRequestInfo);
}
}
private GHPullRequest getPullRequestWithPullReqIdentifier(GHRepository parent) throws IOException {
List pullRequests;
GHUser myself;
try {
pullRequests = parent.getPullRequests(GHIssueState.OPEN);
myself = dockerfileGitHubUtil.getMyself();
} catch (IOException e) {
log.warn("Error occurred while retrieving pull requests for {}", parent.getFullName());
return null;
}
for (GHPullRequest pullRequest : pullRequests) {
GHUser user = pullRequest.getHead().getUser();
if (myself.equals(user) && pullRequest.getBody().equals(Constants.PULL_REQ_ID)) {
return pullRequest;
}
}
return null;
}
}