Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* (C) Copyright IBM Corp. 2019,2020
*
* SPDX-License-Identifier: Apache-2.0
*/
package com.ibm.ta.sdk.spi.plugin;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;
import com.ibm.ta.sdk.spi.collect.*;
import com.ibm.ta.sdk.spi.recommendation.Recommendation;
import com.ibm.ta.sdk.spi.assess.RecommendationJson;
import com.ibm.ta.sdk.spi.recommendation.Target;
import com.ibm.ta.sdk.spi.report.Report;
import com.ibm.ta.sdk.spi.util.Util;
import org.tinylog.Logger;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.stream.Collectors;
public class TADataCollector {
private static final String TADATACOLLECTOR_COMMAND_HELP = "Run 'TADataCollector MIDDLEWARE COMMAND --help' for more information on a command.";
private static final String TADATACOLLECTOR_HELP_USAGE_PREFIX = "Usage: TADataCollector";
private static final String TADATACOLLECTOR_BASE_HELP_USAGE = TADATACOLLECTOR_HELP_USAGE_PREFIX + " MIDDLEWARE COMMAND [OPTIONS]";
public static final String ENVIRONMENT_JSON_FILE = "environment.json";
public static final String RECOMMENDATIONS_JSON_FILE = "recommendations.json";
public static final String ASSESSMENTUNIT_META_JSON_FILE = "metadata.assessmentUnit.json";
public static final String TARGETS_JSON_FILE = "targets.json";
ServiceLoader loader = ServiceLoader.load(PluginProvider.class);
public Iterator getPluginProviders() {
return loader.iterator();
}
public void runCommand(String middleware, List cliArguments) throws TAException, IOException {
PluginProvider provider = getProvider(middleware);
if (provider == null) {
throw new IllegalArgumentException("No plug-in provider found for middleware:" + middleware + ".");
}
Logger.debug("cliArguments:" + cliArguments);
if (cliArguments.isEmpty()) {
throw new IllegalArgumentException("No command was specified.");
}
List providerCommands = new LinkedList<>();
CliInputCommand collectCommand = provider.getCollectCommand();
if (collectCommand != null) {
collectCommand.setName(CliInputCommand.CMD_COLLECT);
collectCommand.setDescription(CliInputCommand.CMD_COLLECT_DESC);
providerCommands.add(collectCommand);
}
CliInputCommand assessCommand = provider.getAssessCommand();
if (assessCommand != null) {
assessCommand.setName(CliInputCommand.CMD_ASSESS);
assessCommand.setDescription(CliInputCommand.CMD_ASSESS_DESC);
providerCommands.add(assessCommand);
}
CliInputCommand reportCommand = provider.getReportCommand();
if (reportCommand != null) {
reportCommand.setName(CliInputCommand.CMD_REPORT);
reportCommand.setDescription(CliInputCommand.CMD_REPORT_DESC);
providerCommands.add(reportCommand);
}
CliInputCommand migrateCommand = provider.getMigrateCommand();
if (migrateCommand != null) {
migrateCommand.setName(CliInputCommand.CMD_MIGRATE);
migrateCommand.setDescription(CliInputCommand.CMD_MIGRATE_DESC);
providerCommands.add(migrateCommand);
// Add target option to migrate command
List migrateOptions = new LinkedList<>();
migrateOptions.addAll(migrateCommand.getOptions());
migrateOptions.add(CliInputOption.buildTargetOption());
migrateCommand.setOptions(migrateOptions);
}
// Add 'run' command which performs collect, assess, and report
// The command does not be be provided by the provided, we could re-use the 'assess' command
if (assessCommand != null) {
CliInputCommand runCommand = new CliInputCommand(CliInputCommand.CMD_RUN, CliInputCommand.CMD_RUN_DESC,
assessCommand.getOptions(), assessCommand.getCommands(), assessCommand.getArgumentDisplayNames());
providerCommands.add(runCommand);
}
// Help command - lists all top level commands available for middleware
if (cliArguments.get(0).equals("help")) {
System.out.println("\n" + getBaseMiddlewareHelp(middleware, providerCommands) + "\n");
return;
}
// version command - lists the version for SDK and the provider
if (cliArguments.get(0).contains("version")||cliArguments.get(0).equals("-v") ) {
System.out.println("\n Transformation Advisor SDK version: "+Util.getSDKVersion());
System.out.println(" - Plugin provider "+provider.getClass()+" version: "+provider.getVersion()+"\n");
return;
}
// Find command and display help for that command
if (cliArguments.contains("--help") || cliArguments.contains("-h")) {
CliInputCommand commandHelp = findMatchingCommandForUsageHelp(cliArguments, providerCommands);
if (commandHelp == null) {
throw new IllegalArgumentException("Cannot display help for command. Option is not supported for the command.");
}
// Display help and exit
System.out.println("\n" + TADATACOLLECTOR_HELP_USAGE_PREFIX + " " + middleware + " " + commandHelp.getUsageHelp() + "\n\n" + TADATACOLLECTOR_COMMAND_HELP + "\n");
return;
}
// Finding matching command
CliInputCommand matchedCommand = findMatchingCommand(cliArguments, providerCommands);
if (matchedCommand == null) {
throw new IllegalArgumentException("Command is not supported for middleware: " + middleware + ".");
}
// Invoke supported commands
if (CliInputCommand.CMD_COLLECT.equals(matchedCommand.getName())) {
runCollect(provider, matchedCommand);
} else if (CliInputCommand.CMD_ASSESS.equals(matchedCommand.getName())) {
runAssess(provider, matchedCommand);
} else if (CliInputCommand.CMD_REPORT.equals(matchedCommand.getName())) {
runReport(provider, matchedCommand);
} else if (CliInputCommand.CMD_MIGRATE.equals(matchedCommand.getName())) {
runMigrate(provider, matchedCommand);
} else if (CliInputCommand.CMD_RUN.equals(matchedCommand.getName())) {
runRun(provider, matchedCommand);
} else {
throw new IllegalArgumentException("Command '" + matchedCommand.getName() + "' is not supported.");
}
System.out.println("Command '" + matchedCommand.getName() + "' completed successfully.\n");
}
public void runCollect(PluginProvider provider, CliInputCommand cliInputCommand) throws TAException, IOException {
collectData(provider, cliInputCommand);
}
private List collectData(PluginProvider provider, CliInputCommand cliInputCommand) throws TAException, IOException {
List dataCollections = provider.getCollection(cliInputCommand);
if (dataCollections == null || dataCollections.size() == 0) {
throw new TAException("Collect failed. No recommendations generated by plugin provider.");
}
for (DataCollection dataCollection : dataCollections) {
// Get environment
Environment environment = dataCollection.getEnvironment();
// Create output dir
String collectionName = environment.getCollectionUnitName();
// collectionName need to be set in the env.json, otherwise cannot find the output dir,
// throw exception here
if (collectionName==null || collectionName.length()==0) {
throw new TAException("Collection unit name isnot set in the environment by the plug-in provider "+provider.getClass());
}
File outputDir = Util.getAssessmentOutputDir(collectionName);
if (!outputDir.exists()) {
outputDir.mkdirs();
}
// Get assessment units
List aus = getAssessmentUnits(dataCollection, outputDir);
List auNameList = aus.stream()
.map(au -> ((AssessmentUnit) au).getName())
.collect(Collectors.toList());
//copy targets.json file to output dir
Util.copyResourceToDir(provider.getMiddleware()+"/targets.json", outputDir);
//copy templates files to output dir
Util.copyResourceToDir(provider.getMiddleware()+"/templates/", outputDir);
// Write environment json to output dir
writeEnvironmentJson(environment, provider.getVersion(), auNameList, outputDir);
}
return dataCollections;
}
private List getAssessmentUnits(DataCollection dataCollection, File outputDir) throws TAException, IOException {
List assessUnits = dataCollection.getAssessmentUnits();
for (AssessmentUnit au : assessUnits) {
// Create sub dir for each assessment unit
File auOutputDir = new File(outputDir, au.getName());
if (!auOutputDir.exists()) {
auOutputDir.mkdirs();
}
// Write assessment unit data file
writeAssessmentDataJson(au, auOutputDir);
// Write assessment unit metadata file
writeAssessmentUnitMetaJson(au, dataCollection.getEnvironment(), auOutputDir);
// Copy assessment files to make them available during recommendations
List configFiles = au.getConfigFiles();
if (configFiles != null) {
List outputConfigFiles = new LinkedList<>();
for (Path file : configFiles) {
File destFile = new File(auOutputDir, file.getFileName().toString());
Path destPath = destFile.toPath();
if (!destFile.getParentFile().exists()) {
destFile.getParentFile().mkdirs();
}
Files.copy(file, destPath, StandardCopyOption.REPLACE_EXISTING);
outputConfigFiles.add(destPath);
// Add files from directory
File dirFile = destPath.toFile();
if (dirFile.isDirectory()) {
outputConfigFiles.addAll(getFilesFromDir(dirFile));
}
}
// Apply mask to content
Logger.debug("Applying content masks");
List contentMasks = au.getContentMasks();
if (contentMasks != null) {
for (Path path : outputConfigFiles) {
for (ContentMask mask : contentMasks) {
for (String contentMaskFile : mask.getFiles()) {
// Use the original path of the file, not the new path where the file is copied to
String origPath = path.toAbsolutePath().toString().replace(auOutputDir.getAbsolutePath(), "");
Logger.debug("Comparing file:" + origPath + " to contentMaskFile:" + contentMaskFile);
if (origPath.matches(contentMaskFile)) {
Logger.info("Applying mask to file:" + path);
// Read lines from file
List lines = Files.readAllLines(path);
// Mask file content
List updatedLines = mask.mask(lines);
// Write updated context to the file
Files.write(path, updatedLines);
// Move on to the next mask
break;
}
}
}
}
}
// Update config files that point to output dir for use in recommendations
configFiles.clear();
configFiles.addAll(outputConfigFiles);
}
}
return assessUnits;
}
private List getFilesFromDir(File dir) {
List dirFiles = new LinkedList<>();
if (!dir.isDirectory()) {
return dirFiles;
}
File[] files = dir.listFiles();
for (File file : files) {
if (file.isDirectory()) {
dirFiles.addAll(getFilesFromDir(file));
} else {
dirFiles.add(file.toPath());
}
}
return dirFiles;
}
public void runAssess(PluginProvider provider, CliInputCommand cliInputCommand) throws TAException, IOException {
provider.validateJsonFiles();
// Run collections
List dataCollections = collectData(provider, cliInputCommand);
if (dataCollections == null || dataCollections.size() == 0) {
throw new TAException("Collect failed. No recommendations generated by plugin provider.");
}
// Generate and write recommendations
List recs = provider.getRecommendation(cliInputCommand);
if (recs == null || recs.size() == 0) {
throw new TAException("Assessment failed. No recommendations generated by plugin provider.");
}
// Get target commandline option to filter out targets
List filterTargets = CliInputOption.getCliOptionValuesByLongName(cliInputCommand.getOptions(), CliInputOption.OPT_TARGET);
for (Recommendation rec : recs) {
String assessmentName = rec.getCollectionUnitName();
Optional dcOp = dataCollections.stream()
.filter(d -> d.getAssessmentName().equals(assessmentName))
.findFirst();
if (!dcOp.isPresent()) {
throw new TAException("Collection not found for assessment:" + assessmentName);
}
DataCollection dc = dcOp.get();
Environment environment = dc.getEnvironment();
List assessUnits = dc.getAssessmentUnits();
RecommendationJson recJson = new RecommendationJson(rec, environment, assessUnits, filterTargets);
File outputDir = Util.getAssessmentOutputDir(assessmentName);
writeRecommendationsJson(recJson, outputDir);
// Add log message to indicate zip does not contain data because plugin collects sensitive data
if (environment.hasSensitiveData()) {
Logger.info("The environment.json file indicates that the collection contains sensitive data. The collection zip archive created will not include any of the collected data files.");
}
// zip output dir
String zipFileName = environment.getCollectionUnitName() + ".zip";
File zipFile = new File(outputDir.getParentFile(), zipFileName);
Util.zipCollection(zipFile.toPath(), outputDir, environment.hasSensitiveData());
}
}
public void runReport(PluginProvider provider, CliInputCommand cliInputCommand) throws TAException, IOException {
// Get a list of assessments by scanning directories in the output directory
List assessmentNames = new ArrayList<>();
File outputDir = Util.getOutputDir();
File[] outputDirFiles = outputDir.listFiles();
for (File file : outputDirFiles) {
if (file.isDirectory()) {
assessmentNames.add(file.getName());
}
}
Logger.debug("Generating reports for assessment units:" + assessmentNames);
// Get report for each assessment
for (String assessmentName : assessmentNames) {
// Read env.json for assessment
File aOutputDir = Util.getAssessmentOutputDir(assessmentName);
File envFile = new File(aOutputDir, ENVIRONMENT_JSON_FILE);
JsonElement envJsonEle = new JsonParser().parse(new FileReader(envFile));
EnvironmentJson envJson = new Gson().fromJson(envJsonEle, new TypeToken(){}.getType());
Environment env = envJson.getEnvironment();
List reports = provider.getReport(assessmentName, cliInputCommand);
for (Report report : reports) {
Target target = report.getTarget();
String reportName = "recommendations_" + target.getTargetId() +
"." + report.getReportType().toString().toLowerCase();
// Report path
File auOutputDir = new File(aOutputDir, report.getAssessmentUnitName());
File recFile = new File(auOutputDir, reportName);
// Write report
Logger.info("Writing report:" + recFile.getAbsolutePath());
writeFile(recFile, report.getReport());
// Update assessment unit zip
String zipFileName = assessmentName + ".zip";
File zipFile = new File(aOutputDir.getParentFile(), zipFileName);
Util.zipCollection(zipFile.toPath(), aOutputDir, env.hasSensitiveData());
}
}
}
public void runMigrate(PluginProvider provider, CliInputCommand cliInputCommand) throws TAException, IOException {
// Get target commandline option to filter out targets
List filterTargets = CliInputOption.getCliOptionValuesByLongName(cliInputCommand.getOptions(), CliInputOption.OPT_TARGET);
provider.getMigrationBundle(cliInputCommand, filterTargets);
}
public void runRun(PluginProvider provider, CliInputCommand cliInputCommand) throws TAException, IOException {
runAssess(provider, cliInputCommand);
runReport(provider, cliInputCommand);
}
private CliInputCommand findMatchingCommand(List cliArguments, List providerCommands) {
Logger.debug("cliArguments:" + cliArguments);
if (cliArguments.size() < 1) {
return null;
}
String cliArg = cliArguments.get(0);
// stop looking for commands when first arg is found
if (cliArg.startsWith("-")) {
return null;
}
CliInputCommand matchedCommand = null;
for (CliInputCommand command : providerCommands) {
if (command.getName().equals(cliArg)) {
matchedCommand = new CliInputCommand(command);
cliArguments.remove(0); // Pop out the first command now that we found a match
// Process subcommand
List subCommandsList = command.getCommands();
if (subCommandsList.size() > 0) {
CliInputCommand subCommand = findMatchingCommand (cliArguments, subCommandsList);
Logger.debug("Matched subCommand:" + subCommand);
if (subCommand != null) {
matchedCommand.setCommands(new ArrayList<>(Arrays.asList(subCommand)));
}
}
if (matchedCommand.getCommands().isEmpty()) {
Logger.debug("No matching subcommands. Adding args and options to command:" + matchedCommand.getName());
// No more subcommands, this is the last subcommand, add options and arguments to this command
List matchedOptions = command.getMatchedOptions(cliArguments);
Logger.debug("Matched options:" + matchedOptions);
matchedCommand.setOptions(matchedOptions);
// Remaining cli arguments are added as arguments to command
Logger.debug("Set arguments:" + cliArguments);
matchedCommand.setArguments(cliArguments);
}
break;
}
}
return matchedCommand;
}
// Finds the last subcommand that matches the commands in the CLI arguments. This CliInputCommand is used to display help.
private CliInputCommand findMatchingCommandForUsageHelp(List cliArguments, List providerCommands) {
if (cliArguments.size() < 1) {
return null;
}
String cliArg = cliArguments.get(0);
Logger.debug("Comparing cliArg:" + cliArg);
// stop looking for commands when first arg is found
if (cliArg.startsWith("-")) {
return null;
}
for (CliInputCommand command : providerCommands) {
if (command.getName().equals(cliArg)) {
cliArguments.remove(0); // Pop out the first command now that we found a match
// Process subcommand
List subCommandsList = command.getCommands();
if (subCommandsList.size() > 0) {
CliInputCommand subCommand = findMatchingCommandForUsageHelp (cliArguments, subCommandsList);
Logger.debug("subCommand:" + subCommand);
if (subCommand != null) {
return subCommand;
}
}
return command;
}
}
return null;
}
/*
* Base help for a specific middleware. Lists all commands a available for that middleware.
*/
private String getBaseMiddlewareHelp(String middleware, List providerCommands) {
String usage = TADATACOLLECTOR_HELP_USAGE_PREFIX + " " + middleware + " COMMAND [OPTIONS]\n\n";
usage += "Commands:\n";
for (CliInputCommand command : providerCommands) {
usage += " " + String.format("%1$-" + 15 + "s", command.getName()) + command.getDescription() + "\n";
}
usage += "\n\n" + TADATACOLLECTOR_COMMAND_HELP;
return usage;
}
/*
* Base help for TADataCollector. Lists all middlewares available to help the user start.
*/
private static String getBaseHelp() {
Iterator itRecP = new TADataCollector().getPluginProviders();
if (!itRecP.hasNext()) {
throw new TARuntimeException("No plug-in available. Add a plug-in to the classpath and run TADataCollector again.");
}
String middleware = null;
while (itRecP.hasNext()) {
PluginProvider recProvider = itRecP.next();
if (middleware == null) {
middleware = recProvider.getMiddleware();
} else {
middleware += " | " + recProvider.getMiddleware();
}
}
String usage = TADATACOLLECTOR_BASE_HELP_USAGE + "\n\n";
usage += "Middleware:\n" + " Plug-ins available for these middleware [ " + middleware + " ]\n\n";
usage += "Commands:\n" + " help Get information on the commands and options available for a middleware";
return usage;
}
// Check cli input args against list of args from plugin provider
private static void validateCliArgs(List providerArgs, Properties cliInputProps) {
for (CliInputOption providerArg : providerArgs) {
}
}
private String getJsonStr(Object recJson) {
GsonBuilder builder = new GsonBuilder();
builder.excludeFieldsWithoutExposeAnnotation();
builder.setPrettyPrinting();
Gson gson = builder.create();
String issuesJsonString = gson.toJson(recJson);
return issuesJsonString;
}
private void writeRecommendationsJson(RecommendationJson recJson, File outputDir) throws TAException {
File rjFile = new File(outputDir, RECOMMENDATIONS_JSON_FILE);
if (rjFile.exists()) {
rjFile.delete();
}
// Convert recommendation to JSON
String recJsonStr = getJsonStr(recJson);
writeFile(rjFile, recJsonStr);
}
private void writeAssessmentDataJson(AssessmentUnit au, File outputDir) throws TAException {
File auFile = new File(outputDir, "data" + ".json");
if (auFile.exists()) {
auFile.delete();
}
Logger.debug("Writing assessment unit data json file:" + auFile);
// Convert recommendation to JSON
String auJsonStr = getJsonStr(au.getAssessmentData());
writeFile(auFile, auJsonStr);
}
private void writeEnvironmentJson(Environment environment, String version, List auNameList, File outputDir) throws TAException {
File envFile = new File(outputDir, ENVIRONMENT_JSON_FILE);
if (envFile.exists()) {
envFile.delete();
}
Logger.debug("Writing env file:" + envFile);
EnvironmentJson envJson = new EnvironmentJson(environment);
envJson.setPluginVersion(version);
envJson.setAssessmentUnits(auNameList);
// check the template directory, and update the containsTemplateFiles flag
File templateDir = new File(outputDir.getAbsolutePath()+
File.separator+envJson.getMiddlewareName()+File.separator+"templates");
if (templateDir.exists() && templateDir.isDirectory()) {
envJson.setContainsTemplateFiles(true);
}
String envJsonStr = getJsonStr(envJson);
writeFile(envFile, envJsonStr);
}
private void writeAssessmentUnitMetaJson(AssessmentUnit au, Environment environment, File outputDir) throws TAException {
File auMetaFile = new File(outputDir, ASSESSMENTUNIT_META_JSON_FILE);
if (auMetaFile.exists()) {
auMetaFile.delete();
}
Logger.debug("Writing assessment unit metadata json file:" + auMetaFile);
AssessmentUnitMetadataJson auMeta = new AssessmentUnitMetadataJson(environment, au.getName());
String auMetaJsonStr = getJsonStr(auMeta.toJsonObject());
writeFile(auMetaFile, auMetaJsonStr);
}
private static void writeFile(File file, String content) throws TAException {
writeFile(file, content.getBytes());
}
private static void writeFile(File file, byte[] content) throws TAException {
try {
Files.write(file.toPath(), content);
} catch (IOException e) {
throw new TAException("Error writing file:" + file.getAbsolutePath(), e);
}
}
private PluginProvider getProvider(String middleware) {
Iterator itRecP = getPluginProviders();
while (itRecP.hasNext()) {
PluginProvider recProvider = itRecP.next();
// Find provider that matches domain
if (recProvider.getMiddleware().equals(middleware)) {
try {
recProvider.validateJsonFiles();
} catch (TAException e) {
Logger.error("validate issues jdon file failed.");
}
return recProvider;
}
}
return null;
}
/*
private static void processInput(String[] args) {
Properties cliArgs = new Properties();
List cliCommands = new LinkedList<>();
for (int i = 0; i < args.length; i++) {
switch (args[i].charAt(0)) {
case '-':
String argKey = args[i];
String argValue = "";
if (i+1 < args.length) {
if (!args[i + 1].startsWith("-")) {
argValue = args[++i];
}
}
if (argKey.startsWith("--")) {
argKey = argKey.replaceFirst("--", "");
if (argKey.length() <= 1) {
throw new IllegalArgumentException("Invalid argument '--" + argKey + "'. Use the help command to get usage information.");
}
} else if (argKey.startsWith("-")) {
argKey = argKey.replaceFirst("-", "");
if (argKey.length() != 1) {
throw new IllegalArgumentException("Invalid argument '-" + argKey + "'. Use the help command to get usage information.");
}
}
cliArgs.setProperty(argKey, argValue);
break;
default:
cliCommands.add(args[i]);
}
}
}
*/
public static void main(String[] args) {
List cliCommands = new LinkedList<>();
for (int i = 0; i < args.length; i++) {
cliCommands.add(args[i]);
}
// No args at all - display help and exit
if (cliCommands.isEmpty()) {
System.out.println(getBaseHelp());
return;
}
// First arg is not middleware
String middleware = cliCommands.get(0);
if (middleware.startsWith("-")) {
System.out.println("\n" + getBaseHelp() + "\n");
return;
}
if (middleware.contains("version")) {
System.out.println("\n Transformation Advisor SDK version: "+Util.getSDKVersion()+"\n");
return;
}
cliCommands.remove(0); // Pop out middleware from CLI args
try {
new TADataCollector().runCommand(middleware, cliCommands);
} catch (IllegalArgumentException e) {
System.out.println(e.getMessage() + "\n\n" + getBaseHelp() + "\n");
} catch (TAException tae) {
Logger.error("Fail to run the command:", tae);
System.out.println("Fail to run the command, check log file for detail information.\n " +tae.getMessage() + "\n");
} catch (IOException ioe) {
Logger.error("Fail to run the command:", ioe);
System.out.println("Fail to run the command, check log file for detail information.\n " +ioe.getMessage() + "\n");
}
}
}
