org.opentripplanner.graph_builder.DataImportIssuesToHTML Maven / Gradle / Ivy
package org.opentripplanner.graph_builder;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.opentripplanner.datastore.CompositeDataSource;
import org.opentripplanner.datastore.DataSource;
import org.opentripplanner.graph_builder.services.GraphBuilderModule;
import org.opentripplanner.routing.graph.Graph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class generates a nice HTML graph import data issue report.
*
* They are created with the help of getHTMLMessage function in {@link DataImportIssue} derived classes.
* @author mabu
*/
public class DataImportIssuesToHTML implements GraphBuilderModule {
private static final Logger LOG = LoggerFactory.getLogger(DataImportIssuesToHTML.class);
//Path to output folder
private final CompositeDataSource reportDirectory;
//If there are more then this number of issues the report are split into multiple files
//This is because browsers aren't made for giant HTML files which can be made with 500k lines
private final int maxNumberOfIssuesPerFile;
//This counts all occurrences of HTML issue type
//If one issue type is split into two files it has two entries in this Multiset
//IT is used to show numbers in HTML files name and links
private final Multiset issueTypeOccurrences = HashMultiset.create();
//List of writers which are used for actual writing issues to HTML
private final List writers = new ArrayList<>();
//Key is classname, value is issue message
//Multimap because there are multiple issues for each classname
private final Multimap issues = ArrayListMultimap.create();
DataImportIssuesToHTML(CompositeDataSource reportDirectory, int maxNumberOfIssuesPerFile) {
this.reportDirectory = reportDirectory;
this.maxNumberOfIssuesPerFile = maxNumberOfIssuesPerFile;
}
@Override
public void checkInputs() { }
@Override
public void buildGraph(
Graph graph,
HashMap, Object> extra,
DataImportIssueStore issueStore
) {
try {
// Delete all files in the report directory if it exist
if (!deleteReportDirectoryAndContent()) { return; }
//Groups issues in multimap according to issue type
for (DataImportIssue it : issueStore.getIssues()) {
//writer.println("" + it.getHTMLMessage() + "
");
// writer.println("" + it.getTypeName()+"");
addIssue(it);
}
LOG.info("Creating data import issue log");
//Creates list of HTML writers. Each writer has whole class of HTML issues
//Or multiple HTML writers can have parts of one class of HTML issues if number
// of issues is larger than maxNumberOfIssuesPerFile.
for (Map.Entry> entry: issues.asMap().entrySet()) {
List issueList;
if (entry.getValue() instanceof List) {
issueList = (List) entry.getValue();
} else {
issueList = new ArrayList<>(entry.getValue());
}
addIssues(entry.getKey(), issueList);
}
//Actual writing to the file is made here since
// this is the first place where actual number of files is known (because it depends on
// the issue count)
for (HTMLWriter writer : writers) {
writer.writeFile(issueTypeOccurrences, false);
}
try {
HTMLWriter indexFileWriter = new HTMLWriter("index", (Multimap)null);
indexFileWriter.writeFile(issueTypeOccurrences, true);
} catch (Exception e) {
LOG.error("Index file coudn't be created:{}", e.getMessage());
}
LOG.info("Data import issue logs are in {}", reportDirectory.path());
}
catch (Exception e) {
// If the issue report fails due to a remote storage or network problem, then we log
// the error an CONTINUE with the graph build process. Preventing OTP from saving the
// Graph might have much bigger consequences than just failing to save the issue report.
LOG.error("OTP failed to save issue report!", e);
}
finally {
closeReportDirectory();
}
}
/**
* Delete report if it exist, and return true if successful. Return {@code false} if the
* {@code reportDirectory} is {@code null} or the directory can NOT be deleted.
*/
private boolean deleteReportDirectoryAndContent() {
if (reportDirectory == null) {
LOG.error("Saving folder is empty!");
return false;
}
if (reportDirectory.exists()) {
//Removes all files from report directory
try {
reportDirectory.delete();
} catch (Exception e) {
LOG.error("Failed to clean HTML report directory: " + reportDirectory.path() + ". HTML report won't be generated!", e);
return false;
}
}
// No need to create directories here, because the 'reportDirectory' is responsible for
// creating paths (it they don´t exist) when saving files.
return true;
}
/**
* Creates file with given type of issues
*
* If number of issues is larger then 'maxNumberOfIssuesPerFile' multiple files are generated.
* And named issueClassName1,2,3 etc.
*
* @param issueTypeName name of import data issue class and then also filename
* @param issues list of all import data issue with that class
*/
private void addIssues(String issueTypeName, List issues) {
HTMLWriter file_writer;
if (issues.size() > 1.2* maxNumberOfIssuesPerFile) {
LOG.debug("Number of issues is very large. Splitting: {}", issueTypeName);
List> partitions = Lists.partition(issues,
maxNumberOfIssuesPerFile
);
for (List partition: partitions) {
issueTypeOccurrences.add(issueTypeName);
int labelCount = issueTypeOccurrences.count(issueTypeName);
file_writer =new HTMLWriter(issueTypeName + labelCount, partition);
writers.add(file_writer);
}
} else {
issueTypeOccurrences.add(issueTypeName);
int labelCount = issueTypeOccurrences.count(issueTypeName);
file_writer = new HTMLWriter(issueTypeName + labelCount, issues);
writers.add(file_writer);
}
}
/**
* Groups issues according to issue type, using the classname as type name.
*
* All issues are saved together in multimap where key is issue classname
* and values are list of issue with that class
*/
private void addIssue(DataImportIssue issue) {
issues.put(issue.getType(), issue.getHTMLMessage());
}
class HTMLWriter {
private final DataSource target;
private final Multimap writerIssues;
private final String issueTypeName;
HTMLWriter(String key, Collection issues) {
LOG.debug("Making file: {}", key);
this.target = reportDirectory.entry(key + ".html");
this.writerIssues = ArrayListMultimap.create();
this.writerIssues.putAll(key, issues);
this.issueTypeName = key;
}
HTMLWriter(String filename, Multimap curMap) {
LOG.debug("Making file: {}", filename);
this.target = reportDirectory.entry(filename + ".html");
this.writerIssues = curMap;
this.issueTypeName = filename;
}
private void writeFile(Multiset classes, boolean isIndexFile) {
try(PrintWriter out = new PrintWriter(target.asOutputStream(), true, StandardCharsets.UTF_8)) {
out.println("Graph report for OTP Graph ");
out.println("\t");
out.println("");
out.println("");
out.println(
"");
String css = "\t\t\n"
+ "";
out.println(css);
out.println("");
out.println(String.format("OpenTripPlanner data import issue log for %s
",
issueTypeName
));
out.println("Graph report for graph.obj
");
out.println("");
//adds links to the other HTML files
for (Multiset.Entry htmlIssueType : classes.entrySet()) {
String label_name = htmlIssueType.getElement();
String label;
int currentCount = 1;
//it needs to add link to every file even if they are split
while (currentCount <= htmlIssueType.getCount()) {
label = label_name + currentCount;
if (label.equals(issueTypeName)) {
out.printf(
"%n",
label_name.toLowerCase(),
IssueColors.rgb(label_name),
label
);
}
else {
out.printf(
"%s%n",
label_name.toLowerCase(),
label,
IssueColors.rgb(label_name),
label
);
}
currentCount++;
}
}
out.println("
");
if (!isIndexFile) {
out.println("");
writeIssues(out);
out.println("
");
}
out.println("");
}
}
/**
* Writes issues as LI html elements
*/
private void writeIssues(PrintWriter out) {
String FMT = "%s ";
for (Map.Entry it: writerIssues.entries()) {
out.printf(FMT, it.getValue());
}
}
}
private void closeReportDirectory() {
try{
reportDirectory.close();
}
catch (IOException e) {
LOG.warn(
"Failed to close report directory: {}, details: {}. ",
reportDirectory.path(), e.getLocalizedMessage(), e
);
}
}
}