org.opentripplanner.graph_builder.DataImportIssuesToHTML Maven / Gradle / Ivy
Show all versions of otp Show documentation
package org.opentripplanner.graph_builder;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.opentripplanner.datastore.api.CompositeDataSource;
import org.opentripplanner.datastore.api.DataSource;
import org.opentripplanner.graph_builder.model.GraphBuilderModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class generates a nice HTML graph import data issue report.
*
* They are created with the help of getHTMLMessage function in {@link DataImportIssue} derived
* classes.
*
* @author mabu
*/
public class DataImportIssuesToHTML implements GraphBuilderModule {
private static final Logger LOG = LoggerFactory.getLogger(DataImportIssuesToHTML.class);
//Path to output folder
private final CompositeDataSource reportDirectory;
//If there are more then this number of issues the report are split into multiple files
//This is because browsers aren't made for giant HTML files which can be made with 500k lines
private final int maxNumberOfIssuesPerFile;
//This counts all occurrences of HTML issue type
//If one issue type is split into two files it has two entries in this Multiset
//IT is used to show numbers in HTML files name and links
private final Multiset issueTypeOccurrences = HashMultiset.create();
//List of writers which are used for actual writing issues to HTML
private final List writers = new ArrayList<>();
//Key is classname, value is issue message
//Multimap because there are multiple issues for each classname
private final Multimap issues = ArrayListMultimap.create();
private final DataImportIssueStore issueStore;
public DataImportIssuesToHTML(
DataImportIssueStore issueStore,
CompositeDataSource reportDirectory,
int maxNumberOfIssuesPerFile
) {
this.issueStore = issueStore;
this.reportDirectory = reportDirectory;
this.maxNumberOfIssuesPerFile = maxNumberOfIssuesPerFile;
}
@Override
public void buildGraph() {
try {
// Delete all files in the report directory if it exist
if (!deleteReportDirectoryAndContent()) {
return;
}
//Groups issues in multimap according to issue type
for (DataImportIssue it : issueStore.getIssues()) {
//writer.println("" + it.getHTMLMessage() + "
");
// writer.println("" + it.getTypeName()+"");
addIssue(it);
}
LOG.info("Creating data import issue log");
//Creates list of HTML writers. Each writer has whole class of HTML issues
//Or multiple HTML writers can have parts of one class of HTML issues if number
// of issues is larger than maxNumberOfIssuesPerFile.
for (Map.Entry> entry : issues.asMap().entrySet()) {
List issueList;
if (entry.getValue() instanceof List) {
issueList = (List) entry.getValue();
} else {
issueList = new ArrayList<>(entry.getValue());
}
addIssues(entry.getKey(), issueList);
}
//Actual writing to the file is made here since
// this is the first place where actual number of files is known (because it depends on
// the issue count)
for (HTMLWriter writer : writers) {
writer.writeFile(issueTypeOccurrences, false);
}
try {
HTMLWriter indexFileWriter = new HTMLWriter("index", (Multimap) null);
indexFileWriter.writeFile(issueTypeOccurrences, true);
} catch (Exception e) {
LOG.error("Index file coudn't be created:{}", e.getMessage());
}
LOG.info("Data import issue logs are in {}", reportDirectory.path());
} catch (Exception e) {
// If the issue report fails due to a remote storage or network problem, then we log
// the error an CONTINUE with the graph build process. Preventing OTP from saving the
// Graph might have much bigger consequences than just failing to save the issue report.
LOG.error("OTP failed to save issue report!", e);
} finally {
closeReportDirectory();
}
}
@Override
public void checkInputs() {}
/**
* Delete report if it exist, and return true if successful. Return {@code false} if the {@code
* reportDirectory} is {@code null} or the directory can NOT be deleted.
*/
private boolean deleteReportDirectoryAndContent() {
if (reportDirectory == null) {
LOG.error("Saving folder is empty!");
return false;
}
if (reportDirectory.exists()) {
//Removes all files from report directory
try {
reportDirectory.delete();
} catch (Exception e) {
LOG.error(
"Failed to clean HTML report directory: " +
reportDirectory.path() +
". HTML report won't be generated!",
e
);
return false;
}
}
// No need to create directories here, because the 'reportDirectory' is responsible for
// creating paths (it they don´t exist) when saving files.
return true;
}
/**
* Creates file with given type of issues
*
* If number of issues is larger then 'maxNumberOfIssuesPerFile' multiple files are generated. And
* named issueClassName1,2,3 etc.
*
* @param issueTypeName name of import data issue class and then also filename
* @param issues list of all import data issue with that class
*/
private void addIssues(String issueTypeName, List issues) {
HTMLWriter file_writer;
if (issues.size() > 1.2 * maxNumberOfIssuesPerFile) {
LOG.debug("Number of issues is very large. Splitting: {}", issueTypeName);
List> partitions = Lists.partition(issues, maxNumberOfIssuesPerFile);
for (List partition : partitions) {
issueTypeOccurrences.add(issueTypeName);
int labelCount = issueTypeOccurrences.count(issueTypeName);
file_writer = new HTMLWriter(issueTypeName + labelCount, partition);
writers.add(file_writer);
}
} else {
issueTypeOccurrences.add(issueTypeName);
int labelCount = issueTypeOccurrences.count(issueTypeName);
file_writer = new HTMLWriter(issueTypeName + labelCount, issues);
writers.add(file_writer);
}
}
/**
* Groups issues according to issue type, using the classname as type name.
*
* All issues are saved together in multimap where key is issue classname and values are list of
* issue with that class
*/
private void addIssue(DataImportIssue issue) {
issues.put(issue.getType(), issue.getHTMLMessage());
}
private void closeReportDirectory() {
try {
reportDirectory.close();
} catch (IOException e) {
LOG.warn(
"Failed to close report directory: {}, details: {}. ",
reportDirectory.path(),
e.getLocalizedMessage(),
e
);
}
}
class HTMLWriter {
private final DataSource target;
private final Multimap writerIssues;
private final String issueTypeName;
HTMLWriter(String key, Collection issues) {
LOG.debug("Making file: {}", key);
this.target = reportDirectory.entry(key + ".html");
this.writerIssues = ArrayListMultimap.create();
this.writerIssues.putAll(key, issues);
this.issueTypeName = key;
}
HTMLWriter(String filename, Multimap curMap) {
LOG.debug("Making file: {}", filename);
this.target = reportDirectory.entry(filename + ".html");
this.writerIssues = curMap;
this.issueTypeName = filename;
}
private void writeFile(Multiset classes, boolean isIndexFile) {
try (
PrintWriter out = new PrintWriter(target.asOutputStream(), true, StandardCharsets.UTF_8)
) {
out.println("Graph report for OTP Graph ");
out.println("\t");
out.println("");
out.println("");
out.println(
""
);
String css =
"\t\t\n" +
"";
out.println(css);
out.println("");
out.println(
String.format("OpenTripPlanner data import issue log for %s
", issueTypeName)
);
out.println("Graph report for graph.obj
");
out.println("");
//adds links to the other HTML files
for (Multiset.Entry htmlIssueType : classes.entrySet()) {
String label_name = htmlIssueType.getElement();
String label;
int currentCount = 1;
//it needs to add link to every file even if they are split
while (currentCount <= htmlIssueType.getCount()) {
label = label_name + currentCount;
if (label.equals(issueTypeName)) {
out.printf(
"%n",
label_name.toLowerCase(),
IssueColors.rgb(label_name),
label
);
} else {
out.printf(
"%s%n",
label_name.toLowerCase(),
label,
IssueColors.rgb(label_name),
label
);
}
currentCount++;
}
}
out.println("
");
if (!isIndexFile) {
out.println("");
writeIssues(out);
out.println("
");
}
out.println("");
}
}
/**
* Writes issues as LI html elements
*/
private void writeIssues(PrintWriter out) {
String FMT = "%s ";
for (Map.Entry it : writerIssues.entries()) {
out.printf(FMT, it.getValue());
}
}
}
}