com.simiacryptus.util.io.MarkdownNotebookOutput Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of java-util Show documentation
Show all versions of java-util Show documentation
Miscellaneous Utilities (Pure Java)
/*
* Copyright (c) 2018 by Andrew Charneski.
*
* The author licenses this file to you under the
* Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance
* with the License. You may obtain a copy
* of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.simiacryptus.util.io;
import com.simiacryptus.util.FileNanoHTTPD;
import com.simiacryptus.util.TableOutput;
import com.simiacryptus.util.Util;
import com.simiacryptus.util.lang.CodeUtil;
import com.simiacryptus.util.lang.TimedResult;
import com.simiacryptus.util.lang.UncheckedSupplier;
import com.simiacryptus.util.test.SysOutInterceptor;
import com.vladsch.flexmark.Extension;
import com.vladsch.flexmark.ext.escaped.character.EscapedCharacterExtension;
import com.vladsch.flexmark.ext.gfm.strikethrough.SubscriptExtension;
import com.vladsch.flexmark.ext.tables.TablesExtension;
import com.vladsch.flexmark.html.HtmlRenderer;
import com.vladsch.flexmark.parser.Parser;
import com.vladsch.flexmark.pdf.converter.PdfConverterExtension;
import com.vladsch.flexmark.util.options.MutableDataSet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* The type Markdown notebook output.
*/
public class MarkdownNotebookOutput implements NotebookOutput {
/**
* The Logger.
*/
static final Logger log = LoggerFactory.getLogger(com.simiacryptus.util.io.MarkdownNotebookOutput.class);
/**
* The constant MAX_OUTPUT.
*/
public static int MAX_OUTPUT = 4 * 1024;
private static int excerptNumber = 0;
private static int imageNumber = 0;
@javax.annotation.Nonnull
private final File reportFile;
private final String name;
@javax.annotation.Nonnull
private final PrintStream primaryOut;
private final List markdownData = new ArrayList<>();
private final List> onComplete = new ArrayList<>();
private final Map frontMatter = new HashMap<>();
private final FileNanoHTTPD httpd;
private final boolean autobrowse;
private int maxImageSize = 1600;
/**
* The Toc.
*/
@javax.annotation.Nonnull
public List toc = new ArrayList<>();
/**
* The Anchor.
*/
int anchor = 0;
@Nullable
private final String baseCodeUrl = CodeUtil.getGitBase();
/**
* Instantiates a new Markdown notebook output.
*
* @param reportFile the file name
* @param name the name
* @throws FileNotFoundException the file not found exception
*/
public MarkdownNotebookOutput(@Nonnull final File reportFile, final String name, boolean autobrowse) throws FileNotFoundException {this(reportFile, name, new Random().nextInt(2 * 1024) + 2 * 1024, autobrowse);}
/**
* Instantiates a new Markdown notebook output.
*
* @param reportFile the file name
* @param name the name
* @param httpPort the http port
* @param autobrowse
* @throws FileNotFoundException the file not found exception
*/
public MarkdownNotebookOutput(@javax.annotation.Nonnull final File reportFile, final String name, final int httpPort, final boolean autobrowse) throws FileNotFoundException {
this.name = name;
reportFile.getAbsoluteFile().getParentFile().mkdirs();
primaryOut = new PrintStream(new FileOutputStream(reportFile));
this.reportFile = reportFile;
this.httpd = new FileNanoHTTPD(reportFile.getParentFile(), httpPort);
this.httpd.addHandler("", "text/html", out -> {
try {
writeHtmlAndPdf(getRoot(), testName());
try (FileInputStream input = new FileInputStream(new File(getRoot(), testName() + ".html"))) {
IOUtils.copy(input, out);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
this.httpd.addHandler("pdf", "application/pdf", out -> {
try {
writeHtmlAndPdf(getRoot(), testName());
try (FileInputStream input = new FileInputStream(new File(getRoot(), testName() + ".pdf"))) {
IOUtils.copy(input, out);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
this.autobrowse = autobrowse;
try {
log.info("Starting server at port " + httpPort);
this.httpd.init();
if (!GraphicsEnvironment.isHeadless() && Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) {
new Thread(() -> {
try {
while (!httpd.isAlive()) Thread.sleep(100);
if (isAutobrowse()) Desktop.getDesktop().browse(new URI(String.format("http://localhost:%d", httpPort)));
} catch (InterruptedException | IOException | URISyntaxException e) {
e.printStackTrace();
}
}).start();
onComplete(file -> {
try {
if (isAutobrowse()) Desktop.getDesktop().browse(new File(file, testName() + ".html").toURI());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
onComplete(file -> {
httpd.stop();
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Wrap frontmatter consumer.
*
* @param fn the fn
* @return the consumer
*/
public static Consumer wrapFrontmatter(@Nonnull final Consumer fn) {
return log -> {
@Nonnull TimedResult time = TimedResult.time(() -> {
try {
fn.accept(log);
log.setFrontMatterProperty("result", "OK");
} catch (Throwable e) {
log.setFrontMatterProperty("result", getExceptionString(e).toString().replaceAll("\n", "
").trim());
throw (RuntimeException) (e instanceof RuntimeException ? e : new RuntimeException(e));
}
});
log.setFrontMatterProperty("execution_time", String.format("%.6f", time.timeNanos / 1e9));
};
}
/**
* Get markdown notebook output.
*
* @return the markdown notebook output
* @param autobrowse
*/
public static com.simiacryptus.util.io.MarkdownNotebookOutput get(final boolean autobrowse) {
try {
final StackTraceElement callingFrame = Thread.currentThread().getStackTrace()[2];
final String className = callingFrame.getClassName();
final String methodName = callingFrame.getMethodName();
@javax.annotation.Nonnull final String fileName = methodName + ".md";
@javax.annotation.Nonnull File path = new File(Util.mkString(File.separator, "reports", className.replaceAll("\\.", "/").replaceAll("\\$", "/")));
path = new File(path, fileName);
path.getParentFile().mkdirs();
return new com.simiacryptus.util.io.MarkdownNotebookOutput(path, methodName, autobrowse);
} catch (@javax.annotation.Nonnull final FileNotFoundException e) {
throw new RuntimeException(e);
}
}
/**
* Get markdown notebook output.
*
* @param source the source
* @param autobrowse
* @return the markdown notebook output
*/
public static MarkdownNotebookOutput get(Object source, final boolean autobrowse) {
try {
StackTraceElement callingFrame = Thread.currentThread().getStackTrace()[2];
String className = null == source ? callingFrame.getClassName() : source.getClass().getCanonicalName();
String methodName = callingFrame.getMethodName();
CharSequence fileName = methodName + ".md";
File path = new File(Util.mkString(File.separator, "reports", className.replaceAll("\\.", "/").replaceAll("\\$", "/"), fileName));
path.getParentFile().mkdirs();
return new MarkdownNotebookOutput(path, methodName, autobrowse);
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
/**
* Gets exception string.
*
* @param e the e
* @return the exception string
*/
@Nonnull
public static CharSequence getExceptionString(Throwable e) {
if (e instanceof RuntimeException && e.getCause() != null && e.getCause() != e)
return getExceptionString(e.getCause());
if (e.getCause() != null && e.getCause() != e)
return e.getClass().getSimpleName() + " / " + getExceptionString(e.getCause());
return e.getClass().getSimpleName();
}
/**
* On complete markdown notebook output.
*
* @param tasks the tasks
* @return the markdown notebook output
*/
@Nonnull
public MarkdownNotebookOutput onComplete(Consumer... tasks) {
Arrays.stream(tasks).forEach(onComplete::add);
return this;
}
@Override
public void close() throws IOException {
if (null != primaryOut) {
primaryOut.close();
}
try (@javax.annotation.Nonnull PrintWriter out = new PrintWriter(new FileOutputStream(reportFile))) {
writeMarkdownWithFrontmatter(out);
}
File root = getRoot();
writeHtmlAndPdf(root, testName());
writeZip(root, testName());
onComplete.stream().forEach(fn -> {
try {
fn.accept(root);
} catch (Throwable e) {
log.info("Error closing log", e);
}
});
}
/**
* Gets root.
*
* @return the root
*/
@Nonnull
public File getRoot() {
return new File(reportFile.getParent());
}
/**
* Test name string.
*
* @return the string
*/
public String testName() {
String[] split = reportFile.getName().split(".");
return 0 == split.length ? reportFile.getName() : split[0];
}
/**
* Write zip.
*
* @param root the root
* @param baseName the base name
* @throws IOException the io exception
*/
public void writeZip(final File root, final String baseName) throws IOException {
try (@Nonnull ZipOutputStream out = new ZipOutputStream(new FileOutputStream(new File(root, baseName + ".zip")))) {
writeArchive(root, root, out, file -> !file.getName().equals(baseName + ".zip") && !file.getName().equals(baseName + ".pdf"));
}
}
/**
* Path to code file path.
*
* @param baseFile
* @param file the file
* @return the path
* @throws IOException the io exception
*/
public static Path pathToCodeFile(final File baseFile, @Nonnull File file) throws IOException {
return baseFile.getCanonicalFile().toPath().relativize(file.getCanonicalFile().toPath());
}
/**
* To string string.
*
* @param list the list
* @return the string
*/
@Nonnull
public String toString(final List list) {
if (list.size() > 0 && list.stream().allMatch(x -> {
if (x.length() > 1) {
char c = x.charAt(0);
return c == ' ' || c == '\t';
}
return false;
})) return toString(list.stream().map(x -> x.subSequence(1, x.length()).toString()).collect(Collectors.toList()));
else return list.stream().reduce((a, b) -> a + "\n" + b).orElse("").toString();
}
/**
* Write archive.
*
* @param root the root
* @param dir the dir
* @param out the out
* @param filter the filter
*/
public void writeArchive(final File root, final File dir, final ZipOutputStream out, final Predicate filter) {
Arrays.stream(dir.listFiles()).filter(filter).forEach(file ->
{
if (file.isDirectory()) {
writeArchive(root, file, out, filter);
}
else {
String absRoot = root.getAbsolutePath();
String absFile = file.getAbsolutePath();
String relativeFile = absFile.substring(absRoot.length());
if (relativeFile.startsWith(File.separator)) relativeFile = relativeFile.substring(1);
try {
out.putNextEntry(new ZipEntry(relativeFile));
IOUtils.copy(new FileInputStream(file), out);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
});
}
/**
* Write markdown with frontmatter.
*
* @param out the out
*/
public void writeMarkdownWithFrontmatter(final PrintWriter out) {
if (!frontMatter.isEmpty()) {
out.println("---");
frontMatter.forEach((key, value) -> {
CharSequence escaped = StringEscapeUtils.escapeJson(String.valueOf(value))
.replaceAll("\n", " ")
.replaceAll(":", ":")
.replaceAll("\\{", "\\{")
.replaceAll("\\}", "\\}");
out.println(String.format("%s: %s", key, escaped));
});
out.println("---");
}
toc.forEach(out::println);
out.print("\n\n");
markdownData.forEach(out::println);
}
public void setFrontMatterProperty(CharSequence key, CharSequence value) {
frontMatter.put(key, value);
}
@Override
public CharSequence getFrontMatterProperty(CharSequence key) {
return frontMatter.get(key);
}
@Override
public CharSequence getName() {
return name;
}
/**
* Anchor string.
*
* @param anchorId the anchor id
* @return the string
*/
public CharSequence anchor(CharSequence anchorId) {
return String.format("", anchorId);
}
/**
* Anchor id string.
*
* @return the string
*/
public CharSequence anchorId() {
return String.format("p-%d", anchor++);
}
/**
* Write html and pdf.
*
* @param root the root
* @param baseName the base name
* @throws IOException the io exception
*/
public void writeHtmlAndPdf(final File root, final String baseName) throws IOException {
MutableDataSet options = new MutableDataSet();
List extensions = Arrays.asList(
TablesExtension.create(),
SubscriptExtension.create(),
EscapedCharacterExtension.create()
);
Parser parser = Parser.builder(options).extensions(extensions).build();
HtmlRenderer renderer = HtmlRenderer.builder(options).extensions(extensions).escapeHtml(false).indentSize(2).softBreak("\n").build();
File htmlFile = new File(root, baseName + ".html");
String html = renderer.render(parser.parse(toString(toc) + "\n\n" + toString(markdownData)));
html = "" + html + "";
try (FileOutputStream out = new FileOutputStream(htmlFile)) {
IOUtils.write(html, out, Charset.forName("UTF-8"));
}
try (FileOutputStream out = new FileOutputStream(new File(root, baseName + ".pdf"))) {
PdfConverterExtension.exportToPdf(out, html, htmlFile.getAbsoluteFile().toURI().toString(), options);
}
}
@javax.annotation.Nonnull
@Override
public OutputStream file(@javax.annotation.Nonnull final CharSequence name) {
try {
return new FileOutputStream(new File(getResourceDir(), name.toString()));
} catch (@javax.annotation.Nonnull final FileNotFoundException e) {
throw new RuntimeException(e);
}
}
@javax.annotation.Nonnull
@Override
public String file(final CharSequence data, final CharSequence caption) {
return file(data, ++com.simiacryptus.util.io.MarkdownNotebookOutput.excerptNumber + ".txt", caption);
}
@javax.annotation.Nonnull
@Override
public CharSequence file(@javax.annotation.Nonnull byte[] data, @javax.annotation.Nonnull CharSequence filename, CharSequence caption) {
return file(new String(data, Charset.forName("UTF-8")), filename, caption);
}
@javax.annotation.Nonnull
@Override
public String file(@Nullable final CharSequence data, @javax.annotation.Nonnull final CharSequence fileName, final CharSequence caption) {
try {
if (null != data) {
IOUtils.write(data, new FileOutputStream(new File(getResourceDir(), fileName.toString())), Charset.forName("UTF-8"));
}
} catch (@javax.annotation.Nonnull final IOException e) {
throw new RuntimeException(e);
}
return "[" + caption + "](etc/" + fileName + ")";
}
/**
* Gets resource dir.
*
* @return the resource dir
*/
@javax.annotation.Nonnull
public File getResourceDir() {
@javax.annotation.Nonnull final File etc = new File(reportFile.getParentFile(), "etc").getAbsoluteFile();
etc.mkdirs();
return etc;
}
@Override
public void h1(@javax.annotation.Nonnull final CharSequence fmt, final Object... args) {
CharSequence anchorId = anchorId();
@javax.annotation.Nonnull CharSequence msg = format(fmt, args);
toc.add(String.format("1. [%s](#%s)", msg, anchorId));
out("# " + anchor(anchorId) + msg);
}
@Override
public void h2(@javax.annotation.Nonnull final CharSequence fmt, final Object... args) {
CharSequence anchorId = anchorId();
@javax.annotation.Nonnull CharSequence msg = format(fmt, args);
toc.add(String.format(" 1. [%s](#%s)", msg, anchorId));
out("## " + anchor(anchorId) + fmt, args);
}
@Override
public void h3(@javax.annotation.Nonnull final CharSequence fmt, final Object... args) {
CharSequence anchorId = anchorId();
@javax.annotation.Nonnull CharSequence msg = format(fmt, args);
toc.add(String.format(" 1. [%s](#%s)", msg, anchorId));
out("### " + anchor(anchorId) + fmt, args);
}
@javax.annotation.Nonnull
@Override
public String image(@Nullable final BufferedImage rawImage, final CharSequence caption) {
if (null == rawImage) return "";
new ByteArrayOutputStream();
final int thisImage = ++com.simiacryptus.util.io.MarkdownNotebookOutput.imageNumber;
@javax.annotation.Nonnull final String fileName = name + "." + thisImage + ".png";
@javax.annotation.Nonnull final File file = new File(getResourceDir(), fileName);
@Nullable final BufferedImage stdImage = Util.maximumSize(rawImage, getMaxImageSize());
try {
if (stdImage != rawImage) {
@javax.annotation.Nonnull final String rawName = name + "_raw." + thisImage + ".png";
ImageIO.write(rawImage, "png", new File(getResourceDir(), rawName));
}
ImageIO.write(stdImage, "png", file);
} catch (IOException e) {
throw new RuntimeException(e);
}
return anchor(anchorId()) + "![" + caption + "](etc/" + file.getName() + ")";
}
@Override
@SuppressWarnings("unchecked")
public T code(@javax.annotation.Nonnull final UncheckedSupplier fn, final int maxLog, final int framesNo) {
try {
final StackTraceElement callingFrame = Thread.currentThread().getStackTrace()[framesNo];
final String sourceCode = CodeUtil.getInnerText(callingFrame);
@javax.annotation.Nonnull final SysOutInterceptor.LoggedResult> result = SysOutInterceptor.withOutput(() -> {
long priorGcMs = ManagementFactory.getGarbageCollectorMXBeans().stream().mapToLong(x -> x.getCollectionTime()).sum();
final long start = System.nanoTime();
try {
@Nullable Object result1 = null;
try {
result1 = fn.get();
} catch (@javax.annotation.Nonnull final RuntimeException e) {
throw e;
} catch (@javax.annotation.Nonnull final Exception e) {
throw new RuntimeException(e);
}
long gcTime = ManagementFactory.getGarbageCollectorMXBeans().stream().mapToLong(x -> x.getCollectionTime()).sum() - priorGcMs;
return new TimedResult