net.dankito.utils.html.FormattingVisitor Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of java-utils Show documentation
Some basic utils needed in many projects
The newest version!
package net.dankito.utils.html;

import org.jsoup.internal.StringUtil;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.NodeVisitor;

/**
 * HTML to plain-text. This example program demonstrates the use of jsoup to convert HTML input to lightly-formatted
 * plain-text. That is divergent from the general goal of jsoup's .text() methods, which is to get clean data from a
 * scrape.
 * 
 * Note that this is a fairly simplistic formatter -- for real world use you'll want to embrace and extend.
 * 
 * 
 * To invoke from the command line, assuming you've downloaded the jsoup jar to your current directory:
 * java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]
 * where url is the URL to fetch, and selector is an optional CSS selector.
 *
 * @author Jonathan Hedley, [email protected]
 *
 * This code has been copied from https://github.com/jhy/jsoup/blob/master/src/main/java/org/jsoup/examples/HtmlToPlainText.java
 */
public class FormattingVisitor implements NodeVisitor {
    private static final int maxWidth = 80;
    private int width = 0;
    private StringBuilder accum = new StringBuilder(); // holds the accumulated text

    // hit when the node is first seen
    public void head(Node node, int depth) {
        String name = node.nodeName();
        if (node instanceof TextNode)
            append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
        else if (name.equals("li"))
            append("\n * ");
        else if (name.equals("dt"))
            append("  ");
        else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr"))
            append("\n");
    }

    // hit when all of the node's children (if any) have been visited
    public void tail(Node node, int depth) {
        String name = node.nodeName();
        if (StringUtil.in(name, "br", "dd", "dt", "p", "h1", "h2", "h3", "h4", "h5"))
            append("\n");
        else if (name.equals("a"))
            append(String.format(" <%s>", node.absUrl("href")));
    }

    // appends text to the string builder with a simple word wrap method
    private void append(String text) {
        if (text.startsWith("\n"))
            width = 0; // reset counter if starts with a newline. only from formats above, not in natural text
        if (text.equals(" ") &&
            (accum.length() == 0 || StringUtil.in(accum.substring(accum.length() - 1), " ", "\n")))
            return; // don't accumulate long runs of empty spaces

        if (text.length() + width > maxWidth) { // won't fit, needs to wrap
            String[] words = text.split("\\s+");
            for (int i = 0; i < words.length; i++) {
                String word = words[i];
                boolean last = i == words.length - 1;
                if (!last) // insert a space if not the last word
                    word = word + " ";
                if (word.length() + width > maxWidth) { // wrap and reset counter
                    accum.append("\n").append(word);
                    width = word.length();
                } else {
                    accum.append(word);
                    width += word.length();
                }
            }
        } else { // fits as is, without need to wrap text
            accum.append(text);
            width += text.length();
        }
    }

    @Override
    public String toString() {
        return accum.toString();
    }

}