Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.eclipse.jdt.internal.formatter.CommentsPreparator Maven / Gradle / Ivy
/*******************************************************************************
* Copyright (c) 2014, 2016 Mateusz Matela and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Mateusz Matela - [formatter] Formatter does not format Java code correctly, especially when max line width is set - https://bugs.eclipse.org/303519
* Mateusz Matela - [formatter] IndexOutOfBoundsException in TokenManager - https://bugs.eclipse.org/462945
* Mateusz Matela - [formatter] Bad line breaking in Eclipse javadoc comments - https://bugs.eclipse.org/348338
* Lars Vogel - Contributions for
* Bug 473178
*******************************************************************************/
package org.eclipse.jdt.internal.formatter;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameNotAToken;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameStringLiteral;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameWHITESPACE;
import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNamepackage;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.eclipse.jdt.core.dom.ASTNode;
import org.eclipse.jdt.core.dom.ASTVisitor;
import org.eclipse.jdt.core.dom.BlockComment;
import org.eclipse.jdt.core.dom.Javadoc;
import org.eclipse.jdt.core.dom.LineComment;
import org.eclipse.jdt.core.dom.MemberRef;
import org.eclipse.jdt.core.dom.MethodRef;
import org.eclipse.jdt.core.dom.QualifiedName;
import org.eclipse.jdt.core.dom.TagElement;
import org.eclipse.jdt.core.formatter.CodeFormatter;
import org.eclipse.jdt.core.formatter.DefaultCodeFormatterConstants;
import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
import org.eclipse.jdt.internal.compiler.parser.ScannerHelper;
import org.eclipse.jdt.internal.formatter.Token.WrapMode;
import org.eclipse.jdt.internal.formatter.Token.WrapPolicy;
public class CommentsPreparator extends ASTVisitor {
public static final int COMMENT_LINE_SEPARATOR_LENGTH = 3;
private final static Pattern NLS_TAG_PATTERN = Pattern.compile("//\\$NON-NLS-([0-9]+)\\$"); //$NON-NLS-1$
private final static Pattern STRING_LITERAL_PATTERN = Pattern.compile("\".*?(\\\\(\\\\\\\\)*\".*?)*\""); //$NON-NLS-1$
private final static Pattern HTML_TAG_PATTERN;
private final static Pattern HTML_ATTRIBUTE_PATTERN;
static {
String formatCodeTags = "(pre)"; //$NON-NLS-1$
String separateLineTags = "(dl|hr|nl|p|ul|ol|table|tr)"; //$NON-NLS-1$
String breakBeforeTags = "(dd|dt|li|td|th|h1|h2|h3|h4|h5|h6|q)"; //$NON-NLS-1$
String breakAfterTags = "(br)"; //$NON-NLS-1$
String noFormatTags = "(code|tt)"; //$NON-NLS-1$
String otherTags = "([^<>&&\\S]++)"; //$NON-NLS-1$
String ws = "(?>[ \\t]++|[\\r\\n]++[ \\t]*+\\*?)"; // whitespace or line break with optional asterisk //$NON-NLS-1$
String attributeValue = "(?>\"[^\"]*\")|(?>\'[^\']*\')|[^/>\"\'&&\\S]++"; //$NON-NLS-1$
String attribute = "(?>" + ws + "+[^=&&\\S]+" + ws + "*(=)" + ws + "*(?>" + attributeValue + "))"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
HTML_TAG_PATTERN = Pattern.compile("<(/)?+(?:" //$NON-NLS-1$
+ formatCodeTags + '|' + separateLineTags + '|' + breakBeforeTags + '|' + breakAfterTags + '|' + noFormatTags + '|' + otherTags + ')'
+ "(" + attribute + "*)" + ws + "*/?>", Pattern.CASE_INSENSITIVE); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
HTML_ATTRIBUTE_PATTERN = Pattern.compile(attribute);
}
private final static Pattern HTML_ENTITY_PATTERN = Pattern
.compile("&(#x[0-9a-fA-F]+)?(#[0-9]+)?(lt)?(gt)?(nbsp)?(amp)?(circ)?(tilde)?(quot)?;"); //$NON-NLS-1$
private final static String HTML_ENTITY_REPLACE = " <> &^~\""; //$NON-NLS-1$
// Param tags list copied from IJavaDocTagConstants in legacy formatter for compatibility.
// There were the following comments:
// TODO (frederic) should have another name than 'param' for the following tags
// TODO (frederic) investigate how and why this list was created
private final static List PARAM_TAGS = Arrays.asList(
TagElement.TAG_PARAM,
TagElement.TAG_EXCEPTION,
TagElement.TAG_SERIALFIELD,
TagElement.TAG_THROWS);
private final static List IMMUTABLE_TAGS = Arrays.asList(TagElement.TAG_CODE, TagElement.TAG_LITERAL);
private final TokenManager tm;
private final DefaultCodeFormatterOptions options;
private final String sourceLevel;
private final String formatDisableTag;
private final String formatEnableTag;
private Token lastLineComment;
private int lastLineCommentPosition;
private Token lastFormatOffComment;
private TokenManager ctm;
private List commentStructure;
private int commentIndent;
/** Index: position within current comment; Value: whether wrapping on special characters is allowed */
private boolean[] allowSubstituteWrapping;
private int noFormatTagOpenStart = -1;
private int formatCodeTagOpenEnd = -1;
private int lastFormatCodeClosingTagIndex = -1;
private Token firstTagToken;
private DefaultCodeFormatter commentCodeFormatter;
public CommentsPreparator(TokenManager tm, DefaultCodeFormatterOptions options, String sourceLevel) {
this.tm = tm;
this.options = options;
this.sourceLevel = sourceLevel;
this.formatDisableTag = options.disabling_tag != null ? new String(options.disabling_tag) : null;
this.formatEnableTag = options.enabling_tag != null ? new String(options.enabling_tag) : null;
}
@Override
public boolean preVisit2(ASTNode node) {
boolean isMalformed = (node.getFlags() & ASTNode.MALFORMED) != 0;
return !isMalformed;
}
@Override
public boolean visit(LineComment node) {
int commentIndex = this.tm.firstIndexIn(node, TokenNameCOMMENT_LINE);
handleLineComment(commentIndex);
return true;
}
public void handleLineComment(int commentIndex) {
Token commentToken = this.tm.get(commentIndex);
boolean isOnFirstColumn = handleWhitespaceAround(commentIndex);
if (handleFormatOnOffTags(commentToken))
return;
if (isOnFirstColumn) {
if (this.options.comment_format_line_comment
&& !this.options.comment_format_line_comment_starting_on_first_column) {
this.lastLineComment = null;
commentToken.setIndent(0);
commentToken.setWrapPolicy(null);
return;
}
if (this.options.never_indent_line_comments_on_first_column) {
commentToken.setIndent(0);
commentToken.setWrapPolicy(null);
}
}
handleNLSTags(commentToken, commentIndex);
int positionInLine = this.tm.findSourcePositionInLine(commentToken.originalStart);
boolean isContinuation = commentIndex > 0 && this.tm.get(commentIndex - 1) == this.lastLineComment
&& (positionInLine >= this.lastLineCommentPosition - this.options.indentation_size + 1)
&& this.tm.countLineBreaksBetween(this.lastLineComment, commentToken) == 1;
boolean isHeader = this.tm.isInHeader(commentIndex);
boolean formattingEnabled = (this.options.comment_format_line_comment && !isHeader)
|| (this.options.comment_format_header && isHeader);
if (!formattingEnabled) {
preserveWhitespace(commentToken, commentIndex);
if (isContinuation) {
WrapPolicy policy = this.lastLineComment.getWrapPolicy();
if (policy == null) {
int lineStart = this.tm.getPositionInLine(this.tm.findFirstTokenInLine(commentIndex - 1));
int commentStart = this.tm.getPositionInLine(commentIndex - 1);
policy = new WrapPolicy(WrapMode.WHERE_NECESSARY, commentIndex - 1, commentStart - lineStart);
}
commentToken.setWrapPolicy(policy);
this.lastLineComment = commentToken;
} else if (commentToken.getLineBreaksBefore() == 0) {
this.lastLineComment = commentToken;
this.lastLineCommentPosition = positionInLine;
}
return;
}
List structure = tokenizeLineComment(commentToken);
if (isContinuation) {
Token first = structure.get(0);
first.breakBefore();
first.setWrapPolicy(
new WrapPolicy(WrapMode.WHERE_NECESSARY, commentIndex - 1, this.lastLineCommentPosition));
// merge previous and current line comment
Token previous = this.lastLineComment;
Token merged = new Token(previous, previous.originalStart, commentToken.originalEnd, previous.tokenType);
this.tm.remove(commentIndex - 1);
this.tm.insert(commentIndex - 1, merged);
this.tm.remove(commentIndex);
List lastStructure = this.lastLineComment.getInternalStructure();
lastStructure.addAll(structure);
merged.setInternalStructure(lastStructure);
this.lastLineComment = merged;
} else {
commentToken.setInternalStructure(structure);
handleCompilerTags(commentToken, commentIndex);
preserveWhitespace(commentToken, commentIndex);
this.lastLineComment = commentToken;
this.lastLineCommentPosition = positionInLine;
}
}
private void preserveWhitespace(Token commentToken, int commentIndex) {
if (this.options.comment_preserve_white_space_between_code_and_line_comments
&& commentToken.getLineBreaksBefore() == 0 && commentIndex > 0) {
commentToken.clearSpaceBefore();
List structure = commentToken.getInternalStructure();
if (structure != null && !structure.isEmpty())
structure.get(0).clearSpaceBefore();
Token previous = this.tm.get(commentIndex - 1);
previous.clearSpaceAfter();
if (previous.originalEnd + 1 >= commentToken.originalStart)
return;
if (structure == null || structure.isEmpty()) {
structure = new ArrayList<>();
structure.add(new Token(previous.originalEnd + 1, commentToken.originalEnd, TokenNameCOMMENT_LINE));
commentToken.setInternalStructure(structure);
} else {
structure.add(0, new Token(previous.originalEnd + 1, commentToken.originalStart - 1,
TokenNameWHITESPACE));
}
}
}
/**
* @return true if the comment contains on/off tag and should not be formatted
*/
private boolean handleFormatOnOffTags(Token commentToken) {
if (!this.options.use_tags)
return false;
String commentString = this.tm.toString(commentToken);
int offIndex = this.formatDisableTag != null ? commentString.lastIndexOf(this.formatDisableTag) : -1;
int onIndex = this.formatEnableTag != null ? commentString.lastIndexOf(this.formatEnableTag) : -1;
if (this.lastFormatOffComment == null) {
if (offIndex > onIndex)
this.lastFormatOffComment = commentToken;
} else {
if (onIndex > offIndex) {
this.tm.addDisableFormatTokenPair(this.lastFormatOffComment, commentToken);
this.lastFormatOffComment = null;
}
}
return offIndex >= 0 || onIndex >= 0;
}
private void handleNLSTags(Token comment, int commentIndex) {
List stringLiterals = findStringLiteralsInLine(commentIndex);
if (stringLiterals.isEmpty())
return;
List commentFragments = new ArrayList<>();
Matcher matcher = NLS_TAG_PATTERN.matcher(this.tm.toString(comment));
int previousMatcherEnd = 0;
boolean nlsFound = false;
while (matcher.find()) {
int nlsNumber = Integer.parseInt(matcher.group(1));
if (nlsNumber > 0 && nlsNumber <= stringLiterals.size()) {
if (matcher.start() > previousMatcherEnd) {
Token fragment = new Token(comment.originalStart + previousMatcherEnd,
comment.originalStart + matcher.start() - 1, TokenNameCOMMENT_LINE);
commentFragments.add(fragment);
}
Token nlsTag = new Token(comment.originalStart + matcher.start(),
comment.originalStart + matcher.end() - 1, TokenNameCOMMENT_LINE);
stringLiterals.get(nlsNumber - 1).setNLSTag(nlsTag);
nlsTag.setNLSTag(stringLiterals.get(nlsNumber - 1));
commentFragments.add(nlsTag);
nlsFound = true;
previousMatcherEnd = matcher.end();
}
}
if (nlsFound) {
comment.setInternalStructure(commentFragments);
if (comment.originalStart + previousMatcherEnd <= comment.originalEnd) {
Token fragment = new Token(comment.originalStart + previousMatcherEnd, comment.originalEnd,
TokenNameCOMMENT_LINE);
commentFragments.add(fragment);
}
}
}
private void handleCompilerTags(Token commentToken, int commentIndex) {
final String commentText = this.tm.toString(commentToken);
final List structure = commentToken.getInternalStructure();
if (commentText.startsWith("//$FALL-THROUGH$") //$NON-NLS-1$
|| commentText.startsWith("//$IDENTITY-COMPARISON$")) { //$NON-NLS-1$
structure.get(1).clearSpaceBefore();
}
if (commentText.contains("//$IDENTITY-COMPARISON$")) { //$NON-NLS-1$
// make sure the whole line is not broken
Token token = commentToken;
for (int i = commentIndex; i > 0; i--) {
Token left = this.tm.get(i - 1);
if (this.tm.countLineBreaksBetween(left, token) > 0)
break;
token.clearLineBreaksBefore();
left.clearLineBreaksAfter();
token.setWrapPolicy(WrapPolicy.DISABLE_WRAP);
token = left;
}
}
}
private List findStringLiteralsInLine(int lastTokenIndex) {
List stringLiterals = new ArrayList<>();
Token previous = this.tm.get(lastTokenIndex);
for (int i = lastTokenIndex - 1; i >= 0; i--) {
Token token = this.tm.get(i);
if (this.tm.countLineBreaksBetween(token, previous) > 0)
break;
if (token.tokenType == TokenNameStringLiteral)
stringLiterals.add(token);
previous = token;
}
Collections.reverse(stringLiterals);
return stringLiterals;
}
private List tokenizeLineComment(Token commentToken) {
List fragments = commentToken.getInternalStructure();
if (fragments == null) {
fragments = Arrays.asList(commentToken);
}
ArrayList result = new ArrayList<>();
for (int i = 0; i < fragments.size(); i++) {
Token token = fragments.get(i);
if (token.hasNLSTag()) {
if (ScannerHelper.isWhitespace(this.tm.charAt(token.originalStart - 1)))
token.spaceBefore();
result.add(token);
continue;
}
int sourcePosition = token.originalStart;
if (sourcePosition == commentToken.originalStart) {
// separate starting slashes
while (sourcePosition <= token.originalEnd && this.tm.charAt(sourcePosition) == '/')
sourcePosition++;
result.add(new Token(commentToken.originalStart, sourcePosition - 1, TokenNameCOMMENT_LINE));
}
int tokenStart = sourcePosition;
while (sourcePosition <= token.originalEnd + 1) {
if (sourcePosition == token.originalEnd + 1
|| ScannerHelper.isWhitespace(this.tm.charAt(sourcePosition))) {
if (tokenStart < sourcePosition) {
Token outputToken = new Token(tokenStart, sourcePosition - 1, TokenNameCOMMENT_LINE);
outputToken.spaceBefore();
result.add(outputToken);
}
tokenStart = sourcePosition + 1;
}
sourcePosition++;
}
}
return result;
}
@Override
public boolean visit(BlockComment node) {
int commentIndex = this.tm.firstIndexIn(node, TokenNameCOMMENT_BLOCK);
handleBlockComment(commentIndex);
return true;
}
public void handleBlockComment(int commentIndex) {
Token commentToken = this.tm.get(commentIndex);
boolean isFirstColumn = handleWhitespaceAround(commentIndex);
if (handleFormatOnOffTags(commentToken))
return;
boolean isHeader = this.tm.isInHeader(commentIndex);
boolean formattingEnabled = (this.options.comment_format_block_comment && !isHeader)
|| (this.options.comment_format_header && isHeader);
formattingEnabled = formattingEnabled && this.tm.charAt(commentToken.originalStart + 2) != '-';
if (formattingEnabled && tokenizeMultilineComment(commentToken)) {
this.commentStructure = commentToken.getInternalStructure();
this.ctm = new TokenManager(this.commentStructure, this.tm);
handleStringLiterals(this.tm.toString(commentToken), commentToken.originalStart);
addSubstituteWraps();
} else {
commentToken.setInternalStructure(commentToLines(commentToken, -1));
}
if (this.options.never_indent_block_comments_on_first_column && isFirstColumn) {
commentToken.setIndent(0);
commentToken.setWrapPolicy(null);
}
}
private boolean handleWhitespaceAround(int commentIndex) {
Token commentToken = this.tm.get(commentIndex);
char charBefore = commentToken.originalStart > 0 ? this.tm.charAt(commentToken.originalStart - 1) : 0;
if (charBefore == ' ' || charBefore == '\t')
commentToken.spaceBefore();
if (commentToken.originalEnd < this.tm.getSourceLength() - 1) {
char charAfter = this.tm.charAt(commentToken.originalEnd + 1);
if (charAfter == ' ' || charAfter == '\t')
commentToken.spaceAfter();
}
Token previous = null, next = null;
int existingBreaksBefore = 2, existingBreaksAfter = 2;
if (commentIndex > 0) {
previous = this.tm.get(commentIndex - 1);
existingBreaksBefore = this.tm.countLineBreaksBetween(previous, commentToken);
if (existingBreaksBefore > 0) {
commentToken.breakBefore();
commentToken.clearSpaceBefore();
}
}
if (commentIndex < this.tm.size() - 1) {
next = this.tm.get(commentIndex + 1);
existingBreaksAfter = this.tm.countLineBreaksBetween(commentToken, next);
if (existingBreaksAfter > 0)
commentToken.breakAfter();
}
if (existingBreaksBefore <= 1
&& (previous.tokenType == TokenNameCOMMENT_LINE || previous.tokenType == TokenNameCOMMENT_BLOCK)) {
commentToken.setWrapPolicy(previous.getWrapPolicy());
} else {
int i = commentIndex + 2;
while (existingBreaksAfter <= 1 && i < this.tm.size()
&& (next.tokenType == TokenNameCOMMENT_LINE || next.tokenType == TokenNameCOMMENT_BLOCK)) {
Token next2 = this.tm.get(i++);
existingBreaksAfter = this.tm.countLineBreaksBetween(next, next2);
next = next2;
}
if (existingBreaksBefore < existingBreaksAfter && previous != null) {
commentToken.putLineBreaksAfter(previous.getLineBreaksAfter());
previous.clearLineBreaksAfter();
} else if (existingBreaksAfter <= existingBreaksBefore && next != null
&& next.tokenType != TokenNamepackage /* doesn't apply to a comment before the package declaration */) {
commentToken.putLineBreaksBefore(next.getLineBreaksBefore());
next.clearLineBreaksBefore();
}
}
boolean isFirstColumn = (charBefore == '\r' || charBefore == '\n' || commentToken.originalStart == 0);
return isFirstColumn;
}
private List commentToLines(Token commentToken, int commentStartPositionInLine) {
List lines = new ArrayList<>();
int tab = this.options.tab_size;
String commentText = this.tm.toString(commentToken);
int commentStartPosition = commentStartPositionInLine;
if (commentStartPosition < 0)
commentStartPosition = this.tm.findSourcePositionInLine(commentToken.originalStart);
int positionInLine = commentStartPosition;
int lineStart = 0;
int breaksBeforeFirstLine = 0;
boolean firstLine = true; // all lines except first will be NotAToken to disable asterisk adding
boolean emptyLine = true;
for (int i = 0; i < commentText.length(); i++) {
char c = commentText.charAt(i);
switch (c) {
case ' ':
if ((lineStart == i && positionInLine < commentStartPosition)
|| (emptyLine && positionInLine == commentToken.getIndent() - 1))
lineStart = i + 1;
positionInLine++;
break;
case '\t':
if ((lineStart == i && positionInLine < commentStartPosition)
|| (emptyLine && positionInLine == commentToken.getIndent() - 1))
lineStart = i + 1;
if (tab > 0)
positionInLine += tab - positionInLine % tab;
break;
case '\r':
case '\n':
if (lineStart < i) {
Token line = new Token(commentToken.originalStart + lineStart,
commentToken.originalStart + i - 1,
firstLine ? commentToken.tokenType : TokenNameNotAToken);
line.breakAfter();
if (lines.isEmpty())
line.putLineBreaksBefore(breaksBeforeFirstLine);
lines.add(line);
} else if (!lines.isEmpty()) {
Token previousLine = lines.get(lines.size() - 1);
previousLine.putLineBreaksAfter(previousLine.getLineBreaksAfter() + 1);
} else {
breaksBeforeFirstLine++;
}
if (i + 1 < commentText.length() && commentText.charAt(i + 1) == (c == '\r' ? '\n' : '\r'))
i++;
lineStart = i + 1;
positionInLine = 0;
firstLine = false;
emptyLine = true;
break;
default:
positionInLine++;
emptyLine = false;
}
}
if (lineStart < commentText.length()) {
Token line = new Token(commentToken.originalStart + lineStart, commentToken.originalEnd,
firstLine ? commentToken.tokenType : TokenNameNotAToken);
line.setWrapPolicy(WrapPolicy.DISABLE_WRAP);
lines.add(line);
}
return lines;
}
@Override
public boolean visit(Javadoc node) {
this.noFormatTagOpenStart = -1;
this.formatCodeTagOpenEnd = -1;
this.lastFormatCodeClosingTagIndex = -1;
this.firstTagToken = null;
this.ctm = null;
int commentIndex = this.tm.firstIndexIn(node, TokenNameCOMMENT_JAVADOC);
Token commentToken = this.tm.get(commentIndex);
if (node.getParent() == null) {
// not a proper javadoc, treat as block comment
handleWhitespaceAround(commentIndex);
}
if (commentIndex < this.tm.size() - 1)
commentToken.breakAfter();
if (handleFormatOnOffTags(commentToken))
return false;
boolean isHeader = this.tm.isInHeader(commentIndex);
boolean formattingEnabled = (this.options.comment_format_javadoc_comment && !isHeader)
|| (this.options.comment_format_header && isHeader);
if (!formattingEnabled || !tokenizeMultilineComment(commentToken)) {
commentToken.setInternalStructure(commentToLines(commentToken, -1));
return false;
}
this.commentStructure = commentToken.getInternalStructure();
this.commentIndent = this.tm.toIndent(commentToken.getIndent(), true);
this.ctm = new TokenManager(commentToken.getInternalStructure(), this.tm);
return true;
}
@Override
public void endVisit(Javadoc node) {
if (this.ctm == null)
return;
if (this.options.comment_insert_empty_line_before_root_tags && this.firstTagToken != null
&& this.ctm.indexOf(this.firstTagToken) > 1) {
this.firstTagToken.putLineBreaksBefore(2);
}
addSubstituteWraps();
}
@Override
public boolean visit(TagElement node) {
String tagName = node.getTagName();
if (tagName == null || tagName.length() <= 1)
return true;
int startIndex = tokenStartingAt(node.getStartPosition());
int nodeEnd = node.getStartPosition() + node.getLength() - 1;
while (ScannerHelper.isWhitespace(this.ctm.charAt(nodeEnd)))
nodeEnd--;
int endIndex = tokenEndingAt(nodeEnd);
this.ctm.get(startIndex + 1).setWrapPolicy(WrapPolicy.DISABLE_WRAP);
if (node.getParent() instanceof Javadoc) {
assert this.ctm.toString(startIndex).startsWith(tagName);
boolean isParamTag = PARAM_TAGS.contains(tagName);
if (isParamTag && this.options.comment_insert_new_line_for_parameter && startIndex < endIndex) {
Token token = this.ctm.get(startIndex + 2);
token.breakBefore();
}
if (this.options.comment_indent_root_tags) {
int indent = this.ctm.getLength(this.ctm.get(startIndex), 0) + 1;
if (isParamTag && this.options.comment_indent_parameter_description)
indent += this.options.indentation_size;
for (int i = startIndex + 1; i <= endIndex; i++) {
Token token = this.ctm.get(i);
token.setIndent(indent);
// indent is used temporarily, tokens that are actually first in line
// will have this changed to align (indent is reserved for code inside tags)
}
}
Token startTokeen = this.ctm.get(startIndex);
if (startIndex > 1)
startTokeen.breakBefore();
int firstTagIndex;
if (this.firstTagToken == null || (firstTagIndex = this.ctm.indexOf(this.firstTagToken)) < 0
|| startIndex < firstTagIndex)
this.firstTagToken = startTokeen;
handleHtml(node);
}
if (node.isNested() && IMMUTABLE_TAGS.contains(tagName) && startIndex < endIndex) {
disableFormatting(startIndex, endIndex);
}
return true;
}
@Override
public void endVisit(TagElement node) {
String tagName = node.getTagName();
if (tagName == null || tagName.length() <= 1)
handleHtml(node);
if (TagElement.TAG_SEE.equals(tagName))
handleStringLiterals(this.tm.toString(node), node.getStartPosition());
}
private void handleHtml(TagElement node) {
if (!this.options.comment_format_html && !this.options.comment_format_source)
return;
String text = this.tm.toString(node);
Matcher matcher = HTML_TAG_PATTERN.matcher(text);
while (matcher.find()) {
int startPos = matcher.start() + node.getStartPosition();
int endPos = matcher.end() - 1 + node.getStartPosition();
boolean isOpeningTag = (matcher.start(1) == matcher.end(1));
if (this.options.comment_format_html) {
// make sure tokens inside the tag are wrapped only as a substitute
int firstTokenIndex = tokenStartingAt(startPos), lastTokenIndex = tokenEndingAt(endPos);
for (int i = firstTokenIndex + 1; i <= lastTokenIndex; i++) {
Token token = this.ctm.get(i);
if (token.getWrapPolicy() == null)
token.setWrapPolicy(WrapPolicy.SUBSTITUTE_ONLY);
}
// allow wraps around equals sign in attributes
String attributesText = matcher.group(8);
Matcher attrMatcher = HTML_ATTRIBUTE_PATTERN.matcher(attributesText);
final int commentStart = this.ctm.get(0).originalStart;
while (attrMatcher.find()) {
int equalPos = node.getStartPosition() + matcher.start(8) + attrMatcher.start(1);
assert this.tm.charAt(equalPos) == '=';
this.allowSubstituteWrapping[equalPos - commentStart] = true;
}
}
int matchedGroups = 0;
for (int i = 2; i <= 7; i++)
if (matcher.start(i) < matcher.end(i))
matchedGroups++;
if (matchedGroups != 1)
continue;
if (matcher.start(2) < matcher.end(2)) {
handleFormatCodeTag(startPos, endPos, isOpeningTag);
}
if (this.options.comment_format_html) {
if (TagElement.TAG_PARAM.equals(node.getTagName())
&& this.ctm.findIndex(startPos, -1, false) == 1 + this.ctm.firstIndexIn(node, -1)) {
continue; // it's a generic class parameter name, not an HTML tag
}
if (matcher.start(3) < matcher.end(3)) {
handleSeparateLineTag(startPos, endPos);
} else if (matcher.start(4) < matcher.end(4)) {
handleBreakBeforeTag(startPos, endPos, isOpeningTag);
} else if (matcher.start(5) < matcher.end(5)) {
handleBreakAfterTag(startPos, endPos);
} else if (matcher.start(6) < matcher.end(6)) {
handleNoFormatTag(startPos, endPos, isOpeningTag);
}
}
}
}
@Override
public boolean visit(MethodRef node) {
handleReference(node);
return true;
}
@Override
public boolean visit(MemberRef node) {
handleReference(node);
return true;
}
@Override
public boolean visit(QualifiedName node) {
handleReference(node);
return false;
}
private void handleReference(ASTNode node) {
ASTNode parent = node.getParent();
if ((parent instanceof TagElement) && ((TagElement) parent).isNested()) {
int firstIndex = tokenStartingAt(node.getStartPosition());
int lastIndex = tokenEndingAt(node.getStartPosition() + node.getLength() - 1);
if (this.ctm.charAt(this.ctm.get(lastIndex + 1).originalStart) == '}')
lastIndex++;
for (int i = firstIndex; i <= lastIndex; i++) {
Token token = this.ctm.get(i);
token.setWrapPolicy(WrapPolicy.DISABLE_WRAP);
}
}
}
private void handleStringLiterals(String text, int textStartPosition) {
Matcher matcher = STRING_LITERAL_PATTERN.matcher(text);
while (matcher.find()) {
int startPosition = textStartPosition + matcher.start();
int startIndex = this.ctm.findIndex(startPosition, -1, false);
int endPosition = textStartPosition + matcher.end() - 1;
int endIndex = this.ctm.findIndex(endPosition, -1, false);
if (startIndex != endIndex) {
startIndex = tokenStartingAt(startPosition);
endIndex = tokenEndingAt(endPosition);
disableFormatting(startIndex, endIndex);
}
noSubstituteWrapping(startPosition, endPosition);
}
}
private void handleSeparateLineTag(int startPos, int endPos) {
int openingTagIndex = tokenStartingAt(startPos);
if (openingTagIndex > 1 && this.lastFormatCodeClosingTagIndex == openingTagIndex - 1) {
Token token = this.ctm.get(openingTagIndex - 1);
assert token.getLineBreaksAfter() == 2;
token.clearLineBreaksAfter();
token.breakAfter();
}
handleBreakBeforeTag(startPos, endPos, true);
handleBreakAfterTag(startPos, endPos);
}
private void handleBreakBeforeTag(int start, int end, boolean isOpeningTag) {
int firstPartIndex = tokenStartingAt(start);
int lastPartIndex = tokenEndingAt(end);
Token firstPartToken = this.ctm.get(firstPartIndex);
firstPartToken.setWrapPolicy(null);
if (isOpeningTag) {
firstPartToken.breakBefore();
this.ctm.get(lastPartIndex + 1).clearSpaceBefore();
} else {
firstPartToken.clearSpaceBefore();
}
}
private void handleBreakAfterTag(int start, int end) {
int tokenIndex = tokenEndingAt(end);
this.ctm.get(tokenIndex).breakAfter();
}
private void handleNoFormatTag(int start, int end, boolean isOpeningTag) {
if (isOpeningTag) {
if (this.noFormatTagOpenStart < 0)
this.noFormatTagOpenStart = start;
} else if (this.noFormatTagOpenStart >= 0) {
int openingTagIndex = tokenStartingAt(this.noFormatTagOpenStart);
int closingTagIndex = tokenEndingAt(end);
if (openingTagIndex < closingTagIndex) {
disableFormatting(openingTagIndex, closingTagIndex);
closingTagIndex = tokenEndingAt(end);
}
cleanupHTMLElement(openingTagIndex, closingTagIndex, false);
this.noFormatTagOpenStart = -1;
}
}
private void handleFormatCodeTag(int startPos, int endPos, boolean isOpeningTag) {
if (!this.options.comment_format_source) {
handleNoFormatTag(startPos, endPos, isOpeningTag);
return;
}
// add empty lines before opening and after closing token
handleSeparateLineTag(startPos, endPos);
if (isOpeningTag) {
int startIndex = tokenStartingAt(startPos);
if (startIndex > 1)
this.ctm.get(startIndex).putLineBreaksBefore(2);
if (this.formatCodeTagOpenEnd < 0)
this.formatCodeTagOpenEnd = endPos;
} else if (this.formatCodeTagOpenEnd >= 0) {
int endTagIndex = tokenEndingAt(endPos);
if (endTagIndex < this.ctm.size() - 2)
this.ctm.get(endTagIndex).putLineBreaksAfter(2);
formatCode(startPos, endPos);
this.formatCodeTagOpenEnd = -1;
this.lastFormatCodeClosingTagIndex = this.ctm.findIndex(startPos, -1, true);
}
}
private void cleanupHTMLElement(int openingTagIndex, int closingTagIndex, boolean formattedCode) {
Token previous = this.ctm.get(openingTagIndex);
int indent = previous.getIndent();
for (int i = openingTagIndex + 1; i < closingTagIndex; i++) {
Token token = this.ctm.get(i);
token.setToEscape(true);
if (formattedCode && (token.getLineBreaksBefore() > 0 || previous.getLineBreaksAfter() > 0))
token.setAlign(indent);
previous = token;
}
for (int i = closingTagIndex; i < this.ctm.size(); i++) {
Token token = this.ctm.get(i);
if (token.getIndent() == 0)
break;
token.setIndent(indent);
previous = token;
}
}
private void disableFormatting(int startIndex, int endIndex) {
Token startToken = this.ctm.get(startIndex), endToken = this.ctm.get(endIndex);
Token noFormatToken = new Token(startToken.originalStart, endToken.originalEnd, TokenNameCOMMENT_JAVADOC);
List tokensToReplace = this.commentStructure.subList(startIndex, endIndex + 1);
if (this.ctm.countLineBreaksBetween(startToken, endToken) == 0) {
tokensToReplace.clear();
tokensToReplace.add(noFormatToken);
} else {
int commentStart = findCommentLineIndent(startIndex);
tokensToReplace.clear();
tokensToReplace.addAll(commentToLines(noFormatToken, commentStart));
}
Token first = tokensToReplace.get(0), last = tokensToReplace.get(tokensToReplace.size() - 1);
if (startToken.isSpaceBefore())
first.spaceBefore();
first.putLineBreaksBefore(startToken.getLineBreaksBefore());
first.setWrapPolicy(startToken.getWrapPolicy());
if (endToken.isSpaceAfter())
last.spaceAfter();
last.putLineBreaksAfter(endToken.getLineBreaksAfter());
for (Token token : tokensToReplace)
if (token.tokenType == TokenNameCOMMENT_JAVADOC)
token.setIndent(startToken.getIndent());
}
private void disableFormattingExclusively(int openingTagIndex, int closingTagIndex) {
Token openingTag = this.ctm.get(openingTagIndex);
int noFormatStart = openingTag.originalEnd + 1;
int noFormatEnd = this.ctm.get(closingTagIndex - 1).originalEnd;
if (noFormatStart <= noFormatEnd) {
Token noFormatToken = new Token(noFormatStart, noFormatEnd, TokenNameCOMMENT_JAVADOC);
int commentStart = findCommentLineIndent(openingTagIndex);
List lines = commentToLines(noFormatToken, commentStart);
List tokensToReplace = this.commentStructure.subList(openingTagIndex + 1, closingTagIndex);
tokensToReplace.clear();
tokensToReplace.addAll(lines);
} else {
this.commentStructure.subList(openingTagIndex + 1, closingTagIndex).clear();
Token closingTag = this.ctm.get(closingTagIndex);
if (this.ctm.countLineBreaksBetween(openingTag, closingTag) == 0) {
openingTag.clearLineBreaksAfter();
closingTag.clearLineBreaksBefore();
}
}
}
private int findCommentLineIndent(int commentFragmentIndex) {
int position = this.ctm.get(commentFragmentIndex).originalStart;
int lastNonWhitespace = position;
while (--position > 0) {
char c = this.ctm.charAt(position);
if (c == '\r' || c == '\n')
break;
if (!ScannerHelper.isWhitespace(c))
lastNonWhitespace = position;
}
if (lastNonWhitespace > 0 && this.ctm.charAt(lastNonWhitespace - 1) == ' ')
lastNonWhitespace--;
return this.ctm.getLength(position, lastNonWhitespace - 1, 0);
}
private int tokenStartingAt(int start) {
int tokenIndex = this.ctm.findIndex(start, -1, false);
Token token = this.ctm.get(tokenIndex);
if (token.originalStart == start)
return tokenIndex;
assert start > token.originalStart && start <= token.originalEnd;
splitToken(token, tokenIndex, start);
return tokenIndex + 1;
}
private int tokenEndingAt(int end) {
int tokenIndex = this.ctm.findIndex(end, -1, true);
Token token = this.ctm.get(tokenIndex);
if (token.originalEnd == end)
return tokenIndex;
assert end < token.originalEnd && end >= token.originalStart;
splitToken(token, tokenIndex, end + 1);
return tokenIndex;
}
private void splitToken(Token token, int tokenIndex, int splitPosition) {
assert splitPosition > token.originalStart && splitPosition <= token.originalEnd;
Token part1 = new Token(token.originalStart, splitPosition - 1, token.tokenType);
Token part2 = new Token(splitPosition, token.originalEnd, token.tokenType);
if (token.isSpaceBefore())
part1.spaceBefore();
part1.putLineBreaksBefore(token.getLineBreaksBefore());
if (token.isSpaceAfter())
part2.spaceAfter();
part2.putLineBreaksAfter(token.getLineBreaksAfter());
part1.setIndent(token.getIndent());
part2.setIndent(token.getIndent());
part1.setWrapPolicy(token.getWrapPolicy());
this.commentStructure.set(tokenIndex, part1);
this.commentStructure.add(tokenIndex + 1, part2);
}
private boolean tokenizeMultilineComment(Token commentToken) {
if (this.allowSubstituteWrapping == null || this.allowSubstituteWrapping.length < commentToken.countChars()) {
this.allowSubstituteWrapping = new boolean[commentToken.countChars()];
}
boolean isJavadoc = commentToken.tokenType == TokenNameCOMMENT_JAVADOC;
Arrays.fill(this.allowSubstituteWrapping, 0, commentToken.countChars(), !isJavadoc);
final boolean cleanBlankLines = isJavadoc ? this.options.comment_clear_blank_lines_in_javadoc_comment
: this.options.comment_clear_blank_lines_in_block_comment;
List structure = new ArrayList<>();
int firstTokenEnd = commentToken.originalStart + 1;
while (firstTokenEnd < commentToken.originalEnd - 1 && this.tm.charAt(firstTokenEnd + 1) == '*')
firstTokenEnd++;
Token first = new Token(commentToken.originalStart, firstTokenEnd, commentToken.tokenType);
first.spaceAfter();
structure.add(first);
int lastTokenStart = commentToken.originalEnd - 1;
while (lastTokenStart - 1 > firstTokenEnd && this.tm.charAt(lastTokenStart - 1) == '*')
lastTokenStart--;
int position = firstTokenEnd + 1;
int lineBreaks = 0;
while (position <= commentToken.originalEnd) {
// find line start
for (int i = position; i < lastTokenStart; i++) {
char c = this.tm.charAt(i);
if (c == '\r' || c == '\n') {
lineBreaks++;
char c2 = this.tm.charAt(i + 1);
if ((c2 == '\r' || c2 == '\n') && c2 != c)
i++;
position = i + 1;
} else if (!ScannerHelper.isWhitespace(c)) {
while (this.tm.charAt(i) == '*' && lineBreaks > 0)
i++;
position = i;
break;
}
}
int tokenStart = position;
while (position <= commentToken.originalEnd + 1) {
char c = 0;
if (position == commentToken.originalEnd + 1 || position == lastTokenStart
|| ScannerHelper.isWhitespace(c = this.tm.charAt(position))) {
if (tokenStart < position) {
Token outputToken = new Token(tokenStart, position - 1, commentToken.tokenType);
outputToken.spaceBefore();
if (lineBreaks > 0) {
if (cleanBlankLines)
lineBreaks = 1;
if (lineBreaks > 1 || !this.options.join_lines_in_comments)
outputToken.putLineBreaksBefore(lineBreaks);
}
if (this.tm.charAt(tokenStart) == '@') {
outputToken.setWrapPolicy(WrapPolicy.DISABLE_WRAP);
if (commentToken.tokenType == TokenNameCOMMENT_BLOCK && lineBreaks == 1
&& structure.size() > 1)
outputToken.putLineBreaksBefore(cleanBlankLines ? 1 : 2);
if (this.tm.charAt(tokenStart + 1) == '@' && lineBreaks > 0 && this.firstTagToken == null) {
// Commons Attributes annotation, see bug 237051
this.firstTagToken = outputToken;
}
}
structure.add(outputToken);
lineBreaks = 0;
}
if (c == '\r' || c == '\n')
break;
tokenStart = position == lastTokenStart ? position : position + 1;
}
position++;
}
}
Token last = structure.get(structure.size() - 1);
boolean newLinesAtBoundries = commentToken.tokenType == TokenNameCOMMENT_JAVADOC
? this.options.comment_new_lines_at_javadoc_boundaries
: this.options.comment_new_lines_at_block_boundaries;
if (newLinesAtBoundries && this.tm.countLineBreaksBetween(first, last) > 0) {
first.breakAfter();
last.breakBefore();
last.setAlign(1);
}
if (structure.size() == 2)
return false;
commentToken.setInternalStructure(structure);
return true;
}
private void noSubstituteWrapping(int from, int to) {
int commentStart = this.ctm.get(0).originalStart;
assert commentStart <= from && from <= to && to <= this.ctm.get(this.ctm.size() - 1).originalEnd;
Arrays.fill(this.allowSubstituteWrapping, from - commentStart, to - commentStart + 1, false);
}
private void addSubstituteWraps() {
int commentStart = this.ctm.get(0).originalStart;
for (int i = 1; i < this.ctm.size() - 1; i++) {
Token token = this.ctm.get(i);
boolean touchesPrevious = token.originalStart == this.ctm.get(i - 1).originalEnd + 1;
if (touchesPrevious && token.getWrapPolicy() == null && token.getLineBreaksBefore() == 0) {
boolean allowWrap = this.allowSubstituteWrapping[token.originalStart - commentStart];
token.setWrapPolicy(allowWrap ? WrapPolicy.SUBSTITUTE_ONLY : WrapPolicy.DISABLE_WRAP);
}
for (int pos = token.originalStart + 1; pos < token.originalEnd; pos++) {
if (!this.allowSubstituteWrapping[pos - commentStart])
continue;
char c = this.ctm.charAt(pos);
if (!ScannerHelper.isJavaIdentifierPart(c)) {
this.ctm.get(tokenStartingAt(pos)).setWrapPolicy(WrapPolicy.SUBSTITUTE_ONLY);
this.ctm.get(tokenStartingAt(pos + 1)).setWrapPolicy(WrapPolicy.SUBSTITUTE_ONLY);
}
}
}
}
private void formatCode(int javadocNoFormatCloseStart, int javadocNoFormatCloseEnd) {
int openingTagLastIndex = tokenEndingAt(this.formatCodeTagOpenEnd);
int closingTagFirstIndex = tokenStartingAt(javadocNoFormatCloseStart);
int codeStartPosition = this.formatCodeTagOpenEnd + 1;
int codeEndPosition = javadocNoFormatCloseStart - 1;
StringBuilder codeBuilder = new StringBuilder(codeEndPosition - codeStartPosition + 1);
int[] positionMapping = new int[codeEndPosition - codeStartPosition + 1];
// ^ index: original source position (minus startPosition), value: position in code string
getCodeToFormat(codeStartPosition, codeEndPosition, codeBuilder, positionMapping);
List formattedTokens = getCommentCodeFormatter().prepareFormattedCode(codeBuilder.toString(),
CodeFormatter.K_UNKNOWN);
if (formattedTokens == null) {
disableFormattingExclusively(openingTagLastIndex, closingTagFirstIndex);
closingTagFirstIndex = tokenStartingAt(javadocNoFormatCloseStart);
cleanupHTMLElement(openingTagLastIndex, closingTagFirstIndex, false);
return;
}
formattedTokens = translateFormattedTokens(codeStartPosition, formattedTokens, positionMapping, null);
// there are too few linebreaks at the start and end
Token start = formattedTokens.get(0);
start.putLineBreaksBefore(start.getLineBreaksBefore() + 1);
Token end = formattedTokens.get(formattedTokens.size() - 1);
end.putLineBreaksAfter(end.getLineBreaksAfter() + 1);
// and there may be too many line breaks before closing tag
this.ctm.get(closingTagFirstIndex).clearLineBreaksBefore();
List tokensToReplace = this.commentStructure.subList(openingTagLastIndex + 1, closingTagFirstIndex);
tokensToReplace.clear();
tokensToReplace.addAll(formattedTokens);
cleanupHTMLElement(openingTagLastIndex, openingTagLastIndex + formattedTokens.size() + 1, true);
}
private DefaultCodeFormatter getCommentCodeFormatter() {
if (this.commentCodeFormatter == null) {
Map options2 = this.options.getMap();
options2.put(DefaultCodeFormatterConstants.FORMATTER_COMMENT_LINE_LENGTH,
String.valueOf(this.options.comment_line_length - this.commentIndent
- COMMENT_LINE_SEPARATOR_LENGTH));
options2.put(DefaultCodeFormatterConstants.FORMATTER_LINE_SPLIT,
String.valueOf(this.options.page_width - this.commentIndent - COMMENT_LINE_SEPARATOR_LENGTH));
options2.put(CompilerOptions.OPTION_Source, this.sourceLevel);
this.commentCodeFormatter = new DefaultCodeFormatter(options2);
}
return this.commentCodeFormatter;
}
private void getCodeToFormat(int startPos, int endPos, StringBuilder sb, int[] posMapping) {
int position = 0; // original source position (minus startPos)
// skip excessive line break at the beginning
char c, c2;
if ((c = this.ctm.charAt(position + startPos)) == '\r' || c == '\n') {
posMapping[position++] = sb.length() - 1;
if (((c2 = this.ctm.charAt(position + startPos)) == '\r' || c2 == '\n') && c2 != c)
posMapping[position++] = sb.length() - 1;
}
while (position + startPos <= endPos) {
int lineStart = position + startPos;
for (int i = lineStart;; i++) {
c = this.ctm.charAt(i);
if (c == '\r' || c == '\n') {
sb.append(c);
lineStart = i + 1;
} else if (!ScannerHelper.isWhitespace(c)) {
if (c == '*')
lineStart = (this.ctm.charAt(i + 1) == ' ') ? i + 2 : i + 1;
break;
}
}
int lineEnd = endPos + 1;
for (int i = lineStart; i <= endPos; i++) {
c = this.ctm.charAt(i);
if (c == '\r' || c == '\n') {
lineEnd = i;
break;
}
}
while (position + startPos < lineStart) {
posMapping[position++] = sb.length() - 1;
}
int htmlEntityStart = -1;
for (int i = lineStart; i < lineEnd; i++) {
c = this.ctm.charAt(i);
sb.append(c);
posMapping[position++] = sb.length() - 1;
if (c == '&') {
htmlEntityStart = i;
} else if (c == ';' && htmlEntityStart >= 0) {
char replacementChar = getHtmlEntityChar(this.ctm.getSource().substring(htmlEntityStart, i + 1));
if (replacementChar != 0) {
sb.setLength(sb.length() - (i + 1 - htmlEntityStart));
sb.append(replacementChar);
for (int k = position - (i + 1 - htmlEntityStart); k < position; k++)
posMapping[k] = sb.length() - 1;
}
htmlEntityStart = -1;
}
}
}
// remove last line if empty
while (sb.length() > 0 && ((c = sb.charAt(sb.length() - 1)) == ' ' || c == '\t'))
sb.deleteCharAt(sb.length() - 1);
if (sb.length() > 0 && ((c = sb.charAt(sb.length() - 1)) == '\r' || c == '\n')) {
sb.deleteCharAt(sb.length() - 1);
if (sb.length() > 0 && ((c2 = sb.charAt(sb.length() - 1)) == '\r' || c2 == '\n') && c2 != c)
sb.deleteCharAt(sb.length() - 1);
}
}
private char getHtmlEntityChar(String entity) {
Matcher matcher = HTML_ENTITY_PATTERN.matcher(entity);
if (matcher.find()) {
char replaceChar = 0;
for (int i = 1; i < HTML_ENTITY_REPLACE.length(); i++) {
int start = matcher.start(i);
int end = matcher.end(i);
if (start == end)
continue; // group not matched
if (replaceChar != 0)
return 0; // more than one group matched
switch (i) {
case 1:
replaceChar = (char) Integer.parseInt(entity.substring(start + 2, end), 16);
break;
case 2:
replaceChar = (char) Integer.parseInt(entity.substring(start + 1, end), 10);
break;
default:
replaceChar = HTML_ENTITY_REPLACE.charAt(i);
}
}
return replaceChar;
}
return 0;
}
private List translateFormattedTokens(int startPosition, List formattedTokens, int[] positionMapping,
HashMap translationMap) {
int previousLineBreaks = 0;
List result = new ArrayList<>();
for (Token token : formattedTokens) {
int newStart = Arrays.binarySearch(positionMapping, token.originalStart);
while (newStart > 0 && positionMapping[newStart - 1] == token.originalStart)
newStart--;
int newEnd = Arrays.binarySearch(positionMapping, token.originalEnd);
while (newEnd + 1 < positionMapping.length && positionMapping[newEnd + 1] == token.originalEnd)
newEnd++;
Token translated = new Token(token, newStart + startPosition, newEnd + startPosition, token.tokenType);
if (translated.getWrapPolicy() == null)
translated.setWrapPolicy(WrapPolicy.DISABLE_WRAP);
if (token.hasNLSTag())
translationMap.put(token, translated);
int lineBreaks = Math.max(previousLineBreaks, token.getLineBreaksBefore());
List structure = token.getInternalStructure();
if (structure != null && !structure.isEmpty()) {
if (translationMap == null)
translationMap = new HashMap<>();
translated.setInternalStructure(translateFormattedTokens(startPosition, structure, positionMapping,
translationMap));
}
translated.putLineBreaksBefore(lineBreaks);
result.add(translated);
previousLineBreaks = token.getLineBreaksAfter();
}
result.get(result.size() - 1).putLineBreaksAfter(previousLineBreaks);
for (Token translated : result) {
if (translated.getNLSTag() != null) {
Token nlsTagToken = translationMap.get(translated.getNLSTag());
translated.setNLSTag(nlsTagToken);
nlsTagToken.setNLSTag(translated);
assert translated.getNLSTag() != null;
}
}
return result;
}
public void finishUp() {
if (this.lastFormatOffComment != null)
this.tm.addDisableFormatTokenPair(this.lastFormatOffComment, this.tm.get(this.tm.size() - 1));
}
}
