net.sf.okapi.common.resource.TextFragmentUtil Maven / Gradle / Ivy
/*===========================================================================
Copyright (C) 2016-2021 by the Okapi Framework contributors
-----------------------------------------------------------------------------
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
===========================================================================*/
package net.sf.okapi.common.resource;
import net.sf.okapi.common.StringUtil;
import net.sf.okapi.common.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import static net.sf.okapi.common.resource.CodeMatchStrategy.STRICT;
public final class TextFragmentUtil {
public static final int MAX_INLINE_CODES = 6127;
// Comparators for finding codes
public static final CodeComparatorOnData CMP_DATA = new CodeComparatorOnData();
public static final CodeComparatorOnId CMP_ID = new CodeComparatorOnId();
public static final CodeComparatorOnTagType CMP_TAG_TYPE = new CodeComparatorOnTagType();
public static final CodeComparatorOnType CMP_TYPE = new CodeComparatorOnType();
public static final CodeComparatorOnCloseType CMP_CLOSE_TYPE = new CodeComparatorOnCloseType();
private static final Logger LOGGER = LoggerFactory.getLogger(TextFragmentUtil.class);
/**
* Update the meta fields in {@link Code} "{@code to}" that match those of
* "{@code from}" But only if the "to" code has empty data (@see{forceCopy}).
* Otherwise, keep the to code data as-is. Codes match if both {@link Code#data}
* and {@link Code#tagType} are the same.
*
* @param from {@link TextFragment} codes are used to match codes in
* "to"
* @param to {@link TextFragment} that has its code id's updated to
* match
* @param addMissingCodes indicates if codes that are in the original source but
* not in the new target should be automatically added at
* the end of the new target copy (even if they are
* removable) if there are references in the original
* source and/or empty codes in the new target.
* @param forceCopy if true always overwrite the target code data.
* WARNING: This will overwrite any code data changes
* in the target and force them to be like the source
* This may be needed for some simplified formats that
* only display the tagType and id. For example: A
* letter-coded format like:
* "<g1>text<x2/></g1><b3/>".
*/
static public CodeMatches alignAndCopyCodeMetadata(TextFragment from, TextFragment to, boolean addMissingCodes,
boolean forceCopy, CodeMatchStrategy strategy) {
// short circuit optimizations
if (from == null || to == null) {
return CodeMatches.NO_CODES;
}
CodeMatches cm = synchronizeCodeIds(from, to, strategy);
// If the codes of the original sources and the matched one are the same: no
// need to copy over
if (cm == CodeMatches.SAME_CODES) {
return cm;
}
// code id's are matched now copy over meta based on conditions
copyCodeMetadata(from, to, forceCopy, cm);
if (addMissingCodes && (cm.hasFromMismatch() || cm.hasToMismatch())) {
addMissingCodes(from, to, cm);
}
// Some codes might now be isolated.
// Rebalance so they are marked properly
to.invalidate();
to.balanceMarkers();
return cm;
}
static public CodeMatches alignAndCopyCodeMetadata(TextFragment from, TextFragment to, boolean addMissingCodes, boolean forceCopy) {
return alignAndCopyCodeMetadata(from, to, addMissingCodes, forceCopy, STRICT);
}
/**
* Log all code mismatch issues
*
* @param matches holds matches
* @param from base or original fragment (aka source)
* @param to new fragment (aka target)
*/
static public void logCodeMismatchErrors(CodeMatches matches, TextFragment from, TextFragment to, String textUnitId) {
// codes not matched in 'to'
if (matches.hasToMismatch()) {
for (Integer index : matches.getToMismatchIterator()) {
String warn = buildErrorString(to, index);
if (warn != null) {
String s = StringUtil.substring(to.toText(), 0, 50);
LOGGER.warn("Can't find matching target Code(s) {} in TU {}\n Segment: \"{}...\"", warn,
textUnitId, s);
}
}
}
// codes not matched in 'from'
if (matches.hasFromMismatch()) {
for (Integer index : matches.getFromMismatchIterator()) {
String warn = buildErrorString(from, index);
if (warn != null) {
String s = StringUtil.substring(from.toText(), 0, 50);
LOGGER.warn("Can't find matching source Code(s) {} in TU {}\n Segment: \"{}...\"", warn,
textUnitId, s);
}
}
}
}
private static String buildErrorString(TextFragment f, Integer index) {
Code c = f.getCode(index);
// don't log an error if the mismatched code is addable, deletable or cloneable
if (c.isAdded() || c.isDeleteable() || c.isCloneable()) {
return null;
}
StringBuilder error = new StringBuilder(String.format("id='%s' originalId='%s' data='%s'", c.id,
c.originalId == null ? "" : c.originalId, getCodeDataOrSimulate(c)));
if (c.getTagType() == TextFragment.TagType.OPENING) {
int mi = f.getIndexForClosing(c.id);
if (mi != -1) {
Code m = f.getCode(mi);
error.append(String.format(", Closing data='%s'", getCodeDataOrSimulate(m)));
}
}
return error.toString();
}
private static String getCodeDataOrSimulate(Code c) {
if (!Util.isEmpty(c.getData()))
return c.getData();
// Otherwise, try building something similar.
StringBuilder b = new StringBuilder();
switch (c.tagType) {
case OPENING:
b.append("<").append(c.type).append(" id=").append(c.id).append(">");
break;
case CLOSING:
b.append("");
break;
case PLACEHOLDER:
b.append("<").append(c.type).append(" id=").append(c.id).append("/>");
break;
}
return b.toString();
}
/**
* Update the meta fields in {@link Code} "{@code to}" that match those of
* "{@code from}" But only if the "to" code has empty data. Otherwise keep the
* to code data as-is. Codes match if both {@link Code#data} and
* {@link Code#tagType} are the same.
*
* @param from {@link TextFragment} codes are used to match codes in "to"
* @param to {@link TextFragment} that has its code id's updated to match
*/
static public CodeMatches alignAndCopyCodeMetadata(TextFragment from, TextFragment to) {
return alignAndCopyCodeMetadata(from, to, false, false, STRICT);
}
/**
* Update the meta fields in {@link Code} "{@code to}" that match those of
* "{@code from}" But only if the "to" code has empty data (@see{forceCopy}).
* Otherwise keep the to code data as-is. Codes match if both {@link Code#data}
* and {@link Code#tagType} are the same.
*
* @param from {@link TextFragment} codes are used to match codes in
* "to"
* @param to {@link TextFragment} that has its code id's updated to
* match
* @param addMissingCodes indicates if codes that are in the original source but
* not in the new target should be automatically added at
* the end of the new target copy (even if they are
* removable) if there are references in the original
* source and/or empty codes in the new target.
*/
static public CodeMatches alignAndCopyCodeMetadata(TextFragment from, TextFragment to, boolean addMissingCodes) {
return alignAndCopyCodeMetadata(from, to, addMissingCodes, false, STRICT);
}
/**
* Add missing leading and trailing codes from source (aka "from") to target (aka 'to').
*
* @param from object which has the missing codes
* @param to object where we need to add the missing codes
* @param codeMatches {@link CodeMatches} shows us what is missing
*/
public static void addMissingCodes(TextFragment from, TextFragment to, CodeMatches codeMatches) {
TextFragment leadingCodes = new TextFragment();
// iterate over all the mismatches in from and add if possible
for (Integer index : codeMatches.getFromMismatchIterator()) {
Code fc = from.codes.get(index);
if (isLeadingCode(fc, from)) {
leadingCodes.append(fc.clone());
} else {
to.append(fc.clone());
}
// mark code as matched so we don't produce errors
codeMatches.setFromMatch(index, CodeMatches.ADDED_MATCH);
}
// add missing leading codes
to.insert(0, leadingCodes, true);
}
/**
* Determine if the code is at the beginning of the {@link TextFragment} without
* intervening text.
*
* @param code the code we are testing.
* @param source the source (aka 'from') fragment we are searching.
* @return true if the code is leading.
*/
public static boolean isLeadingCode(Code code, TextFragment source) {
int index = source.codes.indexOf(code);
if (index == -1)
return false;
String ctext = source.getCodedText();
int pos = ctext.indexOf(String.valueOf(TextFragment.toChar(index)));
if (pos == -1)
return false;
// Remove all codes from the beginning of the string before the pos and see if
// any text remains
String substr = ctext.substring(0, pos - 1);
substr = TextFragment.MARKERS_REGEX.matcher(substr).replaceAll("");
return substr.trim().length() == 0;
}
/**
* Find a match in codes based on the list of {@link Comparator}
*
* @param orig the code we are looking for
* @param codes the code list we are looking in
* @param fromMatches boolean arrays marking the fromMatches already found
* @param cmps the list of {@link Comparator}
* @return the index of the match in from or -1 if already matched or nothing
* found
*/
@SafeVarargs
static public int findMatch(Code orig, List codes, int[] fromMatches, Comparator... cmps) {
int i = -1;
for (final Code c : codes) {
i++;
if ((fromMatches[i] == CodeMatches.NO_MATCH || fromMatches[i] == CodeMatches.ANNOTATION_ONLY) &&
compareAll(orig, c, i, cmps)) {
return i;
}
}
return -1;
}
/**
* Compare from and to based on the list of {@link Comparator}
*
* @param from first object to compare
* @param to second object to compare
* @param cmps the list of {@link Comparator}
* @param The type of the thing we are looking for
* @return true if from and to are the same based on the list of
* {@link Comparator}
*/
@SafeVarargs
static boolean compareAll(T from, T to, int index, Comparator... cmps) {
for (Comparator cmp : cmps) {
if (cmp instanceof CodeComparatorOnIsolated) {
((CodeComparatorOnIsolated) cmp).setFromIndex(index);
}
if (cmp.compare(from, to) != 0) {
return false;
}
}
return true;
}
static public void copyCodeMetadata(TextFragment sf, TextFragment tf, boolean forceCopy, CodeMatches cm) {
if (sf == null || tf == null) {
return;
}
if (sf.codes == null || tf.codes == null) {
return;
}
for (int toIndex = 0; toIndex < tf.codes.size(); toIndex++) {
Code tc = tf.codes.get(toIndex);
int fromIndex = cm.getToMatchIndex(toIndex);
if (fromIndex == CodeMatches.NO_MATCH || fromIndex == CodeMatches.ANNOTATION_ONLY) {
continue;
}
Code sc = sf.codes.get(fromIndex);
copyCodeMetadata(sc, tc, forceCopy);
}
}
static public void copyCodeMetadata(Code sc, Code tc, boolean forceCopy) {
// remove xliff or tmx formatting
tc.setOuterData(null);
// don't overwrite target code data if it exists
// as its content may have changed
// in some cases we always copy if forceCopy=true
if (forceCopy || !tc.hasData() || sc.hasReference()) {
tc.setData(sc.getData());
// must call after setData
tc.setReferenceFlag(sc.hasReference());
}
tc.setOuterData(sc.getOuterData());
tc.setOriginalId(sc.getOriginalId());
tc.setAdded(sc.isAdded());
tc.setCloneable(sc.isCloneable());
tc.setDeleteable(sc.isDeleteable());
tc.setDisplayText(sc.getDisplayText());
tc.setFlag(sc.getFlag());
tc.setMerged(sc.isMerged());
tc.setMarkerMasking(sc.isMarkerMasking());
tc.setMergedData(sc.getMergedData());
tc.setTagType(sc.getTagType());
tc.setType(sc.getType());
// Copy all InlineAnnotations of various types
final Set types = sc.getAnnotationsTypes();
for (final String type : types) {
// WARNING: copy properties with IWithProperties.copy below, skip here
if (Code.PROPERTIES.equals(type))
continue;
final InlineAnnotation a = sc.getAnnotation(type);
tc.setAnnotation(type, a.clone());
}
IWithProperties.copy(sc, tc);
}
/**
* Matches the code IDs of the
* fromto%d equals %s and the target is
* %s equals %d and %s and %d are codes.
* You want their IDs to match for the code with the same content.
*
* @param from the fragment to use as the base for the synchronization.
* @param to the fragment who's id's will be adjusted.
*/
static public CodeMatches synchronizeCodeIds(TextFragment from, TextFragment to, CodeMatchStrategy strategy) {
// short circuit optimizations
if (from == null || to == null) {
return CodeMatches.NO_CODES;
}
// no codes to process
if (!from.hasCode() && !to.hasCode()) {
return CodeMatches.NO_CODES;
}
// If it's the same object, there is no need to match
if (from == to) {
return CodeMatches.SAME_CODES;
}
// catch edge case where codes is null
// replace with empty list to avoid NPE
if (from.codes == null) {
from.setCodes(new ArrayList<>());
}
if (to.codes == null) {
to.setCodes(new ArrayList<>());
}
from.balanceMarkers();
to.balanceMarkers();
// initialize our match object with current codes
// balanceMarkers must be called before to establish isolated codes
CodeMatches codeMatches = new CodeMatches(from, to);
for (int toIndex = 0; toIndex < to.codes.size(); toIndex++) {
Code tc = to.codes.get(toIndex);
// wellformed CLOSING codes require special processing as they have more
// constraints and must match their OPENING mate.
// isolated codes are treated just like PLACEHOLDERS as we are
// processing independent TextFragments
if (tc.tagType == TextFragment.TagType.CLOSING && !codeMatches.isToIsolated(toIndex)) {
continue;
}
// search for 'to' code in 'from' (aka base fragment) using the best quality
// comparisons first. Then use less accurate as needed.
int fromIndex = search(tc, from.codes, codeMatches, strategy);
// no possible match
if (fromIndex == -1) {
continue;
}
// Possible 'to' match found in 'from'. Test it further
Code fc = from.codes.get(fromIndex);
// if tc is a true OPEN (OPENING and not isolated) then we expect a closing
// search recursively till we find it
if ((tc.tagType == TextFragment.TagType.OPENING && fc.tagType == TextFragment.TagType.OPENING) &&
!(codeMatches.isToIsolated(toIndex) || codeMatches.isFromIsolated(fromIndex))) {
if (!matchClosing(codeMatches, to, from, toIndex, fromIndex)) {
LOGGER.error("Cannot find matching closing tag. Malformed TextFragment. id={}, originalId={}, data={}",
tc.getId(), tc.getOriginalId(), tc.getData());
}
} else {
// both PLACEHOLDER or ISOLATED
codeMatches.setToMatch(toIndex, fromIndex);
codeMatches.setFromMatch(fromIndex, toIndex);
tc.setId(fc.id);
}
}
to.invalidate();
to.balanceMarkers();
return codeMatches;
}
static public CodeMatches synchronizeCodeIds(TextFragment from, TextFragment to) {
return synchronizeCodeIds(from, to, STRICT);
}
static boolean matchClosing(CodeMatches codeMatches, TextFragment to, TextFragment from, int toOpen, int fromOpen) {
// find closing code in 'from' and match CLOSE in 'to'
if (fromOpen < 0 || fromOpen >= from.codes.size()) {
return false;
}
Code tc = to.codes.get(toOpen);
Code fc = from.codes.get(fromOpen);
// search for CLOSE mates in both fragments
// note we can't get here if either tag is ISOLATED
int fromClose;
int toClose;
// match closing tag on id's or data only - hopefully these are the same
// between the open and closing codes
toClose = findClosing(tc, to.codes, codeMatches.getToMatches());
fromClose = findClosing(fc, from.codes, codeMatches.getFromMatches());
// if we can't find one or more CLOSE tags then mark the OPEN tags as unmatched
if (toClose == -1 && fromClose == -1) {
return false;
} else if (toClose == -1) {
return false;
} else if (fromClose == -1) {
return false;
} else {
// get the matching closing code
Code ctc = to.codes.get(toClose);
Code cfc = from.codes.get(fromClose);
// flag it as matched and set its id the same as the OPENING
// match for CLOSING
tc.setId(fc.getId());
ctc.setId(cfc.getId());
codeMatches.setToMatch(toOpen, fromOpen);
codeMatches.setToMatch(toClose, fromClose);
codeMatches.setFromMatch(fromOpen, toOpen);
codeMatches.setFromMatch(fromClose, toClose);
return true;
}
}
/**
* Given an OPEN tag find it's CLOSE mate. Prefer {@link Code#id} match but also search for a CLOSE with same
* {@link Code#data} as a fall back
* @param open OPEN tag
* @param codes codes to search
* @param matches previous matches to skip
* @return index of close tag or -1 if not found
*/
static int findClosing(Code open, List codes, int[] matches) {
int close = findMatch(open, codes, matches, CMP_ID, CMP_TYPE, CMP_CLOSE_TYPE);
if (close == -1) {
close = findMatch(open, codes, matches, CMP_ID, CMP_CLOSE_TYPE);
}
return close;
}
/*
* Order matters in search
*/
static int search(Code tc, List codes, CodeMatches codeMatches, CodeMatchStrategy strategy) {
// fast fail if we have nothing to match against
if (!codeMatches.hasFromMismatch()) {
return -1;
}
int fromIndex = -1;
CodeComparatorOnIsolated cmpTagTypeWithIsolated = new CodeComparatorOnIsolated(codeMatches,false);
switch(strategy) {
case LAX:
fromIndex = strictSearch(tc, codes, codeMatches, cmpTagTypeWithIsolated);
if (fromIndex == -1) {
// if we reach here we may be dealing with a leveraged target or partially
// translated segment that doesn't necessarily have the same
// codes, id's, or code data of the source. Match
// on TagType (including ISOLATED marker matching a PLACEHOLDER) in linear order.
fromIndex = findMatch(tc, codes, codeMatches.getFromMatches(), cmpTagTypeWithIsolated);
}
break;
case STRICT:
fromIndex = strictSearch(tc, codes, codeMatches, cmpTagTypeWithIsolated);
break;
}
return fromIndex;
}
static int strictSearch(Code tc, List codes, CodeMatches codeMatches, CodeComparatorOnIsolated cmpTagTypeWithIsolated) {
// Most accurate match. Use case when source and target codes are the same.
int fromIndex = findMatch(tc, codes, codeMatches.getFromMatches(), CMP_ID, CMP_TYPE, CMP_TAG_TYPE, CMP_DATA);
if (fromIndex == -1) {
// Most accurate match. but sometime types might mismatch.
fromIndex = findMatch(tc, codes, codeMatches.getFromMatches(), CMP_ID, CMP_TAG_TYPE, CMP_DATA);
if (fromIndex == -1) {
// may have been reordering, renumbering or extra codes or any case where ids might change
fromIndex = findMatch(tc, codes, codeMatches.getFromMatches(), CMP_DATA, CMP_TAG_TYPE);
if (fromIndex == -1) {
// Mostly cases where simplified codes were used: (, etc..) We assume id's match
// FIXME: we can't really trust the id's in every case, but simplified codes have very little to
// match on
fromIndex = findMatch(tc, codes, codeMatches.getFromMatches(), CMP_ID, CMP_TAG_TYPE);
if (fromIndex == -1) {
// may have been reordering, renumbering or extra codes or any case where ids might change
// because of segmentation PLACEHOLDER may match ISOLATED
fromIndex = findMatch(tc, codes, codeMatches.getFromMatches(), CMP_DATA, cmpTagTypeWithIsolated);
}
}
}
}
return fromIndex;
}
static public boolean moreThanMaxCodes(TextFragment tf) {
List codes = tf.getCodes();
return codes.size() > MAX_INLINE_CODES;
}
static public TextFragment removeMoreThanMaxCodes(TextFragment tf) {
int lastCodeIndex = 0;
int lastCharIndex = 0;
String codedText = tf.getCodedText();
StringBuilder newCodedText = new StringBuilder();
List codes = tf.getCodes();
for (int i = 0; i < codedText.length(); i++) {
int c = codedText.codePointAt(i);
if ((c == TextFragment.MARKER_OPENING || c == TextFragment.MARKER_CLOSING
|| c == TextFragment.MARKER_ISOLATED)) {
if (lastCodeIndex > MAX_INLINE_CODES - 1) {
lastCharIndex = i - 1;
break;
} else {
switch (c) {
case TextFragment.MARKER_OPENING:
newCodedText.append("" + ((char) TextFragment.MARKER_OPENING)).append(codedText.charAt(i + 1));
lastCodeIndex = TextFragment.toIndex(codedText.charAt(i + 1));
i++;
break;
case TextFragment.MARKER_CLOSING:
newCodedText.append("" + ((char) TextFragment.MARKER_CLOSING)).append(codedText.charAt(i + 1));
lastCodeIndex = TextFragment.toIndex(codedText.charAt(i + 1));
i++;
break;
case TextFragment.MARKER_ISOLATED:
newCodedText.append("" + ((char) TextFragment.MARKER_ISOLATED)).append(codedText.charAt(i + 1));
lastCodeIndex = TextFragment.toIndex(codedText.charAt(i + 1));
i++;
break;
}
}
} else {
newCodedText.appendCodePoint(c);
}
}
// remove all remaining codes
for (int i = lastCharIndex; i < codedText.length(); i++) {
int c = codedText.codePointAt(i);
if ((c == TextFragment.MARKER_OPENING || c == TextFragment.MARKER_CLOSING
|| c == TextFragment.MARKER_ISOLATED)) {
// skip code its past the MAX
i++;
} else {
newCodedText.appendCodePoint(c);
}
}
// remove all max codes to the end
return new TextFragment(newCodedText.toString(), codes.subList(0, lastCodeIndex + 1));
}
/**
* Render the {@link TextFragment} including all {@link Code}s. differs from
* TextFragment.toText() by also using outerData.
*
* @param tf The TextFragment to render
* @return the rendered string
*/
static public String toText(TextFragment tf) {
if ((tf.codes == null) || (tf.codes.size() == 0))
return tf.toString();
if (!tf.isBalanced)
tf.balanceMarkers();
StringBuilder tmp = new StringBuilder();
Code code;
for (int i = 0; i < tf.length(); i++) {
switch (tf.charAt(i)) {
case TextFragment.MARKER_OPENING:
case TextFragment.MARKER_CLOSING:
case TextFragment.MARKER_ISOLATED:
code = tf.codes.get(TextFragment.toIndex(tf.charAt(++i)));
tmp.append(code.getOuterData());
break;
default:
tmp.append(tf.charAt(i));
break;
}
}
return tmp.toString();
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy