android.media.TtmlRenderer Maven / Gradle / Ivy
Show all versions of android-all Show documentation
/*
* Copyright (C) 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.media;
import android.content.Context;
import android.graphics.Color;
import android.media.SubtitleTrack.RenderingWidget.OnChangedListener;
import android.text.Layout.Alignment;
import android.text.SpannableStringBuilder;
import android.text.TextUtils;
import android.util.ArrayMap;
import android.util.AttributeSet;
import android.util.Log;
import android.view.Gravity;
import android.view.View;
import android.view.ViewGroup;
import android.view.View.MeasureSpec;
import android.view.ViewGroup.LayoutParams;
import android.view.accessibility.CaptioningManager;
import android.view.accessibility.CaptioningManager.CaptionStyle;
import android.view.accessibility.CaptioningManager.CaptioningChangeListener;
import android.widget.LinearLayout;
import android.widget.TextView;
import com.android.internal.widget.SubtitleView;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.TreeSet;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlPullParserFactory;
/** @hide */
public class TtmlRenderer extends SubtitleController.Renderer {
private final Context mContext;
private static final String MEDIA_MIMETYPE_TEXT_TTML = "application/ttml+xml";
private TtmlRenderingWidget mRenderingWidget;
public TtmlRenderer(Context context) {
mContext = context;
}
@Override
public boolean supports(MediaFormat format) {
if (format.containsKey(MediaFormat.KEY_MIME)) {
return format.getString(MediaFormat.KEY_MIME).equals(MEDIA_MIMETYPE_TEXT_TTML);
}
return false;
}
@Override
public SubtitleTrack createTrack(MediaFormat format) {
if (mRenderingWidget == null) {
mRenderingWidget = new TtmlRenderingWidget(mContext);
}
return new TtmlTrack(mRenderingWidget, format);
}
}
/**
* A class which provides utillity methods for TTML parsing.
*
* @hide
*/
final class TtmlUtils {
public static final String TAG_TT = "tt";
public static final String TAG_HEAD = "head";
public static final String TAG_BODY = "body";
public static final String TAG_DIV = "div";
public static final String TAG_P = "p";
public static final String TAG_SPAN = "span";
public static final String TAG_BR = "br";
public static final String TAG_STYLE = "style";
public static final String TAG_STYLING = "styling";
public static final String TAG_LAYOUT = "layout";
public static final String TAG_REGION = "region";
public static final String TAG_METADATA = "metadata";
public static final String TAG_SMPTE_IMAGE = "smpte:image";
public static final String TAG_SMPTE_DATA = "smpte:data";
public static final String TAG_SMPTE_INFORMATION = "smpte:information";
public static final String PCDATA = "#pcdata";
public static final String ATTR_BEGIN = "begin";
public static final String ATTR_DURATION = "dur";
public static final String ATTR_END = "end";
public static final long INVALID_TIMESTAMP = Long.MAX_VALUE;
/**
* Time expression RE according to the spec:
* http://www.w3.org/TR/ttaf1-dfxp/#timing-value-timeExpression
*/
private static final Pattern CLOCK_TIME = Pattern.compile(
"^([0-9][0-9]+):([0-9][0-9]):([0-9][0-9])"
+ "(?:(\\.[0-9]+)|:([0-9][0-9])(?:\\.([0-9]+))?)?$");
private static final Pattern OFFSET_TIME = Pattern.compile(
"^([0-9]+(?:\\.[0-9]+)?)(h|m|s|ms|f|t)$");
private TtmlUtils() {
}
/**
* Parses the given time expression and returns a timestamp in millisecond.
*
* For the format of the time expression, please refer timeExpression
*
* @param time A string which includes time expression.
* @param frameRate the framerate of the stream.
* @param subframeRate the sub-framerate of the stream
* @param tickRate the tick rate of the stream.
* @return the parsed timestamp in micro-second.
* @throws NumberFormatException if the given string does not match to the
* format.
*/
public static long parseTimeExpression(String time, int frameRate, int subframeRate,
int tickRate) throws NumberFormatException {
Matcher matcher = CLOCK_TIME.matcher(time);
if (matcher.matches()) {
String hours = matcher.group(1);
double durationSeconds = Long.parseLong(hours) * 3600;
String minutes = matcher.group(2);
durationSeconds += Long.parseLong(minutes) * 60;
String seconds = matcher.group(3);
durationSeconds += Long.parseLong(seconds);
String fraction = matcher.group(4);
durationSeconds += (fraction != null) ? Double.parseDouble(fraction) : 0;
String frames = matcher.group(5);
durationSeconds += (frames != null) ? ((double)Long.parseLong(frames)) / frameRate : 0;
String subframes = matcher.group(6);
durationSeconds += (subframes != null) ? ((double)Long.parseLong(subframes))
/ subframeRate / frameRate
: 0;
return (long)(durationSeconds * 1000);
}
matcher = OFFSET_TIME.matcher(time);
if (matcher.matches()) {
String timeValue = matcher.group(1);
double value = Double.parseDouble(timeValue);
String unit = matcher.group(2);
if (unit.equals("h")) {
value *= 3600L * 1000000L;
} else if (unit.equals("m")) {
value *= 60 * 1000000;
} else if (unit.equals("s")) {
value *= 1000000;
} else if (unit.equals("ms")) {
value *= 1000;
} else if (unit.equals("f")) {
value = value / frameRate * 1000000;
} else if (unit.equals("t")) {
value = value / tickRate * 1000000;
}
return (long)value;
}
throw new NumberFormatException("Malformed time expression : " + time);
}
/**
* Applies the
* default space policy to the given string.
*
* @param in A string to apply the policy.
*/
public static String applyDefaultSpacePolicy(String in) {
return applySpacePolicy(in, true);
}
/**
* Applies the space policy to the given string. This applies the
* default space policy with linefeed-treatment as treat-as-space
* or preserve.
*
* @param in A string to apply the policy.
* @param treatLfAsSpace Whether convert line feeds to spaces or not.
*/
public static String applySpacePolicy(String in, boolean treatLfAsSpace) {
// Removes CR followed by LF. ref:
// http://www.w3.org/TR/xml/#sec-line-ends
String crRemoved = in.replaceAll("\r\n", "\n");
// Apply suppress-at-line-break="auto" and
// white-space-treatment="ignore-if-surrounding-linefeed"
String spacesNeighboringLfRemoved = crRemoved.replaceAll(" *\n *", "\n");
// Apply linefeed-treatment="treat-as-space"
String lfToSpace = treatLfAsSpace ? spacesNeighboringLfRemoved.replaceAll("\n", " ")
: spacesNeighboringLfRemoved;
// Apply white-space-collapse="true"
String spacesCollapsed = lfToSpace.replaceAll("[ \t\\x0B\f\r]+", " ");
return spacesCollapsed;
}
/**
* Returns the timed text for the given time period.
*
* @param root The root node of the TTML document.
* @param startUs The start time of the time period in microsecond.
* @param endUs The end time of the time period in microsecond.
*/
public static String extractText(TtmlNode root, long startUs, long endUs) {
StringBuilder text = new StringBuilder();
extractText(root, startUs, endUs, text, false);
return text.toString().replaceAll("\n$", "");
}
private static void extractText(TtmlNode node, long startUs, long endUs, StringBuilder out,
boolean inPTag) {
if (node.mName.equals(TtmlUtils.PCDATA) && inPTag) {
out.append(node.mText);
} else if (node.mName.equals(TtmlUtils.TAG_BR) && inPTag) {
out.append("\n");
} else if (node.mName.equals(TtmlUtils.TAG_METADATA)) {
// do nothing.
} else if (node.isActive(startUs, endUs)) {
boolean pTag = node.mName.equals(TtmlUtils.TAG_P);
int length = out.length();
for (int i = 0; i < node.mChildren.size(); ++i) {
extractText(node.mChildren.get(i), startUs, endUs, out, pTag || inPTag);
}
if (pTag && length != out.length()) {
out.append("\n");
}
}
}
/**
* Returns a TTML fragment string for the given time period.
*
* @param root The root node of the TTML document.
* @param startUs The start time of the time period in microsecond.
* @param endUs The end time of the time period in microsecond.
*/
public static String extractTtmlFragment(TtmlNode root, long startUs, long endUs) {
StringBuilder fragment = new StringBuilder();
extractTtmlFragment(root, startUs, endUs, fragment);
return fragment.toString();
}
private static void extractTtmlFragment(TtmlNode node, long startUs, long endUs,
StringBuilder out) {
if (node.mName.equals(TtmlUtils.PCDATA)) {
out.append(node.mText);
} else if (node.mName.equals(TtmlUtils.TAG_BR)) {
out.append("
");
} else if (node.isActive(startUs, endUs)) {
out.append("<");
out.append(node.mName);
out.append(node.mAttributes);
out.append(">");
for (int i = 0; i < node.mChildren.size(); ++i) {
extractTtmlFragment(node.mChildren.get(i), startUs, endUs, out);
}
out.append("");
}
}
}
/**
* A container class which represents a cue in TTML.
* @hide
*/
class TtmlCue extends SubtitleTrack.Cue {
public String mText;
public String mTtmlFragment;
public TtmlCue(long startTimeMs, long endTimeMs, String text, String ttmlFragment) {
this.mStartTimeMs = startTimeMs;
this.mEndTimeMs = endTimeMs;
this.mText = text;
this.mTtmlFragment = ttmlFragment;
}
}
/**
* A container class which represents a node in TTML.
*
* @hide
*/
class TtmlNode {
public final String mName;
public final String mAttributes;
public final TtmlNode mParent;
public final String mText;
public final List mChildren = new ArrayList();
public final long mRunId;
public final long mStartTimeMs;
public final long mEndTimeMs;
public TtmlNode(String name, String attributes, String text, long startTimeMs, long endTimeMs,
TtmlNode parent, long runId) {
this.mName = name;
this.mAttributes = attributes;
this.mText = text;
this.mStartTimeMs = startTimeMs;
this.mEndTimeMs = endTimeMs;
this.mParent = parent;
this.mRunId = runId;
}
/**
* Check if this node is active in the given time range.
*
* @param startTimeMs The start time of the range to check in microsecond.
* @param endTimeMs The end time of the range to check in microsecond.
* @return return true if the given range overlaps the time range of this
* node.
*/
public boolean isActive(long startTimeMs, long endTimeMs) {
return this.mEndTimeMs > startTimeMs && this.mStartTimeMs < endTimeMs;
}
}
/**
* A simple TTML parser (http://www.w3.org/TR/ttaf1-dfxp/) which supports DFXP
* presentation profile.
*
* Supported features in this parser are:
*
* - content
*
- core
*
- presentation
*
- profile
*
- structure
*
- time-offset
*
- timing
*
- tickRate
*
- time-clock-with-frames
*
- time-clock
*
- time-offset-with-frames
*
- time-offset-with-ticks
*
*
*
* @hide
*/
class TtmlParser {
static final String TAG = "TtmlParser";
// TODO: read and apply the following attributes if specified.
private static final int DEFAULT_FRAMERATE = 30;
private static final int DEFAULT_SUBFRAMERATE = 1;
private static final int DEFAULT_TICKRATE = 1;
private XmlPullParser mParser;
private final TtmlNodeListener mListener;
private long mCurrentRunId;
public TtmlParser(TtmlNodeListener listener) {
mListener = listener;
}
/**
* Parse TTML data. Once this is called, all the previous data are
* reset and it starts parsing for the given text.
*
* @param ttmlText TTML text to parse.
* @throws XmlPullParserException
* @throws IOException
*/
public void parse(String ttmlText, long runId) throws XmlPullParserException, IOException {
mParser = null;
mCurrentRunId = runId;
loadParser(ttmlText);
parseTtml();
}
private void loadParser(String ttmlFragment) throws XmlPullParserException {
XmlPullParserFactory factory = XmlPullParserFactory.newInstance();
factory.setNamespaceAware(false);
mParser = factory.newPullParser();
StringReader in = new StringReader(ttmlFragment);
mParser.setInput(in);
}
private void extractAttribute(XmlPullParser parser, int i, StringBuilder out) {
out.append(" ");
out.append(parser.getAttributeName(i));
out.append("=\"");
out.append(parser.getAttributeValue(i));
out.append("\"");
}
private void parseTtml() throws XmlPullParserException, IOException {
LinkedList nodeStack = new LinkedList();
int depthInUnsupportedTag = 0;
boolean active = true;
while (!isEndOfDoc()) {
int eventType = mParser.getEventType();
TtmlNode parent = nodeStack.peekLast();
if (active) {
if (eventType == XmlPullParser.START_TAG) {
if (!isSupportedTag(mParser.getName())) {
Log.w(TAG, "Unsupported tag " + mParser.getName() + " is ignored.");
depthInUnsupportedTag++;
active = false;
} else {
TtmlNode node = parseNode(parent);
nodeStack.addLast(node);
if (parent != null) {
parent.mChildren.add(node);
}
}
} else if (eventType == XmlPullParser.TEXT) {
String text = TtmlUtils.applyDefaultSpacePolicy(mParser.getText());
if (!TextUtils.isEmpty(text)) {
parent.mChildren.add(new TtmlNode(
TtmlUtils.PCDATA, "", text, 0, TtmlUtils.INVALID_TIMESTAMP,
parent, mCurrentRunId));
}
} else if (eventType == XmlPullParser.END_TAG) {
if (mParser.getName().equals(TtmlUtils.TAG_P)) {
mListener.onTtmlNodeParsed(nodeStack.getLast());
} else if (mParser.getName().equals(TtmlUtils.TAG_TT)) {
mListener.onRootNodeParsed(nodeStack.getLast());
}
nodeStack.removeLast();
}
} else {
if (eventType == XmlPullParser.START_TAG) {
depthInUnsupportedTag++;
} else if (eventType == XmlPullParser.END_TAG) {
depthInUnsupportedTag--;
if (depthInUnsupportedTag == 0) {
active = true;
}
}
}
mParser.next();
}
}
private TtmlNode parseNode(TtmlNode parent) throws XmlPullParserException, IOException {
int eventType = mParser.getEventType();
if (!(eventType == XmlPullParser.START_TAG)) {
return null;
}
StringBuilder attrStr = new StringBuilder();
long start = 0;
long end = TtmlUtils.INVALID_TIMESTAMP;
long dur = 0;
for (int i = 0; i < mParser.getAttributeCount(); ++i) {
String attr = mParser.getAttributeName(i);
String value = mParser.getAttributeValue(i);
// TODO: check if it's safe to ignore the namespace of attributes as follows.
attr = attr.replaceFirst("^.*:", "");
if (attr.equals(TtmlUtils.ATTR_BEGIN)) {
start = TtmlUtils.parseTimeExpression(value, DEFAULT_FRAMERATE,
DEFAULT_SUBFRAMERATE, DEFAULT_TICKRATE);
} else if (attr.equals(TtmlUtils.ATTR_END)) {
end = TtmlUtils.parseTimeExpression(value, DEFAULT_FRAMERATE, DEFAULT_SUBFRAMERATE,
DEFAULT_TICKRATE);
} else if (attr.equals(TtmlUtils.ATTR_DURATION)) {
dur = TtmlUtils.parseTimeExpression(value, DEFAULT_FRAMERATE, DEFAULT_SUBFRAMERATE,
DEFAULT_TICKRATE);
} else {
extractAttribute(mParser, i, attrStr);
}
}
if (parent != null) {
start += parent.mStartTimeMs;
if (end != TtmlUtils.INVALID_TIMESTAMP) {
end += parent.mStartTimeMs;
}
}
if (dur > 0) {
if (end != TtmlUtils.INVALID_TIMESTAMP) {
Log.e(TAG, "'dur' and 'end' attributes are defined at the same time." +
"'end' value is ignored.");
}
end = start + dur;
}
if (parent != null) {
// If the end time remains unspecified, then the end point is
// interpreted as the end point of the external time interval.
if (end == TtmlUtils.INVALID_TIMESTAMP &&
parent.mEndTimeMs != TtmlUtils.INVALID_TIMESTAMP &&
end > parent.mEndTimeMs) {
end = parent.mEndTimeMs;
}
}
TtmlNode node = new TtmlNode(mParser.getName(), attrStr.toString(), null, start, end,
parent, mCurrentRunId);
return node;
}
private boolean isEndOfDoc() throws XmlPullParserException {
return (mParser.getEventType() == XmlPullParser.END_DOCUMENT);
}
private static boolean isSupportedTag(String tag) {
if (tag.equals(TtmlUtils.TAG_TT) || tag.equals(TtmlUtils.TAG_HEAD) ||
tag.equals(TtmlUtils.TAG_BODY) || tag.equals(TtmlUtils.TAG_DIV) ||
tag.equals(TtmlUtils.TAG_P) || tag.equals(TtmlUtils.TAG_SPAN) ||
tag.equals(TtmlUtils.TAG_BR) || tag.equals(TtmlUtils.TAG_STYLE) ||
tag.equals(TtmlUtils.TAG_STYLING) || tag.equals(TtmlUtils.TAG_LAYOUT) ||
tag.equals(TtmlUtils.TAG_REGION) || tag.equals(TtmlUtils.TAG_METADATA) ||
tag.equals(TtmlUtils.TAG_SMPTE_IMAGE) || tag.equals(TtmlUtils.TAG_SMPTE_DATA) ||
tag.equals(TtmlUtils.TAG_SMPTE_INFORMATION)) {
return true;
}
return false;
}
}
/** @hide */
interface TtmlNodeListener {
void onTtmlNodeParsed(TtmlNode node);
void onRootNodeParsed(TtmlNode node);
}
/** @hide */
class TtmlTrack extends SubtitleTrack implements TtmlNodeListener {
private static final String TAG = "TtmlTrack";
private final TtmlParser mParser = new TtmlParser(this);
private final TtmlRenderingWidget mRenderingWidget;
private String mParsingData;
private Long mCurrentRunID;
private final LinkedList mTtmlNodes;
private final TreeSet mTimeEvents;
private TtmlNode mRootNode;
TtmlTrack(TtmlRenderingWidget renderingWidget, MediaFormat format) {
super(format);
mTtmlNodes = new LinkedList();
mTimeEvents = new TreeSet();
mRenderingWidget = renderingWidget;
mParsingData = "";
}
@Override
public TtmlRenderingWidget getRenderingWidget() {
return mRenderingWidget;
}
@Override
public void onData(byte[] data, boolean eos, long runID) {
try {
// TODO: handle UTF-8 conversion properly
String str = new String(data, "UTF-8");
// implement intermixing restriction for TTML.
synchronized(mParser) {
if (mCurrentRunID != null && runID != mCurrentRunID) {
throw new IllegalStateException(
"Run #" + mCurrentRunID +
" in progress. Cannot process run #" + runID);
}
mCurrentRunID = runID;
mParsingData += str;
if (eos) {
try {
mParser.parse(mParsingData, mCurrentRunID);
} catch (XmlPullParserException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
finishedRun(runID);
mParsingData = "";
mCurrentRunID = null;
}
}
} catch (java.io.UnsupportedEncodingException e) {
Log.w(TAG, "subtitle data is not UTF-8 encoded: " + e);
}
}
@Override
public void onTtmlNodeParsed(TtmlNode node) {
mTtmlNodes.addLast(node);
addTimeEvents(node);
}
@Override
public void onRootNodeParsed(TtmlNode node) {
mRootNode = node;
TtmlCue cue = null;
while ((cue = getNextResult()) != null) {
addCue(cue);
}
mRootNode = null;
mTtmlNodes.clear();
mTimeEvents.clear();
}
@Override
public void updateView(Vector activeCues) {
if (!mVisible) {
// don't keep the state if we are not visible
return;
}
if (DEBUG && mTimeProvider != null) {
try {
Log.d(TAG, "at " +
(mTimeProvider.getCurrentTimeUs(false, true) / 1000) +
" ms the active cues are:");
} catch (IllegalStateException e) {
Log.d(TAG, "at (illegal state) the active cues are:");
}
}
mRenderingWidget.setActiveCues(activeCues);
}
/**
* Returns a {@link TtmlCue} in the presentation time order.
* {@code null} is returned if there is no more timed text to show.
*/
public TtmlCue getNextResult() {
while (mTimeEvents.size() >= 2) {
long start = mTimeEvents.pollFirst();
long end = mTimeEvents.first();
List activeCues = getActiveNodes(start, end);
if (!activeCues.isEmpty()) {
return new TtmlCue(start, end,
TtmlUtils.applySpacePolicy(TtmlUtils.extractText(
mRootNode, start, end), false),
TtmlUtils.extractTtmlFragment(mRootNode, start, end));
}
}
return null;
}
private void addTimeEvents(TtmlNode node) {
mTimeEvents.add(node.mStartTimeMs);
mTimeEvents.add(node.mEndTimeMs);
for (int i = 0; i < node.mChildren.size(); ++i) {
addTimeEvents(node.mChildren.get(i));
}
}
private List getActiveNodes(long startTimeUs, long endTimeUs) {
List activeNodes = new ArrayList();
for (int i = 0; i < mTtmlNodes.size(); ++i) {
TtmlNode node = mTtmlNodes.get(i);
if (node.isActive(startTimeUs, endTimeUs)) {
activeNodes.add(node);
}
}
return activeNodes;
}
}
/**
* Widget capable of rendering TTML captions.
*
* @hide
*/
class TtmlRenderingWidget extends LinearLayout implements SubtitleTrack.RenderingWidget {
/** Callback for rendering changes. */
private OnChangedListener mListener;
private final TextView mTextView;
public TtmlRenderingWidget(Context context) {
this(context, null);
}
public TtmlRenderingWidget(Context context, AttributeSet attrs) {
this(context, attrs, 0);
}
public TtmlRenderingWidget(Context context, AttributeSet attrs, int defStyleAttr) {
this(context, attrs, defStyleAttr, 0);
}
public TtmlRenderingWidget(Context context, AttributeSet attrs, int defStyleAttr,
int defStyleRes) {
super(context, attrs, defStyleAttr, defStyleRes);
// Cannot render text over video when layer type is hardware.
setLayerType(View.LAYER_TYPE_SOFTWARE, null);
CaptioningManager captionManager = (CaptioningManager) context.getSystemService(
Context.CAPTIONING_SERVICE);
mTextView = new TextView(context);
mTextView.setTextColor(captionManager.getUserStyle().foregroundColor);
addView(mTextView, LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT);
mTextView.setGravity(Gravity.BOTTOM | Gravity.CENTER_HORIZONTAL);
}
@Override
public void setOnChangedListener(OnChangedListener listener) {
mListener = listener;
}
@Override
public void setSize(int width, int height) {
final int widthSpec = MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY);
final int heightSpec = MeasureSpec.makeMeasureSpec(height, MeasureSpec.EXACTLY);
measure(widthSpec, heightSpec);
layout(0, 0, width, height);
}
@Override
public void setVisible(boolean visible) {
if (visible) {
setVisibility(View.VISIBLE);
} else {
setVisibility(View.GONE);
}
}
@Override
public void onAttachedToWindow() {
super.onAttachedToWindow();
}
@Override
public void onDetachedFromWindow() {
super.onDetachedFromWindow();
}
public void setActiveCues(Vector activeCues) {
final int count = activeCues.size();
String subtitleText = "";
for (int i = 0; i < count; i++) {
TtmlCue cue = (TtmlCue) activeCues.get(i);
subtitleText += cue.mText + "\n";
}
mTextView.setText(subtitleText);
if (mListener != null) {
mListener.onChanged(this);
}
}
}