marytts.modules.PolynomialF0Modeller Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of marytts-runtime Show documentation
The newest version!
/**
 * Copyright 2009 DFKI GmbH.
 * All Rights Reserved.  Use is subject to license terms.
 *
 * This file is part of MARY TTS.
 *
 * MARY TTS is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, version 3 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see .
 *
 */
package marytts.modules;

import java.util.Locale;

import marytts.cart.DirectedGraph;
import marytts.cart.io.DirectedGraphReader;
import marytts.datatypes.MaryData;
import marytts.datatypes.MaryDataType;
import marytts.datatypes.MaryXML;
import marytts.features.FeatureDefinition;
import marytts.features.FeatureProcessorManager;
import marytts.features.TargetFeatureComputer;
import marytts.modules.phonemiser.Allophone;
import marytts.modules.phonemiser.AllophoneSet;
import marytts.modules.synthesis.Voice;
import marytts.server.MaryProperties;
import marytts.unitselection.data.FeatureFileReader;
import marytts.unitselection.select.Target;
import marytts.util.MaryRuntimeUtils;
import marytts.util.MaryUtils;
import marytts.util.dom.MaryDomUtils;
import marytts.util.dom.NameNodeFilter;
import marytts.util.math.ArrayUtils;
import marytts.util.math.Polynomial;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.traversal.DocumentTraversal;
import org.w3c.dom.traversal.NodeFilter;
import org.w3c.dom.traversal.NodeIterator;
import org.w3c.dom.traversal.TreeWalker;

/**
 * Predict f0 contours using polynomial curves predicted from a directed graph per syllable.
 * 
 * @author Marc Schröder
 */

public class PolynomialF0Modeller extends InternalModule {
	protected FeatureFileReader contourFeatures;
	protected DirectedGraph contourGraph;
	protected TargetFeatureComputer featureComputer;
	private String propertyPrefix;
	private FeatureProcessorManager featureProcessorManager;

	/**
	 * Constructor which can be directly called from init info in the config file. Different languages can call this code with
	 * different settings.
	 * 
	 * @param locale
	 *            a locale string, e.g. "en"
	 * @param propertyPrefix
	 *            the prefix to be used when looking up entries in the config files, e.g. "english.f0"
	 * @param featprocClassInfo
	 *            a package name for an instance of FeatureProcessorManager, e.g. "marytts.language.en.FeatureProcessorManager"
	 * @throws Exception
	 *             Exception
	 */
	public PolynomialF0Modeller(String locale, String propertyPrefix, String featprocClassInfo) throws Exception {
		this(MaryUtils.string2locale(locale), propertyPrefix, (FeatureProcessorManager) MaryRuntimeUtils
				.instantiateObject(featprocClassInfo));
	}

	/**
	 * Constructor to be called with instantiated objects.
	 * 
	 * @param locale
	 *            locale
	 * @param propertyPrefix
	 *            the prefix to be used when looking up entries in the config files, e.g. "english.f0"
	 * @param featureProcessorManager
	 *            the manager to use when looking up feature processors.
	 */
	protected PolynomialF0Modeller(Locale locale, String propertyPrefix, FeatureProcessorManager featureProcessorManager) {
		super("PolynomialF0Modeller", MaryDataType.DURATIONS, MaryDataType.ACOUSTPARAMS, locale);
		if (propertyPrefix.endsWith("."))
			this.propertyPrefix = propertyPrefix;
		else
			this.propertyPrefix = propertyPrefix + ".";
		this.featureProcessorManager = featureProcessorManager;
	}

	public void startup() throws Exception {
		super.startup();

		contourFeatures = new FeatureFileReader(MaryProperties.needFilename(propertyPrefix + "contours"));
		contourGraph = new DirectedGraphReader().load(MaryProperties.needFilename(propertyPrefix + "graph"));
		featureComputer = new TargetFeatureComputer(featureProcessorManager, contourGraph.getFeatureDefinition()
				.getFeatureNames());
	}

	public MaryData process(MaryData d) throws Exception {
		Document doc = d.getDocument();
		NodeIterator sentenceIt = ((DocumentTraversal) doc).createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, new NameNodeFilter(
				MaryXML.SENTENCE), false);
		Element sentence = null;
		AllophoneSet allophoneSet = null;
		while ((sentence = (Element) sentenceIt.nextNode()) != null) {
			// Make sure we have the correct voice:
			Element voice = (Element) MaryDomUtils.getAncestor(sentence, MaryXML.VOICE);
			Voice maryVoice = Voice.getVoice(voice);
			if (maryVoice == null) {
				maryVoice = d.getDefaultVoice();
			}
			if (maryVoice == null) {
				// Determine Locale in order to use default voice
				Locale locale = MaryUtils.string2locale(doc.getDocumentElement().getAttribute("xml:lang"));
				maryVoice = Voice.getDefaultVoice(locale);
			}
			FeatureFileReader currentContours = contourFeatures;
			DirectedGraph currentGraph = contourGraph;
			TargetFeatureComputer currentFeatureComputer = featureComputer;
			if (maryVoice != null) {
				DirectedGraph voiceGraph = maryVoice.getF0Graph();
				if (voiceGraph != null) {
					currentGraph = voiceGraph;
					logger.debug("Using voice graph");
					FeatureDefinition voiceFeatDef = voiceGraph.getFeatureDefinition();
					currentFeatureComputer = new TargetFeatureComputer(featureProcessorManager, voiceFeatDef.getFeatureNames());
					FeatureFileReader voiceContourFeatures = maryVoice.getF0ContourFeatures();
					currentContours = voiceContourFeatures;
				}
			}

			TreeWalker tw = ((DocumentTraversal) doc).createTreeWalker(sentence, NodeFilter.SHOW_ELEMENT, new NameNodeFilter(
					MaryXML.SYLLABLE), false);
			Element syllable;
			Element previous = null;
			while ((syllable = (Element) tw.nextNode()) != null) {
				Element vowel = null;
				float sylDur = 0;
				for (Element s = MaryDomUtils.getFirstChildElement(syllable); s != null; s = MaryDomUtils
						.getNextSiblingElement(s)) {
					assert s.getTagName().equals(MaryXML.PHONE) : "expected phone element, found " + s.getTagName();
					if (s.hasAttribute("d")) {
						sylDur += Float.parseFloat(s.getAttribute("d"));
					}
					String phone = s.getAttribute("p");
					if (allophoneSet == null) {
						allophoneSet = MaryRuntimeUtils.determineAllophoneSet(s);
					}
					assert allophoneSet != null;
					Allophone allophone = allophoneSet.getAllophone(phone);
					if (allophone.isVowel()) {
						// found a vowel
						vowel = s;
					}
				}
				// only predict F0 values if we have a vowel:
				if (vowel != null) {
					// Now predict the f0 values using the CARTs:ssh
					String phone = vowel.getAttribute("p");
					Target t = new Target(phone, vowel);
					t.setFeatureVector(currentFeatureComputer.computeFeatureVector(t));
					// double[] coeffs = ArrayUtils.toDoubleArray((float[]) currentGraph.interpret(t));
					int[] leafContours = (int[]) currentGraph.interpret(t);
					if (leafContours == null) { // no prediction :-(
						continue;
					}
					// At this stage, use the mean contour.
					double[] coeffs = getMeanContour(currentContours, leafContours);
					float posInSyl = 0;
					float relStart = 0;
					float relEnd = 0;
					for (Element s = MaryDomUtils.getFirstChildElement(syllable); s != null; s = MaryDomUtils
							.getNextSiblingElement(s)) {
						if (s.hasAttribute("d")) {
							float dur = Float.parseFloat(s.getAttribute("d"));
							relStart = posInSyl / sylDur;
							posInSyl += dur;
							relEnd = posInSyl / sylDur;
							// Now, predict three values for each phone, at beginning, mid, and end of phone.
							double initialLogF0 = Polynomial.getValueAt(coeffs, relStart);
							double midLogF0 = Polynomial.getValueAt(coeffs, (relStart + relEnd) / 2);
							double finalLogF0 = Polynomial.getValueAt(coeffs, relEnd);
							StringBuilder f0String = new StringBuilder();
							if (!Double.isNaN(initialLogF0)) {
								f0String.append("(0,").append((int) Math.exp(initialLogF0)).append(")");
							}
							if (!Double.isNaN(midLogF0)) {
								f0String.append("(50,").append((int) Math.exp(midLogF0)).append(")");
							}
							if (!Double.isNaN(finalLogF0)) {
								f0String.append("(100,").append((int) Math.exp(finalLogF0)).append(")");
							}
							if (f0String.length() > 0) {
								s.setAttribute("f0", f0String.toString());
							}
						}
					}
					assert posInSyl == sylDur;

				}
			}
		}
		MaryData output = new MaryData(outputType(), d.getLocale());
		output.setDocument(doc);
		return output;
	}

	protected double[] getMeanContour(FeatureFileReader currentContours, int[] contourIDs) {
		double[] coeffs = null;
		for (int i = 0; i < contourIDs.length; i++) {
			float[] oneCoeffs = currentContours.getFeatureVector(contourIDs[i]).getContinuousFeatures();
			assert !ArrayUtils.isZero(oneCoeffs) : "Feature vector " + contourIDs[i] + " is zero";
			if (coeffs == null)
				coeffs = new double[oneCoeffs.length];
			for (int j = 0; j < oneCoeffs.length; j++) {
				coeffs[j] += oneCoeffs[j];
			}
		}
		for (int j = 0; j < coeffs.length; j++) {
			coeffs[j] /= contourIDs.length;
		}
		return coeffs;
	}
}