marytts.tools.voiceimport.VoiceCompiler Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of marytts-builder Show documentation
The newest version!
/**
 * 
 */
package marytts.tools.voiceimport;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Pattern;

import marytts.util.MaryUtils;
import marytts.util.io.StreamGobbler;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.text.StrSubstitutor;

import com.twmacinta.util.MD5;

/**
 * @author marc
 *
 */
public class VoiceCompiler extends VoiceImportComponent {

	// constants to access filenames in database component properties and organize file list:

	public static final String CARTFILE = "CARTBuilder.cartFile";

	public static final String DURTREE = "DurationCARTTrainer.durTree";

	public static final String F0LEFTTREE = "F0CARTTrainer.f0LeftTreeFile";

	public static final String F0MIDTREE = "F0CARTTrainer.f0MidTreeFile";

	public static final String F0RIGHTTREE = "F0CARTTrainer.f0RightTreeFile";

	public static final String HALFPHONEFEATSAC = "AcousticFeatureFileWriter.acFeatureFile";

	public static final String HALFPHONEFEATDEFAC = "AcousticFeatureFileWriter.acFeatDef";

	public static final String HALFPHONEUNITS = "HalfPhoneUnitfileWriter.unitFile";

	public static final String JOINCOSTFEATS = "JoinCostFileMaker.joinCostFile";

	public static final String JOINCOSTWEIGHTS = "JoinCostFileMaker.weightsFile";

	public static final String PHONEFEATDEF = "PhoneFeatureFileWriter.weightsFile";

	public static final String WAVETIMELINE = "WaveTimelineMaker.waveTimeline";

	public static final String BASETIMELINE = "BasenameTimelineMaker.timelineFile";

	public final String COMPILEDIR = getName() + ".compileDir";

	public final String MVN = getName() + ".mavenBin";

	protected MavenVoiceCompiler compiler;

	/**
	 * 
	 */
	public VoiceCompiler() {
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see marytts.tools.voiceimport.VoiceImportComponent#compute()
	 */
	@Override
	public boolean compute() throws Exception {

		File compileDir = new File(getProp(COMPILEDIR));
		compiler = createCompiler(compileDir);

		if (!isUnitSelectionVoice()) {
			mapFeatures();
		}

		logger.info("Creating directories");
		compiler.createDirectories();

		logger.info("Copying template files");
		compiler.copyTemplateFiles();

		logger.info("Copying voice files");
		compiler.copyVoiceFiles();

		logger.info("Compiling with Maven");
		compiler.compileWithMaven();

		// logger.info("Creating component description file");
		// compiler.createComponentFile();
		logger.info("done.");

		return true;
	}

	protected void mapFeatures() throws Exception {
		throw new IllegalStateException("This method should not be called for unit selection voices, "
				+ " and hmm-based voices should extend it.");
	}

	protected boolean isUnitSelectionVoice() {
		return true;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see marytts.tools.voiceimport.VoiceImportComponent#getDefaultProps(marytts.tools.voiceimport.DatabaseLayout)
	 */
	@Override
	public SortedMap getDefaultProps(DatabaseLayout db) {
		if (props == null) {
			props = new TreeMap();
			props.put(COMPILEDIR, new File(db.getVoiceFileDir(), "voice-" + getVoiceName(db)).getAbsolutePath());
			props.put(MVN, "/usr/bin/mvn");
		}
		return props;
	}

	protected String getVoiceName(DatabaseLayout db) {
		return db.getVoiceName();
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see marytts.tools.voiceimport.VoiceImportComponent#getName()
	 */
	@Override
	public String getName() {
		return "VoiceCompiler";
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see marytts.tools.voiceimport.VoiceImportComponent#getProgress()
	 */
	@Override
	public int getProgress() {
		return -1;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see marytts.tools.voiceimport.VoiceImportComponent#setupHelp()
	 */
	@Override
	protected void setupHelp() {
		props2Help = new TreeMap();
		props2Help.put(COMPILEDIR, "The directory in which the files for compiling the voice will be copied.");
		props2Help.put(MVN, "The path to the Maven binary (i.e., mvn).");
	}

	protected Map getExtraVariableSubstitutionMap() {
		return null;
	}

	protected File[] getFilesForResources() {
		String[] propsResources = new String[] { CARTFILE, DURTREE, F0LEFTTREE, F0MIDTREE, F0RIGHTTREE, HALFPHONEFEATDEFAC,
				JOINCOSTWEIGHTS };
		File[] filesForResources = new File[propsResources.length];
		for (int i = 0; i < propsResources.length; i++) {
			filesForResources[i] = new File(db.getProperty(propsResources[i]));
		}
		return filesForResources;
	}

	protected File[] getFilesForFilesystem() {
		String[] propsFilesystem = new String[] { HALFPHONEFEATSAC, HALFPHONEUNITS, JOINCOSTFEATS, BASETIMELINE, WAVETIMELINE };
		File[] filesForFilesystem = new File[propsFilesystem.length];
		for (int i = 0; i < propsFilesystem.length; i++) {
			filesForFilesystem[i] = new File(db.getProperty(propsFilesystem[i]));
		}
		return filesForFilesystem;
	}

	protected MavenVoiceCompiler createCompiler(File compileDir) {
		File[] filesForResources = getFilesForResources();
		File[] filesForFilesystem = getFilesForFilesystem();
		Map extraVariablesToSubstitute = getExtraVariableSubstitutionMap();
		return new MavenVoiceCompiler(getProp(MVN), compileDir, getVoiceName(db), db.getMaryVersion(), db.getLocale(),
				db.getGender(), db.getDomain(), db.getSamplingRate(), isUnitSelectionVoice(), filesForResources,
				filesForFilesystem, extraVariablesToSubstitute);
	}

	public static class MavenVoiceCompiler {
		protected File compileDir;
		protected String voiceName;
		protected String voiceVersion;
		protected Locale locale;
		protected String gender;
		protected String domain;
		protected int samplingRate;
		protected boolean isUnitSelectionVoice;
		protected File[] filesForResources;
		protected File[] filesForFilesystem;
		protected Map extraVariablesToSubstitute;

		protected StrSubstitutor substitutor;
		protected File mainJavaDir;
		protected File mainResourcesDir;
		protected File nonPackagedResourcesDir;
		protected File mainDescriptionsDir;
		protected File metaInfDir;
		protected File testJavaDir;
		protected File libVoiceDir;
		protected String mvn;

		/**
		 * @deprecated Use constructor with path to Maven instead.
		 * @param compileDir
		 *            compileDir
		 * @param voiceName
		 *            voiceName
		 * @param voiceVersion
		 *            voiceVersion
		 * @param locale
		 *            locale
		 * @param gender
		 *            gender
		 * @param domain
		 *            domain
		 * @param samplingRate
		 *            samplingRate
		 * @param isUnitSelectionVoice
		 *            isUnitSelectionVoice
		 * @param filesForResources
		 *            filesForResources
		 * @param filesForFilesystem
		 *            filesForFilesystem
		 * @param extraVariablesToSubstitute
		 *            extraVariablesToSubstitute
		 */
		@Deprecated
		public MavenVoiceCompiler(File compileDir, String voiceName, String voiceVersion, Locale locale, String gender,
				String domain, int samplingRate, boolean isUnitSelectionVoice, File[] filesForResources,
				File[] filesForFilesystem, Map extraVariablesToSubstitute) {
			this("mvn", compileDir, voiceName, voiceVersion, locale, gender, domain, samplingRate, isUnitSelectionVoice,
					filesForResources, filesForFilesystem, extraVariablesToSubstitute);
		}

		public MavenVoiceCompiler(String mvn, File compileDir, String voiceName, String voiceVersion, Locale locale,
				String gender, String domain, int samplingRate, boolean isUnitSelectionVoice, File[] filesForResources,
				File[] filesForFilesystem, Map extraVariablesToSubstitute) {
			this.mvn = mvn;
			this.compileDir = compileDir;
			this.voiceName = voiceName.replaceAll("[^\\w\\-]", "");
			this.voiceVersion = voiceVersion;
			this.locale = locale;
			this.gender = gender;
			this.domain = domain;
			this.samplingRate = samplingRate;
			this.isUnitSelectionVoice = isUnitSelectionVoice;
			this.substitutor = new StrSubstitutor(getVariableSubstitutionMap(extraVariablesToSubstitute));

			this.filesForResources = filesForResources;
			this.filesForFilesystem = filesForFilesystem;
		}

		protected Map getVariableSubstitutionMap(Map extra) {
			Map m = new HashMap();
			m.put("MARYVERSION", voiceVersion);
			m.put("VOICENAME", voiceName);
			m.put("LOCALE", MaryUtils.locale2xmllang(locale));
			m.put("LANG", locale.getLanguage());
			m.put("DISPLAYLANG", locale.getDisplayLanguage());
			m.put("GENDER", gender);
			m.put("DOMAIN", domain);
			m.put("SAMPLINGRATE", String.valueOf(samplingRate));
			m.put("PACKAGE", getPackageName());
			m.put("VOICECLASS", isUnitSelectionVoice ? "marytts.unitselection.UnitSelectionVoice" : "marytts.htsengine.HMMVoice");
			if (extra != null) {
				m.putAll(extra);
			}
			return m;
		}

		public void createDirectories() throws IOException {
			if (compileDir.exists()) {
				FileUtils.deleteDirectory(compileDir);
			}
			compileDir.mkdir();
			String packageName = getPackageName();
			mainJavaDir = new File(compileDir.getAbsolutePath() + "/src/main/java/marytts/voice/" + packageName);
			mainJavaDir.mkdirs();
			mainResourcesDir = new File(compileDir.getAbsolutePath() + "/src/main/resources/marytts/voice/" + packageName);
			mainResourcesDir.mkdirs();
			nonPackagedResourcesDir = new File(compileDir.getAbsolutePath() + "/src/non-packaged-resources");
			nonPackagedResourcesDir.mkdirs();
			mainDescriptionsDir = new File(compileDir.getAbsolutePath() + "/src/main/descriptors");
			mainDescriptionsDir.mkdirs();
			metaInfDir = new File(compileDir.getAbsolutePath() + "/src/main/resources/META-INF/services");
			metaInfDir.mkdirs();
			testJavaDir = new File(compileDir.getAbsolutePath() + "/src/test/java/marytts/voice/" + packageName);
			testJavaDir.mkdirs();
			if (filesForFilesystem != null && filesForFilesystem.length > 0) {
				libVoiceDir = new File(compileDir.getAbsolutePath() + "/lib/voices/" + voiceName);
				libVoiceDir.mkdir();
			}
		}

		public void copyTemplateFiles() throws IOException {
			copyWithVarSubstitution("pom.xml", new File(compileDir, "pom.xml"));
			copyWithVarSubstitution("generateComponentFile.groovy", new File(nonPackagedResourcesDir,
					"generateComponentFile.groovy"));
			copyWithVarSubstitution("installable.xml", new File(mainDescriptionsDir, "installable.xml"));
			copyWithVarSubstitution("Config.java", new File(mainJavaDir, "Config.java"));
			copyWithVarSubstitution("LoadVoiceIT.java", new File(testJavaDir, "LoadVoiceIT.java"));
			copyWithVarSubstitution("marytts.config.MaryConfig", new File(metaInfDir, "marytts.config.MaryConfig"));
			if (isUnitSelectionVoice) {
				copyWithVarSubstitution("unitselection-voice.config", getConfigFile());
			} else {
				copyWithVarSubstitution("hsmm-voice.config", getConfigFile());
			}
		}

		public void setFilesForResources(File[] filesForResources) {
			this.filesForResources = filesForResources;
		}

		public File getConfigFile() {
			return new File(mainResourcesDir, "voice.config");
		}

		public File getMainResourcesDir() {
			return mainResourcesDir;
		}

		private void copyWithVarSubstitution(String resourceName, File destination, StrSubstitutor... moreSubstitutors)
				throws IOException {
			String resource = marytts.util.io.FileUtils.getStreamAsString(
					getClass().getResourceAsStream("templates/" + resourceName), "UTF-8");
			String resourceWithReplacements = substitutor.replace(resource);
			for (StrSubstitutor more : moreSubstitutors) {
				resourceWithReplacements = more.replace(resourceWithReplacements);
			}
			PrintWriter out = new PrintWriter(destination, "UTF-8");
			out.print(resourceWithReplacements);
			out.close();
		}

		public void copyVoiceFiles() throws IOException {

			if (filesForResources != null) {
				for (File f : filesForResources) {
					FileUtils.copyFileToDirectory(f, mainResourcesDir);
				}
			}

			if (filesForFilesystem != null) {
				for (File f : filesForFilesystem) {
					FileUtils.copyFileToDirectory(f, libVoiceDir);
				}
			}
		}

		public void compileWithMaven() throws IOException, InterruptedException {
			Process maven = Runtime.getRuntime().exec(this.mvn + " verify", null, compileDir);
			StreamGobbler merr = new StreamGobbler(maven.getErrorStream(), "maven err");
			StreamGobbler mout = new StreamGobbler(maven.getInputStream(), "maven out");
			merr.start();
			mout.start();
			int result = maven.waitFor();
			if (result != 0) {
				throw new IOException("Maven compilation did not succeed -- check console for details.");
			}
		}

		// public void createComponentFile() throws IOException {
		// String zipFileName = substitutor.replace("voice-${VOICENAME}-${MARYVERSION}.zip");
		// File zipFile = new File(compileDir.getAbsolutePath()+"/target/"+zipFileName);
		// String zipFileMd5Hash = MD5.asHex(MD5.getHash(zipFile));
		// Map compMap = new HashMap();
		// compMap.put("MD5", zipFileMd5Hash);
		// compMap.put("FILESIZE", String.valueOf(zipFile.length()));
		// StrSubstitutor compSubst = new StrSubstitutor(compMap);
		// String componentFileName = substitutor.replace("voice-${VOICENAME}-${MARYVERSION}-component.xml");
		// File componentFile = new File(compileDir.getAbsolutePath()+"/target/"+componentFileName);
		// copyWithVarSubstitution("component.xml", componentFile, compSubst);
		// }

		public String getPackageName() {
			return toPackageName(voiceName);
		}
	}

	/**
	 * Convert an arbitrary string into a valid java package name, as follows: - any characters that are not alphanumeric or
	 * underscore are deleted; - if the first character after a deleted one is a letter, it is capitalised. - if the first
	 * character is not a letter, we prepend a "V" for "voice".
	 * 
	 * @param voiceName
	 *            voiceName
	 * @return result in string format
	 */
	public static String toPackageName(String voiceName) {
		String regexLCLetter = "[a-z]";
		String regexLetter = "[a-zA-Z]";
		String invalidChars = "[^_a-zA-Z0-9]";
		String[] parts = voiceName.split(invalidChars);
		StringBuilder result = new StringBuilder();
		for (String part : parts) {
			if (part.isEmpty()) {
				continue;
			}
			String firstChar = part.substring(0, 1);
			if (Pattern.matches(regexLCLetter, firstChar)) {
				result.append(firstChar.toUpperCase()).append(part.substring(1));
			} else {
				result.append(part);
			}
		}
		if (!Pattern.matches(regexLetter, result.subSequence(0, 1))) {
			result.insert(0, "V");
		}
		return result.toString();
	}

}