eu.stratosphere.meteor.QueryParser Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of meteor-meteor Show documentation
The newest version!
package eu.stratosphere.meteor;

import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.IdentityHashMap;
import java.util.Map;

import org.antlr.runtime.ANTLRInputStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;

import eu.stratosphere.core.fs.Path;
import eu.stratosphere.sopremo.SopremoEnvironment;
import eu.stratosphere.sopremo.operator.JsonStream;
import eu.stratosphere.sopremo.operator.Name;
import eu.stratosphere.sopremo.operator.Operator;
import eu.stratosphere.sopremo.operator.SopremoPlan;
import eu.stratosphere.sopremo.packages.IRegistry;
import eu.stratosphere.sopremo.query.ConfObjectInfo;
import eu.stratosphere.sopremo.query.ConfObjectInfo.ConfObjectIndexedPropertyInfo;
import eu.stratosphere.sopremo.query.ConfObjectInfo.ConfObjectPropertyInfo;
import eu.stratosphere.sopremo.query.IConfObjectRegistry;
import eu.stratosphere.sopremo.query.PackageManager;
import eu.stratosphere.sopremo.query.QueryParserException;
import eu.stratosphere.util.StringUtil;

public class QueryParser {
	private File inputDirectory = new File(".");

	private final PackageManager packageManager = new PackageManager(MeteorParserBase.NameChooserProvider);

	/**
	 * Returns the inputDirectory.
	 * 
	 * @return the inputDirectory
	 */
	public File getInputDirectory() {
		return this.inputDirectory;
	}

	/**
	 * Returns the packageManager.
	 * 
	 * @return the packageManager
	 */
	public PackageManager getPackageManager() {
		return this.packageManager;
	}

	public MeteorParser getParser(final CharStream charStream) {
		final MeteorLexer lexer = new MeteorLexer(charStream);
		final CommonTokenStream tokens = new CommonTokenStream();
		tokens.setTokenSource(lexer);
		final MeteorParser parser = new MeteorParser(tokens);
		SopremoEnvironment.getInstance().getEvaluationContext()
			.setWorkingPath(new Path(this.inputDirectory.toURI().toString()));
		parser.getPackageManager().addAll(this.packageManager);
		parser.getPackageManager().addJarPathLocation(this.inputDirectory);
		parser.setTreeAdaptor(new SopremoTreeAdaptor());
		return parser;
	}

	public MeteorParser getParser(final InputStream stream) throws IOException {
		return this.getParser(new ANTLRInputStream(stream));
	}

	public MeteorParser getParser(final String script) {
		return this.getParser(new ANTLRStringStream(script));
	}

	/**
	 * Sets the inputDirectory to the specified value.
	 * 
	 * @param inputDirectory
	 *        the inputDirectory to set
	 */
	public void setInputDirectory(final File inputDirectory) {
		if (inputDirectory == null)
			throw new NullPointerException("inputDirectory must not be null");

		this.inputDirectory = inputDirectory;
	}

	public String toSopremoCode(final InputStream stream) throws IOException, QueryParserException {
		return this.toSopremoCode(new ANTLRInputStream(stream));
	}

	public String toSopremoCode(final String script) throws QueryParserException {
		return this.toSopremoCode(new ANTLRStringStream(script));
	}

	public SopremoPlan tryParse(final CharStream charStream) {
		return this.getParser(charStream).parse();
	}

	public SopremoPlan tryParse(final InputStream stream) throws IOException, QueryParserException {
		return this.tryParse(new ANTLRInputStream(stream));
	}

	public SopremoPlan tryParse(final String script) throws QueryParserException {
		return this.tryParse(new ANTLRStringStream(script));
	}

	/**
	 * Sets the inputDirectory to the specified value.
	 * 
	 * @param inputDirectory
	 *        the inputDirectory to set
	 */
	public QueryParser withInputDirectory(final File inputDirectory) {
		this.setInputDirectory(inputDirectory);
		return this;
	}

	protected > void appendInputProperties(final O op, final JavaRenderInfo renderInfo,
			final ConfObjectInfo info, final O defaultInstance) {
		final IRegistry inputPropertyRegistry = info.getInputPropertyRegistry(op);
		for (final String propertyName : inputPropertyRegistry.keySet())
			for (int index = 0; index < op.getInputs().size(); index++) {
				final ConfObjectIndexedPropertyInfo propertyInfo = inputPropertyRegistry.get(propertyName);
				final Object actualValue = propertyInfo.getValue(op, index);
				final Object defaultValue = propertyInfo.getValue(defaultInstance, index);
				if (!actualValue.equals(defaultValue)) {
					renderInfo.builder.append(renderInfo.getVariableName(op)).
						append(".set").append(StringUtil.upperFirstChar(propertyInfo.getDescriptor().getName())).
						append("(").append(index).append(", ");
					this.appendExpression(actualValue, renderInfo);
					renderInfo.builder.append(");\n");
				}
			}
	}

	protected > void appendInputs(final O op, final JavaRenderInfo renderInfo,
			final O defaultInstance) {
		if (!defaultInstance.getInputs().equals(op.getInputs())) {
			renderInfo.builder.append(renderInfo.getVariableName(op)).append(".setInputs(");
			for (int index = 0; index < op.getInputs().size(); index++) {
				if (index > 0)
					renderInfo.builder.append(", ");
				renderInfo.builder.append(renderInfo.getVariableName(op.getInput(index)));
			}
			renderInfo.builder.append(");\n");
		}
	}

	protected void appendJavaOperator(final Operator op, final JavaRenderInfo renderInfo) {
		final String className = op.getClass().getSimpleName();
		renderInfo.builder.append(String.format("%s %s = new %1$s();\n", className, renderInfo.getVariableName(op)));

		final IConfObjectRegistry> operatorRegistry = renderInfo.parser.getOperatorRegistry();
		final ConfObjectInfo> info = operatorRegistry.get(op.getClass().getAnnotation(Name.class));
		final Operator defaultInstance = info.newInstance();
		this.appendInputs(op, renderInfo, defaultInstance);
		defaultInstance.setInputs(op.getInputs());
		this.appendOperatorProperties(op, renderInfo, info, defaultInstance);
		this.appendInputProperties(op, renderInfo, info, defaultInstance);
	}

	protected > void appendOperatorProperties(final O op,
			final JavaRenderInfo renderInfo, final ConfObjectInfo info, final O defaultInstance) {

		final IRegistry operatorPropertyRegistry = info.getOperatorPropertyRegistry(op);
		for (final String propertyName : operatorPropertyRegistry.keySet()) {
			final ConfObjectPropertyInfo propertyInfo = operatorPropertyRegistry.get(propertyName);
			final Object actualValue = propertyInfo.getValue(op);
			final Object defaultValue = propertyInfo.getValue(defaultInstance);
			if (!actualValue.equals(defaultValue)) {
				renderInfo.builder.append(renderInfo.getVariableName(op)).
					append(".set").append(StringUtil.upperFirstChar(propertyInfo.getDescriptor().getName())).
					append("(");
				this.appendExpression(actualValue, renderInfo);
				renderInfo.builder.append(");\n");
			}
		}
	}

	protected String toSopremoCode(final CharStream input) {
		final MeteorLexer lexer = new MeteorLexer(input);
		final CommonTokenStream tokens = new CommonTokenStream();
		tokens.setTokenSource(lexer);
		final MeteorParser parser = new MeteorParser(tokens);
		final TraceableSopremoTreeAdaptor adaptor = new TraceableSopremoTreeAdaptor();
		parser.setTreeAdaptor(adaptor);
		final SopremoPlan result = parser.parse();
		final JavaRenderInfo info = new JavaRenderInfo(parser, adaptor);
		this.toSopremoCode(result, info);
		return info.builder.toString();
	}

	protected String toSopremoCode(final SopremoPlan result, final JavaRenderInfo info) {
		for (final Operator op : result.getContainedOperators())
			this.appendJavaOperator(op, info);
		return info.builder.toString();
	}

	private void appendExpression(final Object value, final JavaRenderInfo renderInfo) {
		renderInfo.adaptor.addJavaFragment(value, renderInfo.builder);
	}

	public static String getPrefixedName(final String prefix, final String name) {
		return String.format("%s:%s", prefix, name);
	}

	private static class JavaRenderInfo {
		private final MeteorParser parser;

		private final TraceableSopremoTreeAdaptor adaptor;

		private final StringBuilder builder = new StringBuilder();

		private final Map variableNames = new IdentityHashMap();

		private final Object2IntMap> instanceCounter = new Object2IntOpenHashMap>();

		public JavaRenderInfo(final MeteorParser parser, final TraceableSopremoTreeAdaptor adaptor) {
			this.parser = parser;
			this.adaptor = adaptor;
		}

		public String getVariableName(final JsonStream input) {
			final Operator op = input instanceof Operator ? (Operator) input : input.getSource().getOperator();
			String name = this.variableNames.get(op);
			if (name == null) {
				final int counter = this.instanceCounter.getInt(op.getClass()) + 1;
				this.instanceCounter.put(op.getClass(), counter);
				name = String.format("%s%d", StringUtil.lowerFirstChar(op.getClass().getSimpleName()), counter);
				this.variableNames.put(op, name);
			}
			return name;
		}
	}
}