org.apache.mahout.vectorizer.EncodingMapper Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of mahout-mr Show documentation
Scalable machine learning libraries
There is a newer version: 0.13.0
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.mahout.vectorizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.mahout.common.ClassUtils;
import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.SequentialAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder;
import org.apache.mahout.vectorizer.encoders.LuceneTextValueEncoder;

import java.io.IOException;

/**
 * The Mapper that does the work of encoding text
 */
public class EncodingMapper extends Mapper {

  public static final String USE_NAMED_VECTORS = "namedVectors";
  public static final String USE_SEQUENTIAL = "sequential";
  public static final String ANALYZER_NAME = "analyzer";
  public static final String ENCODER_FIELD_NAME = "encoderFieldName";
  public static final String ENCODER_CLASS = "encoderClass";
  public static final String CARDINALITY = "cardinality";
  private boolean sequentialVectors;
  private boolean namedVectors;
  private FeatureVectorEncoder encoder;
  private int cardinality;

  @Override
  protected void setup(Context context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    sequentialVectors = conf.getBoolean(USE_SEQUENTIAL, false);
    namedVectors = conf.getBoolean(USE_NAMED_VECTORS, false);
    String analyzerName = conf.get(ANALYZER_NAME, StandardAnalyzer.class.getName());
    Analyzer analyzer;
    try {
      analyzer = AnalyzerUtils.createAnalyzer(analyzerName);
    } catch (ClassNotFoundException e) {
      //TODO: hmmm, don't like this approach
      throw new IOException("Unable to create Analyzer for name: " + analyzerName, e);
    }

    String encoderName = conf.get(ENCODER_FIELD_NAME, "text");
    cardinality = conf.getInt(CARDINALITY, 5000);
    String encClass = conf.get(ENCODER_CLASS);
    encoder = ClassUtils.instantiateAs(encClass,
            FeatureVectorEncoder.class,
            new Class[]{String.class},
            new Object[]{encoderName});
    if (encoder instanceof LuceneTextValueEncoder) {
      ((LuceneTextValueEncoder) encoder).setAnalyzer(analyzer);
    }
  }

  @Override
  protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
    Vector vector;
    if (sequentialVectors) {
      vector = new SequentialAccessSparseVector(cardinality);
    } else {
      vector = new RandomAccessSparseVector(cardinality);
    }
    if (namedVectors) {
      vector = new NamedVector(vector, key.toString());
    }
    encoder.addToVector(value.toString(), vector);
    context.write(new Text(key.toString()), new VectorWritable(vector));
  }
}