org.elasticsearch.hadoop.serialization.dto.mapping.MappingSet Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch-hadoop-cascading Show documentation
Elasticsearch Hadoop Cascading
The newest version!
package org.elasticsearch.hadoop.serialization.dto.mapping;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;

import org.elasticsearch.hadoop.EsHadoopIllegalArgumentException;
import org.elasticsearch.hadoop.serialization.FieldType;

/**
 * Object representation of all mappings available under a given set of indices. Manages storing each mapping
 * by its index and type for fast lookups as well as resolving multiple mappings down into one combined schema.
 */
public class MappingSet implements Serializable {

    private static final String RESOLVED_MAPPING_NAME = "*";

    private final boolean empty;
    private final Map> indexTypeMap = new HashMap>();
    private final Mapping resolvedSchema;

    public MappingSet(List fields) {
        if (fields.isEmpty()) {
            this.empty = true;
            this.resolvedSchema = new Mapping(RESOLVED_MAPPING_NAME, Field.NO_FIELDS);
        } else {
            this.empty = false;
            for (Field field : fields) {
                String indexName = field.name();
                Field[] mappings = field.properties();
                Map mappingsToSchema = new HashMap();
                this.indexTypeMap.put(indexName, mappingsToSchema);

                for (Field mappingHeader : mappings) {
                    // There's only one mapping Header named "mappings". Unwrap it to get the actual mappings.
                    for (Field mapping : mappingHeader.properties()) {
                        mappingsToSchema.put(mapping.name(), new Mapping(mapping.name(), mapping.properties()));
                    }
                }

            }
            this.resolvedSchema = mergeMappings(fields);
        }
    }

    private static Mapping mergeMappings(List fields) {
        Map fieldMap = new LinkedHashMap();
        for (Field rootField : fields) {
            Field[] props = rootField.properties();
            // handle the common case of mapping by removing the first field (mapping.)
            if (props.length > 0 && props[0] != null && "mappings".equals(props[0].name()) && FieldType.OBJECT.equals(props[0].type())) {
                // can't return the type as it is an object of properties
                Field[] mappings = props[0].properties();

                for (Field mapping : mappings) {
                    // At this point we have the root mapping info
                    for (Field field : mapping.properties()) {
                        addToFieldTable(field, "", fieldMap);
                    }
                }
            }
        }
        Field[] collapsed = collapseFields(fieldMap);
        return new Mapping(RESOLVED_MAPPING_NAME, collapsed);
    }

    @SuppressWarnings("unchecked")
    private static void addToFieldTable(Field field, String parent, Map fieldTable) {
        String fullName = parent + field.name();
        Object[] entry = fieldTable.get(fullName);
        if (entry == null) {
            // Haven't seen field yet.
            if (FieldType.isCompound(field.type())) {
                // visit its children
                Map subTable =  new LinkedHashMap();
                entry = new Object[]{field, subTable};
                String prefix = fullName + ".";
                for (Field subField : field.properties()) {
                    addToFieldTable(subField, prefix, subTable);
                }
            } else {
                // note that we saw it
                entry = new Object[]{field};
            }
            fieldTable.put(fullName, entry);
        } else {
            // We've seen this field before.
            Field previousField = (Field)entry[0];
            // ensure that it doesn't conflict
            if (!previousField.type().equals(field.type())) {
                // Attempt to resolve field type conflicts by upcasting fields to a common "super type"
                FieldType resolvedType = resolveTypeConflict(fullName, previousField.type(), field.type());
                // If successful, update the previous field entry with the updated field type
                if (!previousField.type().equals(resolvedType)) {
                    previousField = new Field(previousField.name(), resolvedType, previousField.properties());
                    entry[0] = previousField;
                }
            }
            // If it does not conflict, visit it's children if it has them
            if (FieldType.isCompound(field.type())) {
                Map subTable = (Map)entry[1];
                String prefix = fullName + ".";
                for (Field subField : field.properties()) {
                    addToFieldTable(subField, prefix, subTable);
                }
            }
        }
    }

    private static FieldType resolveTypeConflict(String fullName, FieldType existing, FieldType incoming) {
        // Prefer to upcast the incoming field to the existing first
        LinkedHashSet incomingSuperTypes = incoming.getCastingTypes();
        if (incomingSuperTypes.contains(existing)) {
            // Incoming can be cast to existing.
            return existing;
        }
        // See if existing can be upcast to the incoming field's type next
        LinkedHashSet existingSuperTypes = existing.getCastingTypes();
        if (existingSuperTypes.contains(incoming)) {
            // Existing can be cast to incoming
            return incoming;
        }
        // Finally, Try to pick the lowest common super type for both fields if it exists
        if (incomingSuperTypes.size() > 0 && existingSuperTypes.size() > 0) {
            LinkedHashSet combined = new LinkedHashSet(incomingSuperTypes);
            combined.retainAll(existingSuperTypes);
            if (combined.size() > 0) {
                return combined.iterator().next();
            }
        }
        // If none of the above options succeed, the fields are conflicting
        throw new EsHadoopIllegalArgumentException("Incompatible types found in multi-mapping: " +
                "Field ["+fullName+"] has conflicting types of ["+existing+"] and ["+
                incoming+"].");
    }

    @SuppressWarnings("unchecked")
    private static Field[] collapseFields(Map fieldTable) {
        List fields = new ArrayList();
        for (Map.Entry fieldInfo : fieldTable.entrySet()) {
            Field currentField = (Field)(fieldInfo.getValue()[0]);
            if (FieldType.isCompound(currentField.type())) {
                Map subTable = (Map)(fieldInfo.getValue()[1]);
                Field[] children = collapseFields(subTable);
                fields.add(new Field(currentField.name(), currentField.type(), children));
            } else {
                fields.add(currentField);
            }
        }
        return fields.size() == 0 ? Field.NO_FIELDS : fields.toArray(new Field[fields.size()]);
    }

    public Mapping getMapping(String index, String type) {
        Mapping mapping = null;
        Map mappings = indexTypeMap.get(index);
        if (mappings != null) {
            mapping = mappings.get(type);
        }
        return mapping;
    }

    /**
     * True if there are no mappings in this mapping set
     */
    public boolean isEmpty() {
        return empty;
    }

    /**
     * Returns the combined schema of all mappings contained within this mapping set.
     */
    public Mapping getResolvedView() {
        return resolvedSchema;
    }

    @Override
    public String toString() {
        return "MappingSet{" +
                "indexTypeMap=" + indexTypeMap +
                ", resolvedSchema=" + resolvedSchema +
                '}';
    }
}