org.elasticsearch.hadoop.serialization.dto.mapping.MappingSet Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of elasticsearch-hadoop-cascading Show documentation
Show all versions of elasticsearch-hadoop-cascading Show documentation
Elasticsearch Hadoop Cascading
package org.elasticsearch.hadoop.serialization.dto.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import org.elasticsearch.hadoop.EsHadoopIllegalArgumentException;
import org.elasticsearch.hadoop.serialization.FieldType;
/**
* Object representation of all mappings available under a given set of indices. Manages storing each mapping
* by its index and type for fast lookups as well as resolving multiple mappings down into one combined schema.
*/
public class MappingSet implements Serializable {
private static final String RESOLVED_MAPPING_NAME = "*";
private final boolean empty;
private final Map> indexTypeMap = new HashMap>();
private final Mapping resolvedSchema;
public MappingSet(List fields) {
if (fields.isEmpty()) {
this.empty = true;
this.resolvedSchema = new Mapping(RESOLVED_MAPPING_NAME, Field.NO_FIELDS);
} else {
this.empty = false;
for (Field field : fields) {
String indexName = field.name();
Field[] mappings = field.properties();
Map mappingsToSchema = new HashMap();
this.indexTypeMap.put(indexName, mappingsToSchema);
for (Field mappingHeader : mappings) {
// There's only one mapping Header named "mappings". Unwrap it to get the actual mappings.
for (Field mapping : mappingHeader.properties()) {
mappingsToSchema.put(mapping.name(), new Mapping(mapping.name(), mapping.properties()));
}
}
}
this.resolvedSchema = mergeMappings(fields);
}
}
private static Mapping mergeMappings(List fields) {
Map fieldMap = new LinkedHashMap();
for (Field rootField : fields) {
Field[] props = rootField.properties();
// handle the common case of mapping by removing the first field (mapping.)
if (props.length > 0 && props[0] != null && "mappings".equals(props[0].name()) && FieldType.OBJECT.equals(props[0].type())) {
// can't return the type as it is an object of properties
Field[] mappings = props[0].properties();
for (Field mapping : mappings) {
// At this point we have the root mapping info
for (Field field : mapping.properties()) {
addToFieldTable(field, "", fieldMap);
}
}
}
}
Field[] collapsed = collapseFields(fieldMap);
return new Mapping(RESOLVED_MAPPING_NAME, collapsed);
}
@SuppressWarnings("unchecked")
private static void addToFieldTable(Field field, String parent, Map fieldTable) {
String fullName = parent + field.name();
Object[] entry = fieldTable.get(fullName);
if (entry == null) {
// Haven't seen field yet.
if (FieldType.isCompound(field.type())) {
// visit its children
Map subTable = new LinkedHashMap();
entry = new Object[]{field, subTable};
String prefix = fullName + ".";
for (Field subField : field.properties()) {
addToFieldTable(subField, prefix, subTable);
}
} else {
// note that we saw it
entry = new Object[]{field};
}
fieldTable.put(fullName, entry);
} else {
// We've seen this field before.
Field previousField = (Field)entry[0];
// ensure that it doesn't conflict
if (!previousField.type().equals(field.type())) {
// Attempt to resolve field type conflicts by upcasting fields to a common "super type"
FieldType resolvedType = resolveTypeConflict(fullName, previousField.type(), field.type());
// If successful, update the previous field entry with the updated field type
if (!previousField.type().equals(resolvedType)) {
previousField = new Field(previousField.name(), resolvedType, previousField.properties());
entry[0] = previousField;
}
}
// If it does not conflict, visit it's children if it has them
if (FieldType.isCompound(field.type())) {
Map subTable = (Map)entry[1];
String prefix = fullName + ".";
for (Field subField : field.properties()) {
addToFieldTable(subField, prefix, subTable);
}
}
}
}
private static FieldType resolveTypeConflict(String fullName, FieldType existing, FieldType incoming) {
// Prefer to upcast the incoming field to the existing first
LinkedHashSet incomingSuperTypes = incoming.getCastingTypes();
if (incomingSuperTypes.contains(existing)) {
// Incoming can be cast to existing.
return existing;
}
// See if existing can be upcast to the incoming field's type next
LinkedHashSet existingSuperTypes = existing.getCastingTypes();
if (existingSuperTypes.contains(incoming)) {
// Existing can be cast to incoming
return incoming;
}
// Finally, Try to pick the lowest common super type for both fields if it exists
if (incomingSuperTypes.size() > 0 && existingSuperTypes.size() > 0) {
LinkedHashSet combined = new LinkedHashSet(incomingSuperTypes);
combined.retainAll(existingSuperTypes);
if (combined.size() > 0) {
return combined.iterator().next();
}
}
// If none of the above options succeed, the fields are conflicting
throw new EsHadoopIllegalArgumentException("Incompatible types found in multi-mapping: " +
"Field ["+fullName+"] has conflicting types of ["+existing+"] and ["+
incoming+"].");
}
@SuppressWarnings("unchecked")
private static Field[] collapseFields(Map fieldTable) {
List fields = new ArrayList();
for (Map.Entry fieldInfo : fieldTable.entrySet()) {
Field currentField = (Field)(fieldInfo.getValue()[0]);
if (FieldType.isCompound(currentField.type())) {
Map subTable = (Map)(fieldInfo.getValue()[1]);
Field[] children = collapseFields(subTable);
fields.add(new Field(currentField.name(), currentField.type(), children));
} else {
fields.add(currentField);
}
}
return fields.size() == 0 ? Field.NO_FIELDS : fields.toArray(new Field[fields.size()]);
}
public Mapping getMapping(String index, String type) {
Mapping mapping = null;
Map mappings = indexTypeMap.get(index);
if (mappings != null) {
mapping = mappings.get(type);
}
return mapping;
}
/**
* True if there are no mappings in this mapping set
*/
public boolean isEmpty() {
return empty;
}
/**
* Returns the combined schema of all mappings contained within this mapping set.
*/
public Mapping getResolvedView() {
return resolvedSchema;
}
@Override
public String toString() {
return "MappingSet{" +
"indexTypeMap=" + indexTypeMap +
", resolvedSchema=" + resolvedSchema +
'}';
}
}