org.molgenis.data.importer.EmxMetaDataParser Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of molgenis-data-import Show documentation
Show all versions of molgenis-data-import Show documentation
Entity importer framework and EMX importer.
The newest version!
package org.molgenis.data.importer;
import static java.util.Objects.requireNonNull;
import static org.molgenis.data.i18n.I18nUtils.getLanguageCode;
import static org.molgenis.data.i18n.I18nUtils.isI18n;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.AGGREGATEABLE;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.DATA_TYPE;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.DEFAULT_VALUE;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.DESCRIPTION;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.ENUM_OPTIONS;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.EXPRESSION;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.LABEL;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.NAME;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.NILLABLE;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.RANGE_MAX;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.RANGE_MIN;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.READ_ONLY;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.REF_ENTITY;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.UNIQUE;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.VALIDATION_EXPRESSION;
import static org.molgenis.data.meta.AttributeMetaDataMetaData.VISIBLE;
import static org.molgenis.data.meta.EntityMetaDataMetaData.ABSTRACT;
import static org.molgenis.data.meta.EntityMetaDataMetaData.BACKEND;
import static org.molgenis.data.meta.EntityMetaDataMetaData.EXTENDS;
import static org.molgenis.data.meta.EntityMetaDataMetaData.PACKAGE;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.molgenis.MolgenisFieldTypes;
import org.molgenis.MolgenisFieldTypes.FieldTypeEnum;
import org.molgenis.data.AttributeMetaData;
import org.molgenis.data.DataService;
import org.molgenis.data.EditableEntityMetaData;
import org.molgenis.data.Entity;
import org.molgenis.data.EntityMetaData;
import org.molgenis.data.MolgenisDataException;
import org.molgenis.data.Package;
import org.molgenis.data.Range;
import org.molgenis.data.Repository;
import org.molgenis.data.RepositoryCollection;
import org.molgenis.data.UnknownEntityException;
import org.molgenis.data.i18n.I18nStringMetaData;
import org.molgenis.data.i18n.I18nUtils;
import org.molgenis.data.i18n.LanguageMetaData;
import org.molgenis.data.importer.MyEntitiesValidationReport.AttributeState;
import org.molgenis.data.meta.AttributeMetaDataMetaData;
import org.molgenis.data.meta.EntityMetaDataMetaData;
import org.molgenis.data.meta.MetaValidationUtils;
import org.molgenis.data.meta.PackageImpl;
import org.molgenis.data.meta.PackageMetaData;
import org.molgenis.data.meta.TagMetaData;
import org.molgenis.data.semantic.TagImpl;
import org.molgenis.data.support.DefaultAttributeMetaData;
import org.molgenis.fieldtypes.CompoundField;
import org.molgenis.fieldtypes.EnumField;
import org.molgenis.fieldtypes.FieldType;
import org.molgenis.fieldtypes.IntField;
import org.molgenis.fieldtypes.LongField;
import org.molgenis.fieldtypes.MrefField;
import org.molgenis.fieldtypes.StringField;
import org.molgenis.fieldtypes.XrefField;
import org.molgenis.framework.db.EntitiesValidationReport;
import org.molgenis.util.DependencyResolver;
import org.springframework.core.convert.ConversionFailedException;
import org.springframework.util.StringUtils;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
/**
* Parser for the EMX metadata. This class is stateless, but it passes state between methods using
* {@link IntermediateParseResults}.
*/
public class EmxMetaDataParser implements MetaDataParser
{
private static final String ID_ATTRIBUTE = "idAttribute";
private static final String LOOKUP_ATTRIBUTE = "lookupAttribute";
private static final String LABEL_ATTRIBUTE = "labelAttribute";
// Sheet names
public static final String ENTITIES = EntityMetaDataMetaData.ENTITY_NAME;
public static final String PACKAGES = PackageMetaData.ENTITY_NAME;
public static final String TAGS = TagMetaData.ENTITY_NAME;
public static final String ATTRIBUTES = AttributeMetaDataMetaData.ENTITY_NAME;
public static final String LANGUAGES = LanguageMetaData.ENTITY_NAME;
public static final String I18NSTRINGS = I18nStringMetaData.ENTITY_NAME;
public static final String ENTITY = "entity";
public static final String PART_OF_ATTRIBUTE = "partOfAttribute";
static final List SUPPORTED_ENTITY_ATTRIBUTES = Arrays.asList(
org.molgenis.data.meta.EntityMetaDataMetaData.LABEL.toLowerCase(),
org.molgenis.data.meta.EntityMetaDataMetaData.DESCRIPTION.toLowerCase(), "name", ABSTRACT.toLowerCase(),
EXTENDS.toLowerCase(), "package", EntityMetaDataMetaData.TAGS, BACKEND);
static final List SUPPORTED_ATTRIBUTE_ATTRIBUTES = Arrays.asList(AGGREGATEABLE.toLowerCase(),
DATA_TYPE.toLowerCase(), DESCRIPTION.toLowerCase(), ENTITY.toLowerCase(), ENUM_OPTIONS.toLowerCase(),
ID_ATTRIBUTE.toLowerCase(), LABEL.toLowerCase(), LABEL_ATTRIBUTE.toLowerCase(),
LOOKUP_ATTRIBUTE.toLowerCase(), NAME, NILLABLE.toLowerCase(), PART_OF_ATTRIBUTE.toLowerCase(),
RANGE_MAX.toLowerCase(), RANGE_MIN.toLowerCase(), READ_ONLY.toLowerCase(), REF_ENTITY.toLowerCase(),
VISIBLE.toLowerCase(), UNIQUE.toLowerCase(), TAGS.toLowerCase(), EXPRESSION.toLowerCase(),
VALIDATION_EXPRESSION.toLowerCase(), DEFAULT_VALUE.toLowerCase());
static final String AUTO = "auto";
private final DataService dataService;
public EmxMetaDataParser(DataService dataService)
{
this.dataService = dataService;
}
/**
* Parses metadata from a collection of repositories.
*
* @param source
* the {@link RepositoryCollection} containing the metadata to parse
* @return {@link IntermediateParseResults} containing the parsed metadata
*/
private IntermediateParseResults getEntityMetaDataFromSource(RepositoryCollection source)
{
// TODO: this task is actually a 'merge' instead of 'import'
// so we need to consider both new metadata and existing ...
IntermediateParseResults intermediateResults = parseTagsSheet(source.getRepository(TAGS));
parsePackagesSheet(source.getRepository(PACKAGES), intermediateResults);
parsePackageTags(source.getRepository(PACKAGES), intermediateResults);
parseEntitiesSheet(source.getRepository(ENTITIES), intermediateResults);
parseAttributesSheet(source.getRepository(ATTRIBUTES), intermediateResults);
reiterateToMapRefEntity(source.getRepository(ATTRIBUTES), intermediateResults);
// languages tab
if (source.hasRepository(LANGUAGES))
{
parseLanguages(source.getRepository(LANGUAGES), intermediateResults);
}
// i18nstrings tab
if (source.hasRepository(I18NSTRINGS))
{
parseI18nStrings(source.getRepository(I18NSTRINGS), intermediateResults);
}
return intermediateResults;
}
/**
* Parses all tags defined in the tags repository.
*
* @param source
* the {@link Repository} that contains the tags entity
* @return Map mapping tag Identifier to tag {@link Entity}, will be empty if no tags repository was found
*/
private IntermediateParseResults parseTagsSheet(Repository tagRepository)
{
IntermediateParseResults result = new IntermediateParseResults();
if (tagRepository != null)
{
for (Entity tag : tagRepository)
{
String id = tag.getString(TagMetaData.IDENTIFIER);
if (id != null)
{
result.addTagEntity(id, tag);
}
}
}
return result;
}
/**
* Load all attributes from the source repository and add it to the {@link IntermediateParseResults}.
*
* @param source
* Repository for the attributes
* @param intermediateResults
* {@link IntermediateParseResults} with the tags already parsed
*/
private void parseAttributesSheet(Repository attributesRepo, IntermediateParseResults intermediateResults)
{
for (AttributeMetaData attr : attributesRepo.getEntityMetaData().getAtomicAttributes())
{
if (!SUPPORTED_ATTRIBUTE_ATTRIBUTES.contains(attr.getName().toLowerCase())
&& !((I18nUtils.isI18n(attr.getName()) && (attr.getName().toLowerCase().startsWith(LABEL)
|| attr.getName().toLowerCase().startsWith(DESCRIPTION)))))
{
throw new IllegalArgumentException("Unsupported attribute metadata: attributes. " + attr.getName());
}
}
Map> attributesMap = new LinkedHashMap>();
// 1st pass: create attribute stubs
int i = 1;// Header
for (Entity attributeEntity : attributesRepo)
{
i++;
String attributeName = attributeEntity.getString(NAME);
if (attributeName == null) throw new IllegalArgumentException("attributes.name is missing on line " + i);
String entityName = attributeEntity.getString(ENTITY);
if (entityName == null) throw new IllegalArgumentException(
"attributes.entity is missing for attribute named: " + attributeName + " on line " + i);
// create attribute
DefaultAttributeMetaData attribute = new DefaultAttributeMetaData(attributeName);
Map entitiesMap = attributesMap.get(entityName);
if (entitiesMap == null)
{
entitiesMap = new LinkedHashMap<>();
attributesMap.put(entityName, entitiesMap);
}
entitiesMap.put(attributeName, new EmxAttribute(attribute));
}
// 2nd pass: set all properties on attribute stubs except for attribute relations
i = 1;// Header
for (Entity attributeEntity : attributesRepo)
{
i++;
String entityName = attributeEntity.getString(ENTITY);
Map entityMap = attributesMap.get(entityName);
String attributeName = attributeEntity.getString(NAME);
EmxAttribute emxAttribute = entityMap.get(attributeName);
DefaultAttributeMetaData attribute = emxAttribute.getAttr();
String attributeDataType = attributeEntity.getString(DATA_TYPE);
String refEntityName = attributeEntity.getString(REF_ENTITY);
if (attributeDataType != null)
{
FieldType t = MolgenisFieldTypes.getType(attributeDataType);
if (t == null) throw new IllegalArgumentException(
"attributes.dataType error on line " + i + ": " + attributeDataType + " unknown data type");
attribute.setDataType(t);
}
else
{
attribute.setDataType(MolgenisFieldTypes.STRING);
}
Boolean attributeNillable = attributeEntity.getBoolean(NILLABLE);
String attributeIdAttribute = attributeEntity.getString(ID_ATTRIBUTE);
Boolean attributeAggregateable = attributeEntity.getBoolean(AGGREGATEABLE);
Boolean lookupAttribute = attributeEntity.getBoolean(LOOKUP_ATTRIBUTE);
Boolean labelAttribute = attributeEntity.getBoolean(LABEL_ATTRIBUTE);
Boolean readOnly = attributeEntity.getBoolean(READ_ONLY);
Boolean unique = attributeEntity.getBoolean(UNIQUE);
String expression = attributeEntity.getString(EXPRESSION);
List tagIds = attributeEntity.getList(TAGS);
String validationExpression = attributeEntity.getString(VALIDATION_EXPRESSION);
String defaultValue = attributeEntity.getString(DEFAULT_VALUE);
if (attributeNillable != null) attribute.setNillable(attributeNillable);
boolean isIdAttr = attributeIdAttribute != null
&& (attributeIdAttribute.equalsIgnoreCase("true") || attributeIdAttribute.equalsIgnoreCase(AUTO));
emxAttribute.setIdAttr(isIdAttr);
if (isIdAttr)
{
if ((attributeNillable != null) && attributeNillable)
{
throw new IllegalArgumentException(
"Attributes error on line " + i + ". Id attributes cannot be nillable");
}
attribute.setNillable(false);
}
String attributeVisible = attributeEntity.getString(VISIBLE);
if (attributeVisible != null)
{
if (attributeVisible.equalsIgnoreCase("true") || attributeVisible.equalsIgnoreCase("false"))
{
attribute.setVisible(Boolean.parseBoolean(attributeVisible));
}
else
{
attribute.setVisibleExpression(attributeVisible);
}
}
if (attributeAggregateable != null) attribute.setAggregateable(attributeAggregateable);
// cannot update ref entities yet, will do so later on
if (readOnly != null) attribute.setReadOnly(readOnly);
if (unique != null) attribute.setUnique(unique);
if (expression != null) attribute.setExpression(expression);
if (validationExpression != null) attribute.setValidationExpression(validationExpression);
if (defaultValue != null) attribute.setDefaultValue(defaultValue);
attribute.setAuto(attributeIdAttribute != null && attributeIdAttribute.equalsIgnoreCase(AUTO));
if ((attributeIdAttribute != null) && !attributeIdAttribute.equalsIgnoreCase("true")
&& !attributeIdAttribute.equalsIgnoreCase("false") && !attributeIdAttribute.equalsIgnoreCase(AUTO))
{
throw new IllegalArgumentException("Attributes error on line " + i
+ ". Illegal idAttribute value. Allowed values are 'TRUE', 'FALSE' or 'AUTO'");
}
if (attribute.isAuto() && !(attribute.getDataType() instanceof StringField))
{
throw new IllegalArgumentException(
"Attributes error on line " + i + ". Auto attributes can only be of data type 'string'");
}
if (lookupAttribute != null)
{
if (lookupAttribute && ((attribute.getDataType() instanceof XrefField)
|| (attribute.getDataType() instanceof MrefField)))
{
throw new IllegalArgumentException(
"attributes.lookupAttribute error on line " + i + " (" + entityName + "." + attributeName
+ "): lookupAttribute cannot be of type " + attribute.getDataType());
}
emxAttribute.setLookupAttr(lookupAttribute);
}
if (labelAttribute != null)
{
if (labelAttribute && ((attribute.getDataType() instanceof XrefField)
|| (attribute.getDataType() instanceof MrefField)))
{
throw new IllegalArgumentException(
"attributes.labelAttribute error on line " + i + " (" + entityName + "." + attributeName
+ "): labelAttribute cannot be of type " + attribute.getDataType());
}
emxAttribute.setLabelAttr(labelAttribute);
}
attribute.setLabel(attributeEntity.getString(LABEL));
for (String attr : attributeEntity.getAttributeNames())
{
if (isI18n(attr))
{
if (attr.startsWith(LABEL))
{
String label = attributeEntity.getString(attr);
if (label != null)
{
String languageCode = getLanguageCode(attr);
attribute.setLabel(languageCode, label);
}
}
else if (attr.startsWith(DESCRIPTION))
{
String description = attributeEntity.getString(attr);
if (description != null)
{
String languageCode = getLanguageCode(attr);
attribute.setDescription(languageCode, description);
}
}
}
}
attribute.setDescription(attributeEntity.getString(DESCRIPTION));
if (attribute.getDataType() instanceof EnumField)
{
List enumOptions = attributeEntity.getList(ENUM_OPTIONS);
if ((enumOptions == null) || enumOptions.isEmpty())
{
throw new IllegalArgumentException("Missing enum options for attribute [" + attribute.getName()
+ "] of entity [" + entityName + "]");
}
attribute.setEnumOptions(enumOptions);
}
if (((attribute.getDataType() instanceof XrefField) || (attribute.getDataType() instanceof MrefField))
&& StringUtils.isEmpty(refEntityName))
{
throw new IllegalArgumentException(
"Missing refEntity on line " + i + " (" + entityName + "." + attributeName + ")");
}
if (((attribute.getDataType() instanceof XrefField) || (attribute.getDataType() instanceof MrefField))
&& attribute.isNillable() && attribute.isAggregateable())
{
throw new IllegalArgumentException(
"attributes.aggregatable error on line " + i + " (" + entityName + "." + attributeName
+ "): aggregatable nillable attribute cannot be of type " + attribute.getDataType());
}
Long rangeMin;
Long rangeMax;
try
{
rangeMin = attributeEntity.getLong(RANGE_MIN);
}
catch (ConversionFailedException e)
{
throw new MolgenisDataException(
"Invalid range rangeMin [" + attributeEntity.getString(RANGE_MIN) + "] value for attribute ["
+ attributeName + "] of entity [" + entityName + "], should be a long");
}
try
{
rangeMax = attributeEntity.getLong(RANGE_MAX);
}
catch (ConversionFailedException e)
{
throw new MolgenisDataException("Invalid rangeMax value [" + attributeEntity.getString(RANGE_MAX)
+ "] for attribute [" + attributeName + "] of entity [" + entityName + "], should be a long");
}
if ((rangeMin != null) || (rangeMax != null))
{
if (!(attribute.getDataType() instanceof IntField) && !(attribute.getDataType() instanceof LongField))
{
throw new MolgenisDataException("Range not supported for [" + attribute.getDataType().getEnumType()
+ "] fields only int and long are supported. (attribute [" + attribute.getName()
+ "] of entity [" + entityName + "])");
}
attribute.setRange(new Range(rangeMin, rangeMax));
}
if (tagIds != null)
{
for (String tagId : tagIds)
{
Entity tagEntity = intermediateResults.getTagEntity(tagId);
if (tagEntity == null)
{
throw new MolgenisDataException(
"Unknown tag: " + tagId + " for attribute [" + attribute.getName() + "] of entity ["
+ entityName + "]). Please specify on the " + TAGS + " sheet.");
}
intermediateResults.addAttributeTag(entityName,
TagImpl. asTag(attribute, tagEntity));
}
}
}
// 3rd pass: validate and create attribute relationships
Map> rootAttributes = new LinkedHashMap>();
i = 1;// Header
for (Entity attributeEntity : attributesRepo)
{
i++;
String entityName = attributeEntity.getString(ENTITY);
Map entityMap = attributesMap.get(entityName);
String attributeName = attributeEntity.getString(NAME);
DefaultAttributeMetaData attribute = entityMap.get(attributeName).getAttr();
// register attribute parent-children relations for compound attributes
String partOfAttribute = attributeEntity.getString(PART_OF_ATTRIBUTE);
if (partOfAttribute != null && !partOfAttribute.isEmpty())
{
DefaultAttributeMetaData compoundAttribute = entityMap.get(partOfAttribute).getAttr();
if (compoundAttribute == null)
{
throw new IllegalArgumentException(
"partOfAttribute [" + partOfAttribute + "] of attribute [" + attributeName + "] of entity ["
+ entityName + "] must refer to an existing compound attribute on line " + i);
}
if (compoundAttribute.getDataType().getEnumType() != FieldTypeEnum.COMPOUND)
{
throw new IllegalArgumentException("partOfAttribute [" + partOfAttribute + "] of attribute ["
+ attributeName + "] of entity [" + entityName + "] must refer to a attribute of type ["
+ FieldTypeEnum.COMPOUND + "] on line " + i);
}
compoundAttribute.addAttributePart(attribute);
}
else
{
Set entityRootAttributes = rootAttributes.get(entityName);
if (entityRootAttributes == null)
{
entityRootAttributes = new LinkedHashSet();
rootAttributes.put(entityName, entityRootAttributes);
}
entityRootAttributes.add(attributeName);
}
}
// store attributes with entities
for (Map.Entry> entry : attributesMap.entrySet())
{
String entityName = entry.getKey();
Map attributes = entry.getValue();
List editableEntityMetaData = new ArrayList<>();
// add root attributes to entity
Set entityAttributeNames = rootAttributes.get(entityName);
if (entityAttributeNames != null)
{
for (EmxAttribute attribute : attributes.values())
{
if (entityAttributeNames.contains(attribute.getAttr().getName()))
{
editableEntityMetaData.add(attribute);
}
}
}
intermediateResults.addAttributes(entityName, editableEntityMetaData);
}
}
/**
* Load all entities (optional)
*
* @param entitiesRepo
* the Repository for the entities
* @param intermediateResults
* {@link IntermediateParseResults} containing the attributes already parsed
*/
private void parseEntitiesSheet(Repository entitiesRepo, IntermediateParseResults intermediateResults)
{
if (entitiesRepo != null)
{
for (AttributeMetaData attr : entitiesRepo.getEntityMetaData().getAtomicAttributes())
{
if (!EmxMetaDataParser.SUPPORTED_ENTITY_ATTRIBUTES.contains(attr.getName().toLowerCase()) && !(I18nUtils
.isI18n(attr.getName())
&& (attr.getName().startsWith(org.molgenis.data.meta.EntityMetaDataMetaData.DESCRIPTION)
|| attr.getName().startsWith(org.molgenis.data.meta.EntityMetaDataMetaData.LABEL))))
{
throw new IllegalArgumentException("Unsupported entity metadata: entities." + attr.getName());
}
}
int i = 1;
for (Entity entity : entitiesRepo)
{
i++;
String entityName = entity.getString("name");
// required
if (entityName == null) throw new IllegalArgumentException("entity.name is missing on line " + i);
String packageName = entity.getString(PACKAGE);
if (packageName != null && !Package.DEFAULT_PACKAGE_NAME.equals(packageName))
{
entityName = packageName + Package.PACKAGE_SEPARATOR + entityName;
}
EditableEntityMetaData md = intermediateResults.getEntityMetaData(entityName);
if (md == null)
{
md = intermediateResults.addEntityMetaData(entityName);
}
String backend = entity.getString(BACKEND);
if (backend != null)
{
if (dataService.getMeta().getBackend(backend) == null)
{
throw new MolgenisDataException("Unknown backend '" + backend + "'");
}
md.setBackend(backend);
}
if (packageName != null)
{
PackageImpl p = intermediateResults.getPackage(packageName);
if (p == null)
{
throw new MolgenisDataException("Unknown package: '" + packageName + "' for entity '"
+ entity.getString("name") + "'. Please specify the package on the " + PACKAGES
+ " sheet and use the fully qualified package and entity names.");
}
md.setPackage(p);
}
md.setLabel(entity.getString(org.molgenis.data.meta.EntityMetaDataMetaData.LABEL));
md.setDescription(entity.getString(org.molgenis.data.meta.EntityMetaDataMetaData.DESCRIPTION));
for (String attributeName : entity.getAttributeNames())
{
if (isI18n(attributeName))
{
if (attributeName.startsWith(org.molgenis.data.meta.EntityMetaDataMetaData.DESCRIPTION))
{
String description = entity.getString(attributeName);
if (description != null)
{
String languageCode = getLanguageCode(attributeName);
md.setDescription(languageCode, description);
}
}
else if (attributeName.startsWith(org.molgenis.data.meta.EntityMetaDataMetaData.LABEL))
{
String label = entity.getString(attributeName);
if (label != null)
{
String languageCode = getLanguageCode(attributeName);
md.setLabel(languageCode, label);
}
}
}
}
if (entity.getBoolean(ABSTRACT) != null) md.setAbstract(entity.getBoolean(ABSTRACT));
List tagIds = entity.getList(TAGS);
String extendsEntityName = entity.getString(EXTENDS);
if (extendsEntityName != null)
{
EntityMetaData extendsEntityMeta = null;
if (intermediateResults.knowsEntity(extendsEntityName))
{
extendsEntityMeta = intermediateResults.getEntityMetaData(extendsEntityName);
}
else
{
extendsEntityMeta = dataService.getMeta().getEntityMetaData(extendsEntityName);
}
if (extendsEntityMeta == null)
{
throw new MolgenisDataException("Missing super entity " + extendsEntityName + " for entity "
+ entityName + " on line " + i);
}
md.setExtends(extendsEntityMeta);
}
if (tagIds != null)
{
for (String tagId : tagIds)
{
Entity tagEntity = intermediateResults.getTagEntity(tagId);
if (tagEntity == null)
{
throw new MolgenisDataException("Unknown tag: " + tagId + " for entity [" + entityName
+ "]). Please specify on the " + TAGS + " sheet.");
}
intermediateResults.addEntityTag(TagImpl. asTag(md, tagEntity));
}
}
}
}
}
/**
* Parses the packages sheet
*
* @param repo
* {@link Repository} for the packages
* @param intermediateResults
* {@link IntermediateParseResults} containing the parsed tag entities
*/
private void parsePackagesSheet(Repository repo, IntermediateParseResults intermediateResults)
{
if (repo == null) return;
// Collect packages
int i = 1;
for (Entity pack : resolvePackages(repo))
{
i++;
String name = pack.getString(NAME);
// required
if (name == null) throw new IllegalArgumentException("package.name is missing on line " + i);
String simpleName = name;
String description = pack.getString(org.molgenis.data.meta.PackageMetaData.DESCRIPTION);
String parentName = pack.getString(org.molgenis.data.meta.PackageMetaData.PARENT);
PackageImpl parent = null;
if (parentName != null)
{
if (!name.toLowerCase().startsWith(parentName.toLowerCase())) throw new MolgenisDataException(
"Inconsistent package structure. Package: '" + name + "', parent: '" + parentName + "'");
simpleName = name.substring(parentName.length() + 1);// subpackage_package
parent = intermediateResults.getPackage(parentName);
}
intermediateResults.addPackage(name, new PackageImpl(simpleName, description, parent));
}
}
private void parseLanguages(Repository repo, IntermediateParseResults intermediateResults)
{
repo.forEach(intermediateResults::addLanguage);
}
private void parseI18nStrings(Repository repo, IntermediateParseResults intermediateResults)
{
repo.forEach(intermediateResults::addI18nString);
}
private void parsePackageTags(Repository repo, IntermediateParseResults intermediateResults)
{
if (repo != null)
{
for (Entity pack : repo)
{
Iterable tagIdentifiers = pack.getList(org.molgenis.data.meta.PackageMetaData.TAGS);
if (tagIdentifiers != null)
{
String name = pack.getString(NAME);
PackageImpl p = intermediateResults.getPackage(name);
if (p == null) throw new IllegalArgumentException("Unknown package '" + name + "'");
for (String tagIdentifier : tagIdentifiers)
{
Entity tagEntity = intermediateResults.getTagEntity(tagIdentifier);
if (tagEntity == null)
{
throw new IllegalArgumentException("Unknown tag '" + tagIdentifier + "'");
}
p.addTag(TagImpl. asTag(p, tagEntity));
}
}
}
}
}
private List resolvePackages(Repository packageRepo)
{
List resolved = new ArrayList<>();
if ((packageRepo == null) || Iterables.isEmpty(packageRepo)) return resolved;
List unresolved = new ArrayList<>();
Map resolvedByName = new HashMap<>();
for (Entity pack : packageRepo)
{
String name = pack.getString(NAME);
String parentName = pack.getString(org.molgenis.data.meta.PackageMetaData.PARENT);
if (parentName == null)
{
resolved.add(pack);
resolvedByName.put(name, pack);
}
else
{
unresolved.add(pack);
}
}
if (resolved.isEmpty()) throw new IllegalArgumentException(
"Missing root package. There must be at least one package without a parent.");
List ready = new ArrayList<>();
while (!unresolved.isEmpty())
{
for (Entity pack : unresolved)
{
Entity parent = resolvedByName.get(pack.getString(org.molgenis.data.meta.PackageMetaData.PARENT));
if (parent != null)
{
String name = pack.getString(NAME);
ready.add(pack);
resolvedByName.put(name, pack);
}
}
if (ready.isEmpty())
throw new IllegalArgumentException("Could not resolve packages. Is there a circular reference?");
resolved.addAll(ready);
unresolved.removeAll(ready);
ready.clear();
}
return resolved;
}
/**
* re-iterate to map the mrefs/xref refEntity (or give error if not found) TODO consider also those in existing db
*
* @param attributeRepo
* the attributes {@link Repository}
* @param intermediateResults
* {@link ParsedMetaData} to add the ref entities to
*/
private void reiterateToMapRefEntity(Repository attributeRepo, IntermediateParseResults intermediateResults)
{
int i = 1;
for (Entity attribute : attributeRepo)
{
final String refEntityName = (String) attribute.get(REF_ENTITY);
final String entityName = attribute.getString(ENTITY);
final String attributeName = attribute.getString(NAME);
i++;
if (refEntityName != null)
{
EntityMetaData defaultEntityMetaData = intermediateResults.getEntityMetaData(entityName);
DefaultAttributeMetaData defaultAttributeMetaData = (DefaultAttributeMetaData) defaultEntityMetaData
.getAttribute(attributeName);
if (intermediateResults.knowsEntity(refEntityName))
{
defaultAttributeMetaData.setRefEntity(intermediateResults.getEntityMetaData(refEntityName));
}
else
{
EntityMetaData refEntityMeta;
try
{
refEntityMeta = dataService.getEntityMetaData(refEntityName);
}
catch (UnknownEntityException e)
{
throw new IllegalArgumentException(
"attributes.refEntity error on line " + i + ": " + refEntityName + " unknown");
}
// allow computed xref attributes to refer to pre-existing entities
defaultAttributeMetaData.setRefEntity(refEntityMeta);
}
}
}
}
@Override
public ParsedMetaData parse(final RepositoryCollection source, String defaultPackage)
{
if (source.getRepository(EmxMetaDataParser.ATTRIBUTES) != null)
{
IntermediateParseResults intermediateResults = getEntityMetaDataFromSource(source);
List entities;
if ((defaultPackage == null) || Package.DEFAULT_PACKAGE_NAME.equalsIgnoreCase(defaultPackage))
{
entities = intermediateResults.getEntities();
}
else
{
entities = putEntitiesInDefaultPackage(intermediateResults, defaultPackage);
}
return new ParsedMetaData(resolveEntityDependencies(entities), intermediateResults.getPackages(),
intermediateResults.getAttributeTags(), intermediateResults.getEntityTags(),
intermediateResults.getLanguages(), intermediateResults.getI18nStrings());
}
else
{
List metadataList = new ArrayList();
for (String name : source.getEntityNames())
{
metadataList.add(dataService.getRepository(name).getEntityMetaData());
}
IntermediateParseResults intermediateResults = parseTagsSheet(source.getRepository(TAGS));
parsePackagesSheet(source.getRepository(PACKAGES), intermediateResults);
parsePackageTags(source.getRepository(PACKAGES), intermediateResults);
if (source.hasRepository(LANGUAGES))
{
parseLanguages(source.getRepository(LANGUAGES), intermediateResults);
}
if (source.hasRepository(I18NSTRINGS))
{
parseI18nStrings(source.getRepository(I18NSTRINGS), intermediateResults);
}
return new ParsedMetaData(resolveEntityDependencies(metadataList), intermediateResults.getPackages(),
intermediateResults.getAttributeTags(), intermediateResults.getEntityTags(),
intermediateResults.getLanguages(), intermediateResults.getI18nStrings());
}
}
/**
* Put the entities that are not in a package in the selected package
*
* @param metaDataList
* @param defaultPackageName
* @return
*/
private List putEntitiesInDefaultPackage(IntermediateParseResults intermediateResults,
String defaultPackageName)
{
PackageImpl p = intermediateResults.getPackage(defaultPackageName);
if (p == null) throw new IllegalArgumentException("Unknown package '" + defaultPackageName + "'");
List entities = new ArrayList<>();
for (EditableEntityMetaData entityMetaData : intermediateResults.getEntities())
{
if (entityMetaData.getPackage() == null)
{
entityMetaData.setPackage(p);
}
entities.add(entityMetaData);
}
return entities;
}
/**
* Puts EntityMetaData in the right import order.
*
* @param metaDataList
* {@link EntityMetaData} to put in the right order
* @return List of {@link EntityMetaData}, in the import order
*/
private List resolveEntityDependencies(List metaDataList)
{
Set allMetaData = Sets.newLinkedHashSet(metaDataList);
Iterable existingMetaData = dataService.getMeta().getEntityMetaDatas();
Iterables.addAll(allMetaData, existingMetaData);
// Use all metadata for dependency resolving
List resolved = DependencyResolver.resolve(allMetaData);
// Only import source
resolved.retainAll(metaDataList);
return resolved;
}
private ImmutableMap getEntityMetaDataMap(DataService dataService,
RepositoryCollection source)
{
if (source.getRepository(EmxMetaDataParser.ATTRIBUTES) != null)
{
return getEntityMetaDataFromSource(source).getEntityMap();
}
else
{
return getEntityMetaDataFromDataService(dataService, source.getEntityNames());
}
}
private ImmutableMap getEntityMetaDataFromDataService(DataService dataService,
Iterable entityNames)
{
ImmutableMap.Builder builder = ImmutableMap. builder();
for (String name : entityNames)
{
builder.put(name, dataService.getRepository(name).getEntityMetaData());
}
return builder.build();
}
@Override
public EntitiesValidationReport validate(RepositoryCollection source)
{
MyEntitiesValidationReport report = new MyEntitiesValidationReport();
Map metaDataMap = getEntityMetaDataMap(dataService, source);
for (EntityMetaData emd : metaDataMap.values())
{
MetaValidationUtils.validateEntityMetaData(emd);
}
for (String sheet : source.getEntityNames())
{
if (PACKAGES.equals(sheet))
{
IntermediateParseResults parseResult = new IntermediateParseResults();
parsePackagesSheet(source.getRepository(sheet), parseResult);
for (String packageName : parseResult.getPackages().keySet())
{
report.addPackage(packageName);
}
}
else if (!ENTITIES.equals(sheet) && !ATTRIBUTES.equals(sheet) && !TAGS.equals(sheet)
&& !LANGUAGES.equals(sheet) && !I18NSTRINGS.equals(sheet))
{
// check if sheet is known
report = report.addEntity(sheet, metaDataMap.containsKey(sheet));
// check the fields
Repository s = source.getRepository(sheet);
EntityMetaData target = metaDataMap.get(sheet);
if (target != null)
{
for (AttributeMetaData att : s.getEntityMetaData().getAttributes())
{
AttributeMetaData attribute = target.getAttribute(att.getName());
boolean known = attribute != null && attribute.getExpression() == null;
report = report.addAttribute(att.getName(),
known ? AttributeState.IMPORTABLE : AttributeState.UNKNOWN);
}
for (AttributeMetaData att : target.getAttributes())
{
if (!(att.getDataType() instanceof CompoundField))
{
if (!att.isAuto() && att.getExpression() == null
&& !report.getFieldsImportable().get(sheet).contains(att.getName()))
{
boolean required = !att.isNillable() && !att.isAuto();
report = report.addAttribute(att.getName(),
required ? AttributeState.REQUIRED : AttributeState.AVAILABLE);
}
}
}
}
}
}
// Add entities without data
for (String entityName : metaDataMap.keySet())
{
if (!report.getSheetsImportable().containsKey(entityName)) report.addEntity(entityName, true);
}
return report;
}
static class EmxAttribute
{
private final DefaultAttributeMetaData attr;
private boolean idAttr;
private boolean labelAttr;
private boolean lookupAttr;
public EmxAttribute(DefaultAttributeMetaData attr)
{
this.attr = requireNonNull(attr);
}
public DefaultAttributeMetaData getAttr()
{
return attr;
}
public boolean isIdAttr()
{
return idAttr;
}
public void setIdAttr(boolean idAttr)
{
this.idAttr = idAttr;
}
public boolean isLabelAttr()
{
return labelAttr;
}
public void setLabelAttr(boolean labelAttr)
{
this.labelAttr = labelAttr;
}
public boolean isLookupAttr()
{
return lookupAttr;
}
public void setLookupAttr(boolean lookupAttr)
{
this.lookupAttr = lookupAttr;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy