org.apache.oodt.opendapps.util.ProfileUtils Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of opendapps Show documentation
Show all versions of opendapps Show documentation
A generic, configurable Apache OODT profile server
implementation that easily connects to OPeNDAP data sources. Connections
are configured via an XML configuration file, providing information on how
to extract and translate datasets from OPeNDAP and THREDDS into OODT profiles.
The newest version!
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.oodt.opendapps.util;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.opendapps.OpendapProfileElementExtractor;
import org.apache.oodt.opendapps.config.ConstantSpec;
import org.apache.oodt.opendapps.config.DatasetMetElem;
import org.apache.oodt.opendapps.config.OpendapConfig;
import org.apache.oodt.opendapps.config.RewriteSpec;
import org.apache.oodt.profile.EnumeratedProfileElement;
import org.apache.oodt.profile.Profile;
import org.apache.oodt.profile.ProfileAttributes;
import org.apache.oodt.profile.ProfileElement;
import org.apache.oodt.profile.ResourceAttributes;
import org.springframework.util.StringUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
import opendap.dap.BaseType;
import opendap.dap.DArray;
import opendap.dap.DConnect;
import opendap.dap.DDS;
import opendap.dap.DGrid;
import static org.apache.oodt.opendapps.config.OpendapConfigMetKeys.*;
//JDK imports
//OPENDAP imports
//OODT imports
/**
*
* Static methods for unraveling and generating {@link ProfileElement}s,
* {@link ProfileAttributes} and {@link ResourceAttributes} for a
* {@link Profile}, derived from a set of OPeNDAP dataset information and an
* {@link OpendapConfig}.
*
*
*/
public class ProfileUtils {
static {
// Note: must override the CAS PathUtils delimiter otherwise every sentence is split at the ',' as different metadata fields.
// The delimiter must be a character that is not commonly used in the metadata values,
// and that it does not a special regular expression character.
// Cannot use '#' as it is used in URL anchors, such as THREDDS urls.
// Cannot user '?', '&' as they are used in URL query strings.
// Cannot use '|' as it is used as multi-part separators in encoding of metadata fields.
PathUtils.DELIMITER = "~";
}
// character separating multiple parts of the same metadata field,
// when more than one piece of information needs to be stored in one field
public final static String CHAR = "|";
// HTTP mime types
public final static String MIME_TYPE_THREDDS = "application/xml+thredds";
public final static String MIME_TYPE_NETCDF = "application/netcdf";
public final static String MIME_TYPE_GRIDFTP = "application/gridftp";
public final static String MIME_TYPE_FTP = "application/ftp";
public final static String MIME_TYPE_LAS = "application/las";
public final static String MIME_TYPE_HTML = "text/html";
public final static String MIME_TYPE_GOOGLE_EARTH = "application/vnd.google-earth.kmz";
public final static String MIME_TYPE_HDF = "application/x-hdf";
public final static String MIME_TYPE_OPENDAP = "application/opendap";
public final static String MIME_TYPE_OPENDAP_DODS = "application/opendap-dods";
public final static String MIME_TYPE_OPENDAP_DAS = "application/opendap-das";
public final static String MIME_TYPE_OPENDAP_DDS = "application/opendap-dds";
public final static String MIME_TYPE_OPENDAP_HTML = "application/opendap-html";
public final static String MIME_TYPE_RSS = "application/rss+xml";
public final static String MIME_TYPE_GIS = "application/gis";
private static final Logger LOG = Logger.getLogger(ProfileUtils.class
.getName());
public static ResourceAttributes getResourceAttributes(OpendapConfig conf,
String opendapUrl, DConnect dConn, Metadata datasetMet) {
ResourceAttributes resAttr = new ResourceAttributes();
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(RES_ATTR_SPEC_TYPE)) {
try {
// first process expanded '[@...]' instructions
List values = multipleEnvVariablesReplacement(spec.getValue(), datasetMet);
// then process standard '[...]' instructions
for (String value : values) {
String _value = PathUtils.replaceEnvVariables(value, datasetMet, true);
if (StringUtils.hasText(_value)) {
setResourceAttributes(resAttr, spec.getName(), _value);
}
}
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Error setting field: [" + spec.getName()
+ "] in resource attributes: Message: " + e.getMessage());
}
}
}
return resAttr;
}
/**
* Utility method to process environment replacement instructions of the form '[@key]'
* resulting in as many output values as there are values for the environment variable 'key'.
* Note that currently only one such pattern '[@key']' can be processed.
*
* @param value
* @param metadata
* @return
*/
private static List multipleEnvVariablesReplacement(String value, Metadata metadata) {
List newValues = new ArrayList();
// regexp matching found > replace values
int start = value.indexOf("[@");
if (start>=0) {
int end = value.indexOf("]",start+2);
// remove '[@',']' to obtain environment variable key
String envKey = value.substring(start+2,end);
List envValues = metadata.getAllMetadata(envKey);
if (envValues!=null) {
for (String envValue : envValues) {
// create new metadata value for this environment replacement
String newValue = value.replaceAll("\\[@"+envKey+"\\]", envValue);
newValues.add(newValue);
}
}
// regexp matching not found > return original value
} else {
newValues.add(value);
}
return newValues;
}
public static ProfileAttributes getProfileAttributes(OpendapConfig conf, Metadata datasetMet) {
ProfileAttributes profAttr = new ProfileAttributes();
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(PROF_ATTR_SPEC_TYPE)) {
setProfileAttributesProperty(profAttr, spec.getName(), PathUtils
.replaceEnvVariables(spec.getValue(), datasetMet, true));
}
}
return profAttr;
}
public static Map getProfileElements(
OpendapConfig conf, DConnect dConn, Metadata datasetMet, Profile profile) throws Exception {
OpendapProfileElementExtractor pe = new OpendapProfileElementExtractor(conf);
Map profElements = new ConcurrentHashMap();
// extracts all variables defined in DDS
try {
DDS dds = dConn.getDDS();
// loop over all variables found
Enumeration variables = dds.getVariables();
while (variables.hasMoreElements()) {
BaseType variable = (BaseType)variables.nextElement();
String varName = variable.getName();
if (variable instanceof DArray) {
LOG.log(Level.FINE, "Extracting Darray variable: "+varName);
} else if (variable instanceof DGrid) {
LOG.log(Level.FINE, "Extracting Dgrid variable: "+varName);
}
RewriteSpec spec = getProfileElementSpec(varName, conf);
if (spec!=null) {
// use configuration to set variable re-name and type
String peName = spec.getRename() != null && !spec.getRename().equals("") ? spec.getRename() : spec.getOrigName();
if (spec.getElementType().equals(RANGED_ELEMENT_TYPE)) {
profElements.put(peName, pe.extractRangedProfileElement(peName, spec.getOrigName(), profile, dConn.getDAS()));
} else if (spec.getElementType().equals(ENUM_ELEMENT_TYPE)) {
profElements.put(peName, pe.extractEnumeratedProfileElement(peName, spec.getOrigName(), profile, dConn.getDAS()));
}
} else {
// if not explicitly configured, assume variable if of RANGED_ELEMENT_TYPE
profElements.put(varName, pe.extractRangedProfileElement(varName, varName, profile, dConn.getDAS()));
}
}
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Error extracting metadata from DDS ("+dConn.URL()+") :" +e.getMessage());
// rethrow the exception so that this dataset is not harvested
throw e;
}
// add profile elements from specification
if (datasetMet != null) {
for (DatasetMetElem datasetSpec : conf.getDatasetMetSpecs()) {
// retrieve values from metadata container
List values = datasetMet.getAllMetadata(datasetSpec.getValue());
addValuesToEnumeratedProfileElement(datasetSpec.getProfileElementName(), values, profile, profElements);
}
}
// add profile elements from specification
for (ConstantSpec spec : conf.getConstSpecs()) {
if (spec.getType().equals(PROF_ELEM_SPEC_TYPE)) {
// retrieve value from XML configuration file, replace with value from metadata container if required,
// split according to delimiter
String replaceVal = PathUtils.replaceEnvVariables(spec.getValue(), datasetMet);
List values = Arrays.asList(replaceVal.split(PathUtils.DELIMITER));
addValuesToEnumeratedProfileElement(spec.getName(), values, profile, profElements);
}
}
return profElements;
}
/**
* Method to add one or more values to an EnumeratedProfileElement, creating the element if not existing already.
* The supplied values are added only if valid.
*/
private static void addValuesToEnumeratedProfileElement(final String name, final List values, Profile profile, Map profElements) {
// try retrieve existing profile element
ProfileElement epe = profElements.get(name);
// or create a new one if not found
if (epe==null) {
epe = new EnumeratedProfileElement(profile);
epe.setName(name);
}
if (values!=null) {
for (String value : values) {
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
epe.getValues().add(value);
}
}
}
// only save profile elements with one or more values
if (epe.getValues().size()>0) {
profElements.put(name, epe);
}
}
private static void setProfileAttributesProperty(ProfileAttributes attr,
String propName, String value) {
if (propName.equals("profId")) {
attr.setID(value);
} else if (propName.equals("profVersion")) {
attr.setVersion(value);
} else if (propName.equals("profType")) {
attr.setType(value);
} else if (propName.equals("profStatusId")) {
attr.setStatusID(value);
} else if (propName.equals("profSecurityType")) {
attr.setSecurityType(value);
} else if (propName.equals("profParentId")) {
attr.setParent(value);
} else if (propName.equals("profRegAuthority")) {
attr.setRegAuthority(value);
} else if (propName.equals("profChildId")) {
attr.getChildren().addAll(Arrays.asList(value.split(PathUtils.DELIMITER)));
} else if (propName.equals("profRevisionNote")) {
attr.getRevisionNotes().addAll(Arrays.asList(value.split(PathUtils.DELIMITER)));
}
}
private static void setResourceAttributes(ResourceAttributes resAttr,
String propName, String value) {
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
if (propName.equals("Identifier")) {
resAttr.setIdentifier(value);
} else if (propName.equals("Title")) {
resAttr.setTitle(value);
} else if (propName.equals("Format")) {
resAttr.getFormats().addAll( parseValues(value) );
} else if (propName.equals("Description")) {
resAttr.setDescription(value);
} else if (propName.equals("Creator")) {
resAttr.getCreators().addAll( parseValues(value) );
} else if (propName.equals("Subject")) {
resAttr.getSubjects().addAll( parseValues(value) );
} else if (propName.equals("Publisher")) {
resAttr.getPublishers().addAll( parseValues(value) );
} else if (propName.equals("Contributor")) {
resAttr.getContributors().addAll( parseValues(value) );
} else if (propName.equals("Date")) {
resAttr.getDates().addAll( parseValues(value) );
} else if (propName.equals("Type")) {
resAttr.getTypes().addAll( parseValues(value) );
} else if (propName.equals("Source")) {
resAttr.getSources().addAll( parseValues(value) );
} else if (propName.equals("Language")) {
resAttr.getLanguages().addAll( parseValues(value) );
} else if (propName.equals("Relation")) {
resAttr.getRelations().addAll( parseValues(value) );
} else if (propName.equals("Coverage")) {
resAttr.getCoverages().addAll( parseValues(value) );
} else if (propName.equals("Rights")) {
resAttr.getRights().addAll( parseValues(value) );
} else if (propName.equals("resContext")) {
resAttr.getResContexts().addAll( parseValues(value) );
} else if (propName.equals("resAggregation")) {
resAttr.setResAggregation(value);
} else if (propName.equals("resClass")) {
resAttr.setResClass(value);
} else if (propName.equals("resLocation")) {
resAttr.getResLocations().addAll( parseValues(value) );
}
}
}
/**
* Utility method to discover the rewrite specification for a named variable, if available.
* @param conf
*/
private static RewriteSpec getProfileElementSpec(String origName, OpendapConfig conf) {
for (RewriteSpec spec : conf.getRewriteSpecs()) {
if (spec.getOrigName().equals(origName)) {
return spec;
}
}
return null;
}
/**
* Utility method to split a metadata field value according to the known delimiter.
* @param value
* @return
*/
public static List parseValues(String value) {
List values = new ArrayList();
for (String val : value.split(PathUtils.DELIMITER)) {
if (StringUtils.hasText(val) && !val.equalsIgnoreCase("null")) {
values.add(val);
}
}
return values;
}
/**
* Method to add a (name,value) pair to the metadata container if the value is not null or empty,
* and doesn't exist already.
* @param met
* @param value
*/
public static void addIfNotNull(Metadata met, String key, String value) {
// do not add a null value
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
// do not add the same value twice for the same metadata key
if (!met.containsKey(key) || !met.getAllMetadata(key).contains(value)) {
met.addMetadata(key, value);
}
}
}
/**
* Method to add multiple (key, value) pairs to the metadata container if not existing already.
*/
public static void addIfNotExisting(Metadata metadata, String key, Enumeration values) {
if (StringUtils.hasText(key) && !metadata.containsKey(key)) {
while (values.hasMoreElements()) {
String value = values.nextElement();
if (StringUtils.hasText(value) && !value.equalsIgnoreCase("null")) {
metadata.addMetadata(key,value);
}
}
}
}
// inspired from ASLv2 code at:
// http://www.java2s.com/Code/Java/Data-Type/ISO8601dateparsingutility.htm
public static String toISO8601(Date date) {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
TimeZone tz = TimeZone.getTimeZone("UTC");
df.setTimeZone(tz);
return df.format(date);
}
}