org.integratedmodelling.engine.geospace.coverage.vector.AbstractVectorCoverage Maven / Gradle / Ivy
The newest version!
/*******************************************************************************
* Copyright (C) 2007, 2015:
*
* - Ferdinando Villa - integratedmodelling.org - any
* other authors listed in @author annotations
*
* All rights reserved. This file is part of the k.LAB software suite, meant to enable
* modular, collaborative, integrated development of interoperable data and model
* components. For details, see http://integratedmodelling.org.
*
* This program is free software; you can redistribute it and/or modify it under the terms
* of the Affero General Public License Version 3 or any later version.
*
* This program is distributed in the hope that it will be useful, but without any
* warranty; without even the implied warranty of merchantability or fitness for a
* particular purpose. See the Affero General Public License for more details.
*
* You should have received a copy of the Affero General Public License along with this
* program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite
* 330, Boston, MA 02111-1307, USA. The license is also available at:
* https://www.gnu.org/licenses/agpl.html
*******************************************************************************/
package org.integratedmodelling.engine.geospace.coverage.vector;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.jcs.access.exception.CacheException;
import org.geotools.data.DataStore;
import org.geotools.data.DefaultTransaction;
import org.geotools.data.FeatureSource;
import org.geotools.data.Transaction;
import org.geotools.data.collection.ListFeatureCollection;
import org.geotools.data.shapefile.ShapefileDataStoreFactory;
import org.geotools.data.simple.SimpleFeatureCollection;
import org.geotools.data.simple.SimpleFeatureStore;
import org.geotools.factory.CommonFactoryFinder;
import org.geotools.factory.GeoTools;
import org.geotools.feature.FeatureCollection;
import org.geotools.feature.FeatureIterator;
import org.geotools.feature.simple.SimpleFeatureBuilder;
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
import org.geotools.filter.text.cql2.CQL;
import org.geotools.filter.text.cql2.CQLException;
import org.geotools.geometry.jts.ReferencedEnvelope;
import org.integratedmodelling.api.knowledge.IConcept;
import org.integratedmodelling.api.metadata.IModelMetadata;
import org.integratedmodelling.api.modelling.IDirectObservation;
import org.integratedmodelling.api.modelling.INumericObserver;
import org.integratedmodelling.api.modelling.IObserver;
import org.integratedmodelling.api.modelling.IPresenceObserver;
import org.integratedmodelling.api.modelling.IState;
import org.integratedmodelling.api.monitoring.IMonitor;
import org.integratedmodelling.api.monitoring.Messages;
import org.integratedmodelling.collections.Pair;
import org.integratedmodelling.common.space.IGeometricShape;
import org.integratedmodelling.common.states.States;
import org.integratedmodelling.common.utils.MiscUtilities;
import org.integratedmodelling.common.utils.NameGenerator;
import org.integratedmodelling.common.utils.NetUtilities;
import org.integratedmodelling.engine.geospace.Geospace;
import org.integratedmodelling.engine.geospace.GeotoolsVectorCoverage;
import org.integratedmodelling.engine.geospace.coverage.ICoverage;
import org.integratedmodelling.engine.geospace.extents.Area;
import org.integratedmodelling.engine.geospace.extents.Grid;
import org.integratedmodelling.engine.geospace.extents.SpaceExtent;
import org.integratedmodelling.engine.geospace.gis.ThinklabRasterizer;
import org.integratedmodelling.engine.geospace.literals.ShapeValue;
import org.integratedmodelling.exceptions.KlabException;
import org.integratedmodelling.exceptions.KlabIOException;
import org.integratedmodelling.exceptions.KlabRuntimeException;
import org.integratedmodelling.exceptions.KlabUnsupportedOperationException;
import org.integratedmodelling.exceptions.KlabValidationException;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.AttributeDescriptor;
import org.opengis.filter.Filter;
import org.opengis.filter.FilterFactory2;
import org.opengis.referencing.crs.CoordinateReferenceSystem;
import com.vividsolutions.jts.geom.Geometry;
public abstract class AbstractVectorCoverage
implements ICoverage, GeotoolsVectorCoverage, Iterable {
public static String ALL_ATTRIBUTES = "__ALL_ATTRIBUTES__";
protected FeatureSource featureSource = null;
CoordinateReferenceSystem crs = null;
String crsCode = null;
String coverageId = null;
int attributeHandle = -1;
protected String layerName = null;
private String valueField = null;
protected String sourceUrl = null;
protected String filterExpression = null;
private String valueDefault;
protected ReferencedEnvelope envelope;
protected String authentication;
protected SimpleFeatureTypeBuilder typeBuilder = null;
// holds the mapping between state formal name and attribute name
Map statenames = new HashMap<>();
// maps attribute names to observer for reporting after write()
Map stateobservers = new HashMap<>();
/*
* this only gets filled explicitly using add(observation); in this case, store and
* feature source
* will be null and for now, this coverage is only good to call write() upon.
*/
protected List observations = new ArrayList<>();
private float fillValue = Float.NaN;
/**
* Bean returned by a write() operation, describing the output files
* and the mapping of feature attributes to names.
*
* @author Ferd
*/
public static class VectorOutput {
List files = new ArrayList<>();
Map attributes = new HashMap<>();
public List getFiles() {
return files;
}
public Map getAttributes() {
return attributes;
}
}
AbstractVectorCoverage() {
}
protected FeatureSource getFeatureSource()
throws KlabException {
if (featureSource == null) {
try {
featureSource = getDataStore().getFeatureSource(coverageId);
} catch (IOException e) {
throw new KlabIOException(e);
}
}
return featureSource;
}
protected FeatureCollection getFeatureCollection(CoordinateReferenceSystem desiredCRS)
throws KlabException {
FeatureCollection featureCollection = null;
try {
// THIS WOULD WORK IF GEOTOOLS WORKED. WITH GT BUGS, THIS MESS IS ON ME
// ANYWAY.
// if (envelope != null) {
// DefaultQuery q = new DefaultQuery();
// q.setCoordinateSystem(getFeatureSource().getSchema().getCoordinateReferenceSystem());
// q.setCoordinateSystemReproject(desiredCRS);
// featureCollection = getFeatureSource().getFeatures(q);
// } else {
featureCollection = getFeatureSource().getFeatures();
// }
} catch (IOException e) {
throw new KlabIOException(e);
}
return featureCollection;
}
protected abstract DataStore getDataStore() throws KlabException;
protected abstract void setDataStore(DataStore store);
// @Override
@Override
public AttributeDescriptor getAttributeDescriptor(String valueId)
throws KlabException {
return getFeatureSource().getSchema().getDescriptor(valueId);
}
static public class CachedDescriptor implements Serializable {
public static final long serialVersionUID = 1453574401114688492L;
public double x1, x2, y1, y2;
public String srs;
public String coverageId;
}
class ShapeIterator implements Iterator {
private FeatureIterator it;
ShapeIterator() throws KlabException {
this.it = getFeatureIterator(null, (String[]) null);
}
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
public ShapeValue next() {
SimpleFeature feature = it.next();
/*
* TODO add all attributes as metadata
*/
return new ShapeValue((Geometry) feature
.getDefaultGeometry(), getCoordinateReferenceSystem());
}
@Override
public void remove() {
// come on
}
}
public AbstractVectorCoverage(URL url, String layerName, String valueField,
String filter,
String authentication)
throws KlabException {
this.valueField = (valueField == null
|| valueField.equals(IModelMetadata.PRESENCE_ATTRIBUTE)) ? null
: valueField;
this.sourceUrl = url.toString();
this.filterExpression = filter;
this.layerName = layerName;
this.authentication = authentication;
if (!readFromCache()) {
if (url.toString().startsWith("http:")
&& !NetUtilities.urlResponds(url.toString())) {
throw new KlabIOException("connection to WFS host failed for layer "
+ layerName);
}
try {
/*
* LEAVE this please - must set the coverageId before the next statement
* in WFS
*/
getDataStore();
if (coverageId == null) {
coverageId = getDataStore().getTypeNames()[0];
}
envelope = new ReferencedEnvelope(getFeatureSource().getBounds());
crs = getFeatureSource().getSchema().getCoordinateReferenceSystem();
crsCode = Geospace.getCRSIdentifier(crs, false);
// if (this instanceof WFSCoverage && crsCode.equals("EPSG:4326")) {
//
// /*
// * invert the fucking envelope as WFS 1.1.0 is guaranteed to return it
// * north-based.
// */
// envelope = new ReferencedEnvelope(envelope.getMinY(),
// envelope.getMaxY(), envelope
// .getMinX(), envelope.getMaxX(), crs);
// }
} catch (Exception e) {
throw new KlabIOException(e);
}
saveToCache();
}
}
private void saveToCache() throws KlabException {
if (sourceUrl.startsWith("file:")) {
return;
}
CachedDescriptor cd = new CachedDescriptor();
String key = sourceUrl + "#" + layerName;
cd.srs = crsCode;
cd.x1 = envelope.getMinX();
cd.x2 = envelope.getMaxX();
cd.y1 = envelope.getMinY();
cd.y2 = envelope.getMaxY();
cd.coverageId = coverageId;
try {
Geospace.get().getWFSCache().put(key, cd);
} catch (CacheException e) {
throw new KlabIOException(e);
}
}
private boolean readFromCache() throws KlabException {
if (sourceUrl.startsWith("file:")) {
return false;
}
String key = sourceUrl + "#" + layerName;
CachedDescriptor cd = (CachedDescriptor) Geospace.get().getWFSCache().get(key);
if (cd != null) {
this.crsCode = cd.srs;
this.crs = Geospace.getCRSFromID(cd.srs);
this.envelope = new ReferencedEnvelope(cd.x1, cd.x2, cd.y1, cd.y2, crs);
this.coverageId = cd.coverageId;
return true;
}
return false;
}
@Override
public String getCoordinateReferenceSystemCode() throws KlabValidationException {
return crsCode;
}
/**
* Create a new raster coverage for the passed extent and return it. It even sounds
* easy.
*
* @param arealExtent
* @return converted coverage
* @throws KlabException
*/
public ICoverage convertToRaster(Grid arealExtent, IObserver observer, IMonitor monitor)
throws KlabException {
return ThinklabRasterizer
.rasterize(this, valueField, this.fillValue, arealExtent, observer, valueDefault, monitor, (this instanceof WFSCoverage));
}
@Override
public ICoverage requireMatch(Area arealExtent, IObserver observer, IMonitor monitor, boolean allowClassChange)
throws KlabException {
ICoverage ret = null;
if (arealExtent instanceof Grid && allowClassChange) {
if (monitor != null) {
monitor.info("rasterizing " + layerName
+ (valueField != null ? (" using attribute " + valueField) : "")
+ "...", Messages.INFOCLASS_MODEL);
}
ret = convertToRaster((Grid) arealExtent, observer, monitor);
// if (monitor != null) {
// monitor.info("finished rasterizing " + layerName, null);
// }
} else {
/*
* TODO reproject and subset if the passed extent is different from ours
*/
ret = this;
}
return ret;
}
@Override
public Object getSubdivisionValue(int subdivisionOrder, Area extent)
throws KlabValidationException {
// TODO Auto-generated method stub
return null;
}
public void setName(String covId) {
layerName = covId;
}
public String[] getAttributeNames() throws KlabException {
String[] ret = new String[getFeatureSource().getSchema().getAttributeCount()];
int i = 0;
for (AttributeDescriptor ad : getFeatureSource().getSchema()
.getAttributeDescriptors()) {
ret[i++] = ad.getLocalName();
}
return ret;
}
@Override
public ReferencedEnvelope getEnvelope() {
return envelope;
}
@Override
public Iterator iterator() {
try {
return new ShapeIterator();
} catch (KlabException e) {
throw new KlabRuntimeException(e);
}
};
/**
* @param envelope
* @param attributes
* If we want the features to retain an attribute other than the geometry,
* pass it here
* @return feature iterator
* @throws KlabException
*/
@Override
public FeatureIterator getFeatureIterator(ReferencedEnvelope envelope, String... attributes)
throws KlabException {
ClassLoader clsl = null;
FeatureIterator ret = null;
try {
// SPI be damned (BTW TODO - probably now unnecessary)
// clsl = KLABEngine.get().swapClassloader();
if (envelope == null) {
FeatureCollection fc = getFeatureCollection(null);
ret = fc.features();
} else {
/*
* get the envelope in the coverage's SRS before we do any transformation
*/
ReferencedEnvelope dEnvelope = null;
try {
dEnvelope = envelope.transform(crs, true);
} catch (Exception e) {
throw new KlabValidationException(e);
}
/*
* TODO/FIXME/CHECK
* if the COVERAGE is in WGS84, WFS will want the bounding box in lat/lon
* order. How do we ensure which box we need to use? Datasource
* properties? For Geoserver, this is OK. For anything else, who knows.
* NOTE: SWAPPING ONLY in WFS. We force WFS protocol to 1.1.0 which SHOULD
* be guaranteed to swap. For inline data, Geoserver's setting of keeping
* XY consistent should get it done.
*/
// SWAPPER
// if (crsCode.equals("EPSG:4326")
// && Geospace.getCRSIdentifier(envelope.getCoordinateReferenceSystem(),
// true)
// .equals("EPSG:4326")
// && this instanceof WFSCoverage) {
// envelope = new ReferencedEnvelope(envelope.getMinY(),
// envelope.getMaxY(), envelope
// .getMinX(), envelope.getMaxX(),
// envelope.getCoordinateReferenceSystem());
//
// dEnvelope = envelope;
// }
FeatureCollection fc = getFeatureCollection(envelope
.getCoordinateReferenceSystem());
/*
* query
*/
FilterFactory2 ff = CommonFactoryFinder
.getFilterFactory2(GeoTools.getDefaultHints());
String geomName = fc.getSchema().getGeometryDescriptor().getLocalName();
Filter filter = ff.bbox(ff.property(geomName), dEnvelope);
// READ ALL THE F'ING ATTRIBUTES. We only do this once, so we don't know
// what we'll need in
// different models.
// if (true /*attributes != null && attributes.length > 0 && attributes[0]
// != null
// && attributes[0].equals(ALL_ATTRIBUTES)*/) {
ArrayList atn = new ArrayList<>();
for (AttributeDescriptor i : fc.getSchema().getAttributeDescriptors()) {
if (!i.getLocalName().equals(geomName))
atn.add(i.getLocalName());
}
attributes = atn.toArray(new String[atn.size()]);
// }
if (filterExpression != null) {
try {
Filter cql = CQL.toFilter(filterExpression);
filter = ff.and(filter, cql);
} catch (CQLException e) {
throw new KlabValidationException(e);
}
}
/*
* attributes to put in the query
*/
String[] attnames = new String[] { geomName };
int ii = 0;
for (String ss : attributes) {
if (ss != null)
ii++;
}
attnames = new String[ii + 1];
attnames[0] = geomName;
if (attributes != null) {
int i = 1;
for (String a : attributes) {
if (a != null)
attnames[i++] = a;
}
}
FeatureCollection feat = fc
.subCollection(filter);
ret = feat.features();
}
} finally {
// KLABEngine.get().resetClassLoader(clsl);
}
return ret;
}
public void add(IDirectObservation observation) {
if (observation.getScale().getSpace() != null) {
/*
* build up the schema and ensure the geometry is compatible
*/
if (typeBuilder == null) {
/*
* build initial schema
*/
typeBuilder = new SimpleFeatureTypeBuilder();
typeBuilder.setName(getLayerName());
typeBuilder.setCRS(((SpaceExtent) observation.getScale().getSpace())
.getCRS());
typeBuilder.add("the_geom", observation.getScale().getSpace().getShape()
.getGeometryClass());
typeBuilder.add("name", String.class);
for (IState state : observation.getStates()) {
String aname = NameGenerator
.getIdentifier(state.getObserver().getObservable()
.getFormalName(), 9, true, statenames.values());
statenames.put(state.getObserver().getObservable()
.getFormalName(), aname);
stateobservers.put(aname, state.getObserver());
if (state.getObserver() instanceof INumericObserver) {
typeBuilder.add(aname, Double.class);
} else if (state.getObserver() instanceof IPresenceObserver) {
typeBuilder.add(aname, Boolean.class);
} else {
typeBuilder.add(aname, String.class);
}
}
} else {
/*
* TODO
* verify schema and drop observation if not compatible
*/
}
observations.add(observation);
}
}
@Override
public VectorOutput write(File file) throws KlabException {
VectorOutput ret = null;
if (!(file.toString().endsWith(".shp"))) {
throw new KlabUnsupportedOperationException("vector coverage: writing: only shapefile format is supported for now");
}
ShapefileDataStoreFactory dataStoreFactory = new ShapefileDataStoreFactory();
Pair, SimpleFeatureType> features = null;
try {
Map params = new HashMap<>();
params.put("url", file.toURI().toURL());
params.put("create spatial index", Boolean.TRUE);
if (featureSource == null) {
/**
* Create features from stored observations
*/
setDataStore(dataStoreFactory
.createNewDataStore(params));
features = createFeatures();
getDataStore().createSchema(features.getSecond());
} else {
/*
* TODO set features = ... from current feature list and schema.
*/
}
if (features == null) {
return ret;
}
/*
* Write the features to the shapefile
*/
Transaction transaction = new DefaultTransaction("create");
String typeName = getDataStore().getTypeNames()[0];
featureSource = getDataStore().getFeatureSource(typeName);
// SimpleFeatureType SHAPE_TYPE = featureSource.getSchema();
/*
* The Shapefile format has a couple limitations: - "the_geom" is always
* first, and used for the geometry attribute name - "the_geom" must be of
* type Point, MultiPoint, MuiltiLineString, MultiPolygon - Attribute names
* are limited in length - Not all data types are supported (example Timestamp
* represented as Date)
* Each data store has different limitations so check the resulting
* SimpleFeatureType.
*/
// System.out.println("SHAPE:"+SHAPE_TYPE);
SimpleFeatureStore featureStore = (SimpleFeatureStore) featureSource;
/*
* SimpleFeatureStore has a method to add features from a
* SimpleFeatureCollection object, so we use the ListFeatureCollection class
* to wrap our list of features.
*/
SimpleFeatureCollection collection = new ListFeatureCollection(features
.getSecond(), features
.getFirst());
featureStore.setTransaction(transaction);
try {
featureStore.addFeatures(collection);
transaction.commit();
} catch (Exception problem) {
transaction.rollback();
throw problem;
} finally {
transaction.close();
}
ret = new VectorOutput();
ret.attributes.putAll(stateobservers);
ret.files.add(file);
ret.files.addAll(getAccessoryFiles(file));
} catch (Exception e) {
throw new KlabIOException(e);
}
return ret;
}
private List getAccessoryFiles(File file) {
List ret = new ArrayList<>();
String fil = MiscUtilities.getFileBasePath(file.toString());
for (String ext : new String[] {"dbf", "fix", "prj", "shx", "shp.xml"}) {
File f = new File(fil + "." + ext);
if (f.exists()) {
ret.add(f);
}
}
return ret;
}
private Pair, SimpleFeatureType> createFeatures() {
if (typeBuilder == null) {
return null;
}
SimpleFeatureType type = typeBuilder.buildFeatureType();
List features = new ArrayList<>();
for (IDirectObservation obs : observations) {
SimpleFeatureBuilder builder = new SimpleFeatureBuilder(type);
builder.add(((IGeometricShape) obs.getScale().getSpace().getShape())
.getGeometry());
builder.set("name", obs.getName());
for (IState state : obs.getStates()) {
String name = statenames
.get(state.getObserver().getObservable().getFormalName());
if (name != null) {
builder.set(name, reduce(state));
}
}
features.add(builder.buildFeature(obs.getName()));
}
return new Pair<>(features, type);
}
private Object reduce(IState state) {
/*
* TODO accommodate for transitions and use collapse() as necessary
*/
Object o = States.get(state, 0);
if (!(state.isConstant() || state.getScale().getMultiplicity() > 1l)) {
// TODO!
}
return convert(o);
}
private Object convert(Object o) {
if (o instanceof Number) {
return ((Number) o).doubleValue();
} else if (o instanceof Boolean) {
return ((Boolean) o) ? "true" : "false";
} else if (o instanceof IConcept) {
return ((IConcept) o).getDefinition();
}
return null;
}
@Override
public String getLayerName() {
return layerName == null ? "no name" : layerName;
}
@Override
public String getSourceUrl() {
return sourceUrl;
}
@Override
public CoordinateReferenceSystem getCoordinateReferenceSystem() {
if (crs == null && crsCode != null) {
crs = Geospace.getCRSFromID(crsCode);
}
return crs;
}
}