org.integratedmodelling.engine.modelling.kbox.ObservableKbox Maven / Gradle / Ivy
The newest version!
/*******************************************************************************
* Copyright (C) 2007, 2015:
*
* - Ferdinando Villa - integratedmodelling.org - any
* other authors listed in @author annotations
*
* All rights reserved. This file is part of the k.LAB software suite, meant to enable
* modular, collaborative, integrated development of interoperable data and model
* components. For details, see http://integratedmodelling.org.
*
* This program is free software; you can redistribute it and/or modify it under the terms
* of the Affero General Public License Version 3 or any later version.
*
* This program is distributed in the hope that it will be useful, but without any
* warranty; without even the implied warranty of merchantability or fitness for a
* particular purpose. See the Affero General Public License for more details.
*
* You should have received a copy of the Affero General Public License along with this
* program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite
* 330, Boston, MA 02111-1307, USA. The license is also available at:
* https://www.gnu.org/licenses/agpl.html
*******************************************************************************/
package org.integratedmodelling.engine.modelling.kbox;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.integratedmodelling.api.auth.IUser;
import org.integratedmodelling.api.knowledge.IConcept;
import org.integratedmodelling.api.modelling.IDirectObserver;
import org.integratedmodelling.api.modelling.INamespace;
import org.integratedmodelling.api.modelling.resolution.IResolutionScope;
import org.integratedmodelling.api.monitoring.IMonitor;
import org.integratedmodelling.common.beans.Metadata;
import org.integratedmodelling.common.configuration.KLAB;
import org.integratedmodelling.common.owl.Knowledge;
import org.integratedmodelling.common.utils.Escape;
import org.integratedmodelling.common.vocabulary.Observables;
import org.integratedmodelling.engine.kbox.sql.SQL;
import org.integratedmodelling.engine.kbox.sql.h2.H2Database;
import org.integratedmodelling.engine.kbox.sql.h2.H2Kbox;
import org.integratedmodelling.engine.kbox.sql.h2.schema.CompoundSchema;
import org.integratedmodelling.exceptions.KlabException;
import org.integratedmodelling.exceptions.KlabRuntimeException;
/**
* Design principles:
*
* * This is a hybrid kbox that depends on the reasoner knowing all concepts in
* it. For this reason, it holds an index of the definitions of any compound
* observable concept ever stored in it, and assigns an integer ID to it. The
* table is read on startup so that all concepts are known to the reasoner, and
* any that are not will not be retrievable.
*
* * The main table is CONCEPTS, containing simply the ID of the type and its
* definition. The method {@link #getCompatibleTypeIds(IConcept)} returns all
* the compatible type IDs for an observable concept that have ever been stored
* in the database. A query for any instance stored by derived kboxes with its
* ID in the set will return compatible observables.
*
* For convenience, the kbox also maintains a METADATA table for POD objects and
* exposes simple methods to store/retrieve/delete metadata beans.
*
* TODO For now the table does not maintain reference counts, so it is possible
* that concepts IDs are referenced that are no longer represented because the
* corresponding observations that have been deleted. Refcounting is easy to
* implement but costly at store/delete, so let's see how problematic this
* becomes.
*
* @author ferdinando.villa
*
*/
public abstract class ObservableKbox extends H2Kbox {
private Map conceptHash = new HashMap<>();
private Map typeHash = new HashMap<>();
private Set obsoleteConcepts = new HashSet<>();
/**
* The kbox version, which is used to create the filesystem storage. Change
* this when incompatible changes are made to force kbox reset.
*/
public static final String KBOX_VERSION = "0910v0";
/*
* exposed to allow preallocating connections in big imports.
*/
public H2Database getDatabase() {
return this.database;
}
public IConcept getType(long id) {
if (typeHash.containsKey(id)) {
return (IConcept) Knowledge.parse(typeHash.get(id));
}
return null;
}
public String getTypeDefinition(long id) {
return typeHash.get(id);
}
protected String joinStringConditions(String field, Collection stringValues, String operator) {
String ret = "";
for (Object o : stringValues) {
ret += (ret.isEmpty() ? "" : (" " + operator + " ")) + field + " = '" + o + "'";
}
return ret;
}
public String getUrnForStoredObservation(long id, IUser user) {
return KLAB.ENGINE.getName() + ":" + user.getUsername() + ":" + getMainTableId() + ":" + id;
}
/**
* Get the ID of the table that contains the "primary" object we provide.
* Used to check for empty database - if this is not there, either nothing
* needs to be done or initialization needs to be performed.
*
* @return
*/
protected abstract String getMainTableId();
/**
* Delete all objects in the passed namespace and return the number of
* objects deleted.
*
* @param namespaceId
* @return
* @throws KlabException
*/
protected abstract int deleteAllObjectsWithNamespace(String namespaceId) throws KlabException;
protected abstract void deleteObjectWithId(long id) throws KlabException;
public int clearNamespace(String namespaceId) throws KlabException {
if (!database.hasTable(getMainTableId())) {
return 0;
}
int n = deleteAllObjectsWithNamespace(namespaceId);
database.execute("DELETE FROM namespaces where id = '" + namespaceId + "';");
return n;
}
/**
* Count the objects in the main table.
*
* @return number of observations
*/
public long count() {
try {
if (!database.hasTable(getMainTableId())) {
return 0;
}
List ret = database.queryIds("SELECT COUNT(*) from " + getMainTableId() + ";");
return ret.size() > 0 ? ret.get(0) : 0l;
} catch (KlabException e) {
throw new KlabRuntimeException(e);
}
}
class ObservableSchema extends CompoundSchema {
public ObservableSchema() {
super(IDirectObserver.class);
}
@Override
public String getCreateSQL() {
return "CREATE TABLE concepts (" + "oid LONG PRIMARY KEY, " + "definition VARCHAR(1024), " + "refcount LONG"
+ "); " + "CREATE TABLE metadata (" + "fid LONG, " + "key VARCHAR(256), " + "value OTHER" + ");"
+ "CREATE INDEX concepts_oid_index ON concepts(oid); "
+ "CREATE INDEX concepts_definition_index ON concepts(definition); "
+ "CREATE INDEX metadata_oid_index ON metadata(fid); ";
}
@Override
public String getTableName() {
return "concepts";
}
}
static public class NamespaceSchema extends CompoundSchema {
public NamespaceSchema(Class cls) {
super(cls);
}
@Override
public String getCreateSQL() {
return "CREATE TABLE namespaces (" + "id VARCHAR(256) PRIMARY KEY, " + "timestamp LONG, "
+ "isscenario BOOLEAN" + "); " + "CREATE INDEX namespace_id_index ON namespaces(id); ";
}
@Override
public String getTableName() {
return "namespaces";
}
}
static class NamespaceSerializer implements Serializer {
@Override
public String serialize(Object o, Schema schema, long primaryKey, long foreignKey) {
String ret = null;
if (o instanceof INamespace) {
INamespace ns = (INamespace) o;
ret = "INSERT INTO namespaces VALUES ('" + ns.getId() + "', " + ns.getTimeStamp() + ", "
+ (ns.isScenario() ? "TRUE" : "FALSE") + ");";
}
return ret;
}
}
/**
* Get the ID correspondent to the passed concept, and if unavailable return
* -1. Does not use the database so it's very fast.
*
* @param c
* @return
*/
public long getConceptId(IConcept c) {
Long ret = conceptHash.get(c.getDefinition());
return ret == null ? -1l : ret;
}
/**
* Check that the passed observable has been inserted, and if not make sure
* it is represented in the database. Return the stable ID to use for
* storing records that use it.
*
* @param observable
* @return
*/
public long requireConceptId(IConcept observable) {
long ret = getConceptId(observable);
if (ret >= 0) {
return ret;
}
try {
final String definition = observable.getDefinition();
ret = database.storeObject(observable, 0, new Serializer() {
@Override
public String serialize(IConcept o, Schema schema, long primaryKey, long foreignKey) {
return "INSERT INTO concepts VALUES (" + primaryKey + ", '" + definition + "', 1);";
}
}, monitor);
conceptHash.put(definition, ret);
typeHash.put(ret, definition);
} catch (KlabException e) {
throw new KlabRuntimeException(e);
}
return ret;
}
/**
* Determine all the compatible MODEL concepts for which observables have
* been stored, and return the set of their IDs.
*
* If the core type is concrete, only that core type is looked up in the
* observable's parents, so that models that observe that type (potentially
* with other traits not adopted by the observable and in any compatible
* context) are found. If the core type is abstract, any child is OK as long
* as trait, roles, inherency and context are compatible.
*
* @param observable
* @return
*/
public Set getCompatibleTypeIds(IConcept observable, IResolutionScope context) {
// KLAB.info("ASKED FOR TYPES COMPATIBLE WITH " + observable + " = " + observable.getDefinition());
Set ret = new HashSet<>();
IConcept main = Observables.getCoreObservable(observable);
if (main == null) {
/*
* not a domain concept or abstract; can't have observables.
*/
return ret;
}
/*
* add self if it's its own core observable.
* This may not be equal just because it's in another ontology but
* with identical definition, so we also add it to the compat list
* below.
*/
if (main.equals(observable)) {
// KLAB.info(" USING " + observable + " ITSELF");
long id = getConceptId(observable);
if (id >= 0) {
ret.add(id);
}
}
/*
* We lookup all possible models if the observable is abstract, or specific ones
* for the core type if concrete.
*/
Set candidates = ((main.isAbstract() || context.isGeneric()) ? KLAB.REASONER.getSemanticClosure(main)
: KLAB.REASONER.getParentClosure(main));
candidates.add(observable);
for (IConcept candidate : candidates) {
if (candidate.isAbstract()) {
continue;
}
if (Observables.isCompatible(candidate, observable,
main.isAbstract() ? 0 : Observables.REQUIRE_SAME_CORE_TYPE)) {
// KLAB.info(" FOUND " + candidate + " = " + candidate.getDefinition());
long id = getConceptId(candidate);
if (id >= 0) {
// KLAB.info(" USED IT");
ret.add(id);
}
}
}
return ret;
}
public ObservableKbox(String name, IMonitor monitor) {
super(name, monitor);
setSchema(IConcept.class, new ObservableSchema());
setSchema(INamespace.class, new NamespaceSchema(INamespace.class));
setSerializer(INamespace.class, new NamespaceSerializer());
try {
loadConcepts();
} catch (KlabException e) {
throw new KlabRuntimeException(e);
}
}
private void loadConcepts() throws KlabException {
if (!database.hasTable("concepts")) {
return;
}
database.query("SELECT oid, definition FROM concepts", new SQL.SimpleResultHandler() {
@Override
public void onRow(ResultSet rs) {
try {
// IKnowledge k = Knowledge.parse(rs.getString(2));
// if (k != null) {
conceptHash.put(rs.getString(2), rs.getLong(1));
typeHash.put(rs.getLong(1), rs.getString(2));
// } else {
// obsoleteConcepts.add(rs.getString(2));
// }
} catch (SQLException e) {
throw new KlabRuntimeException(e);
}
}
});
}
protected Metadata getMetadataFor(long oid) throws KlabException {
class Handler extends SQL.SimpleResultHandler {
Metadata ret = null;
@Override
public void onRow(ResultSet rs) {
try {
String key = rs.getString(2);
Object value = rs.getObject(3);
if (key != null && value != null) {
if (ret == null) {
ret = new Metadata();
}
ret.getData().put(key, value);
}
} catch (SQLException e) {
throw new KlabRuntimeException(e);
}
}
}
;
Handler handler = new Handler();
database.query("SELECT * FROM metadata WHERE fid = " + oid, handler);
return handler.ret;
}
protected void deleteMetadataFor(long oid) throws KlabException {
database.execute("DELETE FROM metadata WHERE fid = " + oid);
}
protected void storeMetadataFor(long oid, Metadata metadata) {
for (String s : metadata.getData().keySet()) {
String sql = " INSERT INTO metadata VALUES (" + oid + ", "// +
// "fid
// LONG,
// "
+ "'" + s + "', "// + "key VARCHAR(256), "
+ "?"// + "value OTHER"
+ ")";
try {
/*
* OK, must execute these right away unfortunately - so if
* something goes wrong with the object's storage these will
* remain in the DB.
*/
PreparedStatement prsql = database.getConnection().prepareStatement(sql);
prsql.setObject(1, metadata.getData().get(s), Types.JAVA_OBJECT);
prsql.executeUpdate();
} catch (Exception e) {
throw new KlabRuntimeException(e);
}
}
}
/**
* Pass the a namespace to check if its objects need to be stored. If the
* stored namespace record does not exist or has a timestamp older than the
* passed one, remove all objects that belong to it and return true. Does
* not store a new namespace record - this should be done when this has
* returned true and there were no errors.
*
* Returns: 0 if no need to refresh, 1 if it must be entirely refreshed and
* every model and namespace record is removed from the kbox, and 2 if the
* models without errors need to be checked again (they may be in or not).
*
*
* @param namespace
* @return result action code
*/
public int removeIfOlder(INamespace namespace) throws KlabException {
if (!database.hasTable("namespaces")) {
return 1;
}
long dbTimestamp = getNamespaceTimestamp(namespace);
long timestamp = namespace.getTimeStamp();
/*
* if we have stored something and we are younger than the stored ns,
* remove all models coming from it so we can add our new ones.
*/
if (timestamp > dbTimestamp) {
if (dbTimestamp > 0) {
monitor.debug("Removing all observations in namespace " + namespace.getId());
int removed = clearNamespace(namespace.getId());
monitor.debug("Removed " + removed + " observations.");
}
monitor.debug("Refreshing observations in " + namespace.getId() + ": stored " + new Date(dbTimestamp)
+ " < " + new Date(timestamp));
return 1;
}
/*
* if we have not changed the source file but models had errors when
* stored, return the conservative mode so we can check model by model
* and only store those that are no longer in error due to external
* reasons.
*/
if (namespace != null && namespace.hasErrors()) {
return 2;
}
return 0;
}
/**
* Return 0 if namespace is not in the kbox, or the (long) timestamp of the
* namespace if it is.
*
* @return result code
* @throws KlabException
*/
public long getNamespaceTimestamp(INamespace namespace) throws KlabException {
if (!database.hasTable("namespaces")) {
return 0l;
}
List ret = database
.queryIds("SELECT timestamp FROM namespaces WHERE id = '" + Escape.forSQL(namespace.getId()) + "';");
return ret.size() > 0 ? ret.get(0) : 0l;
}
protected static String nullify(String string) {
if (string == null || string.isEmpty()) {
return null;
}
return string;
}
}