All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.espertech.esper.epl.variable.VariableServiceImpl Maven / Gradle / Ivy

There is a newer version: 7.1.0
Show newest version
/**************************************************************************************
 * Copyright (C) 2006-2015 EsperTech Inc. All rights reserved.                        *
 * http://www.espertech.com/esper                                                          *
 * http://www.espertech.com                                                           *
 * ---------------------------------------------------------------------------------- *
 * The software in this package is published under the terms of the GPL license       *
 * a copy of which has been included with this distribution in the license.txt file.  *
 **************************************************************************************/
package com.espertech.esper.epl.variable;

import com.espertech.esper.client.EventBean;
import com.espertech.esper.client.EventType;
import com.espertech.esper.client.VariableValueException;
import com.espertech.esper.collection.Pair;
import com.espertech.esper.core.service.StatementExtensionSvcContext;
import com.espertech.esper.core.start.EPStatementStartMethod;
import com.espertech.esper.epl.core.EngineImportException;
import com.espertech.esper.epl.core.EngineImportService;
import com.espertech.esper.event.EventAdapterService;
import com.espertech.esper.schedule.TimeProvider;
import com.espertech.esper.util.JavaClassHelper;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.io.StringWriter;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

/**
 * Variables service for reading and writing variables, and for setting a version number for the current thread to
 * consider variables for.
 * 

* Consider a statement as follows: select * from MyEvent as A where A.val > var1 and A.val2 > var1 and A.val3 > var2 *

* Upon statement execution we need to guarantee that the same atomic value for all variables is applied for all * variable reads (by expressions typically) within the statement. *

* Designed to support: *

    *
  1. lock-less read of the current and prior version, locked reads for older versions *
  2. atomicity by keeping multiple versions for each variable and a threadlocal that receives the current version each call *
  3. one write lock for all variables (required to coordinate with single global version number), * however writes are very fast (entry to collection plus increment an int) and therefore blocking should not be an issue *
*

* As an alternative to a version-based design, a read-lock for the variable space could also be used, with the following * disadvantages: The write lock may just not be granted unless fair locks are used which are more expensive; And * a read-lock is more expensive to acquire for multiple CPUs; A thread-local is still need to deal with * "set var1=3, var2=var1+1" assignments where the new uncommitted value must be visible in the local evaluation. *

* Every new write to a variable creates a new version. Thus when reading variables, readers can ignore newer versions * and a read lock is not required in most circumstances. *

* This algorithm works as follows: *

* A thread processing an event into the engine via sendEvent() calls the "setLocalVersion" method once * before processing a statement that has variables. * This places into a threadlocal variable the current version number, say version 570. *

* A statement that reads a variable has an {@link com.espertech.esper.epl.expression.core.ExprVariableNode} that has a {@link com.espertech.esper.epl.variable.VariableReader} handle * obtained during validation (example). *

* The {@link com.espertech.esper.epl.variable.VariableReader} takes the version from the threadlocal (570) and compares the version number with the * version numbers held for the variable. * If the current version is same or lower (520, as old or older) then the threadlocal version, * then use the current value. * If the current version is higher (571, newer) then the threadlocal version, then go to the prior value. * Use the prior value until a version is found that as old or older then the threadlocal version. *

* If no version can be found that is old enough, output a warning and return the newest version. * This should not happen, unless a thread is executing for very long within a single statement such that * lifetime-old-version time speriod passed before the thread asks for variable values. *

* As version numbers are counted up they may reach a boundary. Any write transaction after the boundary * is reached performs a roll-over. In a roll-over, all variables version lists are * newly created and any existing threads that read versions go against a (old) high-collection, * while new threads reading the reset version go against a new low-collection. *

* The class also allows an optional state handler to be plugged in to handle persistence for variable state. * The state handler gets invoked when a variable changes value, and when a variable gets created * to obtain the current value from persistence, if any. */ public class VariableServiceImpl implements VariableService { private static Log log = LogFactory.getLog(VariableServiceImpl.class); /** * Sets the boundary above which a reader considers the high-version list of variable values. * For use in roll-over when the current version number overflows the ROLLOVER_WRITER_BOUNDARY. */ protected final static int ROLLOVER_READER_BOUNDARY = Integer.MAX_VALUE - 100000; /** * Applicable for each variable if more then the number of versions accumulated, check * timestamps to determine if a version can be expired. */ protected final static int HIGH_WATERMARK_VERSIONS = 50; // Each variable has an index number, a context-partition id, a current version and a list of values private final ArrayList> variableVersionsPerCP; // Each variable and a context-partition id may have a set of callbacks to invoke when the variable changes private final ArrayList>> changeCallbacksPerCP; // Keep the variable list private final Map variables; // Write lock taken on write of any variable; and on read of older versions private final ReadWriteLock readWriteLock; // Thread-local for the visible version per thread private VariableVersionThreadLocal versionThreadLocal = new VariableVersionThreadLocal(); // Number of milliseconds that old versions of a variable are allowed to live private final long millisecondLifetimeOldVersions; private final TimeProvider timeProvider; private final EventAdapterService eventAdapterService; private final VariableStateHandler optionalStateHandler; private volatile int currentVersionNumber; private int currentVariableNumber; /** * Ctor. * @param millisecondLifetimeOldVersions number of milliseconds a version may hang around before expiry * @param timeProvider provides the current time * @param optionalStateHandler a optional plug-in that may store variable state and retrieve state upon creation * @param eventAdapterService event adapters */ public VariableServiceImpl(long millisecondLifetimeOldVersions, TimeProvider timeProvider, EventAdapterService eventAdapterService, VariableStateHandler optionalStateHandler) { this(0, millisecondLifetimeOldVersions, timeProvider, eventAdapterService, optionalStateHandler); } /** * Ctor. * @param startVersion the first version number to start from * @param millisecondLifetimeOldVersions number of milliseconds a version may hang around before expiry * @param timeProvider provides the current time * @param optionalStateHandler a optional plug-in that may store variable state and retrieve state upon creation * @param eventAdapterService for finding event types */ protected VariableServiceImpl(int startVersion, long millisecondLifetimeOldVersions, TimeProvider timeProvider, EventAdapterService eventAdapterService, VariableStateHandler optionalStateHandler) { this.millisecondLifetimeOldVersions = millisecondLifetimeOldVersions; this.timeProvider = timeProvider; this.eventAdapterService = eventAdapterService; this.optionalStateHandler = optionalStateHandler; this.variables = new HashMap(); this.readWriteLock = new ReentrantReadWriteLock(); this.variableVersionsPerCP = new ArrayList>(); this.changeCallbacksPerCP = new ArrayList>>(); currentVersionNumber = startVersion; } public void destroy() { versionThreadLocal = new VariableVersionThreadLocal(); } public synchronized void removeVariableIfFound(String name) { VariableMetaData metaData = variables.get(name); if (metaData == null) { return; } if (log.isDebugEnabled()) { log.debug("Removing variable '" + name + "'"); } variables.remove(name); if (optionalStateHandler != null) { ConcurrentHashMap readers = variableVersionsPerCP.get(metaData.getVariableNumber()); Set cps = Collections.emptySet(); if (readers != null) { cps = readers.keySet(); } optionalStateHandler.removeVariable(name, cps); } int number = metaData.getVariableNumber(); variableVersionsPerCP.set(number, null); changeCallbacksPerCP.set(number, null); } public void setLocalVersion() { versionThreadLocal.getCurrentThread().setVersion(currentVersionNumber); } public void registerCallback(String variableName, int agentInstanceId, VariableChangeCallback variableChangeCallback) { VariableMetaData metaData = variables.get(variableName); if (metaData == null) { return; } Map> cps = changeCallbacksPerCP.get(metaData.getVariableNumber()); if (cps == null) { cps = new HashMap>(); changeCallbacksPerCP.set(metaData.getVariableNumber(), cps); } if (metaData.getContextPartitionName() == null) { agentInstanceId = EPStatementStartMethod.DEFAULT_AGENT_INSTANCE_ID; } Set callbacks = cps.get(agentInstanceId); if (callbacks == null) { callbacks = new CopyOnWriteArraySet(); cps.put(agentInstanceId, callbacks); } callbacks.add(variableChangeCallback); } public void unregisterCallback(String variableName, int agentInstanceId, VariableChangeCallback variableChangeCallback) { VariableMetaData metaData = variables.get(variableName); if (metaData == null) { return; } Map> cps = changeCallbacksPerCP.get(metaData.getVariableNumber()); if (cps == null) { return; } if (metaData.getContextPartitionName() == null) { agentInstanceId = 0; } Set callbacks = cps.get(agentInstanceId); if (callbacks != null) { callbacks.remove(variableChangeCallback); } } public void createNewVariable(String optionalContextName, String variableName, String variableType, boolean constant, boolean array, boolean arrayOfPrimitive, Object value, EngineImportService engineImportService) throws VariableExistsException, VariableTypeException { // Determime the variable type Class primitiveType = JavaClassHelper.getPrimitiveClassForName(variableType); Class type = JavaClassHelper.getClassForSimpleName(variableType); Class arrayType = null; EventType eventType = null; if (type == null) { if (variableType.toLowerCase().equals("object")) { type = Object.class; } if (type == null) { eventType = eventAdapterService.getExistsTypeByName(variableType); if (eventType != null) { type = eventType.getUnderlyingType(); } } if (type == null) { try { type = engineImportService.resolveClass(variableType, false); if (array) { arrayType = JavaClassHelper.getArrayType(type); } } catch (EngineImportException e) { log.debug("Not found '" + type + "': " + e.getMessage(), e); // expected } } if (type == null) { throw new VariableTypeException("Cannot create variable '" + variableName + "', type '" + variableType + "' is not a recognized type"); } if (array && eventType != null) { throw new VariableTypeException("Cannot create variable '" + variableName + "', type '" + variableType + "' cannot be declared as an array type"); } } else { if (array) { if (arrayOfPrimitive) { if (primitiveType == null) { throw new VariableTypeException("Cannot create variable '" + variableName + "', type '" + variableType + "' is not a primitive type"); } arrayType = JavaClassHelper.getArrayType(primitiveType); } else { arrayType = JavaClassHelper.getArrayType(type); } } } if ((eventType == null) && (!JavaClassHelper.isJavaBuiltinDataType(type)) && (type != Object.class) && !type.isArray() && !type.isEnum()) { if (array) { throw new VariableTypeException("Cannot create variable '" + variableName + "', type '" + variableType + "' cannot be declared as an array, only scalar types can be array"); } eventType = eventAdapterService.addBeanType(type.getName(), type, false, false, false); } if (arrayType != null) { type = arrayType; } createNewVariable(variableName, optionalContextName, type, eventType, constant, value); } private synchronized void createNewVariable(String variableName, String optionalContextName, Class type, EventType eventType, boolean constant, Object value) throws VariableExistsException, VariableTypeException { // check type Class variableType = JavaClassHelper.getBoxedType(type); // check if it exists VariableMetaData metaData = variables.get(variableName); if (metaData != null) { throw new VariableExistsException(VariableServiceUtil.getAlreadyDeclaredEx(variableName, false)); } // find empty spot int emptySpot = -1; int count = 0; for (Map entry : variableVersionsPerCP) { if (entry == null) { emptySpot = count; break; } count++; } int variableNumber; if (emptySpot != -1) { variableNumber = emptySpot; variableVersionsPerCP.set(emptySpot, new ConcurrentHashMap()); changeCallbacksPerCP.set(emptySpot, null); } else { variableNumber = currentVariableNumber; variableVersionsPerCP.add(new ConcurrentHashMap()); changeCallbacksPerCP.add(null); currentVariableNumber++; } // check coercion Object coercedValue = value; if (eventType != null) { if ((value != null) && (!JavaClassHelper.isSubclassOrImplementsInterface(value.getClass(), eventType.getUnderlyingType()))) { throw new VariableTypeException("Variable '" + variableName + "' of declared event type '" + eventType.getName() + "' underlying type '" + eventType.getUnderlyingType().getName() + "' cannot be assigned a value of type '" + value.getClass().getName() + "'"); } coercedValue = eventAdapterService.adapterForType(value, eventType); } else if (variableType == java.lang.Object.class) { // no validation } else { // allow string assignments to non-string variables if ((coercedValue != null) && (coercedValue instanceof String)) { try { coercedValue = JavaClassHelper.parse(variableType, (String) coercedValue); } catch (Exception ex){ throw new VariableTypeException("Variable '" + variableName + "' of declared type " + JavaClassHelper.getClassNameFullyQualPretty(variableType) + " cannot be initialized by value '" + coercedValue + "': " + ex.toString()); } } if ((coercedValue != null) && (variableType != coercedValue.getClass())) { // if the declared type is not numeric or the init value is not numeric, fail if ((!JavaClassHelper.isNumeric(variableType)) || (!(coercedValue instanceof Number))) { throw getVariableTypeException(variableName, variableType, coercedValue.getClass()); } if (!(JavaClassHelper.canCoerce(coercedValue.getClass(), variableType))) { throw getVariableTypeException(variableName, variableType, coercedValue.getClass()); } // coerce coercedValue = JavaClassHelper.coerceBoxed((Number)coercedValue, variableType); } } final Object initialState = coercedValue; VariableStateFactory stateFactory = new VariableStateFactoryConst(initialState); metaData = new VariableMetaData(variableName, optionalContextName, variableNumber, variableType, eventType, constant, stateFactory); variables.put(variableName, metaData); } public void allocateVariableState(String variableName, int agentInstanceId, StatementExtensionSvcContext extensionServicesContext, boolean isRecoveringResilient) { VariableMetaData metaData = variables.get(variableName); if (metaData == null) { throw new IllegalArgumentException("Failed to find variable '" + variableName + "'"); } // Check current state - see if the variable exists in the state handler Object initialState = metaData.getVariableStateFactory().getInitialState(); if (optionalStateHandler != null) { Pair priorValue = optionalStateHandler.getHasState(variableName, metaData.getVariableNumber(), agentInstanceId, metaData.getType(), metaData.getEventType(), extensionServicesContext, metaData.isConstant()); if (isRecoveringResilient) { if (priorValue.getFirst()) { initialState = priorValue.getSecond(); } } else { optionalStateHandler.setState(variableName, metaData.getVariableNumber(), agentInstanceId, initialState); } } // create new holder for versions long timestamp = timeProvider.getTime(); VersionedValueList valuePerVersion = new VersionedValueList(variableName, currentVersionNumber, initialState, timestamp, millisecondLifetimeOldVersions, readWriteLock.readLock(), HIGH_WATERMARK_VERSIONS, false); Map cps = variableVersionsPerCP.get(metaData.getVariableNumber()); VariableReader reader = new VariableReader(metaData, versionThreadLocal, valuePerVersion); cps.put(agentInstanceId, reader); } public void deallocateVariableState(String variableName, int agentInstanceId) { VariableMetaData metaData = variables.get(variableName); if (metaData == null) { throw new IllegalArgumentException("Failed to find variable '" + variableName + "'"); } Map cps = variableVersionsPerCP.get(metaData.getVariableNumber()); cps.remove(agentInstanceId); if (optionalStateHandler != null) { optionalStateHandler.removeState(variableName, metaData.getVariableNumber(), agentInstanceId); } } public VariableMetaData getVariableMetaData(String variableName) { return variables.get(variableName); } public VariableReader getReader(String variableName, int agentInstanceIdAccessor) { VariableMetaData metaData = variables.get(variableName); if (metaData == null) { return null; } Map cps = variableVersionsPerCP.get(metaData.getVariableNumber()); if (metaData.getContextPartitionName() == null) { return cps.get(EPStatementStartMethod.DEFAULT_AGENT_INSTANCE_ID); } return cps.get(agentInstanceIdAccessor); } public String isContextVariable(String variableName) { VariableMetaData metaData = variables.get(variableName); if (metaData == null) { return null; } return metaData.getContextPartitionName(); } public void write(int variableNumber, int agentInstanceId, Object newValue) { VariableVersionThreadEntry entry = versionThreadLocal.getCurrentThread(); if (entry.getUncommitted() == null) { entry.setUncommitted(new HashMap>()); } entry.getUncommitted().put(variableNumber, new Pair(agentInstanceId, newValue)); } public ReadWriteLock getReadWriteLock() { return readWriteLock; } public void commit() { VariableVersionThreadEntry entry = versionThreadLocal.getCurrentThread(); if (entry.getUncommitted() == null) { return; } // get new version for adding the new values (1 or many new values) int newVersion = currentVersionNumber + 1; if (currentVersionNumber == ROLLOVER_READER_BOUNDARY) { // Roll over to new collections; // This honors existing threads that will now use the "high" collection in the reader for high version requests // and low collection (new and updated) for low version requests rollOver(); newVersion = 2; } long timestamp = timeProvider.getTime(); // apply all uncommitted changes for (Map.Entry> uncommittedEntry : entry.getUncommitted().entrySet()) { Map cps = variableVersionsPerCP.get(uncommittedEntry.getKey()); VariableReader reader = cps.get(uncommittedEntry.getValue().getFirst()); VersionedValueList versions = reader.getVersionsLow(); // add new value as a new version Object newValue = uncommittedEntry.getValue().getSecond(); Object oldValue = versions.addValue(newVersion, newValue, timestamp); // make a callback that the value changed Map> cpsCallback = changeCallbacksPerCP.get(uncommittedEntry.getKey()); if (cpsCallback != null) { Set callbacks = cpsCallback.get(uncommittedEntry.getValue().getFirst()); if (callbacks != null) { for (VariableChangeCallback callback : callbacks) { callback.update(newValue, oldValue); } } } // Check current state - see if the variable exists in the state handler if (optionalStateHandler != null) { String name = versions.getName(); int agentInstanceId = reader.getVariableMetaData().getContextPartitionName() == null ? EPStatementStartMethod.DEFAULT_AGENT_INSTANCE_ID : uncommittedEntry.getValue().getFirst(); optionalStateHandler.setState(name, uncommittedEntry.getKey(), agentInstanceId, newValue); } } // this makes the new values visible to other threads (not this thread unless set-version called again) currentVersionNumber = newVersion; entry.setUncommitted(null); // clean out uncommitted variables } public void rollback() { VariableVersionThreadEntry entry = versionThreadLocal.getCurrentThread(); entry.setUncommitted(null); } /** * Rollover includes creating a new */ private void rollOver() { for (Map entryCP : variableVersionsPerCP) { for (Map.Entry entry : entryCP.entrySet()) { String name = entry.getValue().getVariableMetaData().getVariableName(); long timestamp = timeProvider.getTime(); // Construct a new collection, forgetting the history VersionedValueList versionsOld = entry.getValue().getVersionsLow(); Object currentValue = versionsOld.getCurrentAndPriorValue().getCurrentVersion().getValue(); VersionedValueList versionsNew = new VersionedValueList(name, 1, currentValue, timestamp, millisecondLifetimeOldVersions, readWriteLock.readLock(), HIGH_WATERMARK_VERSIONS, false); // Tell the reader to use the high collection for old requests entry.getValue().setVersionsHigh(versionsOld); entry.getValue().setVersionsLow(versionsNew); } } } public void checkAndWrite(String variableName, int agentInstanceId, Object newValue) throws VariableValueException { VariableMetaData metaData = variables.get(variableName); int variableNumber = metaData.getVariableNumber(); if (newValue == null) { write(variableNumber, agentInstanceId, null); return; } Class valueType = newValue.getClass(); if (metaData.getEventType() != null) { if ((!JavaClassHelper.isSubclassOrImplementsInterface(newValue.getClass(), metaData.getEventType().getUnderlyingType()))) { throw new VariableValueException("Variable '" + variableName + "' of declared event type '" + metaData.getEventType().getName() + "' underlying type '" + metaData.getEventType().getUnderlyingType().getName() + "' cannot be assigned a value of type '" + valueType.getName() + "'"); } EventBean eventBean = eventAdapterService.adapterForType(newValue, metaData.getEventType()); write(variableNumber, agentInstanceId, eventBean); return; } Class variableType = metaData.getType(); if ((valueType.equals(variableType)) || (variableType == Object.class)) { write(variableNumber, agentInstanceId, newValue); return; } if ((!JavaClassHelper.isNumeric(variableType)) || (!JavaClassHelper.isNumeric(valueType))) { throw new VariableValueException(VariableServiceUtil.getAssigmentExMessage(variableName, variableType, valueType)); } // determine if the expression type can be assigned if (!(JavaClassHelper.canCoerce(valueType, variableType))) { throw new VariableValueException(VariableServiceUtil.getAssigmentExMessage(variableName, variableType, valueType)); } Object valueCoerced = JavaClassHelper.coerceBoxed((Number) newValue, variableType); write(variableNumber, agentInstanceId, valueCoerced); } public String toString() { StringWriter writer = new StringWriter(); for (Map.Entry entryMeta : variables.entrySet()) { int variableNum = entryMeta.getValue().getVariableNumber(); for (Map.Entry entry : variableVersionsPerCP.get(variableNum).entrySet()) { VersionedValueList list = entry.getValue().getVersionsLow(); writer.write("Variable '" + entry.getKey() + "' : " + list.toString() + "\n"); } } return writer.toString(); } public Map getVariableReadersNonCP() { Map result = new HashMap(); for (Map.Entry entryMeta : variables.entrySet()) { int variableNum = entryMeta.getValue().getVariableNumber(); if (entryMeta.getValue().getContextPartitionName() == null) { for (Map.Entry entry : variableVersionsPerCP.get(variableNum).entrySet()) { result.put(entryMeta.getKey(), entry.getValue()); } } } return result; } public ConcurrentHashMap getReadersPerCP(String variableName) { VariableMetaData metaData = variables.get(variableName); return variableVersionsPerCP.get(metaData.getVariableNumber()); } private static VariableTypeException getVariableTypeException(String variableName, Class variableType, Class initValueClass) { return new VariableTypeException("Variable '" + variableName + "' of declared type " + JavaClassHelper.getClassNameFullyQualPretty(variableType) + " cannot be initialized by a value of type " + JavaClassHelper.getClassNameFullyQualPretty(initValueClass)); } }