All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.yamcs.parameter.ArrayParameterCache Maven / Gradle / Ivy

There is a newer version: 5.10.9
Show newest version
package org.yamcs.parameter;

import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.yamcs.logging.Log;
import org.yamcs.protobuf.Yamcs.Value.Type;
import org.yamcs.utils.IntArray;
import org.yamcs.utils.SortedIntArray;
import org.yamcs.utils.TimeEncoding;
import org.yamcs.utils.ValueUtility;
import org.yamcs.xtce.Parameter;

/**
 * This is another implementation of the parameter cache using arrays to store primitive values (instead of storing
 * {@link Value}).
 */
public class ArrayParameterCache implements ParameterCache {
    SimpleParameterIdMap pidMap = new SimpleParameterIdMap();
    final Log log;
    long cacheStartTime = 0;
    ConcurrentHashMap tables = new ConcurrentHashMap<>();
    final ConcurrentHashMap parametersToCache;
    final ParameterCacheConfig cacheConfig;

    ArrayParameterCache(String instance, ParameterCacheConfig cacheConfig) {
        log = new Log(this.getClass(), instance);
        this.cacheConfig = cacheConfig;
        parametersToCache = cacheConfig.cacheAll ? null : new ConcurrentHashMap<>();
    }

    @Override
    public void update(Collection pvs) {

        Map m = new HashMap<>();
        for (ParameterValue pv : pvs) {
            long t = pv.getGenerationTime();
            if (t < cacheStartTime) {
                continue;
            }
            if (!(cacheConfig.cacheAll || parametersToCache.containsKey(pv.getParameter()))) {
                continue;
            }
            SortedParameterList l = m.get(t);
            if (l == null) {
                l = new SortedParameterList(pidMap);
                m.put(t, l);
            }
            l.add(pv);
        }
        long maxTimestamp = -1;
        for (Map.Entry entry : m.entrySet()) {
            long t = entry.getKey();
            SortedParameterList pvList = entry.getValue();
            addToCache(t, pvList);
            if (t > maxTimestamp) {
                maxTimestamp = t;
            }
        }
    }

    private void addToCache(long t, SortedParameterList pvList) {
        SortedIntArray sia = pvList.getParameterIdArray();
        ParameterValueTable table = tables.get(sia);
        if (table == null) {
            table = new ParameterValueTable(sia, cacheConfig.maxDuration, cacheConfig.maxNumEntries);
            ParameterValueTable table1 = tables.putIfAbsent(sia, table);
            if (table1 != null) {
                table = table1;
            }
        }

        table.add(t, pvList.getParameterValueList());
    }

    @Override
    public ParameterValue getLastValue(Parameter pdef) {
        List pidlist = getParameterIds(pdef);
        ParameterValue result = null;
        long tmax = Long.MIN_VALUE;
        for (ParameterId p : pidlist) {
            SortedIntArray sia = findLatestTableContaining(p.id);
            if (sia == null) {
                continue;
            }

            ParameterValueTable table = tables.get(sia);
            long t = table.getLastTime();
            if (t < tmax) {
                continue;
            }

            ParameterValue pv = table.getLastValue(p);
            if (t == tmax) {
                if (result != null && result.getAcquisitionTime() < pv.getAcquisitionTime()) {
                    result = pv;
                }
            } else {
                result = pv;
            }

            tmax = t;
        }

        return result;

    }

    @Override
    public List getValues(List plist) {
        List pidlist = getParameterIds(plist);

        List result = new ArrayList<>(plist.size());

        for (int i = 0; i < pidlist.size(); i++) {
            ParameterId p = pidlist.get(i);
            if (p == null) {
                continue;
            }
            pidlist.set(i, null);

            SortedIntArray sai = findLatestTableContaining(p.id);
            if (sai == null) {
                continue;
            }
            ParameterValueTable table = tables.get(sai);
            List sublist = new ArrayList<>();
            sublist.add(p);
            for (int j = i + 1; j < pidlist.size(); j++) {
                ParameterId p1 = pidlist.get(j);
                if (p1 == null) {
                    continue;
                }
                if (sai.contains(p1.id)) {
                    sublist.add(p1);
                    pidlist.set(j, null);
                }
            }
            table.retrieveLastValues(sublist, result);
        }

        long now = TimeEncoding.getWallclockTime();
        // check expiration
        for (ParameterValue pv : result) {
            if (!pv.status.isExpired() && pv.isExpired(now)) {
                pv.status.setExpired();
            }
        }

        return result;
    }

    private SortedIntArray findLatestTableContaining(int pid) {
        long tmax = Long.MIN_VALUE;
        SortedIntArray result = null;
        for (Map.Entry me : tables.entrySet()) {
            SortedIntArray sai = me.getKey();
            long t = me.getValue().getLastTime();
            if (sai.contains(pid) && t > tmax) {
                result = sai;
                tmax = t;
            }
        }
        return result;
    }

    @Override
    public List getAllValues(Parameter pdef) {
        return getAllValues(pdef, Long.MIN_VALUE, Long.MAX_VALUE);
    }

    @Override
    public List getAllValues(Parameter pdef, long start, long stop) {
        List pidlist = getParameterIds(pdef);
        List result = new ArrayList<>();
        int numTables = 0;
        for (ParameterId p : pidlist) {
            for (Map.Entry me : tables.entrySet()) {
                SortedIntArray sia = me.getKey();
                if (sia.contains(p.id)) {
                    numTables++;
                    me.getValue().retrieveAll(p, start, stop, result);
                }
            }
        }
        // if values are retrieved from multiple tables, we need to sort them by generation time
        // (in reverse order such that the newest is first)
        if (numTables > 1) {
            Collections.sort(result, (pv1, pv2) -> Long.compare(pv2.getGenerationTime(), pv1.getGenerationTime()));
        }
        if (result.isEmpty()) {
            return null;
        }
        return result;
    }

    /**
     * This is the same as above but returns null if the cache does not cover the interval starting with start.
     * 

*/ public List getAllValuesIfCovered(Parameter pdef, long start, long stop) { List pidlist = getParameterIds(pdef); List result = new ArrayList<>(); int numTables = 0; for (ParameterId p : pidlist) { for (Map.Entry me : tables.entrySet()) { SortedIntArray sia = me.getKey(); if (sia.contains(p.id)) { numTables++; if (!me.getValue().retrieveAllIfCovered(p, start, stop, result)) { return null; } } } } // if values are retrieved from multiple tables, we need to sort them by generation time // (in reverse order such that the newest is first) if (numTables > 1) { Collections.sort(result, (pv1, pv2) -> Long.compare(pv2.getGenerationTime(), pv1.getGenerationTime())); } if (result.isEmpty()) { return null; } return result; } public List> getAllValues(List parameters, long start, long stop) { List pidlist = getParameterIds(parameters); List> result = new ArrayList<>(); boolean mergingRequired = false; for (Map.Entry me : tables.entrySet()) { boolean dataFound = me.getValue().retrieveAll(pidlist, start, stop, result); mergingRequired = mergingRequired || dataFound; } // if values are retrieved from multiple tables, we need to sort them by generation time // (in reverse order such that the newest is first) if (mergingRequired) { Collections.sort(result, (pvList1, pvList2) -> Long.compare(pvList2.get(0).getGenerationTime(), pvList1.get(0).getGenerationTime())); } if (result.isEmpty()) { return null; } return result; } public List> getAllValuesIfCovered(List parameters, long start, long stop) { // TODO Auto-generated method stub return null; } private List getParameterIds(List pdefList) { List result = new ArrayList<>(); for (Parameter pdef : pdefList) { Map m = pidMap.get(pdef); if (m == null) { if (!cacheConfig.cacheAll) { parametersToCache.put(pdef, Boolean.TRUE); } } else { for (Map.Entry me : m.entrySet()) { int pid = me.getValue(); int type = me.getKey(); result.add(new ParameterId(pdef, pid, SimpleParameterIdMap.getRawType(type), SimpleParameterIdMap.getEngType(type))); } } } return result; } private List getParameterIds(Parameter pdef) { List result = new ArrayList<>(); Map m = pidMap.get(pdef); if (m == null) { if (!cacheConfig.cacheAll) { parametersToCache.put(pdef, Boolean.TRUE); } } else { for (Map.Entry me : m.entrySet()) { int pid = me.getValue(); int type = me.getKey(); result.add(new ParameterId(pdef, pid, SimpleParameterIdMap.getRawType(type), SimpleParameterIdMap.getEngType(type))); } } return result; } static class ParameterId { Parameter pdef; int id; Type engType; Type rawType; public ParameterId(Parameter p, int pid, Type rawType, Type engType) { this.pdef = p; this.id = pid; this.rawType = rawType; this.engType = engType; } } static class SimpleParameterIdMap { // parameter fqn -> parameter type -> parameter id Map> p2pidCache = new HashMap<>(); AtomicInteger pidGenerator = new AtomicInteger(); public synchronized int createAndGet(Parameter param, Type engType, Type rawType) { int type = numericType(engType, rawType); Map m = p2pidCache.get(param); if (m == null) { m = new HashMap<>(); p2pidCache.put(param, m); } Integer pid = m.get(type); if (pid == null) { pid = pidGenerator.incrementAndGet(); m.put(type, pid); } return pid; } Parameter getParameterForPid(int x) { for (Map.Entry> me : p2pidCache.entrySet()) { for (Map.Entry me1 : me.getValue().entrySet()) { if (x == me1.getValue()) { return me.getKey(); } } } return null; } public Map get(Parameter p) { return p2pidCache.get(p); } // compose a numeric type from engType and rawType (we assume that no more than 2^15 types will ever exist) static int numericType(Type engType, Type rawType) { int et = (engType == null) ? 0xFFFF : engType.getNumber(); int rt = (rawType == null) ? 0xFFFF : rawType.getNumber(); return et << 16 | rt; } static Type getEngType(int numericType) { int et = numericType >> 16; if (et == 0xFFFF) { return null; } else { return Type.forNumber(et); } } static Type getRawType(int numericType) { int rt = numericType & 0xFFFF; if (rt == 0xFFFF) { return null; } else { return Type.forNumber(rt); } } } static class SortedParameterList { final SimpleParameterIdMap parameterIdMap; final SortedIntArray parameterIdArray = new SortedIntArray(); final List sortedPvList = new ArrayList<>(); public SortedParameterList(SimpleParameterIdMap paraId) { this.parameterIdMap = paraId; } public void add(ParameterValue pv) { Value engValue = pv.getEngValue(); Value rawValue = pv.getRawValue(); Type engType = (engValue == null) ? null : engValue.getType(); Type rawType = (rawValue == null) ? null : rawValue.getType(); int parameterId = parameterIdMap.createAndGet(pv.getParameter(), engType, rawType); int pos = parameterIdArray.insert(parameterId); sortedPvList.add(pos, pv); } public int size() { return parameterIdArray.size(); } public SortedIntArray getParameterIdArray() { return parameterIdArray; } public List getParameterValueList() { return sortedPvList; } } /** * Stores values for list of parameters of predefined types * * It's like a big table: * *

     * t0, ev01, rv01, ps01, ev02, rv02, ps02 ... 
     * t1, ev11, rv11, ps11, ev12, rv12, ps12 ... ....
     * 
* * where: t = timestamp ev = engineering value rv = raw value ps = parameter status * * Each column is stored as an array of different type (depending on the parameter type). The array works as a * circular list * */ static class ParameterValueTable { static final int MAX_NUM_ENTRIES = 1024; static final int INITIAL_CAPACITY = 16; long[] generationTimeColumn; final Object[] rawValueColumns; final Object[] engValueColumns; final Object[] statusColumns; final long[][] acquisitionTimeColumns; final int numParams; final long timeToCache; int head = 0; int tail = head; int maxNumEntries = MAX_NUM_ENTRIES; final SortedIntArray pids; long coverageStart = TimeEncoding.INVALID_INSTANT; ReadWriteLock lock = new ReentrantReadWriteLock(); ParameterValueTable(SortedIntArray pids, long timeToCache, int maxNumEntries) { this.numParams = pids.size(); this.pids = pids; this.rawValueColumns = new Object[numParams]; this.engValueColumns = new Object[numParams]; this.statusColumns = new Object[numParams]; this.acquisitionTimeColumns = new long[numParams][]; this.timeToCache = timeToCache; this.maxNumEntries = maxNumEntries; } private void init(List sortedPvList) { this.generationTimeColumn = new long[INITIAL_CAPACITY]; for (int i = 0; i < sortedPvList.size(); i++) { ParameterValue pv = sortedPvList.get(i); Value v = pv.getEngValue(); if (v != null) { engValueColumns[i] = getNewColumn(v.getType()); } Value rawV = pv.getRawValue(); if (rawV != null) { rawValueColumns[i] = getNewColumn(rawV.getType()); } statusColumns[i] = new ParameterStatus[INITIAL_CAPACITY]; acquisitionTimeColumns[i] = new long[INITIAL_CAPACITY]; } } public void add(long t, List sortedPvList) { lock.writeLock().lock(); try { if (numParams != sortedPvList.size()) { throw new IllegalArgumentException("Invalid number of parameters, expected " + sortedPvList.size()); } int _head = head; if (generationTimeColumn == null) { init(sortedPvList); } else if (_head == tail) { long t0 = generationTimeColumn[_head]; if (t < t0) { // parameter older than the last one in the table -> ignore return; } boolean doubled = false; if (t - t0 < timeToCache) { doubled = doubleCapacity(); _head = head; } if (!doubled) { tail = (tail + 1) & (generationTimeColumn.length - 1); } } generationTimeColumn[_head] = t; for (int i = 0; i < numParams; i++) { storeParameter(i, _head, sortedPvList.get(i)); } head = (_head + 1) & (generationTimeColumn.length - 1); } finally { lock.writeLock().unlock(); } } public void retrieveLastValues(List sublist, List result) { lock.readLock().lock(); try { int row = (head - 1) & (generationTimeColumn.length - 1); for (ParameterId p : sublist) { result.add(getParameterValue(row, p)); } } finally { lock.readLock().unlock(); } } public ParameterValue getLastValue(ParameterId p) { lock.readLock().lock(); try { int row = (head - 1) & (generationTimeColumn.length - 1); return getParameterValue(row, p); } finally { lock.readLock().unlock(); } } public void retrieveAll(ParameterId p, long start, long stop, List result) { lock.readLock().lock(); try { // col1 will be different than col2 when there are multiple values for the same parameter int col2 = pids.higherBound(p.id); int col1 = col2; while (col1 > 0 && pids.get(col1 - 1) == p.id) { col1--; } int _tail = tail; int _head = head; int n = generationTimeColumn.length - 1; int row = _head; do { row = (row - 1) & n; for (int col = col2; col >= col1; col--) { if (generationTimeColumn[row] > start && generationTimeColumn[row] <= stop) { result.add(getParameterValue(row, col, p)); } } } while (row != _tail); } finally { lock.readLock().unlock(); } } // return true if any data has been retrieved public boolean retrieveAll(List plist, long start, long stop, List> result) { lock.readLock().lock(); try { // Precompute column bounds for all requested ParameterIds IntArray bounds = new IntArray(); for (ParameterId p : plist) { int col2 = pids.higherBound(p.id); int col1 = col2; while (col1 > 0 && pids.get(col1 - 1) == p.id) { col1--; } bounds.add(col1); // Add the lower bound bounds.add(col2); // Add the upper bound } if (bounds.isEmpty()) { return false; } int _tail = tail; int _head = head; int n = generationTimeColumn.length - 1; int row = _head; // Iterate over the rows of the circular buffer do { row = (row - 1) & n; // Only process rows within the specified time range if (generationTimeColumn[row] > start && generationTimeColumn[row] <= stop) { List rowValues = new ArrayList<>(); // Collect parameter values for all requested ParameterIds in this row for (int i = 0; i < plist.size(); i++) { ParameterId p = plist.get(i); int col1 = bounds.get(i * 2); // Retrieve lower bound int col2 = bounds.get(i * 2 + 1); // Retrieve upper bound for (int col = col2; col >= col1; col--) { rowValues.add(getParameterValue(row, col, p)); } } // Add the collected values for the current row to the result result.add(rowValues); } } while (row != _tail); } finally { lock.readLock().unlock(); } return true; } /** * This is the same as above but returns false if the timestamp of the first entry in the table is older than * start. *

* If this is the case, it means that the interval [start, stop) is not completely covered by this cache entry * (so the code calling this may decide to perform a reply instead) *

* If the method returns false, the result will not be modified */ public boolean retrieveAllIfCovered(ParameterId p, long start, long stop, List result) { lock.readLock().lock(); try { if (start + 1 < generationTimeColumn[tail]) { return false; } // col1 will be different than col2 when there are multiple values for the same parameter int col2 = pids.higherBound(p.id); int col1 = col2; while (col1 > 0 && pids.get(col1 - 1) == p.id) { col1--; } int _tail = tail; int _head = head; int n = generationTimeColumn.length - 1; int row = _head; do { row = (row - 1) & n; for (int col = col2; col >= col1; col--) { if (generationTimeColumn[row] > start && generationTimeColumn[row] <= stop) { result.add(getParameterValue(row, col, p)); } } } while (row != _tail); return true; } finally { lock.readLock().unlock(); } } private ParameterValue getParameterValue(int row, ParameterId p) { int col = pids.search(p.id); return getParameterValue(row, col, p); } private ParameterValue getParameterValue(int row, int col, ParameterId p) { ParameterValue pv = new ParameterValue(p.pdef); if (p.rawType != null) { pv.setRawValue(getValue(rawValueColumns[col], p.rawType, row)); } if (p.engType != null) { pv.setEngValue(getValue(engValueColumns[col], p.engType, row)); } pv.setGenerationTime(generationTimeColumn[row]); pv.setAcquisitionTime(acquisitionTimeColumns[col][row]); pv.setStatus((ParameterStatus) ((Object[]) statusColumns[col])[row]); return pv; } private Value getValue(Object o, Type type, int idx) { switch (type) { case BOOLEAN: return ValueUtility.getBooleanValue(((BitSet) o).get(idx)); case DOUBLE: return ValueUtility.getDoubleValue(((double[]) o)[idx]); case FLOAT: return ValueUtility.getFloatValue(((float[]) o)[idx]); case SINT32: return ValueUtility.getSint32Value(((int[]) o)[idx]); case UINT32: return ValueUtility.getUint32Value(((int[]) o)[idx]); case SINT64: return ValueUtility.getSint64Value(((long[]) o)[idx]); case UINT64: return ValueUtility.getUint64Value(((long[]) o)[idx]); case TIMESTAMP: return ValueUtility.getTimestampValue(((long[]) o)[idx]); case STRING: return ValueUtility.getStringValue((String) ((Object[]) o)[idx]); case BINARY: return ValueUtility.getBinaryValue((byte[]) (((Object[]) o)[idx])); case AGGREGATE: case ARRAY: case ENUMERATED: return (Value) (((Object[]) o)[idx]); default: throw new IllegalStateException("Unknown type " + type); } } public long getLastTime() { lock.readLock().lock(); try { int row = (head - 1) & (generationTimeColumn.length - 1); return generationTimeColumn[row]; } finally { lock.readLock().unlock(); } } private void storeParameter(int col, int row, ParameterValue pv) { Value v = pv.getEngValue(); if (v != null) { storeValue(engValueColumns[col], row, v); } v = pv.getRawValue(); if (v != null) { storeValue(rawValueColumns[col], row, v); } ParameterStatus status = pv.getStatus(); if (row > 0) { // avoid filling up memory with identical ParameterStatus ParameterStatus prevStatus = (ParameterStatus) ((Object[]) statusColumns[col])[row - 1]; if (prevStatus.equals(status)) { status = prevStatus; } } ((Object[]) statusColumns[col])[row] = status; acquisitionTimeColumns[col][row] = pv.getAcquisitionTime(); } private void storeValue(Object o, int pos, Value v) { Type type = v.getType(); switch (type) { case BOOLEAN -> ((BitSet) o).set(pos, v.getBooleanValue()); case DOUBLE -> ((double[]) o)[pos] = v.getDoubleValue(); case FLOAT -> ((float[]) o)[pos] = v.getFloatValue(); case SINT32 -> ((int[]) o)[pos] = v.getSint32Value(); case UINT32 -> ((int[]) o)[pos] = v.getUint32Value(); case SINT64 -> ((long[]) o)[pos] = v.getSint64Value(); case UINT64 -> ((long[]) o)[pos] = v.getUint64Value(); case TIMESTAMP -> ((long[]) o)[pos] = v.getTimestampValue(); case STRING -> { Object[] objArray = (Object[]) o; String stringValue = v.getStringValue(); if (pos > 0 && stringValue.equals(objArray[pos - 1])) { objArray[pos] = objArray[pos - 1]; } else { objArray[pos] = stringValue; } } case BINARY -> { Object[] objArray = (Object[]) o; byte[] binaryValue = v.getBinaryValue(); if (pos > 0 && binaryValue.equals(objArray[pos - 1])) { objArray[pos] = objArray[pos - 1]; } else { objArray[pos] = binaryValue; } } case AGGREGATE, ARRAY, ENUMERATED -> { Object[] objArray = (Object[]) o; if (pos > 0 && v.equals(objArray[pos - 1])) { objArray[pos] = objArray[pos - 1]; } else { objArray[pos] = v; } } default -> throw new IllegalStateException("Unknown type " + type); } } private Object getNewColumn(Type type) { switch (type) { case BOOLEAN: return new BitSet(INITIAL_CAPACITY); case DOUBLE: return new double[INITIAL_CAPACITY]; case FLOAT: return new float[INITIAL_CAPACITY]; case SINT32: case UINT32: return new int[INITIAL_CAPACITY]; case SINT64: case UINT64: case TIMESTAMP: return new long[INITIAL_CAPACITY]; case STRING: case BINARY: case AGGREGATE: case ARRAY: case ENUMERATED: return new Object[INITIAL_CAPACITY]; default: throw new IllegalStateException("Unknown type " + type); } } private boolean doubleCapacity() { int capacity = generationTimeColumn.length; if (capacity >= maxNumEntries) { return false; } int newCapacity = 2 * capacity; long[] o2 = new long[newCapacity]; System.arraycopy(generationTimeColumn, head, o2, 0, capacity - head); System.arraycopy(generationTimeColumn, 0, o2, capacity - head, head); generationTimeColumn = o2; for (int i = 0; i < numParams; i++) { Object c = engValueColumns[i]; if (c != null) { engValueColumns[i] = growCapacity(c, newCapacity); } c = rawValueColumns[i]; if (c != null) { rawValueColumns[i] = growCapacity(c, newCapacity); } c = statusColumns[i]; if (c != null) { statusColumns[i] = growCapacity(c, newCapacity); } c = acquisitionTimeColumns[i]; acquisitionTimeColumns[i] = (long[]) growCapacity(c, newCapacity); } tail = 0; head = capacity; return true; } private Object growCapacity(Object o, int newCapacity) { if (o instanceof int[]) { int[] o1 = (int[]) o; int[] o2 = new int[newCapacity]; System.arraycopy(o1, head, o2, 0, o1.length - head); System.arraycopy(o1, 0, o2, o1.length - head, head); return o2; } else if (o instanceof double[]) { double[] o1 = (double[]) o; double[] o2 = new double[newCapacity]; System.arraycopy(o1, head, o2, 0, o1.length - head); System.arraycopy(o1, 0, o2, o1.length - head, head); return o2; } else if (o instanceof float[]) { float[] o1 = (float[]) o; float[] o2 = new float[newCapacity]; System.arraycopy(o1, head, o2, 0, o1.length - head); System.arraycopy(o1, 0, o2, o1.length - head, head); return o2; } else if (o instanceof long[]) { long[] o1 = (long[]) o; long[] o2 = new long[newCapacity]; System.arraycopy(o1, head, o2, 0, o1.length - head); System.arraycopy(o1, 0, o2, o1.length - head, head); return o2; } else if (o instanceof Object[]) { Object[] o1 = (Object[]) o; Object[] o2 = new Object[newCapacity]; System.arraycopy(o1, head, o2, 0, o1.length - head); System.arraycopy(o1, 0, o2, o1.length - head, head); return o2; } else if (o instanceof BitSet) { return o; } else { throw new IllegalArgumentException("Cannot double objects of type " + o.getClass()); } } } @Override public void clear() { tables.clear(); } @Override public boolean caching(ParameterWithId pid) { // TODO Auto-generated method stub return false; } }





© 2015 - 2025 Weber Informatics LLC | Privacy Policy