thredds.inventory.DatasetCollectionManager Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of netcdf Show documentation
Show all versions of netcdf Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files,
as well as to many other types of scientific data formats.
/*
* Copyright (c) 1998 - 2009. University Corporation for Atmospheric Research/Unidata
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package thredds.inventory;
import net.jcip.annotations.ThreadSafe;
import ucar.nc2.units.TimeUnit;
import ucar.nc2.util.CancelTask;
import java.util.*;
import java.io.IOException;
import thredds.inventory.filter.*;
import thredds.inventory.bdb.MetadataManager;
import ucar.nc2.util.ListenerManager;
/**
* Manage one or more Directory Scanners that find MFiles
* Keep track of when they need to be rescanned.
* Used in:
*
* - replaces older version in ncml.Aggregation
*
- ucar.nc2.ft.point.collection.TimedCollectionImpl
*
*
* looks like we need to be thread safe, for InvDatasetFeatureCollection
*
* @author caron
* @since Jul 8, 2009
*/
@ThreadSafe
public class DatasetCollectionManager implements CollectionManager {
static public final String CATALOG = "catalog:";
static public final String RESCAN = "rescan";
static public final String PROTO = "proto";
static private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetCollectionManager.class);
static private MController controller;
// explicit enabling - allows deleteOld to work first time
static private boolean enableMetadataManager = false;
static public void enableMetadataManager() {
enableMetadataManager = true;
}
/**
* Set the MController used by scan. Defaults to thredds.filesystem.ControllerOS() if not set.
* @param _controller use this MController
*/
static public void setController(MController _controller) {
controller = _controller;
}
// called from Aggregation - deprecated
static public DatasetCollectionManager open(String collection, String olderThan, Formatter errlog) throws IOException {
if (collection.startsWith(CATALOG))
return new DatasetCollectionFromCatalog(collection);
else
return new DatasetCollectionManager(collection, olderThan, errlog);
}
////////////////////////////////////////////////////////////////////
// these actually dont change
protected String collectionName;
protected DateExtractor dateExtractor;
// these are final
private final CollectionSpecParser sp;
private final List scanList = new ArrayList(); // an MCollection is a collection of managed files
private final TimeUnit recheck; // how often to recheck
private final double olderThanFilterInSecs;
private final FeatureCollectionConfig.ProtoChoice protoChoice;
// this can change
private Map map; // current map of MFile in the collection
private long lastScanned; // last time scanned
private long lastChanged; // last time the set of files changed
private ListenerManager lm;
// simplified version called from DatasetCollectionManager.open()
private DatasetCollectionManager(String collectionSpec, String olderThan, Formatter errlog) {
this.sp = new CollectionSpecParser(collectionSpec, errlog);
this.collectionName = collectionSpec; // StringUtil.escape(collectionSpec, "");
this.recheck = null;
this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default
List filters = new ArrayList(2);
if (null != sp.getFilter())
filters.add(new WildcardMatchOnName(sp.getFilter()));
olderThanFilterInSecs = getOlderThanFilter(filters, olderThan);
dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true);
scanList.add(new MCollection(sp.getTopDir(), sp.getTopDir(), sp.wantSubdirs(), filters, null));
createListenerManager();
}
/* public DatasetCollectionManager(CollectionSpecParser sp, Formatter errlog) {
this.collectionName = sp.getSpec();
this.sp = sp;
MFileFilter mfilter = (null == sp.getFilter()) ? null : new WildcardMatchOnName(sp.getFilter());
this.dateExtractor = (sp.getDateFormatMark() == null) ? null : new DateExtractorFromName(sp.getDateFormatMark(), true);
scanList.add(new MCollection(sp.getTopDir(), sp.getTopDir(), sp.wantSubdirs(), mfilter, null));
} */
public DatasetCollectionManager(FeatureCollectionConfig config, Formatter errlog) {
this.sp = new CollectionSpecParser(config.spec, errlog);
this.collectionName = config.name != null ? config.name : config.spec;
List filters = new ArrayList(3);
if (null != sp.getFilter())
filters.add( new WildcardMatchOnName(sp.getFilter()));
olderThanFilterInSecs = getOlderThanFilter(filters, config.olderThan);
dateExtractor = (sp.getDateFormatMark() == null) ? new DateExtractorNone() : new DateExtractorFromName(sp.getDateFormatMark(), true);
scanList.add(new MCollection(sp.getTopDir(), sp.getTopDir(), sp.wantSubdirs(), filters, null));
this.recheck = makeRecheck(config.recheckAfter);
protoChoice = config.protoConfig.choice;
createListenerManager();
}
private double getOlderThanFilter(List filters, String olderThan) {
if (olderThan != null) {
try {
TimeUnit tu = new TimeUnit(olderThan);
double olderThanV = tu.getValueInSeconds();
filters.add( new LastModifiedLimit((long) (1000 * olderThanV)));
return olderThanV;
} catch (Exception e) {
logger.error(collectionName + ": Invalid time unit for olderThan = {}", olderThan);
}
}
return -1;
}
private TimeUnit makeRecheck(String recheckS) {
if (recheckS != null) {
try {
return new TimeUnit(recheckS);
} catch (Exception e) {
logger.error(collectionName+": Invalid time unit for recheckEvery = {}", recheckS);
}
}
return null;
}
public void close() {
if (mm != null) mm.close();
}
// for subclasses
protected DatasetCollectionManager() {
this.recheck = null;
this.olderThanFilterInSecs = -1;
this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate; // default
this.sp = null;
createListenerManager();
}
////////////////////////////////////////////////////////////////////
/**
* For retrofitting to Aggregation
* Must also call addDirectoryScan one or more times
* @param recheckS a undunit time unit, specifying how often to rscan
*/
public DatasetCollectionManager(String recheckS) {
this.recheck = makeRecheck(recheckS);
this.olderThanFilterInSecs = -1;
this.protoChoice = FeatureCollectionConfig.ProtoChoice.Penultimate;
this.sp = null;
createListenerManager();
}
public void setDateExtractor( DateExtractor dateExtractor) {
this.dateExtractor = dateExtractor;
}
/**
* Add a directory scan to the collection
* @param dirName scan this directory
* @param suffix require this suffix (overriddden by regexp), may be null
* @param regexpPatternString if present, use this reqular expression to filter files , may be null
* @param subdirsS if "true", descend into subdirectories, may be null
* @param olderS udunit time unit - files must be older than this amount of time (now - lastModified > olderTime), may be null
// * @param dateFormatString dateFormatMark string, may be null
* @param auxInfo attach this object to any MFile found by this scan
*/
public void addDirectoryScan(String dirName, String suffix, String regexpPatternString, String subdirsS, String olderS, Object auxInfo) {
List filters = new ArrayList(3);
if (null != regexpPatternString)
filters.add(new RegExpMatchOnName(regexpPatternString));
else if (suffix != null)
filters.add(new WildcardMatchOnPath("*" + suffix+"$"));
if (olderS != null) {
try {
TimeUnit tu = new TimeUnit(olderS);
filters.add(new LastModifiedLimit((long) (1000 * tu.getValueInSeconds())));
} catch (Exception e) {
logger.error(collectionName+": Invalid time unit for olderThan = {}", olderS);
}
}
boolean wantSubdirs = true;
if ((subdirsS != null) && subdirsS.equalsIgnoreCase("false"))
wantSubdirs = false;
MFileFilter filter = (filters.size() == 0) ? null : ((filters.size() == 1) ? filters.get(0) : new Composite(filters));
MCollection mc = new thredds.inventory.MCollection(dirName, dirName, wantSubdirs, filter, auxInfo);
// create name
StringBuilder sb = new StringBuilder(dirName);
if (wantSubdirs)
sb.append("**/");
if (null != regexpPatternString)
sb.append(regexpPatternString);
else if (suffix != null)
sb.append(suffix);
else
sb.append("noFilter");
collectionName = sb.toString();
scanList.add(mc);
}
////////////////////////////////////////////////////////////////////
@Override
public String getCollectionName() {
return collectionName;
}
@Override
public String getRoot() {
return (sp == null) ? null : sp.getTopDir();
}
public CollectionSpecParser getCollectionSpecParser() {
return sp;
}
public double getOlderThanFilterInSecs() {
return olderThanFilterInSecs;
}
/**
* Get how often to rescan
*
* @return time dureation of rescan period, or null if none.
*/
@Override
public TimeUnit getRecheck() {
return recheck;
}
/**
* Get the last time scanned
*
* @return msecs since 1970
*/
@Override
public long getLastScanned() {
return lastScanned;
}
/**
* Get the last time files changed
*
* @return msecs since 1970
*/
@Override
public long getLastChanged() {
return lastChanged;
}
/**
* Scan the directory(ies) and create MFile objects.
* Get the results from getFiles()
*
* @param cancelTask allow user to cancel
* @throws java.io.IOException if io error
*/
public void scan(CancelTask cancelTask) throws IOException {
Map newMap = new HashMap();
if (!hasScans()) {
map = newMap;
return;
}
scan(newMap, cancelTask);
deleteOld(newMap);
// LOOK how often ??
synchronized(this) {
map = newMap;
this.lastScanned = System.currentTimeMillis();
this.lastChanged = this.lastScanned;
}
if (logger.isDebugEnabled()) logger.debug(collectionName+": initial scan found n datasets = "+map.keySet().size());
}
public boolean rescanIfNeeded() throws IOException {
if (isRescanNeeded())
return rescan();
return false;
}
protected boolean hasScans() {
return !scanList.isEmpty();
}
/**
* Compute if rescan is needed, based on the recheckEvery parameter.
* True if scanList not empty, recheckEvery not null, and recheckEvery time has passed since last scan
*
* @return true if rescan is needed
*/
@Override
public boolean isRescanNeeded() {
if (!hasScans()) {
if (logger.isDebugEnabled()) logger.debug(collectionName+": rescan not needed, no scanners");
return false;
}
// see if we need to recheck
if (recheck == null) {
if (logger.isDebugEnabled()) logger.debug(collectionName+": rescan not needed, recheck null");
return false;
}
Date now = new Date();
Date lastCheckedDate = new Date(lastScanned);
Date need = recheck.add(lastCheckedDate);
if (now.before(need)) {
if (logger.isDebugEnabled()) logger.debug(collectionName+": rescan not needed, last= " + lastCheckedDate + " now = " + now);
return false;
}
return true;
}
/**
* Rescan the collection. Files may have been deleted or added.
* If the MFile already exists in the current list, leave it in the list.
* If returns true, get the results from getFiles(), otherwise nothing has changed.
*
* @return true if anything actually changed.
* @throws IOException on I/O error
*/
@Override
public boolean rescan() throws IOException {
if (map == null) {
scan(null);
return hasScans();
}
// rescan
Map oldMap = map;
Map newMap = new HashMap();
scan(newMap, null);
// replace with previous datasets if they exist
int nnew = 0;
for (MFile newDataset : newMap.values()) {
String path = newDataset.getPath();
MFile oldDataset = oldMap.get(path);
if (oldDataset != null) {
newMap.put(path, oldDataset);
if (logger.isDebugEnabled()) logger.debug(collectionName+": rescan retains old Dataset= " + path);
} else {
nnew++;
if (logger.isDebugEnabled()) logger.debug(collectionName+": rescan found new Dataset= " + path);
}
}
// check for deletions
int ndelete = 0;
for (MFile oldDataset : oldMap.values()) {
String path = oldDataset.getPath();
MFile newDataset = newMap.get(path);
if (newDataset == null) {
ndelete++;
if (logger.isDebugEnabled()) logger.debug(collectionName+": rescan found deleted Dataset= " + path);
}
}
boolean changed = (nnew > 0) || (ndelete > 0);
if (changed) {
//if (logger.isInfoEnabled()) logger.info(collectionName+": rescan found changes new = "+nnew+" delete= "+ndelete);
synchronized (this) {
map = newMap;
this.lastScanned = System.currentTimeMillis();
this.lastChanged = this.lastScanned;
}
} else {
synchronized (this) {
this.lastScanned = System.currentTimeMillis();
}
}
if (changed)
lm.sendEvent(new TriggerEvent(RESCAN)); // LOOK watch out for infinite loop
if (logger.isInfoEnabled()) logger.info(collectionName+": rescan at " + new Date()+": nnew = "+nnew+" ndelete = "+ndelete);
return changed;
}
/**
* Get the current collection of MFile, since last scan or rescan.
* Must call scan() or rescan() at least once
* @return current list of MFile, sorted by name
*/
@Override
public List getFiles() {
if (map == null) return null; // never scanned
List result = new ArrayList(map.values());
Collections.sort(result);
return result;
}
@Override
public Date extractRunDate(MFile mfile) {
return (dateExtractor == null) ? null : dateExtractor.getDate(mfile);
}
public boolean hasDateExtractor() {
return (dateExtractor != null) && !(dateExtractor instanceof DateExtractorNone);
}
protected void scan(java.util.Map map, CancelTask cancelTask) throws IOException {
if (null == controller) controller = new thredds.filesystem.ControllerOS(); // default
// run through all scanners and collect MFile instances into the Map
for (MCollection mc : scanList) {
Iterator iter = (mc.wantSubdirs()) ? controller.getInventory(mc) : controller.getInventoryNoSubdirs(mc);
if (iter == null) {
logger.error(collectionName+": DatasetCollectionManager Invalid collection= " + mc);
continue;
}
while (iter.hasNext()) {
MFile mfile = iter.next();
mfile.setAuxInfo(mc.getAuxInfo());
map.put(mfile.getPath(), mfile);
}
if ((cancelTask != null) && cancelTask.isCancel())
return;
}
}
@Override
public String toString() {
Formatter f = new Formatter();
f.format("DatasetCollectionManager{ collectionName='%s' recheck=%s ", collectionName, recheck);
for (MCollection mc : scanList) {
f.format("%n dir=%s filter=%s", mc.getDirectoryName(), mc.getFileFilter());
}
return f.toString();
}
////////////////////////////////////////////////////
// proto dataset handling - LOOK
public int getProtoIndex() {
int n = map.values().size();
int protoIdx = 0;
switch (protoChoice) {
case First:
protoIdx = 0;
break;
case Random:
Random r = new Random(System.currentTimeMillis());
protoIdx = r.nextInt(n - 1);
break;
case Penultimate:
protoIdx = Math.max(n - 2, 0);
break;
case Latest:
protoIdx = Math.max(n - 1, 0);
break;
}
return protoIdx;
}
public void resetProto() { // LOOK
lm.sendEvent(new TriggerEvent(PROTO));
}
////////////////////////////////////////////////////
// events; keep the code from getting too coupled
private void createListenerManager() {
lm = new ListenerManager(
"thredds.inventory.DatasetCollectionManager$TriggerListener",
"thredds.inventory.DatasetCollectionManager$TriggerEvent",
"handleCollectionEvent");
}
public void addEventListener(TriggerListener l) {
lm.addListener(l);
}
public void removeEventListener(TriggerListener l) {
lm.removeListener(l);
}
public class TriggerEvent extends java.util.EventObject {
private String message;
TriggerEvent(String message) {
super(DatasetCollectionManager.this);
this.message = message;
}
public String getMessage() {
return message;
}
}
public static interface TriggerListener {
public void handleCollectionEvent(TriggerEvent event);
}
/////////////////////////////////////////////////////////////////
// use bdb to manage metadata associated with the collection. currently, only DatasetInv.xml files
private MetadataManager mm;
private void initMM() {
if (collectionName == null) return; // eg no scan in ncml
try {
mm = new MetadataManager(collectionName);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
// clean up deleted files in metadata manager
private void deleteOld(Map newMap) {
if (mm == null && enableMetadataManager) initMM();
if (mm != null) mm.delete(newMap);
}
@Override
public void putMetadata(MFile file, String key, byte[] value) {
if (mm == null) initMM();
if (mm != null) mm.put(file.getPath()+"#"+key, value);
}
@Override
public byte[] getMetadata(MFile file, String key) {
if (mm == null) initMM();
return (mm == null) ? null : mm.getBytes(file.getPath()+"#"+key);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy