thredds.catalog.InvDatasetFeatureCollection Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
The newest version!
/*
 * Copyright (c) 1998 - 2010. University Corporation for Atmospheric Research/Unidata
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package thredds.catalog;

import net.jcip.annotations.GuardedBy;
import net.jcip.annotations.ThreadSafe;
import org.slf4j.Logger;
import thredds.crawlabledataset.CrawlableDataset;
import thredds.crawlabledataset.CrawlableDatasetFilter;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.featurecollection.FeatureCollectionType;
import thredds.inventory.*;
import thredds.inventory.MCollection;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dt.GridDataset;
import ucar.nc2.ft.FeatureDataset;
import ucar.nc2.time.CalendarDateRange;
import ucar.nc2.units.DateRange;
import ucar.nc2.util.log.LoggerFactory;
import ucar.nc2.util.log.LoggerFactoryImpl;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Abstract superclass for Feature Collection InvDatasets.
 * This is a InvCatalogRef subclass. So the reference is placed in the parent, but
 * the catalog itself isnt constructed until the following call from DataRootHandler.makeDynamicCatalog():
 * match.dataRoot.featCollection.makeCatalog(match.remaining, path, baseURI);
 * 
 * The InvDatasetFeatureCollection object is created once and held in the DataRootHandler's collection
 * of DataRoots.
 *
 * @author caron
 * @since Mar 3, 2010
 */
@ThreadSafe
public abstract class InvDatasetFeatureCollection extends InvCatalogRef implements CollectionUpdateListener {
  static protected final String LATEST_DATASET_CATALOG = "latest.xml";
  static protected final String LATEST_SERVICE = "latest";
  static protected final String VARIABLES = "?metadata=variableMap";
  static protected final String FILES = "files";
  static protected final String Virtual_Services = "VirtualServices"; // exclude HTTPServer

  static private String catalogServletName = "/catalog";            // LOOK
  static protected String context = "/thredds";                     // LOOK
  static private String cdmrFeatureServiceUrlPath = "/cdmrFeature"; // LOOK
  static private LoggerFactory loggerFactory = new LoggerFactoryImpl();
  static private org.slf4j.Logger classLogger = org.slf4j.LoggerFactory.getLogger(InvDatasetFeatureCollection.class);

  static public void setContext(String c) {
    context = c;
  }

  static public void setCatalogServletName(String catServletName) {
    catalogServletName = catServletName;
  }

  static protected String buildCatalogServiceHref(String path) {
    return context + (catalogServletName == null ? "" : catalogServletName) + "/" + path + "/catalog.xml";
  }

  static public void setCdmrFeatureServiceUrlPath(String urlPath) {
    cdmrFeatureServiceUrlPath = urlPath;
  }

  static public void setLoggerFactory(LoggerFactory fac) {
    loggerFactory = fac;
  }

  static private InvService makeCdmrFeatureService() {
    return new InvService("cdmrFeature", "cdmrFeature", context + cdmrFeatureServiceUrlPath, null, null);
  }

  static public InvDatasetFeatureCollection factory(InvDatasetImpl parent, String name, String path, FeatureCollectionType fcType, FeatureCollectionConfig config) {
    InvDatasetFeatureCollection result;
    if (fcType == FeatureCollectionType.FMRC)
      result = new InvDatasetFcFmrc(parent, name, path, fcType, config);

    else if (fcType == FeatureCollectionType.GRIB1 || fcType == FeatureCollectionType.GRIB2) {
      // use reflection to decouple from grib.jar
      try {
        Class c = InvDatasetFeatureCollection.class.getClassLoader().loadClass("thredds.catalog.InvDatasetFcGrib");
        // public InvDatasetFcGrib(InvDatasetImpl parent, String name, String path, FeatureType featureType, FeatureCollectionConfig config) {
        Constructor ctor = c.getConstructor(InvDatasetImpl.class, String.class, String.class, FeatureCollectionType.class, FeatureCollectionConfig.class);
        result = (InvDatasetFeatureCollection) ctor.newInstance(parent, name, path, fcType, config);

      } catch (Throwable e) {
        classLogger.error("Failed to open " + name + " path=" + path, e);
        return null;
      }

    } else {
      result = new InvDatasetFcPoint(parent, name, path, fcType, config);
    }

    if (result != null) {
      result.finishConstruction(); // stuff that shouldnt be done in a constructor
    }

    return result;
  }

  /////////////////////////////////////////////////////////////////////////////
  // heres how we manage state changes in a thread-safe way
  protected class State {
    // catalog metadata
    protected ThreddsMetadata.Variables vars;
    protected ThreddsMetadata.GeospatialCoverage coverage;
    protected CalendarDateRange dateRange;

    protected InvDatasetImpl top;        // top dataset
    protected long lastInvChange;        // last time dataset inventory was changed
    protected long lastProtoChange;      // last time proto dataset was changed

    protected State(State from) {
      if (from != null) {
        this.vars = from.vars;
        this.coverage = from.coverage;
        this.dateRange = from.dateRange;
        this.lastProtoChange = from.lastProtoChange;

        this.top = from.top;
        this.lastInvChange = from.lastInvChange;
      }
    }

    protected State copy() {  // allow override
      return new State(this);
    }
  }

  /////////////////////////////////////////////////////////////////////////////
  // not changed after first call
  protected InvService orgService, virtualService;
  protected InvService cdmrService;  // LOOK why do we need to specify this seperately ??
  protected org.slf4j.Logger logger;

  // from the config catalog
  protected final String path;
  protected final FeatureCollectionType fcType;
  protected final FeatureCollectionConfig config;
  protected String topDirectory;
  protected MCollection datasetCollection; // defines the collection of datasets in this feature collection, actually final
  protected String collectionName;

  @GuardedBy("lock")
  protected State state;
  @GuardedBy("lock")
  protected boolean first = true;
  protected final Object lock = new Object();

  protected InvDatasetFeatureCollection(InvDatasetImpl parent, String name, String path, FeatureCollectionType fcType, FeatureCollectionConfig config) {
    super(parent, name, buildCatalogServiceHref(path));
    this.path = path;
    this.fcType = fcType;
    this.config = config;

    this.getLocalMetadataInheritable().setDataType(fcType.getFeatureType());

    this.collectionName = CollectionAbstract.cleanName(config.name != null ? config.name : name);
    config.name = collectionName;
    this.logger = loggerFactory.getLogger("fc." + collectionName); // seperate log file for each feature collection (!!)

    this.logger.info("FeatureCollection added = {}", getConfig());
  }

  protected void makeCollection() {

    Formatter errlog = new Formatter();
    if (config.spec.startsWith(MFileCollectionManager.CATALOG)) { // LOOK CHANGE THIS
      datasetCollection = new CollectionManagerCatalog(config.name, config.spec, null, errlog);
    } else {
      datasetCollection = new MFileCollectionManager(config, errlog, this.logger);
    }
    topDirectory = datasetCollection.getRoot();
    String errs = errlog.toString();
    if (errs.length() > 0) logger.warn("MFileCollectionManager parse error = {} ", errs);
  }

  // stuff that shouldnt be done in a constructor - eg dont let 'this' escape
  // LOOK maybe not best design to start tasks from here
  // LOOK we want to get notified of events, but no longer initiate changes.
  protected void finishConstruction() {
    CollectionUpdater.INSTANCE.scheduleTasks(config, this, logger);
  }

  public String getCollectionName() {
    return collectionName;
  }

  @Override
  // DatasetCollectionManager was changed asynchronously
  public void sendEvent(CollectionUpdateType type) {
    update(type);

/*     if (event == CollectionUpdateListener.TriggerType.updateNocheck)

    else if (event == CollectionUpdateListener.TriggerType.update)
      update(CollectionManager.Force.always);

    else if (event == CollectionUpdateListener.TriggerType.resetProto)
      updateProto();  */
  }

  /**
   * update the proto dataset being used.
   * called by CollectionUpdater via handleCollectionEvent, so in a quartz scheduler thread
   */
  abstract protected void updateProto();

  // localState is synched, may be directly changed
  abstract protected void updateCollection(State localState, CollectionUpdateType force);

  abstract protected void makeDatasetTop(State localState);

  // this allows us to put warnings into the catalogInit.log
  @Override
  boolean check(StringBuilder out, boolean show) {
    boolean isValid = true;

    // no longer need default service for FC 3/11/14
    /* if (getServiceDefault() == null) {
      out.append("**Warning: Dataset (").append(getFullName()).append("): has no default service\n");
      isValid = false;
    }  */

    return isValid && super.check(out, show);
  }

  ////////////////////////////////////////////////////////////////////////////////////////////

  protected String getCatalogHref(String what) {
    return buildCatalogServiceHref(path + "/" + what);
  }

  // call this first time a request comes in
  protected void firstInit() {
    this.orgService = getServiceDefault();
    if (this.orgService == null) {
      this.orgService = makeServiceDefault();
    }
    this.virtualService = makeServiceVirtual(this.orgService);
    this.cdmrService = makeCdmrFeatureService(); // WTF ??
  }

  /**
   * A request has come in, check that the state has been initialized.
   * this is called from the request thread.
   *
   * @return a copy of the State
   */
  protected State checkState() {
    State localState;

    synchronized (lock) {
      if (first) {
        firstInit();
        updateCollection(state, config.updateConfig.updateType);
        makeDatasetTop(state);
        first = false;
      }
      localState = state.copy();
    }

    return localState;
  }

  /**
   * Collection was changed, update internal objects.
   * called by CollectionUpdater, trigger via handleCollectionEvent, so in a quartz scheduler thread
   *
   * @param force test : update index if anything changed or nocheck - use index if it exists
   */
  protected void update(CollectionUpdateType force) {  // this may be called from a background thread
    State localState;

    synchronized (lock) {
      if (first) {
        state = checkState();
        state.lastInvChange = System.currentTimeMillis();
        return;
      }
      // do the update in a local object
      localState = state.copy();
    }

    updateCollection(localState, force);
    makeDatasetTop(localState);
    localState.lastInvChange = System.currentTimeMillis();

    // switch to live
    synchronized (lock) {
      state = localState;
    }

  }

  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  public String getPath() {
    return path;
  }

  public String getLatestFileName() {
    if (config.gribConfig.latestNamer != null) {
      return config.gribConfig.latestNamer;
    } else {
      return "Latest "+name+" File";
    }
  }

  public String getTopDirectoryLocation() {
    return topDirectory;
  }

  public FeatureCollectionConfig getConfig() {
    return config;
  }

  public MCollection getDatasetCollectionManager() {
    return datasetCollection;
  }

  public Logger getLogger() {
    return logger;
  }

  @Override
  public java.util.List getDatasets() { // probably not used
    State localState;
    try {
      localState = checkState();
    } catch (Exception e) {
      logger.error("Error in checkState", e);
      return new ArrayList<>(0);
    }
    List tops = new ArrayList<>(1);
    tops.add(localState.top);
    return tops;
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////////
  protected InvService makeServiceDefault() {

    // LOOK need (thredds.server.config.AllowableService)
    InvService result = new InvService("Default", ServiceType.COMPOUND.toString(), null, null, null);
    result.addService(InvService.opendap);
    result.addService(InvService.fileServer);
    result.addService(InvService.wms);
    result.addService(InvService.wcs);
    result.addService(InvService.ncss);
    result.addService(InvService.cdmremote);
    result.addService(InvService.ncml);
    result.addService(InvService.uddc);
    result.addService(InvService.iso);
    return result;
  }

  protected InvService makeServiceVirtual(InvService org) {
    if (org.getServiceType() != ServiceType.COMPOUND) return org;

    InvService result = new InvService(Virtual_Services, ServiceType.COMPOUND.toString(), null, null, null);
    for (InvService service : org.getServices()) {
      if (service.getServiceType() != ServiceType.HTTPServer) {
        result.addService(service);
      }
    }
    return result;
  }

  /**
   * Get one of the catalogs contained in this collection,
   * called by DataRootHandler.makeDynamicCatalog()
   *
   * @param match   match.remaining
   * @param orgPath the path for the request.
   * @param catURI  the base URI for the catalog to be made, used to resolve relative URLs.
   * @return containing catalog
   */
  abstract public InvCatalogImpl makeCatalog(String match, String orgPath, URI catURI);

  /**
   * Make the containing catalog of this feature collection
   * "http://server:port/thredds/catalog/path/catalog.xml"
   *
   * @param catURI     base URI of the request
   * @param localState current state to use
   * @return the top FMRC catalog
   * @throws java.io.IOException         on I/O error
   * @throws java.net.URISyntaxException if path is misformed
   */
  protected InvCatalogImpl makeCatalogTop(URI catURI, State localState) throws IOException, URISyntaxException {
    InvCatalogImpl parentCatalog = (InvCatalogImpl) getParentCatalog();
    InvCatalogImpl mainCatalog = new InvCatalogImpl(getName(), parentCatalog.getVersion(), catURI);

    mainCatalog.addDataset(localState.top);
    mainCatalog.addService(InvService.latest);  // in case its needed
    mainCatalog.addService(virtualService);
    // top.getLocalMetadataInheritable().setServiceName(virtualService.getName());  //??
    mainCatalog.finish();
    return mainCatalog;
  }

  // this catalog lists the individual files comprising the collection.
  protected InvCatalogImpl makeCatalogFiles(URI catURI, State localState, List filenames, boolean addLatest) throws IOException {

    //String collectionName = gc.getName();
    InvCatalogImpl parent = (InvCatalogImpl) getParentCatalog();
    //URI myURI = baseURI.resolve(getCatalogHref(collectionName));
    //URI myURI = baseURI.resolve(getCatalogHref(FILES));
    InvCatalogImpl result = new InvCatalogImpl(getFullName(), parent.getVersion(), catURI);
    InvDatasetImpl top = new InvDatasetImpl(this);
    top.setParent(null);
    top.transferMetadata((InvDatasetImpl) this.getParent(), true); // make all inherited metadata local
    top.setName(FILES);

    // add Variables, GeospatialCoverage, TimeCoverage
    ThreddsMetadata tmi = top.getLocalMetadataInheritable();
    if (localState.coverage != null) tmi.setGeospatialCoverage(localState.coverage);
    tmi.setTimeCoverage((DateRange) null); // LOOK

    result.addDataset(top);

    // services need to be local
    result.addService(orgService);
    top.getLocalMetadataInheritable().setServiceName(orgService.getName());

    //String id = getID();
    //if (id == null) id = getPath();

    if (addLatest) {
      InvDatasetImpl ds = new InvDatasetImpl(this, getLatestFileName());
      ds.setUrlPath(LATEST_DATASET_CATALOG);
      // ds.setID(getPath() + "/" + FILES + "/" + LATEST_DATASET_CATALOG);
      ds.setServiceName(LATEST_SERVICE);
      ds.finish();
      top.addDataset(ds);
      result.addService(InvService.latest);
    }

    //sort copy of files
    List sortedFilenames = new ArrayList(filenames);
    Collections.sort(sortedFilenames, String.CASE_INSENSITIVE_ORDER);

    // if not increasing (i.e. we WANT newest file listed first), reverse sort
    if (this.config.gribConfig != null && !this.config.gribConfig.filesSortIncreasing) {
      Collections.reverse(sortedFilenames);
    }

    for (String f : sortedFilenames) {
      if (!f.startsWith(topDirectory))
        logger.warn("File {} doesnt start with topDir {}", f, topDirectory);

      String fname = f.substring(topDirectory.length() + 1);
      InvDatasetImpl ds = new InvDatasetImpl(this, fname);
      String lpath = getPath() + "/" + FILES + "/" + fname;
      ds.setUrlPath(lpath);
      ds.setID(lpath);
      ds.tmi.addVariableMapLink(makeMetadataLink(lpath, VARIABLES));
      File file = new File(f);
      ds.tm.setDataSize(file.length());
      ds.finish();
      top.addDataset(ds);
    }

    result.finish();
    return result;
  }

  protected String makeMetadataLink(String datasetName, String metadata) {
    return context + "/metadata/" + datasetName + metadata;
  }

  // called by DataRootHandler.makeDynamicCatalog()
  public InvCatalogImpl makeLatest(String matchPath, String reqPath, URI catURI) throws IOException {
    checkState();

    InvCatalogImpl parent = (InvCatalogImpl) getParentCatalog();
    InvCatalogImpl result = new InvCatalogImpl(getFullName(), parent.getVersion(), catURI);
    InvDatasetImpl top = new InvDatasetImpl(this);
    top.setParent(null);
    top.transferMetadata((InvDatasetImpl) this.getParent(), true); // make all inherited metadata local
    top.setName(getLatestFileName());

    // add Variables, GeospatialCoverage, TimeCoverage
    // ThreddsMetadata tmi = top.getLocalMetadataInheritable();
    //if (localState.gc != null) tmi.setGeospatialCoverage(localState.gc);
    //if (localState.dateRange != null) tmi.setTimeCoverage(localState.dateRange);

    result.addDataset(top);

    // services need to be local
    // result.addService(InvService.latest);
    if (orgService != null) {
      result.addService(orgService);
      top.getLocalMetadataInheritable().setServiceName(orgService.getName());
    }

    MFile mfile = datasetCollection.getLatestFile();  // LOOK - assumes dcm is up to date
    if (mfile == null) return null;

    String mpath = mfile.getPath();
    if (!mpath.startsWith(topDirectory))
      logger.warn("File {} doesnt start with topDir {}", mpath, topDirectory);
    String fname = mpath.substring(topDirectory.length() + 1);

    String path = FILES + "/" + fname;
    top.setUrlPath(this.path + "/" + path);
    top.setID(this.path + "/" + path);
    top.tmi.addVariableMapLink(makeMetadataLink(this.path + "/" + path, VARIABLES));
    top.tm.setDataSize(mfile.getLength());

    result.finish();
    return result;
  }

  /////////////////////////////////////////////////////////////////////////

  /**
   * Get the associated Grid Dataset, if any. called by DatasetHandler.openGridDataset()
   *
   * @param matchPath match.remaining
   * @return Grid Dataset, or null if n/a
   * @throws IOException on error
   */
  public ucar.nc2.dt.grid.GridDataset getGridDataset(String matchPath) throws IOException {
    int pos = matchPath.indexOf('/');
    String type = (pos > -1) ? matchPath.substring(0, pos) : matchPath;
    String name = (pos > -1) ? matchPath.substring(pos + 1) : "";

    // this assumes that these are files. also might be remote datasets from a catalog
    if (type.equals(FILES)) {
      if (topDirectory == null) return null;

      String filename = topDirectory + (topDirectory.endsWith("/") ? "" : "/") + name;
      NetcdfDataset ncd = NetcdfDataset.acquireDataset(null, filename, null, -1, null, null); // no enhancement
      return new ucar.nc2.dt.grid.GridDataset(ncd);
    }

    return null;
  }

  abstract public FeatureDataset getFeatureDataset();

  ///////////////////////////////////////////////////////////////////////////////
  // handle individual files

  /**
   * Get the dataset named by the path.
   * called by DatasetHandler.getNetcdfFile()
   *
   * @param matchPath remaining path from match
   * @return requested dataset
   * @throws IOException if read error
   */
  public NetcdfDataset getNetcdfDataset(String matchPath) throws IOException {
    int pos = matchPath.indexOf('/');
    String type = (pos > -1) ? matchPath.substring(0, pos) : matchPath;
    String name = (pos > -1) ? matchPath.substring(pos + 1) : "";

    // this assumes that these are files. also might be remote datasets from a catalog
    if (type.equals(FILES)) {
      if (topDirectory == null) return null;

      String filename = new StringBuilder(topDirectory)
              .append(topDirectory.endsWith("/") ? "" : "/")
              .append(name).toString();
      return NetcdfDataset.acquireDataset(null, filename, null, -1, null, null); // no enhancement
    }

    GridDataset gds = getGridDataset(matchPath); // LOOK cant be right
    return (gds == null) ? null : (NetcdfDataset) gds.getNetcdfFile();
  }

  // called by DataRootHandler.getCrawlableDatasetAsFile()
  // may have to remove the extra "files" from the path
  // this says that a File URL has to be topDirectory + [FILES/ +] + match.remaining
  public File getFile(String remaining) {
    if (null == topDirectory) return null;
    int pos = remaining.indexOf(FILES);
    StringBuilder fname = new StringBuilder(topDirectory);
    if (!topDirectory.endsWith("/"))
      fname.append("/");
    fname.append((pos > -1) ? remaining.substring(pos + FILES.length() + 1) : remaining);
    return new File(fname.toString());
  }

  // specialized filter handles olderThan and/or filename pattern matching
  // for InvDatasetScan
  static class ScanFilter implements CrawlableDatasetFilter {
    private final Pattern p;
    private final long olderThan;

    public ScanFilter(Pattern p, long olderThan) {
      this.p = p;
      this.olderThan = olderThan;
    }

    @Override
    public boolean accept(CrawlableDataset dataset) {
      if (dataset.isCollection()) return true;

      if (p != null) {
        java.util.regex.Matcher matcher = p.matcher(dataset.getName());
        if (!matcher.matches()) return false;
      }

      if (olderThan > 0) {
        Date lastModDate = dataset.lastModified();
        if (lastModDate != null) {
          long now = System.currentTimeMillis();
          if (now - lastModDate.getTime() <= olderThan)
            return false;
        }
      }

      return true;
    }

    @Override
    public Object getConfigObject() {
      return null;
    }
  }

}