thredds.cataloggen.CatalogGen Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
The newest version!
/*
 * Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
 *
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */
// $Id: CatalogGen.java 63 2006-07-12 21:50:51Z edavis $

package thredds.cataloggen;

import thredds.catalog.*;
import thredds.catalog.parser.jdom.InvCatalogFactory10;
import thredds.cataloggen.config.CatGenConfigMetadataFactory;
import thredds.cataloggen.config.CatalogGenConfig;
import thredds.cataloggen.config.DatasetSource;
import ucar.nc2.units.DateType;

import java.io.*;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;


/**
 * CatalogGen crawls dataset sources given in a CatalogGenConfig file
 * to produce THREDDS catalogs.
 *
 * To generate a catalog from a config file:
 *  *   String inFileName = "file:/home/edavis/testCatGenConfig.xml";
 *   String outFileName = "/home/edavis/testCatGenConfig-results.xml";
 *   StringBuffer log = new StringBuffer();
 *   CatalogGen catGen = new CatalogGen( inFileName);
 *   if ( catGen.isValid( log))
 *   {
 *     catGen.expand();
 *     catGen.writeCatalog( outFileName);
 *   }
 * 
 *
 *
 * @author Ethan Davis
 * @version $Ver$
 */
public class CatalogGen
{
  //private static Log log = LogFactory.getLog( CatalogGen.class );
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(CatalogGen.class);

  /** The catalog: initially as a CatGen config file, until expanded. */
  private InvCatalog catalog = null;

  /** The catalog factory that knows about CatalogGenConfig metadata. */
  protected InvCatalogFactory catFactory = null;

  public List getCatalogRefInfoList()
  {
    return catalogRefInfoList;
  }

  private List catalogRefInfoList = new ArrayList();


  /**
   * Constructs the CatalogGen for the given config document.
   *
   * @param configDocURL - the URL of the configuration document
   */
  public CatalogGen( URL configDocURL)
  {
    // Create a InvCatalogFactory with CATALOG_GEN_CONFIG MetadataType registered.
    log.debug( "CatalogGen(URL): create catalog and CatalogGenConfig converter." );
    this.catFactory = InvCatalogFactory.getDefaultFactory( true );
    this.catFactory.registerMetadataConverter( MetadataType.CATALOG_GEN_CONFIG.toString(),
                                             new CatGenConfigMetadataFactory());

    // Read the given XML config file.
    log.debug( "CatalogGen(URL): reading the config doc <" + configDocURL.toString() + ">.");
    this.catalog = this.catFactory.readXML( configDocURL.toString());
    log.debug( "CatalogGen(URL): done.");
  }

  /**
   * Constructs the CatalogGen for the given config document InputStream.
   *
   * @param configDocInputStream - the InputStream from which to read the config document.
   * @param configDocURL - the URL for the config document.
   */
  public CatalogGen( InputStream configDocInputStream, URL configDocURL )
  {
    // Create a InvCatalogFactory with CATALOG_GEN_CONFIG MetadataType registered.
    log.debug( "CatalogGen(InputStream): create catalog and CatalogGenConfig converter." );
    this.catFactory = new InvCatalogFactory( "default", true );
    this.catFactory.registerMetadataConverter( MetadataType.CATALOG_GEN_CONFIG.toString(),
                                               new CatGenConfigMetadataFactory() );

    // Read the given XML config file.
    log.debug( "CatalogGen(InputStream): reading the config doc <" + configDocURL.toString() + ">." );
    this.catalog = this.catFactory.readXML( configDocInputStream, URI.create( configDocURL.toExternalForm()) );
    log.debug( "CatalogGen(InputStream): CatalogGenConfig doc <" + this.catalog.getName() + "> read.");

  }

  /**
   * Checks the validity of the configuration file.
   * @param out - a StringBuffer with validity error and warning messages.
   * @return - true if no errors, false if errors exist
   */
  public boolean isValid( StringBuilder out)
  {
    log.debug( "isValid(): start");
    return( this.catalog.check( out));
  }

  /**
   * Expand the catalog. Each of the CatalogGenConfig metadata elements
   * is expanded into its constituent datasets.
   */
  public InvCatalog expand()
  {
    CatalogGenConfig tmpCgc = null;
    List cgcList = null;
    DatasetSource dss = null;

    // Find and loop through each CatGenConfigMetadata object.
    List mdataList = findCatGenConfigMdata( this.catalog.getDatasets());
    for ( int i = 0; i < mdataList.size(); i++)
    {
      InvMetadata curMdata = (InvMetadata) mdataList.get( i);
      InvDatasetImpl curParentDataset = ( (InvDatasetImpl) curMdata.getParentDataset());

      // Loop through the CatalogGenConfig objects in current InvMetadata.
      cgcList = (List) curMdata.getContentObject();
      for ( int j = 0; j < cgcList.size(); j++)
      {
        tmpCgc = (CatalogGenConfig) cgcList.get( j);
        log.debug( "expand(): mdata # " + i + " and catGenConfig # " + j + "." );
        dss = tmpCgc.getDatasetSource();
        InvCatalog generatedCat = null;
        try
        {
          generatedCat = dss.fullExpand();
        }
        catch ( IOException e )
        {
          String tmpMsg = "Error: IOException on fullExpand() of DatasetSource <" + dss.getName() + ">: " + e.getMessage();
          log.error( "expand(): " + tmpMsg);
          curParentDataset.addDataset( new InvDatasetImpl( curParentDataset, tmpMsg));
          break;
        }

        catalogRefInfoList.addAll( dss.getCatalogRefInfoList());

        // Always a single top-level dataset in catalog returned by DatasetSource.fullExpand()
        InvDataset genTopDs = (InvDataset) generatedCat.getDatasets().get( 0);

        // Add all services in the generated catalog to the parent catalog.
        for( Iterator it = generatedCat.getServices().iterator(); it.hasNext(); )
        {
          ( (InvCatalogImpl) curParentDataset.getParentCatalog() ).addService( (InvService) it.next() );
        }

        // Add the generated catalog to the parent datasets, i.e., add all the
        // datasets that are children of the generated catalogs' top-level dataset.
        for ( Iterator it = genTopDs.getDatasets().iterator(); it.hasNext(); )
        {
          InvDatasetImpl curGenDataset = (InvDatasetImpl) it.next();
          if ( curGenDataset.hasNestedDatasets())
          {
            // If the dataset has children datasets, add serviceName and make it inherited.
            ThreddsMetadata tm = new ThreddsMetadata( false);
            tm.setServiceName( genTopDs.getServiceDefault().getName());
            InvMetadata md = new InvMetadata( genTopDs, null, XMLEntityResolver.CATALOG_NAMESPACE_10, "", true, true, null, tm);
            curGenDataset.getLocalMetadata().addMetadata( md);
          }
          else
          {
            // Otherwise, add serviceName not inherited.
            curGenDataset.getLocalMetadata().setServiceName(  genTopDs.getServiceDefault().getName());
          }
          curParentDataset.addDataset( curGenDataset);
        }

        curParentDataset.finish();
      }
//      log.debug( "expand(): List datasets that are siblinks of current metadata record CGCM(" + i + ").");
//      List list = curMdata.getParentDataset().getDatasets();
//      Iterator it = list.iterator();
//      while ( it.hasNext())
//      {
//        InvDatasetImpl curDs = (InvDatasetImpl) it.next();
//        log.debug( "Dataset URL is " + curDs.getUrlPath() + ".");
//        log.debug( "Dataset name is " + curDs.getName() + ".");
//      }

      // Remove the current metadata element from its parent dataset.
      log.debug( "expand(): Remove metadata record CGCM(" + i + ")." );
      curParentDataset.removeLocalMetadata( curMdata);
      // *****
    }

    // Finish this catalog now that done building it.
    ((InvCatalogImpl) this.catalog).finish();

    return( this.catalog);
  }

  public void setCatalogExpiresDate( DateType expiresDate )
  {
    ((InvCatalogImpl) this.catalog).setExpires( expiresDate);
  }

  /**
   * Writes the catalog as XML. The catalog is written to the file given
   * in outFileName. If outFileName is null, the catalog
   * is written to standard out.
   *
   * @param outFileName - the pathname of the output file.
   *
   * @throws IOException if can't write catalog
   */
  public void writeCatalog( String outFileName)
          throws IOException
  {
    log.debug( "writeCatalog(): writing catalog to " + outFileName + ".");

    String invCatDTD = "http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.0.6.dtd";
    log.debug( "writeCatalog(): set the catalogs DTD (" + invCatDTD + ").");
    // Set the catalogs DTD.
    // ( (InvCatalogImpl) catalog).setDTDid( invCatDTD);  LOOK !!

    // Print the catalog as an XML document.
    if ( outFileName == null)
    {
      log.debug( "writeCatalog(): write catalog to System.out.");
      this.catFactory.writeXML( (InvCatalogImpl) catalog, System.out);
    }
    else
    {
      log.debug( "writeCatalog(): try writing catalog to the output file (" + outFileName + ").");

      if ( ! this.catalog.getVersion().equals( "1.0" ) )
      {
        this.catFactory.writeXML( (InvCatalogImpl) catalog, outFileName );
      }
      else
      {
        // Override default output catalog version. (kludge for IDV backward compatibility)
        InvCatalogFactory10 fac10 = (InvCatalogFactory10) this.catFactory.getCatalogConverter( XMLEntityResolver.CATALOG_NAMESPACE_10 );
        fac10.setVersion( this.catalog.getVersion() );
        BufferedOutputStream osCat = new BufferedOutputStream( new FileOutputStream( outFileName ) );
        fac10.writeXML( (InvCatalogImpl) catalog, osCat );
        osCat.close();
      }

      log.debug( "writeCatalog(): catalog written to " + outFileName + "." );
    }

    return;
  }

  InvCatalog getCatalog() { return( this.catalog ); }

  private List findCatGenConfigMdata( List datasets)
  {
    List mdataList = new ArrayList();
    if ( datasets == null) return( mdataList );

    // Iterate through list of datasets.
    Iterator it = datasets.iterator();
    InvDataset curDataset = null;
    while ( it.hasNext() )
    {
      curDataset = (InvDataset) it.next();

      // Get all the local metadata for the given dataset.
      ThreddsMetadata tm = ((InvDatasetImpl) curDataset).getLocalMetadata();

      // Iterate over the local InvMetadata checking for CatalogGenConfig metadata.
      Iterator itMdata = tm.getMetadata().iterator();
      InvMetadata curMetadata = null;
      while ( itMdata.hasNext())
      {
        curMetadata = (InvMetadata) itMdata.next();
        if ( (curMetadata.getMetadataType() != null) &&
             curMetadata.getMetadataType().equals( MetadataType.CATALOG_GEN_CONFIG.toString() ) )
        {
          mdataList.add( curMetadata );
        }
        else if ( (curMetadata.getNamespaceURI() != null) &&
                  curMetadata.getNamespaceURI().equals( CatalogGenConfig.CATALOG_GEN_CONFIG_NAMESPACE_URI_0_5 ))
        {
          mdataList.add( curMetadata );
        }
      }

      // Recurse through nested datasets and find CatalogGenConfig metadata.
      mdataList.addAll( this.findCatGenConfigMdata( curDataset.getDatasets() ) );
    }

    return( mdataList);
  }
}