thredds.cataloggen.CatalogGen Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of netcdf Show documentation
Show all versions of netcdf Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files,
as well as to many other types of scientific data formats.
The newest version!
/*
* Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
*
* Portions of this software were developed by the Unidata Program at the
* University Corporation for Atmospheric Research.
*
* Access and use of this software shall impose the following obligations
* and understandings on the user. The user is granted the right, without
* any fee or cost, to use, copy, modify, alter, enhance and distribute
* this software, and any derivative works thereof, and its supporting
* documentation for any purpose whatsoever, provided that this entire
* notice appears in all copies of the software, derivative works and
* supporting documentation. Further, UCAR requests that the user credit
* UCAR/Unidata in any publications that result from the use of this
* software or in any product that includes this software. The names UCAR
* and/or Unidata, however, may not be used in any advertising or publicity
* to endorse or promote any products or commercial entity unless specific
* written permission is obtained from UCAR/Unidata. The user also
* understands that UCAR/Unidata is not obligated to provide the user with
* any support, consulting, training or assistance of any kind with regard
* to the use, operation and performance of this software nor to provide
* the user with any updates, revisions, new versions or "bug fixes."
*
* THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
* INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
* FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
* NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
* WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
*/
// $Id: CatalogGen.java 63 2006-07-12 21:50:51Z edavis $
package thredds.cataloggen;
import thredds.catalog.*;
import thredds.catalog.parser.jdom.InvCatalogFactory10;
import thredds.cataloggen.config.CatGenConfigMetadataFactory;
import thredds.cataloggen.config.CatalogGenConfig;
import thredds.cataloggen.config.DatasetSource;
import ucar.nc2.units.DateType;
import java.io.*;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
* CatalogGen crawls dataset sources given in a CatalogGenConfig file
* to produce THREDDS catalogs.
*
* To generate a catalog from a config file:
*
* String inFileName = "file:/home/edavis/testCatGenConfig.xml";
* String outFileName = "/home/edavis/testCatGenConfig-results.xml";
* StringBuffer log = new StringBuffer();
* CatalogGen catGen = new CatalogGen( inFileName);
* if ( catGen.isValid( log))
* {
* catGen.expand();
* catGen.writeCatalog( outFileName);
* }
*
*
*
* @author Ethan Davis
* @version $Ver$
*/
public class CatalogGen
{
//private static Log log = LogFactory.getLog( CatalogGen.class );
static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(CatalogGen.class);
/** The catalog: initially as a CatGen config file, until expanded. */
private InvCatalog catalog = null;
/** The catalog factory that knows about CatalogGenConfig metadata. */
protected InvCatalogFactory catFactory = null;
public List getCatalogRefInfoList()
{
return catalogRefInfoList;
}
private List catalogRefInfoList = new ArrayList();
/**
* Constructs the CatalogGen for the given config document.
*
* @param configDocURL - the URL of the configuration document
*/
public CatalogGen( URL configDocURL)
{
// Create a InvCatalogFactory with CATALOG_GEN_CONFIG MetadataType registered.
log.debug( "CatalogGen(URL): create catalog and CatalogGenConfig converter." );
this.catFactory = InvCatalogFactory.getDefaultFactory( true );
this.catFactory.registerMetadataConverter( MetadataType.CATALOG_GEN_CONFIG.toString(),
new CatGenConfigMetadataFactory());
// Read the given XML config file.
log.debug( "CatalogGen(URL): reading the config doc <" + configDocURL.toString() + ">.");
this.catalog = this.catFactory.readXML( configDocURL.toString());
log.debug( "CatalogGen(URL): done.");
}
/**
* Constructs the CatalogGen for the given config document InputStream.
*
* @param configDocInputStream - the InputStream from which to read the config document.
* @param configDocURL - the URL for the config document.
*/
public CatalogGen( InputStream configDocInputStream, URL configDocURL )
{
// Create a InvCatalogFactory with CATALOG_GEN_CONFIG MetadataType registered.
log.debug( "CatalogGen(InputStream): create catalog and CatalogGenConfig converter." );
this.catFactory = new InvCatalogFactory( "default", true );
this.catFactory.registerMetadataConverter( MetadataType.CATALOG_GEN_CONFIG.toString(),
new CatGenConfigMetadataFactory() );
// Read the given XML config file.
log.debug( "CatalogGen(InputStream): reading the config doc <" + configDocURL.toString() + ">." );
this.catalog = this.catFactory.readXML( configDocInputStream, URI.create( configDocURL.toExternalForm()) );
log.debug( "CatalogGen(InputStream): CatalogGenConfig doc <" + this.catalog.getName() + "> read.");
}
/**
* Checks the validity of the configuration file.
* @param out - a StringBuffer with validity error and warning messages.
* @return - true if no errors, false if errors exist
*/
public boolean isValid( StringBuilder out)
{
log.debug( "isValid(): start");
return( this.catalog.check( out));
}
/**
* Expand the catalog. Each of the CatalogGenConfig metadata elements
* is expanded into its constituent datasets.
*/
public InvCatalog expand()
{
CatalogGenConfig tmpCgc = null;
List cgcList = null;
DatasetSource dss = null;
// Find and loop through each CatGenConfigMetadata object.
List mdataList = findCatGenConfigMdata( this.catalog.getDatasets());
for ( int i = 0; i < mdataList.size(); i++)
{
InvMetadata curMdata = (InvMetadata) mdataList.get( i);
InvDatasetImpl curParentDataset = ( (InvDatasetImpl) curMdata.getParentDataset());
// Loop through the CatalogGenConfig objects in current InvMetadata.
cgcList = (List) curMdata.getContentObject();
for ( int j = 0; j < cgcList.size(); j++)
{
tmpCgc = (CatalogGenConfig) cgcList.get( j);
log.debug( "expand(): mdata # " + i + " and catGenConfig # " + j + "." );
dss = tmpCgc.getDatasetSource();
InvCatalog generatedCat = null;
try
{
generatedCat = dss.fullExpand();
}
catch ( IOException e )
{
String tmpMsg = "Error: IOException on fullExpand() of DatasetSource <" + dss.getName() + ">: " + e.getMessage();
log.error( "expand(): " + tmpMsg);
curParentDataset.addDataset( new InvDatasetImpl( curParentDataset, tmpMsg));
break;
}
catalogRefInfoList.addAll( dss.getCatalogRefInfoList());
// Always a single top-level dataset in catalog returned by DatasetSource.fullExpand()
InvDataset genTopDs = (InvDataset) generatedCat.getDatasets().get( 0);
// Add all services in the generated catalog to the parent catalog.
for( Iterator it = generatedCat.getServices().iterator(); it.hasNext(); )
{
( (InvCatalogImpl) curParentDataset.getParentCatalog() ).addService( (InvService) it.next() );
}
// Add the generated catalog to the parent datasets, i.e., add all the
// datasets that are children of the generated catalogs' top-level dataset.
for ( Iterator it = genTopDs.getDatasets().iterator(); it.hasNext(); )
{
InvDatasetImpl curGenDataset = (InvDatasetImpl) it.next();
if ( curGenDataset.hasNestedDatasets())
{
// If the dataset has children datasets, add serviceName and make it inherited.
ThreddsMetadata tm = new ThreddsMetadata( false);
tm.setServiceName( genTopDs.getServiceDefault().getName());
InvMetadata md = new InvMetadata( genTopDs, null, XMLEntityResolver.CATALOG_NAMESPACE_10, "", true, true, null, tm);
curGenDataset.getLocalMetadata().addMetadata( md);
}
else
{
// Otherwise, add serviceName not inherited.
curGenDataset.getLocalMetadata().setServiceName( genTopDs.getServiceDefault().getName());
}
curParentDataset.addDataset( curGenDataset);
}
curParentDataset.finish();
}
// log.debug( "expand(): List datasets that are siblinks of current metadata record CGCM(" + i + ").");
// List list = curMdata.getParentDataset().getDatasets();
// Iterator it = list.iterator();
// while ( it.hasNext())
// {
// InvDatasetImpl curDs = (InvDatasetImpl) it.next();
// log.debug( "Dataset URL is " + curDs.getUrlPath() + ".");
// log.debug( "Dataset name is " + curDs.getName() + ".");
// }
// Remove the current metadata element from its parent dataset.
log.debug( "expand(): Remove metadata record CGCM(" + i + ")." );
curParentDataset.removeLocalMetadata( curMdata);
// *****
}
// Finish this catalog now that done building it.
((InvCatalogImpl) this.catalog).finish();
return( this.catalog);
}
public void setCatalogExpiresDate( DateType expiresDate )
{
((InvCatalogImpl) this.catalog).setExpires( expiresDate);
}
/**
* Writes the catalog as XML. The catalog is written to the file given
* in outFileName. If outFileName is null, the catalog
* is written to standard out.
*
* @param outFileName - the pathname of the output file.
*
* @throws IOException if can't write catalog
*/
public void writeCatalog( String outFileName)
throws IOException
{
log.debug( "writeCatalog(): writing catalog to " + outFileName + ".");
String invCatDTD = "http://www.unidata.ucar.edu/projects/THREDDS/xml/InvCatalog.0.6.dtd";
log.debug( "writeCatalog(): set the catalogs DTD (" + invCatDTD + ").");
// Set the catalogs DTD.
// ( (InvCatalogImpl) catalog).setDTDid( invCatDTD); LOOK !!
// Print the catalog as an XML document.
if ( outFileName == null)
{
log.debug( "writeCatalog(): write catalog to System.out.");
this.catFactory.writeXML( (InvCatalogImpl) catalog, System.out);
}
else
{
log.debug( "writeCatalog(): try writing catalog to the output file (" + outFileName + ").");
if ( ! this.catalog.getVersion().equals( "1.0" ) )
{
this.catFactory.writeXML( (InvCatalogImpl) catalog, outFileName );
}
else
{
// Override default output catalog version. (kludge for IDV backward compatibility)
InvCatalogFactory10 fac10 = (InvCatalogFactory10) this.catFactory.getCatalogConverter( XMLEntityResolver.CATALOG_NAMESPACE_10 );
fac10.setVersion( this.catalog.getVersion() );
BufferedOutputStream osCat = new BufferedOutputStream( new FileOutputStream( outFileName ) );
fac10.writeXML( (InvCatalogImpl) catalog, osCat );
osCat.close();
}
log.debug( "writeCatalog(): catalog written to " + outFileName + "." );
}
return;
}
InvCatalog getCatalog() { return( this.catalog ); }
private List findCatGenConfigMdata( List datasets)
{
List mdataList = new ArrayList();
if ( datasets == null) return( mdataList );
// Iterate through list of datasets.
Iterator it = datasets.iterator();
InvDataset curDataset = null;
while ( it.hasNext() )
{
curDataset = (InvDataset) it.next();
// Get all the local metadata for the given dataset.
ThreddsMetadata tm = ((InvDatasetImpl) curDataset).getLocalMetadata();
// Iterate over the local InvMetadata checking for CatalogGenConfig metadata.
Iterator itMdata = tm.getMetadata().iterator();
InvMetadata curMetadata = null;
while ( itMdata.hasNext())
{
curMetadata = (InvMetadata) itMdata.next();
if ( (curMetadata.getMetadataType() != null) &&
curMetadata.getMetadataType().equals( MetadataType.CATALOG_GEN_CONFIG.toString() ) )
{
mdataList.add( curMetadata );
}
else if ( (curMetadata.getNamespaceURI() != null) &&
curMetadata.getNamespaceURI().equals( CatalogGenConfig.CATALOG_GEN_CONFIG_NAMESPACE_URI_0_5 ))
{
mdataList.add( curMetadata );
}
}
// Recurse through nested datasets and find CatalogGenConfig metadata.
mdataList.addAll( this.findCatGenConfigMdata( curDataset.getDatasets() ) );
}
return( mdataList);
}
}