thredds.cataloggen.config.DatasetSource Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of cdm Show documentation
The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.
The newest version!
/*
 * Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
 *
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */
// $Id: DatasetSource.java 63 2006-07-12 21:50:51Z edavis $

package thredds.cataloggen.config;

import thredds.catalog.*;
import thredds.cataloggen.config.CatalogRefInfo;
import thredds.cataloggen.DatasetEnhancer1;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

// @todo Add state machine to control what can happen when.
// State machine: 1) constructed and modified, 2) expanded, 3) named,
// 4) metadata-ified, and 5) sorted.

// @todo Implement DatasetSource type using Strategy pattern rather than subclassing.
// I.e., extract abstract methods into interface, config (type given to
// DatasetSource.factory()) determines the implementation used.
// Current subclassing is cumbersome and not so flexible as the factory must be
// modified when new implementation is added.

/**
 * Represents a source from which a collection of datasets can be found and
 * expanded into an InvCatalog. Besides expanding the DatasetSource, the datasets
 * in the resulting catalog can be named, sorted, and have metadata added to them.
 *
 * 
 * Currently an abstract class, three abstract methods must be implemented to
 * make a concrete subclass. These methods are getTopLevelDataset():InvDataset,
 * isCollection():boolean, and expandThisLevel(InvDataset):List. (In previous
 * versions, expandThisType():void was the only abstract method.)
 * 
 *
 * @author Ethan Davis
 * @version $Id: DatasetSource.java 63 2006-07-12 21:50:51Z edavis $
 */
abstract public class DatasetSource
{
  static private org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetSource.class);

  /** The catalog that results from expanding this DatasetSource.  */
  protected InvCatalog resultingCatalog;

  /** Top-level dataset of the catalog generated by this DatasetSource. */
  protected InvDataset accessPointDataset;

  /** Information on the catalogRefs in the resulting catalog. */
  protected List catalogRefInfoList = new ArrayList();

  // attributes of the datasetSource element
  private String name;
  protected DatasetSourceType type;
  private DatasetSourceStructure structure;
  private boolean flatten = false;
  private String accessPoint;

  private String prefixUrlPath;

  // If true, create CatalogRefs for all collection datasets, otherwise
  // recurse into all collection datasets (default false).
  protected boolean createCatalogRefs = false;
//  protected boolean expandCatalogRef = false;
// @todo Decide if expandCatalogRef is done here in DatasetSource or by CatalogGen (or other DatasetSource caller).


  // Required child ResultService
  private ResultService resultService;

  // list of optional children of the datasetNamer element
  protected List datasetNamerList = new ArrayList();
  protected List datasetFilterList = new ArrayList();
  protected DatasetSorter datasetSorter = null;

  protected List datasetEnhancerList = new ArrayList();
  protected boolean addDatasetSize;

  // for validation
  protected boolean isValid = true;
  protected StringBuffer msgLog = new StringBuffer();

  /** Default Constructor */
  protected DatasetSource(){}

  /**
   * Factory method for this abstract class.
   * @param name
   * @param type
   * @param structure
   * @param accessPoint
   *
   * @throws IllegalArgumentException if DatasetSource type is null or unsupported.
   */
  public static final DatasetSource newDatasetSource( String name, DatasetSourceType type,
                                             DatasetSourceStructure structure,
                                             String accessPoint,
                                             ResultService resultService )
  {
    if ( type == null)
    {
      String tmpMsg = "DatasetSource type cannot be null";
      logger.error( "newDatasetSource(): " + tmpMsg);
      throw new IllegalArgumentException( tmpMsg);
    }

    DatasetSource tmpDsSource = null;

    if ( type == DatasetSourceType.getType( "Local"))
    {
      tmpDsSource = new LocalDatasetSource();
    }
    else if ( type == DatasetSourceType.getType( "DodsDir"))
    {
      tmpDsSource = new DodsDirDatasetSource();
    }
    else if ( type == DatasetSourceType.getType( "DodsFileServer"))
    {
      tmpDsSource = new DodsFileServerDatasetSource();
    }
    else if ( type == DatasetSourceType.getType( "GrADSDataServer"))
    {
      tmpDsSource = new GrADSDataServerDatasetSource();
    }
    else
    {
      String tmpMsg = "Unsupported DatasetSource type <" + type.toString() + ">.";
      logger.error( "newDatasetSource(): " + tmpMsg);
      throw new IllegalArgumentException( tmpMsg);
    }

    tmpDsSource.setName( name);
    tmpDsSource.setStructure( structure);
    tmpDsSource.setAccessPoint( accessPoint);
    tmpDsSource.setResultService( resultService);
    // Test validity and append messages to log.

    logger.debug( "DatasetSource(): constructor done.");
    StringBuilder log = new StringBuilder();
    if ( tmpDsSource.validate( log))
    {
      logger.debug( "DatasetSource(): new DatasetSource is valid: {}", log.toString());
    }
    else
    {
      logger.debug( "DatasetSource(): new DatasetSource is invalid: {}", log.toString());
    }

    return( tmpDsSource);
  }

  public InvCatalog getResultingCatalog() { return( this.resultingCatalog); }

  public List getCatalogRefInfoList()
  {
    return catalogRefInfoList;
  }


  public String getName() { return( this.name); }
  public void setName( String name) { this.name = name; }

  public DatasetSourceType getType() { return( this.type); }

  public DatasetSourceStructure getStructure() { return( this.structure); }
  public void setStructure( DatasetSourceStructure structure)
  {
    this.structure = structure;
    if ( this.structure == DatasetSourceStructure.FLAT)
      flatten = true;
    else
      flatten = false;
  }

  public boolean isFlatten() { return( this.flatten); }

  public String getAccessPoint() { return( this.accessPoint); }
  public void setAccessPoint( String accessPoint)
  { this.accessPoint = accessPoint; }

  public String getPrefixUrlPath() { return( this.prefixUrlPath); }
  public void setPrefixUrlPath( String prefixUrlPath )
  { this.prefixUrlPath = prefixUrlPath; }

  public ResultService getResultService() { return( this.resultService); }
  public void setResultService( ResultService resultService)
  { this.resultService = resultService; }

  public boolean isCreateCatalogRefs()
  { return( this.createCatalogRefs); }
  public void setCreateCatalogRefs( boolean createCatalogRefs)
  { this.createCatalogRefs = createCatalogRefs; }

//  public boolean isExpandCatalogRef()
//  {
//    return expandCatalogRef;
//  }
//
//  public void setExpandCatalogRef( boolean expandCatalogRef )
//  {
//    this.expandCatalogRef = expandCatalogRef;
//  }

  public java.util.List getDatasetNamerList() { return( this.datasetNamerList); }
  public void addDatasetNamer( DatasetNamer datasetNamer)
  { this.datasetNamerList.add( datasetNamer); }

  public java.util.List getDatasetFilterList() { return( this.datasetFilterList); }
  public void addDatasetFilter( DatasetFilter datasetFilter)
  { this.datasetFilterList.add( datasetFilter); }

  public DatasetSorter getDatasetSorter() { return ( this.datasetSorter ); }
  public void setDatasetSorter( DatasetSorter datasetSorter )
  { this.datasetSorter = datasetSorter; }

  public java.util.List getDatasetEnhancerList() { return( this.datasetEnhancerList ); }
  public void addDatasetEnhancer( DatasetEnhancer1 dsEnhancer)
  {
    if ( this.datasetEnhancerList == null) this.datasetEnhancerList = new ArrayList();
    this.datasetEnhancerList.add( dsEnhancer);
  }

  public boolean isAddDatasetSize() { return( this.addDatasetSize); }
  public void setAddDatasetSize( boolean addDatasetSize )
  { this.addDatasetSize = addDatasetSize; }

  public boolean validate( StringBuilder out)
  {
    this.isValid = true;

    // If log from construction has content, append to validation output msg.
    if (this.msgLog.length() > 0)
    {
      out.append( this.msgLog);
    }

    // Check that name is not null (it can be an empty string).
    if ( this.getName() == null)
    {
      this.isValid = false;
      out.append(" ** DatasetSource (5): null value for name is not valid.");
    }
    // Check that type is not null.
    if ( this.getType() == null)
    {
      this.isValid = false;
      out.append(" ** DatasetSource (6): null value for type is not valid (set with bad string?).");
    }
    // Check that structure is not null.
    if ( this.getStructure() == null)
    {
      this.isValid = false;
      out.append(" ** DatasetSource (7): null value for structure is not valid (set with bad string?).");
    }

    // Validate ResultService child element.
    if ( this.getResultService() != null )
    {
      this.isValid &= this.getResultService().validate( out);
    }

    // Validate DatasetNamer child elements.
    java.util.Iterator dsnIter = this.getDatasetNamerList().iterator();
    while (dsnIter.hasNext())
    {
      this.isValid &= ((DatasetNamer) dsnIter.next()).validate( out);
    }

    // Validate DatasetFilters child elements.
    java.util.Iterator dsfIter = this.getDatasetFilterList().iterator();
    while (dsfIter.hasNext())
    {
      this.isValid &= ((DatasetFilter) dsfIter.next()).validate( out);
    }

    return( this.isValid);
  }

  /** string representation */
  public String toString()
  {
    StringBuffer tmp = new StringBuffer();
    tmp.append( "DatasetSource[name:<" + this.getName() +
                "> type:<" + this.getType() +
                "> structure:<" + this.getStructure() +
                "> accessPoint:<" + this.getAccessPoint() +
                "> and children - " +
                "ResultService(" + this.getResultService().getName() + ") - " +
                "DatasetNamer(" + this.getDatasetNamerList().size() + ") - " +
                "DatasetFilter(" + this.getDatasetFilterList().size() + ")]");
    return( tmp.toString());
  }

  /**
   * Crawl this DatasetSource and generate a new InvCatalog, return the
   * top-level InvDataset.
   *
   * Each object found on the DatasetSource becomes an InvDataset if it is
   * accepted by at least one DatasetFilter. The catalog reflects the
   * heirarchical structure of the DatasetSource. All datasets are named
   * with the location of the object they represent on the dataset source.
   *
   * @return the top-level InvDataset in the generated InvCatalog.
   *
   * @throws IOException if the accessPoint for this DatasetSource is not a container dataset.
   */
  public InvDataset expand() throws IOException
  {
    // Get the new catalog being generated and its top-level dataset.
    this.resultingCatalog = this.createSkeletonCatalog( prefixUrlPath );
    this.accessPointDataset = (InvDataset) this.resultingCatalog.getDatasets().get( 0);

    // IOException thrown by createSkeletonCatalog() so this check should not be necessary.
    if ( ! this.isCollection( this.accessPointDataset) )
    {
      String tmpMsg = "The access point dataset <" + this.accessPointDataset.getName() + "> must be a collection dataset.";
      logger.warn( "expand(): {}", tmpMsg);
      throw new IOException( tmpMsg);
    }

    // Recurse into directory structure and expand.
    expandRecursive( this.accessPointDataset);

    // Finish the catalog.
    ((InvCatalogImpl) this.resultingCatalog).finish();

    // Remove empty collection datasets. @todo HACK - should use filters instead.
    this.recursivelyRemoveEmptyCollectionDatasets( this.accessPointDataset);

    // Return the top-level dataset.
    return( this.accessPointDataset);
  }

  /**
   *
   * @param accessPoints
   * @return the top-level dataset
   * @throws IOException if the main access point for this DatasetSource is not a container dataset or any of the given access points do not exist or are not under the main access point.
   */
  public InvDataset expand( List accessPoints) throws IOException
  {
    // Get the new catalog being generated and its top-level dataset.
    this.resultingCatalog = this.createSkeletonCatalog( prefixUrlPath );
    this.accessPointDataset = (InvDataset) this.resultingCatalog.getDatasets().get( 0);

    for ( Iterator it = accessPoints.iterator(); it.hasNext(); )
    {
      InvDataset curDs = this.createDataset( (String) it.next(), this.prefixUrlPath );
      // Recurse into directory structure and expand.
      expandRecursiveCollection( this.accessPointDataset, curDs);
    }

    // Finish the catalog.
    ((InvCatalogImpl) this.resultingCatalog).finish();

    // Remove empty collection datasets. @todo HACK - should use filters instead.
    this.recursivelyRemoveEmptyCollectionDatasets( this.accessPointDataset);

    // Return the top-level dataset.
    return( this.accessPointDataset);
  }

  private void expandRecursive( InvDataset collectionDataset)
  {
    // Get all datasets at this level.
    List listAllDatasets = this.expandThisLevel( collectionDataset, prefixUrlPath );
    InvDataset curChildDs = null;

    for( Iterator i = listAllDatasets.iterator(); i.hasNext(); )
    {
      curChildDs = (InvDataset) i.next();

      // Filter out unwanted datasets.
      if ( ! DatasetFilter.acceptDatasetByFilterGroup( this.getDatasetFilterList(), curChildDs, this.isCollection( curChildDs)))
        continue;

      // Expand each collection dataset.
      if ( this.isCollection( curChildDs))
      {
        expandRecursiveCollection( collectionDataset, curChildDs );
      }
      // Otherwise, add dataset to collection.
      else
      {
        // If not flattening the directory structure, add current dataset to parent dataset.
        if ( ! this.isFlatten())
        {
          ((InvDatasetImpl) collectionDataset).addDataset( (InvDatasetImpl) curChildDs);
        }
        // Otherwise, add current dataset to top-level dataset.
        else
        {
          ((InvDatasetImpl) this.accessPointDataset).addDataset( (InvDatasetImpl) curChildDs);
        }
      }
    }
  }

  private void expandRecursiveCollection( InvDataset collectionDataset, InvDataset childDs )
  {
    // If not creating a catalogRef, expand the current dataset.
    // @todo Make this determination based on current dataset.
    if ( ! this.createCatalogRefs)
    {
      // If not flattening the directory structure, add current collection dataset to parent dataset.
      if ( ! this.isFlatten())
      {
        ((InvDatasetImpl) collectionDataset).addDataset( (InvDatasetImpl) childDs);
      }
      this.expandRecursive( childDs);
    }
    // Otherwise, add current dataset as a catalogRef.
    else
    {
      // @todo Determine title and docName based on createCatalogRef object
      String title = childDs.getName();
      String docName = (childDs.getName() == null || childDs.getName().equals( ""))
                       ? "catalog.xml"
                       : childDs.getName() + "/catalog.xml";
//      if ( ! this.accessPoint.equals( this.resultService.getAccessPointHeader()))
//      {
//        // @todo HACK, HACK not always Files
//        String ap = new File( this.accessPoint).toURI().toString();
//        String aph = new File( this.resultService.getAccessPointHeader()).toURI().toString();
//        String relPath = ap.substring( aph.length() );
//        docName = relPath + ((relPath.endsWith( "/")) ? "" : "/") + docName;
//      }

      // Add catalogRef to the current dataset.
      InvCatalogRef curCatRef = new InvCatalogRef( (InvDatasetImpl) collectionDataset, title, docName);
      ((InvDatasetImpl) collectionDataset).addDataset( curCatRef);

      // Create a DatasetSource for the catalogRef and add info to the catalogRefInfoList.
      DatasetSource catRefDsSrc = DatasetSource.newDatasetSource( childDs.getName(), this.getType(), this.getStructure(),
                                                                  childDs.getName(), new ResultService( this.getResultService()));
      this.catalogRefInfoList.add( new CatalogRefInfo( title, docName, childDs, catRefDsSrc));
    }
  }

  /**
   * Crawl this DatasetSource and generate a new InvCatalog with all datasets
   * named, sorted, and organized as defined by this DatasetSource, return the
   * newly generated InvCatalog.
   *
   * @return the generated InvCatalog.
   *
   * @throws IOException if DatasetSource does not reference a container dataset.
   */
  public InvCatalog fullExpand() throws IOException
  {
    logger.debug( "fullExpand(): expanding DatasetSource named \"{}\"", this.getName());

    InvDataset topDs = this.expand();
    InvCatalog generatedCat = topDs.getParentCatalog();

    // Add metadata to all datasets.
    for ( Iterator it = this.getDatasetEnhancerList().iterator(); it.hasNext(); )
    {
      DatasetEnhancer1 dsE = (DatasetEnhancer1) it.next();
      dsE.addMetadata( topDs);
    }

    // Name all datasets.
    logger.debug( "fullExpand(): naming the datasets.");
    this.nameDatasets( (InvDatasetImpl) topDs );

    // Sort all datasets
    logger.debug( "fullExpand(): sorting the datasets.");
    this.sortDatasets( topDs);

    // Return the generated catalog
    ((InvCatalogImpl) generatedCat).finish();
    return( generatedCat);

  }

  /**
   * Creates an InvDataset to represent the dataset at the given location
   * on this DatasetSource.
   *
   * @param datasetLocation a String indicating the location of a dataset.
   * @param prefixUrlPath a path name to be prefixed on the urlPath
   * @return an InvDataset for the given dataset location.
   * @throws IOException if the dataset location does not correspond to an actual dataset or is not below the accessPointHeader directory.
   * @throws NullPointerException if the given dataset location is null.
   */
  abstract protected InvDataset createDataset( String datasetLocation, String prefixUrlPath )
          throws IOException;

  /**
   * Return true if the given dataset is a collection dataset, false otherwise.
   *
   * @param dataset - the InvDataset to test for being a collection dataset.
   * @return true if the given dataset is a collection dataset, false otherwise.
   * @throws NullPointerException if the given InvDataset is null.
   * @throws ClassCastException if the given InvDataset is not a LocalInvDataset.
   */
  abstract protected boolean isCollection( InvDataset dataset);

  /**
   * Return a skeleton InvCatalog for this DatasetSource. The skeleton catalog
   * is unnamed with a single top-level dataset representing the DatasetSource
   * accessPoint and a service built from the DatasetSource ResultService.
   *
   * @param prefixUrlPath
   * @return a skeleton InvCatalog for this DatasetSource.
   *
   * @throws IOException if top-level dataset does not exist or is not a collection dataset.
   */
  abstract protected InvCatalog createSkeletonCatalog( String prefixUrlPath ) throws IOException;

  /**
   * Return a list of the InvDatasets contained in the given collection dataset
   * on this DatasetSource.
   *
   * @param collectionDataset - the collection dataset to be expanded.
   * @param prefixUrlPath
   * @return A list of the InvDatasets contained in the given collection dataset.
   * @throws IllegalArgumentException when given dataset is not a collection dataset.
   * @throws NullPointerException if given dataset is null.
   * @throws ClassCastException if the given InvDataset is not a LocalInvDataset.
   */
  abstract protected List expandThisLevel( InvDataset collectionDataset, String prefixUrlPath );

  /**
   * Use the list of dsNamers to name the given list of datasets.
   *
   * @param datasetContainer - a InvDatasetImpl that contains one or
   *  more datasets to be named.
   */
  private void nameDatasets( InvDatasetImpl datasetContainer)
  {
    if ( this.getDatasetNamerList().isEmpty()) return;
    if ( this.isFlatten())
    {
      logger.debug( "nameDatasets(): structure is FLAT calling nameDatasetList()");
      this.nameDatasetList( datasetContainer);
    }
    else
    {
      logger.debug( "nameDatasets(): structure is DIRECTORY_TREE calling" +
              " nameDatasetTree() on each dataset in dataset container");
      InvDatasetImpl curDs = null;
      for ( int j = 0; j < datasetContainer.getDatasets().size(); j++)
      {
        curDs = (InvDatasetImpl) datasetContainer.getDatasets().get( j);
        this.nameDatasetTree( curDs);
      }
    }
    return;
  }

  /** Name the datasets contained in the given dataset.
   *  The given dataset contains a flat list of datasets. */
  private void nameDatasetList( InvDatasetImpl dataset)
  {
    // Create temporary dataset in which to hold named datasets.
    InvDatasetImpl namedDs = new InvDatasetImpl( dataset,
          "nameDatastList() temp dataset", null, null, null);
    // InvDatasetImpl(parentDs, name, dataType, serviceName, urlPath)
    dataset.addDataset( namedDs);

    // Loop through the DatasetNamers
    DatasetNamer curNamer = null;
    for ( int i = 0; i < this.datasetNamerList.size(); i++)
    {
      curNamer = (DatasetNamer) this.datasetNamerList.get( i);
      logger.debug( "nameDatasetList(): trying namer ({})", curNamer.getName());

      // If the current DatasetNamer adds a new level, create a new dataset.
      InvDatasetImpl addLevelDs = null;
      if ( curNamer.getAddLevel())
      {
        addLevelDs = new InvDatasetImpl( null, curNamer.getName(),
                                         null, null, null );
      }

      // Iterate over remaining unnamed datasets.
      InvDatasetImpl curDs = null;
      java.util.Iterator dsIter = dataset.getDatasets().iterator();
      while ( dsIter.hasNext())
      {
        curDs = (InvDatasetImpl) dsIter.next();
        logger.debug( "nameDatasetList(): try namer on this ds ({}-{})", curDs.getName(), curDs.getUrlPath() );

        // Try to name the current dataset.
        if ( curNamer.nameDataset( curDs))
        {
          logger.debug( "nameDatasetList(): ds named ({})", curDs.getName());
          // If adding a level, add named datasets to the added level dataset.
          if ( curNamer.getAddLevel())
          {
            addLevelDs.addDataset( curDs);
          }
          // Otherwise, add the named datasets to namedDs.
          else
          {
            namedDs.addDataset( curDs);
          }

          // Remove the now-named dataset from list of unnamed datasets.
          dsIter.remove();
        }
      } // END - InvDatasetImpl loop

      // If the namer added a level and a dataset was named by this namer, add the
      // new level to the list of named datasets.
      if ( curNamer.getAddLevel())
      {
        if ( addLevelDs.hasNestedDatasets())
        {
          namedDs.addDataset( addLevelDs);
        }
      }

    } // END - DatasetNamer loop
    namedDs.finish();

    // Once all datasets are named (or unnamable with these DatasetNamers),
    // add all the datasets in namedDs back into the given containerDataset.
    if (logger.isDebugEnabled()) {
      logger.debug( "nameDatasetList(): number of unnamed datasets is " + dataset.getDatasets().size() + ".");
      logger.debug( "nameDatasetList(): add named datasets back to container.");
    }
    for ( int i = 0; i < namedDs.getDatasets().size(); i++)
    {
      dataset.addDataset( (InvDatasetImpl) namedDs.getDatasets().get( i));
    }
    dataset.removeDataset( namedDs);

    return;
  }

  /** Name the datasets in the given dataset hierarchy using this
   *  DatasetSource's list of datasetNamers. */
  private void nameDatasetTree( InvDatasetImpl dataset)
  {
    // If dataset does not have a name, try naming it with dsNamers.
    // @todo Rethink naming of directories (look at how DatasetFilter deals with collection vs atomic datasets).
    if ( dataset.getName().equals("") || ! dataset.hasAccess())
    {
      logger.debug( "nameDatasetTree(): naming dataset ({})...", dataset.getUrlPath());
      DatasetNamer dsN = null;
      for ( int i = 0; i < this.datasetNamerList.size(); i++)
      {
        dsN = (DatasetNamer) this.datasetNamerList.get( i);
        if ( dsN.nameDataset( dataset))
        {
          logger.debug( "nameDatasetTree(): ... used namer ({})", dsN.getName());
          break;
        }
      }
    }

    // Try to name any child datasets.
    InvDatasetImpl curDs = null;
    for ( int j = 0; j < dataset.getDatasets().size(); j++)
    {
      curDs = (InvDatasetImpl) dataset.getDatasets().get( j);
      logger.debug( "nameDatasetTree(): recurse to name child dataset ({})", curDs.getUrlPath());
      this.nameDatasetTree( curDs);
    }

    return;
  }

  private void sortDatasets( InvDataset dataset)
  {
    if ( this.getDatasetSorter() == null)
    {
      // Create default dataset Comparator (matches pre-DatasetSorter behaviour).
      DatasetSorter defaultSorter = new DatasetSorter(
              new java.util.Comparator()
              {
                public int compare( Object obj1, Object obj2 )
                {
                  InvDataset ds1 = (InvDataset) obj1;
                  InvDataset ds2 = (InvDataset) obj2;
                  return ( -ds1.getName().compareTo( ds2.getName() ) );
                }
              }
      );
      defaultSorter.sortNestedDatasets( dataset);
    }
    else
    {
      this.getDatasetSorter().sortNestedDatasets( dataset);
    }

    return;
  }

  private void recursivelyRemoveEmptyCollectionDatasets( InvDataset parentDataset)
  {
    InvDataset curDs = null;
    for ( Iterator it = parentDataset.getDatasets().iterator(); it.hasNext(); )
    {
      curDs = (InvDataset) it.next();

      // Do not remove accessible dataset.
      if ( curDs.hasAccess()) continue;

      // Do not remove catalogRef datasets.
      if ( curDs instanceof InvCatalogRef) continue;

      // Recurse into collection datasets children.
      if ( curDs.hasNestedDatasets())
      {
        this.recursivelyRemoveEmptyCollectionDatasets( curDs);
      }
      // Remove any empty collection datasets.
      else
      {
        it.remove();
      }
    }
  }

  public boolean equals( Object o )
  {
    if ( this == o ) return true;
    if ( !( o instanceof DatasetSource ) ) return false;

    final DatasetSource datasetSource = (DatasetSource) o;

    if ( createCatalogRefs != datasetSource.createCatalogRefs ) return false;
//    if ( expandCatalogRef != datasetSource.expandCatalogRef ) return false;
    if ( flatten != datasetSource.flatten ) return false;
    if ( accessPoint != null ? !accessPoint.equals( datasetSource.accessPoint ) : datasetSource.accessPoint != null ) return false;
    if ( datasetFilterList != null ? !datasetFilterList.equals( datasetSource.datasetFilterList ) : datasetSource.datasetFilterList != null ) return false;
    if ( datasetNamerList != null ? !datasetNamerList.equals( datasetSource.datasetNamerList ) : datasetSource.datasetNamerList != null ) return false;
    if ( name != null ? !name.equals( datasetSource.name ) : datasetSource.name != null ) return false;
    if ( resultService != null ? !resultService.equals( datasetSource.resultService ) : datasetSource.resultService != null ) return false;
    if ( type != null ? !type.equals( datasetSource.type ) : datasetSource.type != null ) return false;

    return true;
  }

  public int hashCode()
  {
    if ( hashCode == 0)
    {
      int result = 17;
      result = ( name != null ? name.hashCode() : 0 );
      result = 29 * result + ( type != null ? type.hashCode() : 0 );
      result = 29 * result + ( flatten ? 1 : 0 );
      result = 29 * result + ( accessPoint != null ? accessPoint.hashCode() : 0 );
      result = 29 * result + ( createCatalogRefs ? 1 : 0 );
//      result = 29 * result + ( expandCatalogRef ? 1 : 0 );
      result = 29 * result + ( resultService != null ? resultService.hashCode() : 0 );
      result = 29 * result + ( datasetNamerList != null ? datasetNamerList.hashCode() : 0 );
      result = 29 * result + ( datasetFilterList != null ? datasetFilterList.hashCode() : 0 );
      hashCode = result;
    }
    return hashCode;
  }
  private volatile int hashCode = 0;
}