All Downloads are FREE. Search and download functionalities are using the official Maven repository.

thredds.catalog.dl.DIFWriter Maven / Gradle / Ivy

Go to download

The NetCDF-Java Library is a Java interface to NetCDF files, as well as to many other types of scientific data formats.

The newest version!
/*
 * Copyright 1998-2009 University Corporation for Atmospheric Research/Unidata
 *
 * Portions of this software were developed by the Unidata Program at the
 * University Corporation for Atmospheric Research.
 *
 * Access and use of this software shall impose the following obligations
 * and understandings on the user. The user is granted the right, without
 * any fee or cost, to use, copy, modify, alter, enhance and distribute
 * this software, and any derivative works thereof, and its supporting
 * documentation for any purpose whatsoever, provided that this entire
 * notice appears in all copies of the software, derivative works and
 * supporting documentation.  Further, UCAR requests that the user credit
 * UCAR/Unidata in any publications that result from the use of this
 * software or in any product that includes this software. The names UCAR
 * and/or Unidata, however, may not be used in any advertising or publicity
 * to endorse or promote any products or commercial entity unless specific
 * written permission is obtained from UCAR/Unidata. The user also
 * understands that UCAR/Unidata is not obligated to provide the user with
 * any support, consulting, training or assistance of any kind with regard
 * to the use, operation and performance of this software nor to provide
 * the user with any updates, revisions, new versions or "bug fixes."
 *
 * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
 * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
 * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
 */
package thredds.catalog.dl;

import thredds.catalog.*;
import thredds.catalog.crawl.CatalogCrawler;
import ucar.nc2.constants.CDM;
import ucar.nc2.units.DateRange;
import ucar.nc2.units.DateType;

import org.jdom2.*;
import org.jdom2.output.*;

import java.io.*;
import java.util.*;
import java.net.URI;

import ucar.nc2.units.TimeDuration;
import ucar.unidata.geoloc.*;
import ucar.unidata.util.StringUtil2;

public class DIFWriter {
  static private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(DIFWriter.class);

  static private final Namespace defNS = Namespace.getNamespace("http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/");
  static private String schemaLocation ="http://gcmd.nasa.gov/Aboutus/xml/dif/dif.xsd";

  private String fileDir;
  private StringBuilder messBuffer;
  private boolean debug = false;

  public DIFWriter() {
  }

  /**
   * Write all harvestable datasets to DIF records that have at least the minimum metadata.
   * Call isDatasetUseable() to find out.
   *
   * @param cat harvest the datasets starting from here
   * @param fileDir write records to this directory. The dataset id is used as the filename, appending "dif.xml"
   * @param mess status messages are appended here
   */
  public void writeDatasetEntries( InvCatalogImpl cat, String fileDir, StringBuilder mess) {
    this.fileDir = fileDir;
    this.messBuffer = mess;

    File dir = new File(fileDir);
    if (!dir.exists()) {
      boolean ret = dir.mkdirs();
      assert ret;
    }

    CatalogCrawler.Listener listener = new CatalogCrawler.Listener() {
      public void getDataset(InvDataset ds, Object context) {
        doOneDataset(ds);
      }
      public boolean getCatalogRef(InvCatalogRef dd, Object context) { return true; }
      
    };

    ByteArrayOutputStream bis = new ByteArrayOutputStream();
    PrintStream ps = new PrintStream( bis);
    CatalogCrawler crawler = new CatalogCrawler( CatalogCrawler.USE_ALL, true, listener);
    crawler.crawl(cat, null, ps, null);
    mess.append("\n*********************\n");
    mess.append(new String(bis.toByteArray(), CDM.utf8Charset));
  }

  /**
   * Write a DIF record for a specific dataset
   * @param ds use this dataset
   */
  public void doOneDataset( InvDataset ds) {
    if (debug) System.out.println("doDataset "+ds.getName());

    if (isDatasetUseable( ds, messBuffer)) {
      String id = StringUtil2.replace(ds.getID(), "/", "-");
      String fileOutName = fileDir+"/"+id+".dif.xml";
      try {
        OutputStream out = new BufferedOutputStream(new FileOutputStream( fileOutName));
        // writeOneEntry(ds, System.out, mess);
        writeOneEntry(ds, out, messBuffer);
        out.close();
        messBuffer.append(" OK on Write\n");
      } catch (IOException ioe) {
        messBuffer.append("DIFWriter failed on write " + ioe.getMessage() + "\n");
        log.error("DIFWriter failed on write "+ioe.getMessage(), ioe);
      }
    }
  }

   /**
   * Write a DIF record for a specific dataset
   * @param ds use this dataset
   * @param fileDir write records to this directory. The dataset id is used as the filename, appending "dif.xml"
   * @param mess status messages are appended here
   */
  public void doOneDataset( InvDataset ds, String fileDir, StringBuilder mess) {
    if (debug) System.out.println("doDataset "+ds.getName());

    if (isDatasetUseable( ds, mess)) {
      String id = StringUtil2.replace(ds.getID(), "/", "-");
      String fileOutName = fileDir+"/"+id+".dif.xml";
      try {
        OutputStream out = new BufferedOutputStream(new FileOutputStream( fileOutName));
        // writeOneEntry(ds, System.out, mess);
        writeOneEntry(ds, out, mess);
        out.close();
        mess.append(" OK on Write\n");
      } catch (IOException ioe) {
        mess.append("DIFWriter failed on write " + ioe.getMessage() + "\n");
        log.error("DIFWriter failed on write "+ioe.getMessage(), ioe);
      }
    }
  }

  /**
   * See if a dataset is harvestable to a DIF record.
   *
   * @param ds check this dataset.
   * @param sbuff  put status messages here.
   * @return true if a DIF record can be written
   */
  public boolean isDatasetUseable(InvDataset ds, StringBuilder sbuff) {
    boolean ok = true;
    sbuff.append("Dataset " + ds.getName() + " id = " + ds.getID() + ": ");

    if (!ds.isHarvest()) {
      ok = false;
      sbuff.append("Dataset " + ds.getName() + " id = " + ds.getID() + " has harvest = false\n");
    }

    if (ds.getName() == null) {
      ok = false;
      sbuff.append(" missing Name field\n");
    }

    if (ds.getUniqueID() == null) {
      ok = false;
      sbuff.append(" missing ID field\n");
    }

    ThreddsMetadata.Variables vs = ds.getVariables("DIF");
    if ((vs == null) || (vs.getVariableList().size() == 0))
      vs = ds.getVariables("GRIB-1");
    if ((vs == null) || (vs.getVariableList().size() == 0))
      vs = ds.getVariables("GRIB-2");
    if ((vs == null) || (vs.getVariableList().size() == 0)) {
      ok = false;
      sbuff.append(" missing Variables with DIF or GRIB compatible vocabulary\n");
    }

    List list = ds.getPublishers();
    if ((list == null) || (list.size() == 0)) {
      ok = false;
      sbuff.append(" must have publisher element that defines the data center\n");
    }

    String summary = ds.getDocumentation("summary");
    if (summary == null) {
      ok = false;
      sbuff.append(" must have documentation element of type summary\n");
    }

    sbuff.append(" useable= "+ok+"\n");
    return ok;
  }


  private void writeOneEntry( InvDataset ds, OutputStream out, StringBuilder mess) throws IOException {
    Element rootElem = new Element("DIF", defNS);
    Document doc = new Document(rootElem);
    writeDataset( ds, rootElem, mess);
    rootElem.addNamespaceDeclaration(defNS);
    rootElem.addNamespaceDeclaration(XMLEntityResolver.xsiNS);
    rootElem.setAttribute("schemaLocation", defNS.getURI()+" "+schemaLocation, XMLEntityResolver.xsiNS);

    // Output the document, use standard formatter
    XMLOutputter fmt = new XMLOutputter( Format.getPrettyFormat());
    fmt.output( doc, out);
  }

  private Iterator translateGribVocabulary(ThreddsMetadata.Variables vs, boolean isGrib1, StringBuilder mess) {
    if (vs == null)
      return null;

    VocabTranslator vt;
    try {
      vt = isGrib1 ? (VocabTranslator) Grib1toDIF.getInstance() : (VocabTranslator) Grib2toDIF.getInstance();
    } catch (IOException e) {
      log.error("DIFWriter failed opening GribtoDIF VocabTranslator ", e);
      return null;
    }

    // hash on DIF names to eliminate duplicates
    Map hash = new HashMap(100);
    for (ThreddsMetadata.Variable v : vs.getVariableList()) {
      String fromVocabId = v.getVocabularyId();
      if (fromVocabId == null) {
        mess.append("** no id for " + v.getName() + "\n");
        continue;
      }

      String toVocabName = vt.translate(fromVocabId);
      if (toVocabName == null) {
        mess.append("** no translation for " + fromVocabId + " == " + v.getVocabularyName() + "\n");
        continue;
      }

      // do we already have it ?
      if (hash.get(toVocabName) == null) {
        ThreddsMetadata.Variable transV = new ThreddsMetadata.Variable(v.getName(), v.getDescription(), toVocabName,
                v.getUnits(), fromVocabId);
        hash.put(toVocabName, transV);
      }
    }
    return hash.values().iterator();
  }


  private void writeDataset(InvDataset ds, Element rootElem, StringBuilder mess) {
    String entryId = StringUtil2.allow(ds.getUniqueID(), "_-.", '-');
    rootElem.addContent( new Element("Entry_ID", defNS).addContent(entryId));
    rootElem.addContent( new Element("Entry_Title", defNS).addContent(ds.getFullName()));

    // parameters : look for DIF
    ThreddsMetadata.Variables vs = ds.getVariables("DIF");
    boolean hasVocab = (vs != null) && (vs.getVariableList().size() != 0);
    if (hasVocab) {
      for (ThreddsMetadata.Variable v : vs.getVariableList()) {
        writeVariable( rootElem, v);
      }
    } else {
      // look for GRIB-1
      vs = ds.getVariables("GRIB-1");
      if ((vs != null) && (vs.getVariableList().size() != 0)) {
        Iterator iter = translateGribVocabulary(vs, true, mess);
        while (iter.hasNext()) {
          ThreddsMetadata.Variable v = (ThreddsMetadata.Variable) iter.next();
          writeVariable( rootElem, v);
        }
      } else {
        // look for GRIB-2
        vs = ds.getVariables("GRIB-2");
        if ((vs != null) && (vs.getVariableList().size() != 0)) {
          Iterator iter = translateGribVocabulary(vs, false, mess);
          while ((iter != null) && iter.hasNext()) {
            ThreddsMetadata.Variable v = (ThreddsMetadata.Variable) iter.next();
            writeVariable( rootElem, v);
          }
        }
      }
    }

    // keywords
    List list = ds.getKeywords();
    if (list.size() > 0) {
      for (int i=0; i 0) {
          String reletiveTime = "RELATIVE_START_DATE: "+((int)-ndays);
          rootElem.addContent( new Element("Keyword", defNS).addContent(reletiveTime));
        }
      }
    }

    // LOOK KLUDGE - these need to be added to the catalog !!  see http://gcmd.nasa.gov/Resources/valids/sources.html
    Element platform = new Element("Source_Name", defNS);
    rootElem.addContent(platform);
    platform.addContent( new Element("Short_Name", defNS).addContent("MODELS"));

    if (tm != null) {
      Element tmElem = new Element("Temporal_Coverage", defNS);
      rootElem.addContent(tmElem);

      tmElem.addContent(new Element("Start_Date",
                        defNS).addContent(tm.getStart().toString()));
      tmElem.addContent(new Element("Stop_Date",
                        defNS).addContent(tm.getEnd().toString()));
    }

    //geospatial
    ThreddsMetadata.GeospatialCoverage geo = ds.getGeospatialCoverage();
    if (geo != null) {
      Element geoElem = new Element("Spatial_Coverage", defNS);
      rootElem.addContent(geoElem);

      double eastNormal = LatLonPointImpl.lonNormal(geo.getLonEast());
      double westNormal = LatLonPointImpl.lonNormal(geo.getLonWest());

      geoElem.addContent(new Element("Southernmost_Latitude",
                         defNS).addContent(Double.toString(geo.getLatSouth())));
      geoElem.addContent(new Element("Northernmost_Latitude",
                         defNS).addContent(Double.toString(geo.getLatNorth())));
      geoElem.addContent(new Element("Westernmost_Longitude",
                         defNS).addContent(Double.toString(westNormal)));
      geoElem.addContent(new Element("Easternmost_Longitude",
                         defNS).addContent(Double.toString(eastNormal)));
    }

    /* LOOK
    "\n" +
            "    12 Km\n" +
            "    12 Km\n" +
            "    10 km - < 50 km or approximately .09 degree - < .5 degree\n" +
            "    6 Hours\n" +
            "    Hourly - < Daily\n" +
            "   "
    */


    String rights = ds.getDocumentation("rights");
    if (rights != null)
      rootElem.addContent( new Element("Use_Constraints", defNS).addContent(rights));

    // data center
    List plist = ds.getPublishers();
    if (plist.size() > 0) {
      for (ThreddsMetadata.Source p : plist) {
        if (p.getNameVocab().getVocabulary().equalsIgnoreCase("DIF")) {
          Element dataCenter = new Element("Data_Center", defNS);
          rootElem.addContent(dataCenter);
          writeDataCenter(p, dataCenter);
          break;
        }
      }
    }

    String summary = ds.getDocumentation("summary");
    if (summary != null) {
      String summaryLines = StringUtil2.breakTextAtWords(summary, "\n", 80);
      rootElem.addContent( new Element("Summary", defNS).addContent(summaryLines));
    }

    URI uri;
    String href;
    if (ds instanceof InvCatalogRef) {      // LOOK !!
      InvCatalogRef catref = (InvCatalogRef) ds;
      uri = catref.getURI();
      href = uri.toString();
      int pos = href.lastIndexOf('.');
      href = href.substring(0,pos)+".html";

    } else {
      InvCatalogImpl cat = (InvCatalogImpl) ds.getParentCatalog();
      uri = cat.getBaseURI();
      String catURL = uri.toString();
      int pos = catURL.lastIndexOf('.');
      href = catURL.substring(0,pos)+".html";
      if (ds.hasAccess())
        href = href+"?dataset="+ds.getID();
    }

    rootElem.addContent( makeRelatedURL("GET DATA", "THREDDS CATALOG", uri.toString()));
    rootElem.addContent( makeRelatedURL("GET DATA", "THREDDS DIRECTORY", href));

    InvAccess access;
    if (null != (access = ds.getAccess(ServiceType.OPENDAP))) {
      rootElem.addContent( makeRelatedURL("GET DATA", "OPENDAP DATA", access.getStandardUrlName()));
    }
    if (null != (access = ds.getAccess(ServiceType.DAP4))) {
      rootElem.addContent( makeRelatedURL("GET DATA", "DAP4 DATA", access.getStandardUrlName()));
    }

    rootElem.addContent(new Element("Metadata_Name", defNS).addContent("CEOS IDN DIF"));
    rootElem.addContent(new Element("Metadata_Version", defNS).addContent("9.4"));
    DateType today = new DateType(false, new Date());
    rootElem.addContent(new Element("DIF_Creation_Date", defNS).addContent(today.toDateTimeStringISO()));
  }

  private Element makeRelatedURL(String type, String subtype, String url) {
    Element elem = new Element("Related_URL", defNS);
    Element uctElem = new Element("URL_Content_Type", defNS);
    elem.addContent(uctElem);
    uctElem.addContent( new Element("Type", defNS).addContent(type));
    uctElem.addContent( new Element("Subtype", defNS).addContent(subtype));
    elem.addContent( new Element("URL", defNS).addContent(url));
    return elem;
  }

  private void writeDataCenter(ThreddsMetadata.Source p, Element dataCenter) {
    Element name = new Element("Data_Center_Name", defNS);
    dataCenter.addContent( name);
    // dorky
    StringTokenizer stoker = new StringTokenizer(p.getName(), ">");
    int n = stoker.countTokens();
    if (n == 2) {
      name.addContent( new Element("Short_Name", defNS).addContent(stoker.nextToken().trim()));
      name.addContent( new Element("Long_Name", defNS).addContent(stoker.nextToken().trim()));
    } else {
      name.addContent( new Element("Short_Name", defNS).addContent(p.getName()));
    }

    if ((p.getUrl() != null) && p.getUrl().length() > 0)
      dataCenter.addContent( new Element("Data_Center_URL", defNS).addContent(p.getUrl()));

    Element person = new Element("Personnel", defNS);
    dataCenter.addContent( person);
    person.addContent( new Element("Role", defNS).addContent("DATA CENTER CONTACT"));
    person.addContent( new Element("Last_Name", defNS).addContent("Any"));
    person.addContent( new Element("Email", defNS).addContent(p.getEmail()));
  }

  private void writeVariable( Element rootElem, ThreddsMetadata.Variable v) {
    String vname = v.getVocabularyName();
    StringTokenizer stoker = new StringTokenizer(vname,">");
    int n = stoker.countTokens();
    if (n < 3) return; // gottta have the first 3

    Element param = new Element("Parameters", defNS);
    rootElem.addContent(param);

    if (stoker.hasMoreTokens())
      param.addContent( new Element("Category", defNS).addContent(stoker.nextToken().trim()));
    if (stoker.hasMoreTokens())
      param.addContent( new Element("Topic", defNS).addContent(stoker.nextToken().trim()));
    if (stoker.hasMoreTokens())
      param.addContent( new Element("Term", defNS).addContent(stoker.nextToken().trim()));
    if (stoker.hasMoreTokens())
      param.addContent( new Element("Variable", defNS).addContent(stoker.nextToken().trim()));
    if (stoker.hasMoreTokens())
      param.addContent( new Element("Detailed_Variable", defNS).addContent(stoker.nextToken().trim()));
  }

  // test
  private static void doCatalog( InvCatalogFactory fac, String url) {
    System.out.println("***read "+url);
    try {
      InvCatalogImpl cat = fac.readXML(url);
      StringBuilder buff = new StringBuilder();
      boolean isValid = cat.check( buff, false);
      System.out.println("catalog <" + cat.getName()+ "> "+ (isValid ? "is" : "is not") + " valid");
      System.out.println(" validation output=\n" + buff);
      System.out.println(" catalog=\n" + fac.writeXML(cat));

      DIFWriter w = new DIFWriter();
      StringBuilder sbuff = new StringBuilder();
      w.writeDatasetEntries( cat, "C:/temp/dif2", sbuff);
      System.out.println(" messages=\n"+sbuff);
    } catch (Exception e) {
      e.printStackTrace();
    }

  }

   /** testing */
  public static void main (String[] args) throws Exception {
    InvCatalogFactory catFactory = InvCatalogFactory.getDefaultFactory(true);

    doCatalog(catFactory, "http://thredds.ucar.edu:9080/thredds/idd/models.xml");
  }

}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy