All Downloads are FREE. Search and download functionalities are using the official Maven repository.

ucar.nc2.ft.fmrc.FmrcInv Maven / Gradle / Ivy

The newest version!
/*
 * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
 * See LICENSE for license information.
 */

package ucar.nc2.ft.fmrc;

import java.util.stream.Collectors;
import ucar.nc2.time.CalendarDate;
import ucar.nc2.time.CalendarPeriod;
import javax.annotation.concurrent.Immutable;
import java.util.*;

/**
 * Inventory for a Forecast Model Run Collection = a series of Forecast Model Runs.
 *
 * Create rectangular representation of var(runtime, time) of data(ens, vert, x, y).
 * For each Grid, the vert, time and ens coordinates are created as the union of the components.
 * Make sure to share coordinates across grids where they are equivilent.
 * 

* We are thus making a rectangular array var(runtime, time, ens, level). * So obviously we have to tolerate missing data. * Keeps track of what inventory exists, and where it is. *

* * @author caron * @since Jan 11, 2010 */ @Immutable public class FmrcInv { private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(FmrcInv.class); /* * public static CalendarDate addHour(CalendarDate d, double hour) { * long msecs = d.getTime(); * msecs += hour * 3600 * 1000; * return new Date(msecs); * } */ ////////////////////////////////////////////////////////////////////////////////////////////////////// private final String name; // name of ForecastModelRunCollection // this is where the time coordinates are normalized and missing data may occur private final List runSeqs = new ArrayList<>(); // list of unique EnsCoord private final List ensCoords = new ArrayList<>(); // list of unique VertCoord private final List vertCoords = new ArrayList<>(); // the list of runs private final List fmrList; // the list of grids private final List uberGridList; // sorted list of UberGrid // all run times private final List runTimeList; // sorted list of Date: all run times // track offsets and bounds private final CalendarDate baseDate; // first runtime : offsetsAll calculated from here // all forecast times private final List forecastTimeList; // sorted list of Date : all forecast times // use on motherlode to regularize the missing inventory private final boolean regularize; ///////////////////////////////////////////////////// /** * Construct the inventory from a set of FmrInv, one for each model run. * * @param name name of collection * @param fmrList the component runs FmrInv * @param regularize regularize time coords based on offset from 0Z */ FmrcInv(String name, List fmrList, boolean regularize) { this.name = name; this.regularize = regularize; this.fmrList = new ArrayList<>(fmrList); runTimeList = new ArrayList<>(); CalendarDate firstDate = null; Map uvHash = new HashMap<>(); Set offsetHash = new HashSet<>(); Set intervalHash = new HashSet<>(); Set forecastTimeHash = new HashSet<>(); for (FmrInv fmrInv : fmrList) { runTimeList.add(fmrInv.getRunDate()); if (firstDate == null) firstDate = fmrInv.getRunDate(); // hour of this runDate int hour = fmrInv.getRunDate().getHourOfDay(); // for each GridVariable, add to the UberGrid for (FmrInv.GridVariable fmrGrid : fmrInv.getGrids()) { UberGrid uv = uvHash.get(fmrGrid.getName()); if (uv == null) { uv = new UberGrid(fmrGrid.getName()); uvHash.put(fmrGrid.getName(), uv); } uv.addGridVariable(fmrGrid, hour); } // track overall list of times, offsets, and intervals for (TimeCoord tc : fmrInv.getTimeCoords()) { if (tc.isInterval()) { double[] bounds1 = tc.getBound1(); double[] bounds2 = tc.getBound2(); for (int i = 0; i < bounds1.length; i++) { CalendarDate date1 = fmrInv.getRunDate().add(bounds1[i], CalendarPeriod.Field.Hour); CalendarDate date2 = fmrInv.getRunDate().add(bounds2[i], CalendarPeriod.Field.Hour); forecastTimeHash.add(date2); // second is used as the forecast date double b1 = getOffsetInHours(firstDate, date1); double b2 = getOffsetInHours(firstDate, date2); intervalHash.add(new TimeCoord.Tinv(b1, b2)); } } else { // regular single time offset - add to offsetHash for (double offset : tc.getOffsetTimes()) { CalendarDate fcDate = fmrInv.getRunDate().add(offset, CalendarPeriod.Field.Hour); forecastTimeHash.add(fcDate); // track all forecast times double d = getOffsetInHours(firstDate, fcDate); offsetHash.add(d); // track all offset hours, calculated from baseDate } } } } baseDate = firstDate; // create the overall list of variables and coordinates uberGridList = new ArrayList<>(uvHash.values()); Collections.sort(uberGridList); for (UberGrid uv : uberGridList) { uv.finish(); } // LOOK runSeqs is alwats empty, according to InteliJ! // assign sequence ids int seqno = 0; for (RunSeq seq : runSeqs) { seq.id = seqno++; } // create the overall list of forecast times forecastTimeList = forecastTimeHash.stream().sorted().collect(Collectors.toList()); // create the overall list of offsets - may be zero List offsetsAll = offsetHash.stream().sorted().collect(Collectors.toList()); int counto = 0; double[] offs = new double[offsetsAll.size()]; for (double off : offsetsAll) offs[counto++] = off; tcOffAll = new TimeCoord(baseDate); tcOffAll.setOffsetTimes(offs); // create the overall list of offsets - may be zero List intervalAll = intervalHash.stream().sorted().collect(Collectors.toList()); tcIntAll = new TimeCoord(baseDate); tcIntAll.setBounds(intervalAll); } // not really needed private final TimeCoord tcOffAll; // all offsets in this collection, relative to baseDate private final TimeCoord tcIntAll; // all intervals in this collection, relative to baseDate // public for debugging public List getFmrList() { return fmrList; } /* * private UberGrid findVar(String varName) { * for (UberGrid uv : uberGridList) { * if (uv.gridName.equals(varName)) * return uv; * } * return null; * } * * public int findFmrIndex(Date runDate) { * for (int i = 0; i < fmrList.size(); i++) { * FmrInv fmr = fmrList.get(i); * if (fmr.getRunDate().equals(runDate)) return i; * } * return -1; * } * * public List getRunTimes() { * return runTimeList; * } * */ public String getName() { return name; } public List getRunSeqs() { return runSeqs; } public List getEnsCoords() { return ensCoords; } // list of unique VertCoord for FmrcInv public List getVertCoords() { return vertCoords; } public List getUberGrids() { return uberGridList; } public UberGrid findUberGrid(String name) { for (UberGrid grid : uberGridList) if (grid.getName().equals(name)) return grid; return null; } public List getForecastTimes() { return forecastTimeList; } public List getFmrInv() { return fmrList; } public CalendarDate getBaseDate() { return baseDate; } /* * public TimeCoord getTimeCoordsAll() { * return tcAll; * } */ /** * Find the difference between two dates in hours * * @param base date1 * @param forecast date2 * @return (forecast minus base) difference in hours */ public static double getOffsetInHours(CalendarDate base, CalendarDate forecast) { long diff = forecast.getDifferenceInMsecs(base); double result = diff / 1000.0 / 60.0 / 60.0; // LOOK why convert to double? precision may be lost ?? long testRoundoff = (long) (result * 1000.0 * 60.0 * 60.0); if (diff != testRoundoff) log.debug("getOffsetInHours"); return result; } /** * Create a date from base and hour offset * * @param base base date * @param offset hourss * @return base + offset as a Date */ public static CalendarDate makeOffsetDate(CalendarDate base, double offset) { return base.add(offset, CalendarPeriod.Field.Hour); } //////////////////////////////////////////////////////////////////// // The collection of GridVariable for one variable, across all runs in the collection // immutable after finish() is called. public class UberGrid implements Comparable { private final String gridName; private final List runs = new ArrayList<>(); private VertCoord vertCoordUnion; private RunSeq runSeq; UberGrid(String name) { this.gridName = name; } void addGridVariable(FmrInv.GridVariable grid, int hour) { runs.add(grid); } public String getName() { return gridName; } public String toString() { return gridName; } // the union of all offset hours, ignoring rundate public TimeCoord getUnionTimeCoord() { return runSeq.getUnionTimeCoord(); } public boolean isInterval() { return getUnionTimeCoord().isInterval(); } public String getTimeCoordName() { return runSeq.getName(); } public String getVertCoordName() { return (vertCoordUnion == null) ? "" : vertCoordUnion.getName(); } public List getRuns() { return runs; } public int compareTo(UberGrid o) { return gridName.compareTo(o.gridName); } public int countTotal() { int total = 0; for (FmrInv.GridVariable grid : runs) total += grid.countTotal(); return total; } public int countExpected() { int nvert = (vertCoordUnion == null) ? 1 : vertCoordUnion.getSize(); int ntimes = 0; for (FmrInv.GridVariable grid : runs) { // maybe should use the runseq ??? TimeCoord exp = grid.getTimeExpected(); ntimes += exp.getNCoords(); } return ntimes * nvert; } void finish() { if (runs.size() == 1) { FmrInv.GridVariable grid = runs.get(0); vertCoordUnion = VertCoord.findVertCoord(getVertCoords(), grid.vertCoordUnion); // timeCoordUnion = TimeCoord.findTimeCoord(getTimeCoords(), grid.timeCoordUnion); // timeCoordUnion.addGridVariable(this); grid.timeExpected = grid.timeCoordUnion; // if only one, not much else to do this.runSeq = findRunSeq(runs); this.runSeq.addVariable(this); return; } // run over all ensCoords and construct the union List ensList = new ArrayList<>(); EnsCoord ec_union = null; for (FmrInv.GridVariable grid : runs) { EnsCoord ec = grid.ensCoordUnion; if (ec == null) continue; if (ec_union == null) ec_union = new EnsCoord(ec); else if (!ec_union.equalsData(ec)) ensList.add(ec); } if (ec_union != null) { if (!ensList.isEmpty()) EnsCoord.normalize(ec_union, ensList); // add the other coords } // run over all vertCoords and construct the union List vertList = new ArrayList<>(); VertCoord vc_union = null; for (FmrInv.GridVariable grid : runs) { VertCoord vc = grid.vertCoordUnion; if (vc == null) continue; if (vc_union == null) vc_union = new VertCoord(vc); else if (!vc_union.equalsData(vc)) { // log.warn(name+" Grid "+ gridName +" has different vert coords in run " + grid.getRunDate()); vertList.add(vc); } } if (vc_union != null) { VertCoord.normalize(vc_union, vertList); // add the other coords vertCoordUnion = VertCoord.findVertCoord(getVertCoords(), vc_union); // now find unique within collection } // optionally calculate expected inventory based on matching run hours if (regularize) { // create groups of runs with the same runtime hour (integer offset from 0Z) Map hourMap = new HashMap<>(); for (FmrInv.GridVariable grid : runs) { CalendarDate runDate = grid.getRunDate(); int hour = runDate.getHourOfDay(); HourGroup hg = hourMap.get(hour); if (hg == null) { hg = new HourGroup(hour); hourMap.put(hour, hg); } hg.runs.add(grid); } // assume each hour group should have the same set of forecast time coords, as represented by their offset for (HourGroup hg : hourMap.values()) { // run over all timeCoords in this group and construct the union List timeListExp = new ArrayList<>(); for (FmrInv.GridVariable run : hg.runs) timeListExp.add(run.timeCoordUnion); // note that in this case, the baseDates of the TimeCoords in timeListExp are not the same // we are just using this routine to get the union of offset hours or intervals hg.expected = TimeCoord.makeUnion(timeListExp, baseDate); // add the other coords if (hg.expected.isInterval()) { // we discard the resulting TimeCoord and just use the union of intervals. for (FmrInv.GridVariable grid : hg.runs) { grid.timeExpected = new TimeCoord(grid.getRunDate()); grid.timeExpected.setBounds(hg.expected.getBound1(), hg.expected.getBound2()); } } else { // we discard the resulting TimeCoord and just use the union of offsets for (FmrInv.GridVariable grid : hg.runs) grid.timeExpected = new TimeCoord(grid.getRunDate(), hg.expected.getOffsetTimes()); } } // now find the RunSeq, based on the timeExpected this.runSeq = findRunSeq(runs); this.runSeq.addVariable(this); } else { // otherwise expected == actual for (FmrInv.GridVariable grid : runs) grid.timeExpected = grid.timeCoordUnion; // now find the RunSeq, based on the timeExpected this.runSeq = findRunSeq(runs); this.runSeq.addVariable(this); /* * run over all timeCoords and construct the union * List timeList = new ArrayList(); * for (FmrInv.GridVariable grid : runs) * timeList.add(grid.timeCoordUnion); * TimeCoord tc_union = TimeCoord.makeUnion(timeList, baseDate); // create the union of all time coords used by * this grid * this.timeCoordUnion = TimeCoord.findTimeCoord(getRunSeqs(), tc_union); // track unique ones in the FmrcInv */ } } } // end UberGrid // immutable after UberGrid.finish() is called. private static class HourGroup { final int hour; final List runs = new ArrayList<>(); private TimeCoord expected; HourGroup(int hour) { this.hour = hour; } } private RunSeq findRunSeq(List runs) { for (RunSeq seq : runSeqs) { if (seq.equalsData(runs)) return seq; } // make a new one RunSeq result = new RunSeq(runs); runSeqs.add(result); return result; } /** * Represents a sequence of Runs, each run has a particular TimeCoord. * this is where the time coordinates may be regularized * keep track of all the Variables with the same RunSeq */ // immutable after UberGrid.finish() is called. public class RunSeq { private final HashMap coordMap; // runDate, timeExpected private final List vars = new ArrayList<>(); // list of UberGrid that use this private int id; private List timeList; // timeList has differing runDates private TimeCoord timeCoordUnion; // union of all offset hours private boolean isInterval; RunSeq(List runs) { this.coordMap = new HashMap<>(2 * runs.size()); // make sure every date has a slot for (CalendarDate d : runTimeList) this.coordMap.put(d, TimeCoord.EMPTY); // overwrite with actual coords boolean first = true; for (FmrInv.GridVariable grid : runs) { this.coordMap.put(grid.getRunDate(), grid.getTimeExpected()); // match on timeExpected // check intervals are consistent if (first) isInterval = grid.getTimeCoord().isInterval(); else if (isInterval != grid.getTimeCoord().isInterval()) { log.error("mixed intervals for grid " + grid.getName()); throw new IllegalArgumentException("mixed intervals for grid " + grid.getName()); } first = false; } } public boolean isInterval() { return isInterval; } public List getTimes() { if (timeList == null) getUnionTimeCoord(); return timeList; } public String getName() { return id == 0 ? "time" : "time" + id; } public int getNTimeOffsets() { int n = 0; for (TimeCoord tc : coordMap.values()) n = Math.max(n, tc.getNCoords()); return n; } // the union of all offset hours, ignoring rundate, so its the rectangularization of the // offsets for each run // has the side effect of constructing timeList, the list of expected TimeCoords, one for each run public TimeCoord getUnionTimeCoord() { if (timeCoordUnion == null) { // deferred creation // eliminate the empties timeList = new ArrayList<>(); for (TimeCoord tc : coordMap.values()) { if ((tc != null) && (tc != TimeCoord.EMPTY)) timeList.add(tc); } // sort by run Date timeList.sort((o1, o2) -> { if (o1 == null || o1.getRunDate() == null) return -1; if (o2 == null || o2.getRunDate() == null) return 1; return o1.getRunDate().compareTo(o2.getRunDate()); }); // here again the timeList has differing runDates timeCoordUnion = TimeCoord.makeUnion(timeList, baseDate); // create the union of all offsets used by this grid } return timeCoordUnion; } /** * Decide if this RunSeq is equivalent to the list of Runs. * * @param oruns list of FmrInv.GridVariable, use their expected times to match * @return true if it has an equivalent set of runs. */ boolean equalsData(List oruns) { List okAdd = null; // may need to fill in the gaps for (FmrInv.GridVariable grid : oruns) { TimeCoord run = coordMap.get(grid.getRunDate()); if (run == null) { if (okAdd == null) okAdd = new ArrayList<>(); okAdd.add(grid); } else { TimeCoord orune = grid.getTimeExpected(); if (!run.equalsData(orune)) return false; } } // everything else matches, ok to expand this runSeq with missing runs if (okAdd != null) for (FmrInv.GridVariable grid : okAdd) this.coordMap.put(grid.getRunDate(), grid.getTimeExpected()); return true; } // keep track of all the Variables that have this RunSeq void addVariable(UberGrid uv) { vars.add(uv); } public List getUberGrids() { return vars; } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /* * debugging - Fmrc2 Panel * * * public void showRuntimeOffsetMatrix(Formatter out) { * out.format(" Forecast Time Offset %n"); * out.format(" RunTime "); * for (double offsetHour : tcOffAll.getOffsetTimes()) { * out.format("%10.0f", offsetHour); * } * out.format("%n"); * * int count = 0; * for (int i = 0; i < fmrList.size(); i++) { * FmrInv fmr = fmrList.get(i); * Date runDate = fmr.getRunDate(); * out.format("%2d %s ", i, dateFormatter.toDateTimeStringISO(runDate)); * for (double offsetHour : tcOffAll.getOffsetTimes()) { * Inventory inv = getInventory(fmr, offsetHour); * count += inv.showInventory( out); * } * out.format("%n"); * } * out.format("%ntotal inventory count = %d%n", count); * } * * // for a given offset hour and Fmr, find the expected and actual inventory over all grids * private Inventory getInventory(FmrInv fmr, double hour) { * int actual = 0, expected = 0; * for (FmrInv.GridVariable grid : fmr.getGrids()) { * TimeCoord tExpect = grid.getTimeExpected(); * if (tExpect.findIndex(hour) >= 0) * expected += grid.getNVerts(); * * for (GridDatasetInv.Grid inv : grid.getInventory()) { * TimeCoord tc = inv.getTimeCoord(); * if (tc.findIndex(hour) >= 0) * actual += inv.getVertCoordLength(); * } * * } * return new Inventory(actual, expected); * } * * @Immutable * public static class Inventory { * public final int actual; * public final int expected; * * public Inventory(int actual, int expected) { * this.actual = actual; * this.expected = expected; * } * * public int showInventory(Formatter out) { * if (actual != expected) { * out.format("%10s", actual + "/" + expected); * } else if (actual == 0) * out.format(" "); // blank * else * out.format("%10d", actual); * * return actual; * } * * } * * private class TimeInventory { * final TimeCoord tc; // all the TimeCoords possible * final FmrcInv.UberGrid ugrid; // for this grid * final FmrInv.GridVariable[] useGrid; // use this run and grid for ith offset * final int[] runIndex; // the index of the run in the fmrList. ie findFmrIndex() * * TimeInventory(TimeCoord tc, FmrcInv.UberGrid ugrid) { * this.tc = tc; * this.ugrid = ugrid; * this.useGrid = new FmrInv.GridVariable[tc.getNCoords()]; * this.runIndex = new int[tc.getNCoords()]; * } * * private void setOffset(double offsetHour, FmrInv.GridVariable grid, int runIndex) { * int offsetIndex = tc.findIndex(offsetHour); * if (offsetIndex < 0) * throw new IllegalStateException("FmrSnapshot cant find hour " + offsetHour + " in " + tc); * this.useGrid[offsetIndex] = grid; * this.runIndex[offsetIndex] = runIndex; * } * } * * * ////////////////////////////////////// * // 1D subsets * * public void showBest(Formatter out) { * out.format("%nRun used in best dataset%n"); * out.format(" Forecast Time Offset %n"); * out.format("Grid "); * for (double offsetHour : tcOffAll.getOffsetTimes()) * out.format("%4.0f ", offsetHour); * out.format("%n"); * * for (FmrcInv.UberGrid ugrid : getUberGrids()) { * TimeInventory inv = makeBestInventory(ugrid); * String name = ugrid.getName(); * if (name.length() > 27) name = name.substring(0, 27); * out.format(" %-27s ", name); * showInv(inv, out); * out.format("%n"); * } * } * * public void showBest2(Formatter out) { * out.format("%nRun used in best dataset by RunSeq%n"); * out.format("Seq Forecast Time Offset %n "); * for (double offsetHour : tcOffAll.getOffsetTimes()) * out.format("%4.0f ", offsetHour); * out.format("%n"); * * int count = 0; * for (RunSeq seq : getRunSeqs()) { * out.format("%3d ", count++); * List ugrids = seq.getUberGrids(); * TimeInventory inv = makeBestInventory(ugrids.get(0)); * showInv(inv, out); * for (FmrcInv.UberGrid ugrid : seq.getUberGrids()) { * TimeInventory inv2 = makeBestInventory(ugrid); * out.format(", %s ", ugrid.getName()); * if (!testInv(inv, inv2)) out.format("BAD "); * } * out.format("%n"); * } * } * * // select best inventory for each forecast time * private TimeInventory makeBestInventory(FmrcInv.UberGrid ugrid) { * return ugrid.isInterval() ? makeBestInventoryFromIntervals(ugrid) : makeBestInventoryFromOffsets( ugrid); * } * * private TimeInventory makeBestInventoryFromIntervals(FmrcInv.UberGrid ugrid) { * TimeInventory inv = new TimeInventory(tcIntAll, ugrid); * for (FmrInv.GridVariable grid : ugrid.getRuns()) { * double forecastOffset = getOffsetInHours(baseDate, grid.getRunDate()); * int runIndex = findFmrIndex(grid.getRunDate()); * TimeCoord tc = grid.getTimeCoord(); // forecast times for this run * for (double offset : tc.getOffsetTimes()) { * inv.setOffset(forecastOffset + offset, grid, runIndex); // later ones override * } * } * return inv; * } * * // select best inventory for each forecast time * private TimeInventory makeBestInventoryFromOffsets(FmrcInv.UberGrid ugrid) { * TimeInventory inv = new TimeInventory(tcOffAll, ugrid); * for (FmrInv.GridVariable grid : ugrid.getRuns()) { * double forecastOffset = getOffsetInHours(baseDate, grid.getRunDate()); * int runIndex = findFmrIndex(grid.getRunDate()); * TimeCoord tc = grid.getTimeCoord(); // forecast times for this run * for (double offset : tc.getOffsetTimes()) { * inv.setOffset(forecastOffset + offset, grid, runIndex); // later ones override * } * } * return inv; * } * * // select best inventory for each forecast time * public void showBest(FmrcInv.UberGrid ugrid, Formatter out) { * out.format("%nRun used in best dataset for grid %s %n", ugrid.getName()); * out.format(" Forecast Time Offset %n"); * out.format("Run "); * for (double offsetHour : tcOffAll.getOffsetTimes()) { * out.format("%3.0f ", offsetHour); * } * out.format("%n"); * * TimeInventory inv = new TimeInventory(tcOffAll, ugrid); * for (FmrInv.GridVariable grid : ugrid.getRuns()) { * double forecastOffset = getOffsetInHours(baseDate, grid.getRunDate()); * int runIndex = findFmrIndex(grid.getRunDate()); * TimeCoord tc = grid.getTimeCoord(); // forecast times for this run * TimeInventory invRun = new TimeInventory(tcOffAll, ugrid); * for (double offset : tc.getOffsetTimes()) { * inv.setOffset(forecastOffset + offset, grid, runIndex); // later ones override * invRun.setOffset(forecastOffset + offset, grid, runIndex); // later ones override * } * out.format(" %-27s ", dateFormatter.toDateTimeString(grid.getRunDate())); * showInv(invRun, out); * out.format("%n"); * } * * String name = ugrid.getName(); * if (name.length() > 27) name = name.substring(0, 27); * out.format(" %-27s ", "result"); * showInv(inv, out); * out.format("%n"); * } * * private void showInv(TimeInventory tinv, Formatter out) { * for (int i = 0; i < tinv.useGrid.length; i++) { * FmrInv.GridVariable grid = tinv.useGrid[i]; * if (grid == null) * out.format(" "); * else * out.format("%4d ", tinv.runIndex[i]); * } * } * * private boolean testInv(TimeInventory tinv1, TimeInventory tinv2) { * if (tinv1.useGrid.length != tinv2.useGrid.length) return false; * * for (int i = 0; i < tinv1.useGrid.length; i++) { * if (tinv1.runIndex[i] != tinv2.runIndex[i]) * return false; * } * return true; * } */ }





© 2015 - 2025 Weber Informatics LLC | Privacy Policy