Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* $Id:$
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would be
* appreciated but is not required.
*
* 2. Altered source versions must be plainly marked as such, and must not be
* misrepresented as being the original software.
*
* 3. This notice may not be removed or altered from any source
* distribution.
*/
package oms3.io;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.StringReader;
import java.lang.reflect.Array;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Arrays;
import javax.swing.event.TableModelListener;
import javax.swing.table.AbstractTableModel;
import javax.swing.table.TableModel;
import oms3.Conversions;
/** Data Input/Output management.
*
* @author Olaf David
*/
public class DataIO {
private static final String P = "@";
public static final String TABLE = P + "T";
public static final String HEADER = P + "H";
public static final String PROPERTIES = P + "S";
public static final String PROPERTY = P + "P";
public static final String TABLE1 = P + "Table";
public static final String HEADER1 = P + "Header";
public static final String PROPERTIES1 = P + "Properties";
public static final String PROPERTY1 = P + "Property";
//
//
public static final String CSPROPERTIES_EXT = "csp";
public static final String CSTABLE_EXT = "cst";
//
private static final String ROOT_ANN = "___root___";
private static final String COMMENT = "#";
private static final Map NOINFO = Collections.unmodifiableMap(new HashMap());
private static final Pattern varPattern = Pattern.compile("\\$\\{([^$}]+)\\}");
/* some static helpers, might have to go somewhere else */
private static final String ISO8601 = "yyyy-MM-dd'T'hh:mm:ss";
//
// all meta data keys
public static final String KEY_CONVERTED_FROM = "converted_from";
public static final String DATE_FORMAT = "date_format";
public static final String DATE_START = "date_start";
public static final String DATE_END = "date_end";
public static final String KEY_CREATED_AT = "created_at";
public static final String KEY_CREATED_BY = "created_by";
public static final String KEY_UNIT = "unit";
public static final String KEY_FORMAT = "format";
public static final String KEY_TYPE = "type";
public static final String KEY_NAME = "name";
public static final String KEY_MISSING_VAL = "missing_value";
public static final String KEY_FC_START = "forecast_start";
public static final String KEY_FC_DAYS = "forecast_days";
public static final String KEY_HIST_YEAR = "historical_year";
public static final String KEY_DIGEST = "digest";
public static final String VAL_DATE = "Date";
//TimeStep Enumerations
public static final int DAILY = 0;
public static final int MEAN_MONTHLY = 1;
public static final int MONTHLY_MEAN = 2;
public static final int ANNUAL_MEAN = 3;
public static final int PERIOD_MEAN = 4;
public static final int PERIOD_MEDIAN = 5;
public static final int PERIOD_STANDARD_DEVIATION = 6;
public static final int PERIOD_MIN = 7;
public static final int PERIOD_MAX = 8;
public static double[] getColumnDoubleValuesInterval(Date start, Date end, CSTable t, String columnName, int timeStep) {
int col = columnIndex(t, columnName);
if (col == -1) {
throw new IllegalArgumentException("No such column: " + columnName);
}
DateFormat fmt = lookupDateFormat(t, 1);
boolean useOrigDaily = false;
switch (timeStep) {
case DAILY:
if (useOrigDaily) {
List l = new ArrayList();
for (String[] row : t.rows()) {
try {
Date d = fmt.parse(row[1]);
if ((d.equals(start) || d.after(start)) && (d.equals(end) || d.before(end))) {
l.add(new Double(row[col]));
}
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
double[] arr = new double[l.size()];
for (int i = 0; i < arr.length; i++) {
arr[i] = l.get(i);
}
return arr;
}
case ANNUAL_MEAN:
case MONTHLY_MEAN:
case PERIOD_MEAN: {
int previousMonth = -1;
int previousYear = -1;
int previousDay = -1;
boolean previousValid = false;
boolean lastRow = false;
boolean useYear = (timeStep == DAILY) || (timeStep == MONTHLY_MEAN) || (timeStep == ANNUAL_MEAN);
boolean useMonth = (timeStep == DAILY) || (timeStep == MONTHLY_MEAN);
boolean useDay = (timeStep == DAILY);
List l = new ArrayList();
double sum = 0;
int count = 0;
for (String[] row : t.rows()) {
try {
Date d = fmt.parse(row[1]);
if ((d.equals(start) || d.after(start)) && (d.equals(end) || d.before(end))) {
int month = d.getMonth();
int year = d.getYear();
int day = d.getDay();
double data = Double.parseDouble(row[col]);
boolean newEntry = (previousValid && ((useYear && (year != previousYear))
|| (useMonth && (month != previousMonth))
|| (useDay && (day != previousDay))));
if (newEntry) {
l.add(sum / count);
sum = 0;
count = 0;
}
sum += data;
count++;
previousValid = true;
previousDay = day;
previousMonth = month;
previousYear = year;
}
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
l.add(sum / count); // add the final entry which wasn't yet added
// since it never hit a newEntry.
// Copy the List to the output array.
double[] arr = new double[l.size()];
for (int i = 0; i < arr.length; i++) {
arr[i] = l.get(i);
}
return arr;
// break;
}
case MEAN_MONTHLY: {
double[] arr = new double[12]; // 1 per month
int[] count = new int[12];
for (int i = 0; i < 12; i++) {
arr[i] = 0; // initialize data to 0
count[i] = 0;
}
for (String[] row : t.rows()) {
try {
Date d = fmt.parse(row[1]);
if ((d.equals(start) || d.after(start)) && (d.equals(end) || d.before(end))) {
int month = d.getMonth();
double data = Double.parseDouble(row[col]);
arr[month] = arr[month] + data;
count[month] = count[month] + 1;
if (month > 11) {
throw new RuntimeException("Month > 11 = " + month);
}
}
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
for (int i = 0; i < 12; i++) {
arr[i] = arr[i] / count[i];
}
return arr;
// break;
}
case PERIOD_MIN:
case PERIOD_MAX: {
double min = -1;
double max = -1;
boolean previousValid = false;
for (String[] row : t.rows()) {
try {
Date d = fmt.parse(row[1]);
if ((d.equals(start) || d.after(start)) && (d.equals(end) || d.before(end))) {
double data = Double.parseDouble(row[col]);
if (!previousValid) {
min = data;
max = data;
} else if ((timeStep == PERIOD_MIN) && (data < min)) {
min = data;
} else if ((timeStep == PERIOD_MAX) && (data > max)) {
max = data;
}
previousValid = true;
}
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
double[] arr = new double[1];
arr[0] = (timeStep == PERIOD_MIN) ? min : max;
return arr;
// break;
}
case PERIOD_MEDIAN: {
// Put entire table into ArrayList
List l = new ArrayList();
for (String[] row : t.rows()) {
try {
Date d = fmt.parse(row[1]);
if ((d.equals(start) || d.after(start)) && (d.equals(end) || d.before(end))) {
l.add(new Double(row[col]));
}
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
//Copy to array of double
int lSize = l.size();
if (lSize == 0) {
throw new RuntimeException("No data in file matched the specified period " + start + " to " + end);
}
double[] arr = new double[lSize];
for (int i = 0; i < arr.length; i++) {
arr[i] = l.get(i);
}
l.clear(); // Don't need l anymore so clear it.
// Sort the Array
Arrays.sort(arr);
double median;
// Pull out the Median
if (lSize % 2 == 1) {
median = arr[(lSize + 1) / 2 - 1];
} else {
double lower = arr[(lSize / 2) - 1];
double upper = arr[lSize / 2];
median = (lower + upper) / 2.0;
}
// return as an array with 1 entry.
double[] arr2 = new double[1];
arr2[0] = median;
return arr2;
//break;
}
case PERIOD_STANDARD_DEVIATION: {
// kmolson TODO- is it better to read the file twice, or read
// it once and store the data for the 2nd pass while
// computing the mean?
// Read file to get mean.
double sum = 0;
double sq_sum = 0;
double data = 0;
int count = 0;
for (String[] row : t.rows()) {
try {
Date d = fmt.parse(row[1]);
if ((d.equals(start) || d.after(start)) && (d.equals(end) || d.before(end))) {
data = Double.parseDouble(row[col]);
sum += data;
sq_sum += (data * data);
count++;
}
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
double mean = sum / count;
double variance = sq_sum / count - (mean * mean);
double standardDeviation = Math.sqrt(variance);
double[] arr = new double[1];
arr[0] = standardDeviation;
return arr;
}
default: {
throw new IllegalArgumentException("timeStep " + timeStep + "not supported.");
}
}
}
public static SimpleDateFormat lookupDateFormat(CSTable table, int col) {
if (col < 0 || col > table.getColumnCount()) {
throw new IllegalArgumentException("invalid column: " + col);
}
String format = table.getColumnInfo(col).get(KEY_FORMAT);
if (format == null) {
format = table.getInfo().get(DATE_FORMAT);
}
if (format == null) {
format = Conversions.ISO().toPattern();
}
return new SimpleDateFormat(format);
}
public static int findRowByDate(Date date, int dateColumn, CSTable table) {
String type = table.getColumnInfo(dateColumn).get(KEY_TYPE);
if ((type == null) || !type.equalsIgnoreCase(VAL_DATE)) {
throw new IllegalArgumentException();
}
DateFormat fmt = lookupDateFormat(table, dateColumn);
int rowNo = 0;
for (String[] row : table.rows()) {
try {
Date d = fmt.parse(row[dateColumn]);
if (d.equals(date)) {
return rowNo;
}
rowNo++;
} catch (ParseException ex) {
throw new RuntimeException(ex);
}
}
throw new IllegalArgumentException(date.toString());
}
public static CSTable synthESPInput(CSTable table, Date iniStart, Date iniEnd, int fcDays, int year) {
int dateColumn = 1;
DateFormat hfmt = lookupDateFormat(table, dateColumn);
// Forecast start = end of initialzation + 1 day
Calendar fcStartCal = new GregorianCalendar();
fcStartCal.setTime(iniEnd);
fcStartCal.add(Calendar.DATE, 1);
Date fcStart = fcStartCal.getTime();
// get the initialization period
MemoryTable t = new MemoryTable(table);
int iniStartRow = findRowByDate(iniStart, dateColumn, t);
int iniEndRow = findRowByDate(iniEnd, dateColumn, t);
List iniRows = t.getRows(iniStartRow, iniEndRow);
// set the historical date to the forcast date, but use the
// historical year.
Calendar histStart = new GregorianCalendar();
histStart.setTime(fcStart);
histStart.set(Calendar.YEAR, year);
// get the historical data
int histStartRow = findRowByDate(histStart.getTime(), dateColumn, t);
int histEndRow = histStartRow + (fcDays - 1);
List histRows = t.getRows(histStartRow, histEndRow);
// create the new Table.
MemoryTable espTable = new MemoryTable(table);
espTable.getInfo().put(DATE_START, hfmt.format(iniStart));
espTable.getInfo().put(KEY_FC_START, hfmt.format(fcStart));
espTable.getInfo().put(KEY_FC_DAYS, Integer.toString(fcDays));
espTable.getInfo().put(KEY_HIST_YEAR, Integer.toString(year));
espTable.clearRows();
espTable.addRows(iniRows);
espTable.addRows(histRows);
// historical date -> forecast date.
Calendar fcCurrent = new GregorianCalendar();
fcCurrent.setTime(fcStart);
List espRows = espTable.getRows();
int start = iniRows.size();
for (int i = start; i <= start + (fcDays - 1); i++) {
espRows.get(i)[1] = hfmt.format(fcCurrent.getTime());
fcCurrent.add(Calendar.DATE, 1);
}
fcCurrent.add(Calendar.DATE, -1);
espTable.getInfo().put(DATE_END, hfmt.format(fcCurrent.getTime()));
return espTable;
}
/** Get a slice of rows out of the table matching the time window
*
* @param table
* @param timeCol
* @param start
* @param end
* @return the first and last row that matches the time window start->end
*/
public static int[] sliceByTime(CSTable table, int timeCol, Date start, Date end) {
if (end.before(start)) {
throw new IllegalArgumentException("end rows = new ArrayList();
for (String[] row : src.rows()) {
rows.add(row);
}
return new TableModel() {
@Override
public int getColumnCount() {
return src.getColumnCount();
}
@Override
public String getColumnName(int column) {
return src.getColumnName(column);
}
@Override
public int getRowCount() {
return rows.size();
}
@Override
public Class getColumnClass(int columnIndex) {
return String.class;
}
@Override
public boolean isCellEditable(int rowIndex, int columnIndex) {
return false;
}
@Override
public Object getValueAt(int rowIndex, int columnIndex) {
return rows.get(rowIndex)[columnIndex];
}
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
// rows.get(rowIndex)[columnIndex] = (String) aValue;
}
@Override
public void addTableModelListener(TableModelListener l) {
}
@Override
public void removeTableModelListener(TableModelListener l) {
}
};
}
/**
* Get the KVP as table.
* @param p
* @return an AbstractTableModel for properties (KVP)
*/
public static AbstractTableModel getProperties(final CSProperties p) {
return new AbstractTableModel() {
@Override
public int getRowCount() {
return p.keySet().size();
}
@Override
public int getColumnCount() {
return 2;
}
@Override
public Object getValueAt(int rowIndex, int columnIndex) {
if (columnIndex == 0) {
return " " + p.keySet().toArray()[rowIndex];
} else {
return p.values().toArray()[rowIndex];
}
}
@Override
public boolean isCellEditable(int rowIndex, int columnIndex) {
return columnIndex == 1;
}
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
if (columnIndex == 1) {
String[] keys = p.keySet().toArray(new String[0]);
p.put(keys[rowIndex], aValue.toString());
}
}
@Override
public String getColumnName(int column) {
return column == 0 ? "Name" : "Value";
}
@Override
public Class getColumnClass(int columnIndex) {
return String.class;
}
};
}
public static AbstractTableModel get2DBounded(final CSProperties p, final String pname) throws ParseException {
String m = p.getInfo(pname).get("bound");
String[] dims = m.split(",");
final int rows = DataIO.getInt(p, dims[0].trim());
final int cols = DataIO.getInt(p, dims[1].trim());
return new AbstractTableModel() {
@Override
public int getRowCount() {
return rows;
}
@Override
public int getColumnCount() {
return cols;
}
@Override
public boolean isCellEditable(int rowIndex, int columnIndex) {
return true;
}
@Override
public Object getValueAt(int row, int col) {
String[][] d = Conversions.convert(p.get(pname), String[][].class);
return d[row][col].trim();
}
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
String[][] d = Conversions.convert(p.get(pname), String[][].class);
d[rowIndex][columnIndex] = aValue.toString().trim();
String s = toArrayString(d);
p.put(pname, s);
}
@Override
public String getColumnName(int column) {
return Integer.toString(column);
}
@Override
public Class getColumnClass(int columnIndex) {
return String.class;
}
};
}
static public boolean playsRole(final CSProperties p, String key, String role) {
String r = p.getInfo(key).get("role");
if (r == null) {
return false;
}
return r.contains(role);
}
static public boolean isBound(final CSProperties p, String key, int dim) {
String bound = p.getInfo(key).get("bound");
if (bound == null) {
return false;
}
StringTokenizer t = new StringTokenizer(bound, ",");
if (t.countTokens() == dim) {
return true;
}
return false;
}
public static CSTable getTable(final CSProperties p, String boundName) {
MemoryTable m = new MemoryTable();
List arr = keysByMeta(p, "bound", boundName);
for (String a : arr) {
}
return m;
}
// 1D arrays
public static AbstractTableModel getBoundProperties(final CSProperties p, String boundName) throws ParseException {
final int rows = DataIO.getInt(p, boundName);
final List arr = keysByMeta(p, "bound", boundName);
return new AbstractTableModel() {
@Override
public int getRowCount() {
return rows;
}
@Override
public int getColumnCount() {
return arr.size();
}
@Override
public Object getValueAt(int rowIndex, int columnIndex) {
String colname = arr.get(columnIndex);
String[] d = Conversions.convert(p.get(colname), String[].class);
return d[rowIndex].trim();
}
@Override
public boolean isCellEditable(int rowIndex, int columnIndex) {
return true;
}
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
String colname = arr.get(columnIndex);
String[] d = Conversions.convert(p.get(colname), String[].class);
d[rowIndex] = aValue.toString().trim();
String s = toArrayString(d);
p.put(colname, s);
}
@Override
public String getColumnName(int column) {
return arr.get(column);
}
@Override
public Class getColumnClass(int columnIndex) {
return String.class;
}
};
}
// unbound
public static AbstractTableModel getUnBoundProperties(final CSProperties p) throws ParseException {
final List arr = keysByNotMeta(p, "bound");
return new AbstractTableModel() {
@Override
public int getRowCount() {
return arr.size();
}
@Override
public int getColumnCount() {
return 2;
}
@Override
public Object getValueAt(int rowIndex, int columnIndex) {
if (columnIndex == 0) {
return arr.get(rowIndex);
} else {
return p.get(arr.get(rowIndex));
}
}
@Override
public boolean isCellEditable(int rowIndex, int columnIndex) {
return columnIndex == 1;
}
@Override
public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
p.put(arr.get(rowIndex), aValue.toString());
}
@Override
public String getColumnName(int column) {
return (column == 0) ? "Key" : "Value";
}
@Override
public Class getColumnClass(int columnIndex) {
return String.class;
}
};
}
/**
* Create array string.
*
* @param arr
* @return an array String.
*/
public static String toArrayString(String[] arr) {
StringBuilder b = new StringBuilder();
b.append('{');
for (int i = 0; i < arr.length; i++) {
b.append(arr[i]);
if (i < arr.length - 1) {
b.append(',');
}
}
b.append('}');
return b.toString();
}
public static String toArrayString(String[][] arr) {
StringBuilder b = new StringBuilder();
b.append('{');
for (int i = 0; i < arr.length; i++) {
b.append('{');
for (int j = 0; j < arr[i].length; j++) {
b.append(arr[i][j]);
if (j < arr[i].length - 1) {
b.append(',');
}
}
b.append('}');
if (i < arr.length - 1) {
b.append(',');
}
}
b.append('}');
return b.toString();
}
/** Returns a r/o table from a CSP file
*
* @param p
* @param dim
* @return a table model for properties with dimension.
*/
public static TableModel fromCSP(CSProperties p, final int dim) {
List dims = keysByMeta(p, "role", "dimension");
if (dims.isEmpty()) {
return null;
}
for (String d : dims) {
if (Integer.parseInt(p.get(d).toString()) == dim) {
final List bounds = keysByMeta(p, "bound", d);
final List