org.sherpa.NcExporter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of sherpa Show documentation
Sherpa is a Java library that provides support to the integration of Java simulators with Everest through its REST APIs.
The newest version!
package org.sherpa;

import org.joda.time.DateTime;
import ucar.ma2.*;
import ucar.nc2.Attribute;
import ucar.nc2.Group;
import ucar.nc2.NetcdfFileWriter;
import ucar.nc2.Structure;

import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;

import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.*;
import static java.util.stream.IntStream.range;
import static ucar.ma2.DataType.*;
import static ucar.nc2.NetcdfFileWriter.Version.netcdf4;

public class NcExporter {

	private final NetcdfFileWriter ncFile;
	private final Map dimensions = new HashMap<>();
	private final Map> members = new HashMap<>();
	private final Map timeScales = new HashMap<>();
	private final Map> timeStamps = new HashMap<>();
	private final Map groups = new HashMap<>();
	private final Map variables = new HashMap<>();
	private final List facts = new ArrayList<>();
	private final Group results;

	private Structure structure;
	private String simulator = "unknown", author = "unknown", comment = "none";
	private int dateSize = 32, authorSize = 50, softwareSize = 50, commentSize = 200;

	/**
	 * Create an exporter for registering the output of a simulation
	 * @param modelId identifier of the model according to Everest
	 * @param caseId identifier of the case id that are related to the results stored
	 * @param urbanArea urban area of the simulation results
	 */

	public NcExporter(String modelId, String caseId, String urbanArea) {
		System.setProperty("jna.library.path", "C:\\Program Files (x86)\\netCDF 4.3.3.1\\bin");
		ncFile = createNcFile(modelId, caseId, urbanArea);
		results = createResultsGroup(caseId);
	}

	/**
	 * Sets the simulator name as software version in the changelog
	 * @param simulator name of the software version to be set
	 */
	public void simulator(String simulator) {
		this.simulator = simulator;
	}

	/**
	 * Sets the author of the nc file in the changelog
	 * @param author author of the nc file
	 */
	public void author(String author) {
		this.author = author;
	}

	/**
	 * Sets the comments of the nc file in the changelog
	 * @param comment comment of the nc file
	 */
	public void comment(String comment) {
		this.comment = comment;
	}

	private NetcdfFileWriter createNcFile(String modelId, String caseId, String urbanArea) {
		try {
			NetcdfFileWriter ncFile = NetcdfFileWriter.createNew(netcdf4, "res/" + modelId + " " + caseId + ".nc");
			ncFile.addGroupAttribute(null, new Attribute("CITYCDF-VERSION", 2));
			ncFile.addGroupAttribute(null, new Attribute("URBAN-AREA-NAME", urbanArea));
			return ncFile;
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	}

	private Group createResultsGroup(String caseId) {
		Group results = ncFile.addGroup(ncFile.addGroup(null, ""), "Results");
		ncFile.addGroupAttribute(results, new Attribute("CREATED-DATE", LocalDateTime.now().toString()));
		ncFile.addGroupAttribute(results, new Attribute("CURTIS-SIMULATION-CASE-ID", caseId));
		ncFile.addGroupAttribute(results, new Attribute("DATE-SOURCE-INFO", simulator));
		ncFile.addGroupAttribute(results, new Attribute("DATE-SOURCE-TYPE", "SIMULATION"));
		return results;
	}

	/**
	 * Register a member of the simulation (an entity)
	 * @param type type of the member that is registered (e.g. City)
	 * @param member a member containing the Everest Id and the name
	 */
	public void registerMember(String type, Member member) {
		if (!dimensions.containsKey(type)) createDimension(type);
		member.dimension = dimensions.get(type);
		if (!members.containsKey(dimensions.get(type)))
			members.put(dimensions.get(type), new ArrayList<>());
		members.get(dimensions.get(type)).add(member);
	}

	private Dimension createDimension(String type) {
		Group group = ncFile.addGroup(results, type);
		groups.put(type, group);
		ncFile.addGroupAttribute(group, new Attribute("CREATED-DATE", LocalDateTime.now().toString()));
		return dimensions.put(type, new Dimension(type));
	}

	/**
	 * Registers a timescale by a name and a pattern
	 * @param name name of the time scale
	 * @param pattern the pattern that will convert a LocalDateTime into a String. Pattern should be compliant with DateTimeFormatter.
	 */
	public void registerTimeScale(String name, String pattern) {
		createTimeScale(name, DateTimeFormatter.ofPattern(pattern));
	}

	/**
	 * Register a timestamp inside a timescale
	 * @param timeScale name of the time scale
	 * @param timeStamp time stamp to be registered inside the timescale
	 */
	public void registerTimestamp(String timeScale, LocalDateTime timeStamp) {
		if (!timeScales.containsKey(timeScale)) throw new RuntimeException("Time scale " + timeScale + " not defined.");
		if (!timeStamps.containsKey(timeScales.get(timeScale)))
			timeStamps.put(timeScales.get(timeScale), new ArrayList<>());
		timeStamps.get(timeScales.get(timeScale)).add(new TimeStamp(timeScales.get(timeScale).formatter.format(timeStamp)));
	}

	private TimeScale createTimeScale(String type, DateTimeFormatter formatter) {
		return timeScales.put(type, new TimeScale(type, formatter));
	}

	/**
	 * Register a variable (type of fact) for a given type of member (e.g. City) and time scale
	 * @param name name of the variable
	 * @param type type of member that is related to this variable
	 * @param timeScale time scale in which the facts of this variable will be recorded
	 */
	public void registerVariable(String name, String type, String timeScale) {
		registerVariable(name, type, timeScale, "", "", "");
	}

	/**
	 * Register a variable (type of fact) for a given type of member (e.g. City) and time scale
	 * @param name name of the variable
	 * @param type type of member that is related to this variable
	 * @param timeScale time scale in which the facts of this variable will be recorded
	 * @param label label for this variable
	 * @param description description of this variable
	 * @param unit unit that is used for registering values of this variable (e.g. watts)
	 */
	public void registerVariable(String name, String type, String timeScale, String label, String description, String unit) {
		if (!dimensions.containsKey(type) || !timeScales.containsKey(timeScale))
			throw new RuntimeException("Type and/or time scale " + type + ", " + timeScale + " have not been defined");
		variables.put(fullVariableName(name, type), new Variable(name, dimensions.get(type), timeScales.get(timeScale), label, description, unit));
	}

	/**
	 * Creates the file and prepares it to be filled. This step is mandatory before calling to register fact
	 */
	public void init() {
		createDimensions();
		createTimeScales();
		createVariables();
		createChangelog();
		try {
			ncFile.create();
		} catch (IOException e) {
			e.printStackTrace();
		}
		fillDimensions();
		fillTimeScales();
		fillChangelog();
	}

	/**
	 * Register a fact of a variable with a given time, member id and value. This shouldn't be called before init.
	 * @param variableName name of the variable in which the fact is registered
	 * @param timeStamp time stamp at which the fact occurred
	 * @param memberId id of the member involved in the fact
	 * @param value value of the fact for this variable, timestamp and member
	 */
	public void registerFact(String variableName, LocalDateTime timeStamp, String memberId, double value) {
		String format = variables.get(fullVariableName(variableName, findMember(memberId).dimension.name)).timeScale.formatter.format(timeStamp);
		facts.add(new Fact(variableName, format, findMember(memberId), value));
	}

	private Member findMember(String id) {
		return members.values().stream().flatMap(Collection::stream).filter(m -> m.id.equals(id)).findFirst().get();
	}

	/**
	 * Writes the nc file
	 */
	public void write() {
		facts.stream()
				.collect(groupingBy(f -> fullVariableName(f.variableName, f.member.dimension.name), mapping(f -> f, toList())))
				.forEach(this::writeFacts);
		try {
			ncFile.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private String fullVariableName(String variableName, String dimension) {
		return variableName + "@" + dimension;
	}

	private void writeFacts(String destination, List facts) {
		if (facts.isEmpty()) return;
		Map factMap = facts.stream().collect(toMap(f -> f.timeStamp + "@" + f.member, f -> f.value));
		List timeStamps = this.timeStamps.get(variables.get(destination).timeScale);
		List members = this.members.get(variables.get(destination).dimension);
		ArrayDouble.D2 data = new ArrayDouble.D2(timeStamps.size(), members.size());
		range(0, timeStamps.size()).forEach(i -> range(0, members.size()).forEach(j -> {
			String key = timeStamps.get(i) + "@" + members.get(j);
			data.set(i, j, factMap.containsKey(key) ? factMap.get(key) : Double.NaN);
		}));
		try {
			ncFile.write(variables.get(destination).variable, data);
		} catch (IOException | InvalidRangeException e) {
			e.printStackTrace();
		}
	}

	private void createDimensions() {
		dimensions.values().forEach(dimension -> {
			dimension.ncDimension = ncFile.addDimension(null, "PID-" + dimension.name, members.get(dimension).size());
			ncFile.addVariable(null, "PID-" + dimension.name, STRING, singletonList(dimension.ncDimension))
					.addAttribute(new Attribute("CREATED-DATE", LocalDateTime.now().toString()));
			ncFile.addVariable(null, "NAME-" + dimension.name, STRING, singletonList(dimension.ncDimension))
					.addAttribute(new Attribute("CREATED-DATE", LocalDateTime.now().toString()));
		});
	}

	private void createTimeScales() {
		timeScales.values().forEach(timeScale -> {
			timeScale.ncDimension = ncFile.addDimension(null, "TIMESCALE-" + timeScale.name, timeStamps.get(timeScale).size());
			ncFile.addVariable(null, "TIMESCALE-" + timeScale.name, STRING, singletonList(timeScale.ncDimension))
					.addAttribute(new Attribute("CREATED-DATE", LocalDateTime.now().toString()));
		});
	}

	private void createVariables() {
		variables.values().forEach(variable -> {
			variable.variable = ncFile.addVariable(groups.get(variable.dimension.name),
					variable.name,
					DOUBLE,
					asList(variable.timeScale.ncDimension, variable.dimension.ncDimension));
			variable.variable.addAttribute(new Attribute("CREATED-DATE", LocalDateTime.now().toString()));
			variable.variable.addAttribute(new Attribute("DATE-SOURCE-INFO", "Tafat"));
			variable.variable.addAttribute(new Attribute("DATE-SOURCE-TYPE", "SIMULATION"));
			variable.variable.addAttribute(new Attribute("DESCRIPTION", variable.description));
			variable.variable.addAttribute(new Attribute("LABEL", variable.label));
			variable.variable.addAttribute(new Attribute("UNIT", variable.unit));
		});
	}

	private void createChangelog() {
		structure = (Structure) ncFile.addVariable(null, "CHANGELOG", STRUCTURE, singletonList(ncFile.addDimension(null, "CHANGELOG", 1, true, false, false)));
		ncFile.addStructureMember(structure, "DATE", CHAR, null).setDimensions(singletonList(new ucar.nc2.Dimension(null, dateSize, false)));
		ncFile.addStructureMember(structure, "AUTHOR", CHAR, null).setDimensions(singletonList(new ucar.nc2.Dimension(null, authorSize, false)));
		ncFile.addStructureMember(structure, "SOFTWARE-VERSION", CHAR, null).setDimensions(singletonList(new ucar.nc2.Dimension(null, softwareSize, false)));
		ncFile.addStructureMember(structure, "COMMENT", CHAR, null).setDimensions(singletonList(new ucar.nc2.Dimension(null, commentSize, false)));
		structure.calcElementSize();
	}

	private void fillDimensions() {
		dimensions.values().forEach(dimension -> {
			List members = this.members.get(dimension);
			try {
				ArrayString.D1 pidData = new ArrayString.D1(members.size());
				range(0, members.size()).forEach(i -> pidData.set(i, members.get(i).id));
				ncFile.write(ncFile.findVariable("PID-" + dimension.name), pidData);
				ArrayString.D1 nameData = new ArrayString.D1(members.size());
				range(0, members.size()).forEach(i -> nameData.set(i, members.get(i).name));
				ncFile.write(ncFile.findVariable("NAME-" + dimension.name), nameData);
			} catch (IOException | InvalidRangeException e) {
				e.printStackTrace();
			}
		});
	}

	private void fillTimeScales() {
		timeScales.values().forEach(timeScale -> {
			List timeStamps = this.timeStamps.get(timeScale);
			try {
				ArrayString.D1 timeScaleData = new ArrayString.D1(timeStamps.size());
				range(0, timeStamps.size()).forEach(i -> timeScaleData.set(i, timeStamps.get(i).id));
				ncFile.write(ncFile.findVariable("TIMESCALE-" + timeScale.name), timeScaleData);
			} catch (IOException | InvalidRangeException e) {
				e.printStackTrace();
			}
		});
	}

	private void fillChangelog() {
		ArrayStructureMA arrayStructure = new ArrayStructureMA(structure.makeStructureMembers(), structure.getShape());
		arrayStructure.setMemberArray("DATE", toArrayChar(DateTime.now().toString(), dateSize));
		arrayStructure.setMemberArray("AUTHOR", toArrayChar(author, authorSize));
		arrayStructure.setMemberArray("SOFTWARE-VERSION", toArrayChar(simulator, softwareSize));
		arrayStructure.setMemberArray("COMMENT", toArrayChar(comment, commentSize));
		try {
			ncFile.write(structure, arrayStructure);
		} catch (IOException | InvalidRangeException e) {
			e.printStackTrace();
		}
	}

	private ArrayChar.D2 toArrayChar(String value, int size){
		char[] valueInChars = value.toCharArray();
		ArrayChar.D2 result = new ArrayChar.D2(1, size);
		for (int i = 0; i < valueInChars.length; i++) result.setChar(i, valueInChars[i]);
		return result;
	}

	static class Dimension {
		String name;
		ucar.nc2.Dimension ncDimension;

		public Dimension(String name) {
			this.name = name;
		}
	}

	static class Variable {
		String name;
		Dimension dimension;
		TimeScale timeScale;
		String label;
		String description;
		String unit;
		ucar.nc2.Variable variable;

		public Variable(String name, Dimension dimension, TimeScale timeScale, String label, String description, String unit) {
			this.name = name;
			this.dimension = dimension;
			this.timeScale = timeScale;
			this.label = label;
			this.description = description;
			this.unit = unit;
		}
	}

	static class TimeScale extends Dimension {
		private final DateTimeFormatter formatter;

		public TimeScale(String name, DateTimeFormatter formatter) {
			super(name);
			this.formatter = formatter;
		}
	}

	public static class Member {
		String id;
		String name;
		Dimension dimension;

		public Member(String id, String name) {
			this.id = id;
			this.name = name;
		}

		@Override
		public String toString() {
			return id;
		}
	}

	public static class TimeStamp {
		String id;

		public TimeStamp(String id) {
			this.id = id;
		}

		@Override
		public String toString() {
			return id;
		}
	}

	public static class Fact {
		String variableName;
		String timeStamp;
		Member member;
		double value;

		public Fact(String variableName, String timeStamp, Member pid, double value) {
			this.variableName = variableName;
			this.timeStamp = timeStamp;
			this.member = pid;
			this.value = value;
		}
	}

}