nl.tno.bim.nmd.services.Nmd3DataService Maven / Gradle / Ivy

Go to download
package nl.tno.bim.nmd.services;

import java.io.IOException;
import java.nio.file.Path;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;

import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;
import org.apache.log4j.Logger;
import org.bimserver.shared.reflector.KeyValuePair;
import org.eclipse.jdt.core.compiler.InvalidInputException;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.TextNode;

import nl.tno.bim.nmd.config.NmdConfig;
import nl.tno.bim.nmd.config.NmdConfigImpl;
import nl.tno.bim.nmd.domain.NlsfbCode;
import nl.tno.bim.nmd.domain.NmdElement;
import nl.tno.bim.nmd.domain.NmdElementImpl;
import nl.tno.bim.nmd.domain.NmdFaseProfielImpl;
import nl.tno.bim.nmd.domain.NmdMilieuCategorie;
import nl.tno.bim.nmd.domain.NmdProductCard;
import nl.tno.bim.nmd.domain.NmdProductCardImpl;
import nl.tno.bim.nmd.domain.NmdProfileSet;
import nl.tno.bim.nmd.domain.NmdProfileSetImpl;
import nl.tno.bim.nmd.domain.NmdReferenceResources;
import nl.tno.bim.nmd.scaling.NmdScaler;
import nl.tno.bim.nmd.scaling.NmdScalerFactory;

/**
 * Implementation of the NmdDataService to retrieve data from the NMD see :
 * https://www.milieudatabase.nl/ for mor information
 * 
 * @author vijj
 *
 */
public class Nmd3DataService extends AuthorizedRestDataService implements BaseNmdDataService {
// ToDo: implement class AUthorizedBAseDatabaseSession

	private final static Logger logger = Logger.getLogger(Nmd3DataService.class);
	private static final DateFormat dbDateFormat = new SimpleDateFormat("yyyyMMdd");
	
	private String apiPath = "/NMD_30_API_v1.1/api/NMD30_Web_API/";
	private Calendar requestDate;
	private NmdConfig config = null;
	private NmdReferenceResources resources;
	private Calendar tokenExpirationTime = null;
	
	NmdScalerFactory scalerFactory = new NmdScalerFactory();

	private List data;

	public Nmd3DataService(NmdConfig config) {
		setParameters(config);
	}

	private Nmd3DataService() {
		setParameters(new NmdConfigImpl());
		this.preLoadData();
	}
	
	private static Nmd3DataService instance = null;
	
    public static synchronized Nmd3DataService getInstance() {
        if(instance == null) {
            instance = new Nmd3DataService();
        }
        return instance;
    }
    
    public static synchronized Nmd3DataService getInstance(Path contextPath) {
        if(instance == null) {
            instance = new Nmd3DataService(new NmdConfigImpl(contextPath));
        }
        return instance;
    }
    
	private void setParameters(NmdConfig config) {
		this.setPort(-1);
		this.setHost("www.Milieudatabase-datainvoer.nl");
		this.setScheme("https");
		this.config = config;
		this.requestDate = Calendar.getInstance();
		requestDate.set(2019, 8, 15, 12, 00, 00);
		this.data = new ArrayList();
	}

	@Override
	public Calendar getRequestDate() {
		return this.requestDate;
	}

	@Override
	public void setRequestDate(Calendar newDate) {
		this.requestDate = newDate;
	}

	@Override
	public Boolean getIsConnected() {
		return tokenExpirationTime != null
				&& Calendar.getInstance().getTimeInMillis() + 1e4 <= this.tokenExpirationTime.getTimeInMillis();
	}

	@Override
	public List getData() {
		return data;
	}

	@Override
	public void login() {
		this.login(true);
	}

	@SuppressWarnings("null")
	public void login(boolean preloadResources) {
		// Try add a Post request to the base class.
		HttpPost httppost = new HttpPost("https://www.milieudatabase-datainvoer.nl/NMD_30_AuthenticationServer/"
				+ "NMD_30_API_Authentication/getToken");

		httppost.addHeader("refreshToken", config.getToken());
		httppost.addHeader("API_ID", "1");
		httppost.addHeader("Content-Type", "application/x-www-form-urlencoded");

		List params = new ArrayList();
		params.add(new BasicNameValuePair("grant_type", "client_credentials"));

		// Execute and get the response.
		HttpResponse response = null;
		try {
			httppost.setEntity(new UrlEncodedFormEntity(params));
			response = httpClient.execute(httppost);
			JsonNode responseNode = this.responseToJson(response);
			JsonNode tokenNode = responseNode.get("TOKEN");
			this.setToken(tokenNode.asText());
			this.tokenExpirationTime = Calendar.getInstance();
			this.tokenExpirationTime.add(Calendar.MINUTE, 30);

			httppost.releaseConnection();
		} catch (IOException e1) {
			this.tokenExpirationTime = null;
			logger.error("authentication failed: " + response.getStatusLine().toString());
		}

		if (preloadResources) {
			loadResources();
		}
	}

	@Override
	public void preLoadData() {
		this.data = this.getAllElements();
	}

	@Override
	public void logout() {
		this.setToken("");
		this.tokenExpirationTime = null;
		this.resources = null;
	}

	private void loadResources() {
		if (this.resources == null) {
			this.loadReferenceResources();
		}
	}

	public NmdReferenceResources getResources() {
		loadResources();
		return this.resources;
	}

	public void setResources(NmdReferenceResources resources) {
		this.resources = resources;
	}

	/*
	 * Get the reference resources to map database ids to meaningful fieldnames
	 */
	public void loadReferenceResources() {
		List params = new ArrayList();
		params.add(new KeyValuePair("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime())));
		resources = new NmdReferenceResources();

		try {
			performResourceRequest("Fasen", params, this::convertJsonToLifeCycleFasen);
			performResourceRequest("MilieuCategorien", params, this::convertJsonToMilieuCategorieen);
			performResourceRequest("Eenheden", params, this::convertJsonToUnits);
			performResourceRequest("CUAScategorien", params, this::convertJsonToCUASCodes);
			performResourceRequest("SchalingsFormules", params, this::convertJsonToScaling);
		} catch (Exception e) {
			logger.error("failed to retrieve reference resources");
		}
	}

	/**
	 * Generic request to add resource data to the overall resources
	 * 
	 * @param                   type of the resource to be stored
	 * @param resourceName         API name of the resource
	 * @param params               params for the api request
	 * @param resourceInsertMethod Consumer method for specific implementation how
	 *                             to convert Json to resource object
	 */
	private  void performResourceRequest(String resourceName, List params,
			BiConsumer> resourceInsertMethod) {
		// rekenregels reference data
		HttpResponse response = this.performGetRequestWithParams(apiPath + resourceName, params);
		JsonNode resp = this.responseToJson(response);

		if (resp.hasNonNull("results")) {
			JsonNode node = resp.get("results");
			Map resource = new HashMap();
			resourceInsertMethod.accept(node, resource);
		}
	}

	private void convertJsonToScaling(JsonNode node, Map resource) {
		node.forEach(formula -> {
			resource.putIfAbsent(TryParseJsonNode(formula.get("SchalingsFormuleID"), -1),
					TryParseJsonNode(formula.get("SoortFormule"), "") + " : "
							+ TryParseJsonNode(formula.get("Formule"), ""));
		});
		resources.setScalingFormula(resource);
	}

	private void convertJsonToCUASCodes(JsonNode node, Map resource) {
		node.forEach(cuas_code -> {
			resource.putIfAbsent(TryParseJsonNode(cuas_code.get("ID"), -1),
					TryParseJsonNode(cuas_code.get("CUAS_code"), ""));
		});
		resources.setCuasCategorieMapping(resource);
	}

	private void convertJsonToUnits(JsonNode node, Map resource) {
		node.forEach(eenheid -> {

			resource.putIfAbsent(TryParseJsonNode(eenheid.get("EenheidID"), -1),
					TryParseJsonNode(eenheid.get("Code"), ""));
		});
		resources.setUnitMapping(resource);
	}

	private void convertJsonToMilieuCategorieen(JsonNode node, Map resource) {
		node.forEach(categorie -> {
			NmdMilieuCategorie factor = new NmdMilieuCategorie(categorie.get("Milieueffect").asText(),
					categorie.get("Eenheid").asText(),
					categorie.get("Wegingsfactor") != null ? categorie.get("Wegingsfactor").asDouble() : 0.0);
			resource.putIfAbsent(Integer.parseInt(categorie.get("MilieuCategorieID").asText()), factor);
		});
		resources.setMilieuCategorieMapping(resource);
	}

	private void convertJsonToLifeCycleFasen(JsonNode node, Map resource) {
		node.forEach(fase -> {
			resource.putIfAbsent(Integer.parseInt(fase.get("FaseID").asText()), fase.get("FaseNaam").asText());
		});
		resources.setFaseMapping(resource);
	}

	@Override
	public List getAllElements() {
		if (this.data.size() > 0) { return data;}

		if (!this.getIsConnected()) {
			login();
		}

		List params = new ArrayList();
		params.add(new KeyValuePair("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime())));

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "NLsfB_RAW_Elementen", params);
		// do something with the entity to get the token
		JsonNode respNode = this.responseToJson(response);

		// convert reponseNode to NmdProductCard info.
		List results = new ArrayList();

		// parse the results and get the product cards
		if (respNode.hasNonNull("results")) {
			respNode.get("results").forEach(f -> results.add(this.getElementDataFromJson(f)));
			for (NmdElement el : results) {
				el.addProductCards(this.getProductsForElement(el));
			}
		}

		return results;
	}

	/**
	 * get child elements (deelelementen) for a list of parent elements
	 * 
	 * @param parentIds any possible parent element ids
	 * @return a list of child elements to these parent elements and any recurrent
	 *         children.
	 */
	private List getChildElements(List parentIds) {

		List results = new ArrayList();

		parentIds.forEach(id -> {
			results.addAll(this.getChildElement(id));
		});

		return results;
	}

	/**
	 * Call the MD REST api to call a single Element Onderdeel
	 * 
	 * @param id: unique identifier of a (possible) parent element
	 * @return a list of the possible child elements (and its children)
	 */
	private List getChildElement(Integer id) {

		List results = new ArrayList();
		try {
			List params_el = new ArrayList();
			params_el.add(new KeyValuePair("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime())));
			params_el.add(new KeyValuePair("ElementID", id));

			// load the json
			HttpResponse responseEl = this.performGetRequestWithParams(this.apiPath + "ElementOnderdelen", params_el);
			JsonNode respNodeEl = this.responseToJson(responseEl);
			if (respNodeEl != null) {
				// parse json to objects
				List newResults = new ArrayList();
				respNodeEl.get("results").forEach(f -> newResults.add(this.getElementDataFromJson(f)));
				if (newResults.size() > 0) {
					results.addAll(newResults);

					// repeat above process for new finds.
					List newResultIds = newResults.stream().map(r -> r.getElementId())
							.collect(Collectors.toList());
					results.addAll(getChildElements(newResultIds));
				}
			}

		} catch (NullPointerException e) {
			logger.error("error encountered while querying ElementOnderdelen with id: " + id.toString());
		}
		return results;
	}

	@Override
	public List getProductsForElement(NmdElement element) {
		if (!this.getIsConnected()) {
			login();
		}

		List params = new ArrayList();
		params.add(new KeyValuePair("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime())));
		params.add(new KeyValuePair("ElementID", element.getElementId()));

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "ProductenBijElement", params);

		JsonNode resp_node = this.responseToJson(response);
		if (resp_node == null) {
			return new ArrayList();
		}

		JsonNode producten = resp_node.get("results");
		List products = new ArrayList<>();
		if (producten != null) {
			producten.forEach(p -> {

				// only get items that are relevant for us.
				NmdProductCardImpl product = this.getProductCardDataFromJson(p);
				if (TryParseJsonNode(p.get("ProfielSetGekoppeld"), false) && product.getUnit() != null
						&& product.getLifetime() > 0) {
					if (this.getAdditionalProfileDataForCard(product)) {
						products.add(product);
						product.setNlsfbCode(element.getNlsfbCode());
					}
				}
			});
		}

		return products;
	}

	public HashMap> getQuantitiesOfProfileSetsForProduct(Integer productId) {
		if (!this.getIsConnected()) {
			login();
		}

		List params = new ArrayList();
		params.add(new KeyValuePair("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime())));
		params.add(new KeyValuePair("ProductID", productId.toString()));
		params.add(new KeyValuePair("includeNULLs", true));

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "ProfielsetsEnSchalingBijProduct",
				params);

		HashMap> res = new HashMap<>();
		// do something with the entity to get the token
		JsonNode resp_node = this.responseToJson(response);

		JsonNode psNodes = resp_node.get("results");
		if (psNodes != null) {
			psNodes.forEach(ps -> {
				NmdScaler scaler = getScalerFromJson(ps);
				Integer id = ps.get("ProfielSetID").asInt(-1);
				Double q = ps.get("Hoeveelheid").asDouble(1.0);
				q = q > 0.0 ? q : 1.0;
				res.put(id, new MutablePair(q, scaler));
			});
		}

		return res;
	}

	/*
	 * Add missing data to the NmdProfileSet
	 */
	@Override
	public Boolean getAdditionalProfileDataForCard(NmdProductCard c) {
		// check if data has already been loaded
		if (c.getProfileSets().size() > 0) {
			return true;
		}

		try {
			HashMap setData = getProfileSetsByIds(Arrays.asList(c.getProductId()));
			if (setData.size() > 0) {
				c.addProfileSets(setData.values());
				return true;
			}
		} catch (Exception e) {
			logger.warn("Could not find profile data for product with ID: " + c.getProductId().toString());
		}
		return false;
	}

	@Override
	public HashMap getProfileSetsByIds(List ids) {
		if (!this.getIsConnected()) {
			login();
		}
		HashMap profileSets = new HashMap<>();
		
		// First get the profileset data that is independent of the product card
		String idsString = String.join(",", ids.stream().map(id -> id.toString()).collect(Collectors.toList()));
		List params = new ArrayList();
		params.add(new KeyValuePair("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime())));
		params.add(new KeyValuePair("ProductIDs", idsString));
		params.add(new KeyValuePair("includeNULLs", true));

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "ProductenProfielWaarden", params);

		if (response == null) return profileSets;
		// do something with the entity to get the token
		JsonNode resp_node = this.responseToJson(response);

		if (resp_node.get("results") != null) {
			if (!(resp_node.get("results") instanceof TextNode)) {
				JsonNode profielSetNodes = resp_node.get("results").get(0).get("ProfielSet");
				profielSetNodes.forEach(profielSetNode -> {

					Integer profielSetId = TryParseJsonNode(profielSetNode.get("ProfielSetID"), -1);

					// load set specific data
					NmdProfileSetImpl set = this.getDetailedProfielSetData(profielSetNode);

					profileSets.put(profielSetId, set);
				});
			}
		}

		// per product card it also needs the quantity and scaler information
		if (profileSets.size() > 0) {
			HashMap> ratios = this.getQuantitiesOfProfileSetsForProduct(ids.get(0));
			profileSets.values().forEach(ps -> {
				Integer id = ps.getProfielId();
				if (ratios.containsKey(id)) {
					((NmdProfileSetImpl) ps).setQuantity(ratios.get(id).getKey());
					((NmdProfileSetImpl) ps).setScaler(ratios.get(id).getValue());
					((NmdProfileSetImpl) ps).setIsScalable(true);
				}
			});
		}
		return profileSets;
	}

	private void loadFaseProfielDataForSet(JsonNode profielSetNode, NmdProfileSetImpl set) {
		// laad faseprofiel specifieke data
		JsonNode profielen = profielSetNode.get("Profiel");
		if (profielen != null) {
			profielen.forEach(p -> {
				Integer fase = TryParseJsonNode(p.get("FaseID"), -1);
				String faseName = this.getResources().getFaseMapping().get(fase);

				NmdFaseProfielImpl profiel = new NmdFaseProfielImpl(faseName, this.getResources());

				p.get("ProfielMilieuEffecten").forEach(val -> {
					Integer catId = TryParseJsonNode(val.get("MilieuCategorieID"), -1);
					Double catVal = TryParseJsonNode(val.get("MilieuWaarde"), Double.NaN);

					profiel.setProfielCoefficient(
							this.getResources().getMilieuCategorieMapping().get(catId).getDescription(), catVal);
				});

				set.addFaseProfiel(faseName, profiel);
			});
		}
	}

	private NmdProfileSetImpl getDetailedProfielSetData(JsonNode psNode) {
		NmdProfileSetImpl set = new NmdProfileSetImpl();
		set.setProfileLifetime(TryParseJsonNode(psNode.get("Levensduur"), -1));
		set.setUnit(this.getResources().getUnitMapping().get(TryParseJsonNode(psNode.get("ProfielSetEenheidID"), -1)));

		set.setProfielId(TryParseJsonNode(psNode.get("ProfielSetID"), -1));
		set.setName(TryParseJsonNode(psNode.get("ProfielSetNaam"), ""));

		this.loadFaseProfielDataForSet(psNode, set);

		return set;
	}

	/*
	 * Try to get a set of profiel set data from the json node to populate the
	 * NmdProfielSetobject
	 */
	private NmdProductCardImpl getProductCardDataFromJson(JsonNode prodNode) {
		NmdProductCardImpl prod = new NmdProductCardImpl();

		prod.setLifetime(TryParseJsonNode(prodNode.get("Levensduur"), -1));
		prod.setUnit(
				this.getResources().getUnitMapping().get(TryParseJsonNode(prodNode.get("FunctioneleEenheidID"), -1)));
		prod.setProductId(TryParseJsonNode(prodNode.get("ProductID"), -1));
		prod.setParentProductId(TryParseJsonNode(prodNode.get("OuderProductID"), -1));

		prod.setIsTotaalProduct(TryParseJsonNode(prodNode.get("IsElementDekkend"), false));
		prod.setDescription(TryParseJsonNode(prodNode.get("ProductNaam"), ""));
		prod.setCategory(TryParseJsonNode(prodNode.get("CategorieID"), 3));

		return prod;
	}

	/*
	 * Get the essential product card info from an element JsonNode. With this info
	 * profielSet info can be retrieved
	 */
	private NmdElement getElementDataFromJson(JsonNode elementInfo) {
		NmdElementImpl newElement = new NmdElementImpl();
		newElement.setElementId(TryParseJsonNode(elementInfo.get("ElementID"), -1));

		// TODO: Omit any GWW elements?
		//	if (!TryParseJsonNode(elementInfo.get("ElementCode"), "").startsWith("gww")) { return null}
		if (elementInfo.has("PureElementCode")) {
			newElement.setNlsfbCode(new NlsfbCode(TryParseJsonNode(elementInfo.get("PureElementCode"), "")));
		} else {
			String elementCode = TryParseJsonNode(elementInfo.get("ElementCode"), "");
			String cleanedCode = elementCode.substring(0, elementCode.lastIndexOf("."));
			newElement.setNlsfbCode(new NlsfbCode(cleanedCode));
		}

		newElement.setElementName(TryParseJsonNode(elementInfo.get("ElementNaam"), ""));
		newElement.setParentId(TryParseJsonNode(elementInfo.get("OuderID"), -1));
		newElement.setIsMandatory(TryParseJsonNode(elementInfo.get("Verplicht"), false));

		return newElement;
	}
	
	private NmdScaler getScalerFromJson(JsonNode psNode) {
		int scalerType = TryParseJsonNode(psNode.get("SchalingsFormuleID"), -1);
		if (scalerType > 0) {
			String scalerDescription = TryParseJsonNode(psNode.get("OmschrijvingSchalingsmaat"), "");
			double scalerDim1 = TryParseJsonNode(psNode.get("SchalingsMaat_X1"), Double.NaN);
			double scalerDim2 = TryParseJsonNode(psNode.get("SchalingsMaat_X2"), Double.NaN);
			int scalerUnit = TryParseJsonNode(psNode.get("EenheidID_SchalingsMaat"), -1);

			double scalerMinDim1 = TryParseJsonNode(psNode.get("SchalingMinX1"), Double.NaN);
			double scalerMinDim2 = TryParseJsonNode(psNode.get("SchalingMinX2"), Double.NaN);
			double scalerMaxDim1 = TryParseJsonNode(psNode.get("SchalingMaxX1"), Double.NaN);
			double scalerMaxDim2 = TryParseJsonNode(psNode.get("SchalingMaxX2"), Double.NaN);

			JsonNode profNode = psNode.get("ProfielSetInfo").get(0);
			double scalerCoeffA = TryParseJsonNode(profNode.get("SchalingsFormuleA"), Double.NaN);
			double scalerCoeffB = TryParseJsonNode(profNode.get("SchalingsFormuleB"), Double.NaN);
			double scalerCoeffC = TryParseJsonNode(profNode.get("SchalingsFormuleC"), Double.NaN);

			String scalerTypeName = this.getResources().getScalingFormula().get(scalerType);
			String scalerUnitName = this.getResources().getUnitMapping().get(scalerUnit);
			NmdScaler scaler = null;
			try {
				scaler = scalerFactory.create(scalerTypeName, scalerDescription, scalerUnitName,
						new Double[] { scalerCoeffA, scalerCoeffB, scalerCoeffC },
						new Double[] { scalerMinDim1, scalerMaxDim1, scalerMinDim2, scalerMaxDim2 },
						new Double[] { scalerDim1, scalerDim2 });

			} catch (InvalidInputException e) {
				logger.error("encountered invalid input combinations in scaler creation");
			}
			return scaler;
		}
		return null;
	}
}