nl.tno.bim.nmd.services.Nmd3DataService Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of bimnmdservice Show documentation
provides a REST api for retrieving nmd data from various data sources
The newest version!
package nl.tno.bim.nmd.services;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashMap;
import java.util.InputMismatchException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;

import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.TextNode;

import nl.tno.bim.nmd.config.NmdConfig;
import nl.tno.bim.nmd.config.NmdConfigImpl;
import nl.tno.bim.nmd.domain.NlsfbCode;
import nl.tno.bim.nmd.domain.Nmd2ProductCard;
import nl.tno.bim.nmd.domain.NmdElement;
import nl.tno.bim.nmd.domain.NmdElementImpl;
import nl.tno.bim.nmd.domain.NmdFaseProfielImpl;
import nl.tno.bim.nmd.domain.NmdMilieuCategorie;
import nl.tno.bim.nmd.domain.NmdProductCard;
import nl.tno.bim.nmd.domain.NmdBaseProductCard;
import nl.tno.bim.nmd.domain.NmdProfileSet;
import nl.tno.bim.nmd.domain.NmdProfileSetImpl;
import nl.tno.bim.nmd.domain.NmdReferenceResources;
import nl.tno.bim.nmd.scaling.NmdBaseScaler;
import nl.tno.bim.nmd.scaling.NmdScaler;
import nl.tno.bim.nmd.scaling.NmdScalerFactory;

/**
 * Implementation of the NmdDataService to retrieve data from the NMD see :
 * https://www.milieudatabase.nl/ for mor information
 * 
 * @author vijj
 *
 */
public class Nmd3DataService extends AuthorizedRestDataService implements BaseNmdDataService {
// ToDo: implement class AUthorizedBAseDatabaseSession

	private final static Logger logger = LoggerFactory.getLogger(Nmd3DataService.class);
	private static final DateFormat dbDateFormat = new SimpleDateFormat("yyyyMMdd");
	
	private String apiPath = "/NMD_30_API_v1.1/api/NMD30_Web_API/";
	private Calendar requestDate;
	private NmdConfig config = null;
	private NmdReferenceResources resources;
	private Calendar tokenExpirationTime = null;
	private Boolean useCache = true;
	
	NmdScalerFactory scalerFactory = new NmdScalerFactory();

	private List data;
	private Map productTypeLookup = new HashMap<>();

	public Nmd3DataService(NmdConfig config) {
		setParameters(config);
	}

	private Nmd3DataService() throws FileNotFoundException {
		setParameters(new NmdConfigImpl());
		this.preLoadData();
	}
	
	private static Nmd3DataService instance = null;
	
    public static synchronized Nmd3DataService getInstance() throws FileNotFoundException {
        if(instance == null) {
            instance = new Nmd3DataService();
        }
        return instance;
    }
        
	private void setParameters(NmdConfig config) {
		this.setPort(-1);
		this.setHost("www.Milieudatabase-datainvoer.nl");
		this.setScheme("https");
		this.config = config;
		this.requestDate = Calendar.getInstance();
		requestDate.set(2019, 11, 15, 12, 00, 00);
		this.data = new ArrayList();
	}
	
	/**
	 * Return a KVP list that is required for every get or post request
	 * @return stand LIst containing a pre-generated SearchDate parameter
	 */
	private Map getBaseParamsForRequest() {
		Map params = new HashMap<>();
		params.put("ZoekDatum", dbDateFormat.format(this.getRequestDate().getTime()));
		return params;
	}

	@Override
	public Calendar getRequestDate() {
		return this.requestDate;
	}

	@Override
	public void setRequestDate(Calendar newDate) {
		this.requestDate = newDate;
	}

	@Override
	public Boolean getIsConnected() {
		return tokenExpirationTime != null
				&& Calendar.getInstance().getTimeInMillis() + 1e4 <= this.tokenExpirationTime.getTimeInMillis();
	}

	@Override
	public List getData() {
		return data;
	}

	@Override
	public void login() {
		this.login(true);
	}

	@SuppressWarnings("null")
	public void login(boolean preloadResources) {
		// Try add a Post request to the base class.
		HttpPost httppost = new HttpPost("https://www.milieudatabase-datainvoer.nl/NMD_30_AuthenticationServer/"
				+ "NMD_30_API_Authentication/getToken");

		httppost.addHeader("refreshToken", config.getToken());
		httppost.addHeader("API_ID", "1");
		httppost.addHeader("Content-Type", "application/x-www-form-urlencoded");

		List params = new ArrayList();
		params.add(new BasicNameValuePair("grant_type", "client_credentials"));

		// Execute and get the response.
		HttpResponse response = null;
		try {
			httppost.setEntity(new UrlEncodedFormEntity(params));
			response = httpClient.execute(httppost);
			JsonNode responseNode = this.responseToJson(response);
			JsonNode tokenNode = responseNode.get("TOKEN");
			this.setToken(tokenNode.asText());
			this.tokenExpirationTime = Calendar.getInstance();
			this.tokenExpirationTime.add(Calendar.MINUTE, 30);

			httppost.releaseConnection();
		} catch (IOException e1) {
			this.tokenExpirationTime = null;
			logger.error("authentication failed: " + response.getStatusLine().toString());
		}

		if (preloadResources) {
			loadResources();
		}
	}

	@Override
	public void preLoadData() {
		this.data = this.getAllElements();
	}

	@Override
	public void logout() {
		this.setToken("");
		this.tokenExpirationTime = null;
		this.resources = null;
	}

	private void loadResources() {
		if (this.resources == null) {
			this.loadReferenceResources();
		}
	}

	public NmdReferenceResources getResources() {
		loadResources();
		return this.resources;
	}

	public void setResources(NmdReferenceResources resources) {
		this.resources = resources;
	}

	/*
	 * Get the reference resources to map database ids to meaningful fieldnames
	 */
	public void loadReferenceResources() {
		Map params = this.getBaseParamsForRequest();
		resources = new NmdReferenceResources();

		try {
			performResourceRequest("Fasen", params, this::convertJsonToLifeCycleFasen);
			performResourceRequest("MilieuCategorien", params, this::convertJsonToMilieuCategorieen);
			performResourceRequest("Eenheden", params, this::convertJsonToUnits);
			performResourceRequest("CUAScategorien", params, this::convertJsonToCUASCodes);
			performResourceRequest("SchalingsFormules", params, this::convertJsonToScaling);
			
			getNmd23ScalingTypes();
		} catch (Exception e) {
			logger.error("failed to retrieve reference resources");
		}
	}
	
	/**
	 * Generic request to add resource data to the overall resources
	 * 
	 * @param                   type of the resource to be stored
	 * @param resourceName         API name of the resource
	 * @param params               params for the api request
	 * @param resourceInsertMethod Consumer method for specific implementation how
	 *                             to convert Json to resource object
	 */
	private  void performResourceRequest(String resourceName, Map params,
			BiConsumer> resourceInsertMethod) {
		// rekenregels reference data
		HttpResponse response = this.performGetRequestWithParams(apiPath + resourceName, params, this.useCache);
		JsonNode resp = this.responseToJson(response);

		if (resp.hasNonNull("results")) {
			JsonNode node = resp.get("results");
			Map resource = new HashMap();
			resourceInsertMethod.accept(node, resource);
		}
	}

	private void convertJsonToScaling(JsonNode node, Map resource) {
		node.forEach(formula -> {
			resource.putIfAbsent(TryParseJsonNode(formula.get("SchalingsFormuleID"), -1),
					TryParseJsonNode(formula.get("SoortFormule"), "") + " : "
							+ TryParseJsonNode(formula.get("Formule"), ""));
		});
		resources.setScalingFormula(resource);
	}

	private void convertJsonToCUASCodes(JsonNode node, Map resource) {
		node.forEach(cuas_code -> {
			resource.putIfAbsent(TryParseJsonNode(cuas_code.get("ID"), -1),
					TryParseJsonNode(cuas_code.get("CUAS_code"), ""));
		});
		resources.setCuasCategorieMapping(resource);
	}

	private void convertJsonToUnits(JsonNode node, Map resource) {
		node.forEach(eenheid -> {

			resource.putIfAbsent(TryParseJsonNode(eenheid.get("EenheidID"), -1),
					TryParseJsonNode(eenheid.get("Code"), ""));
		});
		resources.setUnitMapping(resource);
	}

	private void convertJsonToMilieuCategorieen(JsonNode node, Map resource) {
		node.forEach(categorie -> {
			NmdMilieuCategorie factor = new NmdMilieuCategorie(categorie.get("Milieueffect").asText(),
					categorie.get("Eenheid").asText(),
					categorie.get("Wegingsfactor") != null ? categorie.get("Wegingsfactor").asDouble() : 0.0);
			resource.putIfAbsent(Integer.parseInt(categorie.get("MilieuCategorieID").asText()), factor);
		});
		resources.setMilieuCategorieMapping(resource);
	}

	private void convertJsonToLifeCycleFasen(JsonNode node, Map resource) {
		node.forEach(fase -> {
			resource.putIfAbsent(Integer.parseInt(fase.get("FaseID").asText()), fase.get("FaseNaam").asText());
		});
		resources.setFaseMapping(resource);
	}
	
	/**
	 * Retrieve a table with which products are scaled in which manner. This will only contain NMD23
	 * product cards. any other products have the scaling data contained in the profielset and/or 
	 * quantity/scaler endpoint
	 */
	private void getNmd23ScalingTypes() {
		String endpoint = "SchalingsTypenBijNMD23_Producten";
		try {
			Map params = this.getBaseParamsForRequest();
			HttpResponse response = this.performGetRequestWithParams(this.apiPath + endpoint, params,
					this.useCache);
			// do something with the entity to get the token
			JsonNode respNode = this.responseToJson(response);
			
			if (respNode.hasNonNull("results")) {
				JsonNode node = respNode.get("results");
				node.forEach(entry -> {
					// 1 scaling type is no scaling, 2 is linear scaling, 3 is MVT, nvt is scaling according to NMD3
					String scalingType = TryParseJsonNode(entry.get("NMD23_SchalingsType"), "nvt");
					Integer productId = TryParseJsonNode(entry.get("ProductID"), -1);
					if (productId > 0 && !scalingType.equals("nvt")) {
						this.productTypeLookup.put(productId, Integer.parseInt(scalingType));
					}
				});
			}
		} catch (Exception e) {
			logger.error(String.format("failed to retrieve data from endpoint %s with message: %s", endpoint, e.getMessage()));
		}
	}
	
	/**
	 * Main method to retrieve all NMD elements and each product card in those elements. 
	 * Will do a recursive search to also find partial elements.
	 */
	@Override
	public List getAllElements() {
		if (this.data.size() > 0) { return data;}

		if (!this.getIsConnected()) {
			login();
		}

		Map params = this.getBaseParamsForRequest();
		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "NLsfB_RAW_Elementen", params, this.useCache);
		// do something with the entity to get the token
		JsonNode respNode = this.responseToJson(response);

		// convert reponseNode to NmdProductCard info.
		List results = new ArrayList();

		// parse the results and get the product cards
		if (respNode.hasNonNull("results")) {
			respNode.get("results").forEach(f -> results.add(this.getElementDataFromJson(f)));
		}
		
		// load child elements for each main element
		//results.addAll(getChildElements(results.stream()
		//		.map(el -> el.getElementId()).collect(Collectors.toList())));
				
		// load product cards for all elements
		results.forEach(el -> el.addProductCards(this.getProductsForElement(el)));
		
		// last but not least: load all table sclaing parameters (requires all products to be loaded)
		//ERROR: Duplicate keys!
		List allTableProducts = results.stream().flatMap(el -> el.getProducts().stream())
				.filter(pc -> productTypeLookup.getOrDefault(pc.getProductId(), -1) == 3)
				.collect(Collectors.toList());
		
		allTableProducts.stream()
			.forEach(pc ->{
				this.loadTableScalersForProduct((Nmd2ProductCard)pc, allTableProducts);
			});
		
		logger.info("All Elements loaded");
		return results;
	}

	/**
	 * get child elements (deelelementen) for a list of parent elements
	 * 
	 * @param parentIds any possible parent element ids
	 * @return a list of child elements to these parent elements and any recurrent
	 *         children.
	 */
	private List getChildElements(List parentIds) {

		List results = new ArrayList();

		parentIds.forEach(id -> {
			results.addAll(this.getChildElement(id));
		});

		return results;
	}

	/**
	 * Call the MD REST api to call a single Element Onderdeel
	 * 
	 * @param id: unique identifier of a (possible) parent element
	 * @return a list of the possible child elements (and its children)
	 */
	private List getChildElement(Integer id) {

		List results = new ArrayList();
		try {
			Map params_el = this.getBaseParamsForRequest();
			params_el.put("ElementID", id);

			// load the json
			HttpResponse responseEl = this.performGetRequestWithParams(this.apiPath + "ElementOnderdelen", params_el, this.useCache);
			JsonNode respNodeEl = this.responseToJson(responseEl);
			if (respNodeEl != null) {
				// parse json to objects
				List newResults = new ArrayList();
				respNodeEl.get("results").forEach(f -> newResults.add(this.getElementDataFromJson(f)));
				if (newResults.size() > 0) {
					results.addAll(newResults);

					// repeat above process for new finds.
					List newResultIds = newResults.stream().map(r -> r.getElementId())
							.collect(Collectors.toList());
					results.addAll(getChildElements(newResultIds));
				}
			}

		} catch (NullPointerException e) {
			logger.error("error encountered while querying ElementOnderdelen with id: " + id.toString());
		}
		return results;
	}

	@Override
	public List getProductsForElement(NmdElement element) {
		if (!this.getIsConnected()) {
			login();
		}

		Map params = this.getBaseParamsForRequest();
		params.put("ElementID", element.getElementId());

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "ProductenBijElement", params, this.useCache);

		JsonNode resp_node = this.responseToJson(response);
		if (resp_node == null) {
			return new ArrayList();
		}

		JsonNode productNodes = resp_node.get("results");
		List products = new ArrayList<>();
		if (productNodes != null) {
			productNodes.forEach(p -> {

				// only get items that are relevant for us.
				NmdProductCard product = this.getProductCardDataFromJson(p, element.getNlsfbCode());
				if (TryParseJsonNode(p.get("ProfielSetGekoppeld"), false) && product.getUnit() != null
						&& product.getLifetime() > 0) {
					if (this.getAdditionalProfileDataForCard(product)) {
						products.add(product);
					}
				}
				
				// load additional scaling information as this is a NMD2.3 product card
				Integer pId = product.getProductId();
				if (this.productTypeLookup.containsKey(pId) && product instanceof Nmd2ProductCard) {
					this.getNMD23ScalingInfoForProduct((Nmd2ProductCard)product, this.productTypeLookup.get(pId));
				}				
			});
		}

		return products;
	}

	private void getNMD23ScalingInfoForProduct(Nmd2ProductCard product, Integer scaleType) {
		Map params = this.getBaseParamsForRequest();
		params.put("ProductID", product.getProductId().toString());
		switch (scaleType) {
			case 1:
				product.getProfileSets().forEach(ps -> ((NmdProfileSetImpl)ps).setIsScalable(false));
				break;
			case 2:
				HttpResponse response = this.performGetRequestWithParams(this.apiPath + "NMD23_Type2_SchalingInfoBijProduct",
						params, this.useCache);
				JsonNode node = responseToJson(response);
				NmdScaler scaler = this.createLinearNMD23ScalerFromJson(node);
				product.setScaler(scaler);
				break;
			case 3:
				// defer table scale loading and parsing until all productcards are loadded to avoid infinite loops
				break;
			default:
				logger.warn(String.format("Could not parse scale type for product %d", product.getProductId()));
				break;
		}
	}

	private void loadTableScalersForProduct(Nmd2ProductCard prod, List allProducts) {
		Map params = this.getBaseParamsForRequest();
		params.put("ProductID", prod.getProductId().toString());
		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "NMD23_MVT_infoBijProduct",
				params, this.useCache);
		JsonNode node = responseToJson(response);
		JsonNode scaleNode = node.get("results").get(0);
		
		if (scaleNode != null) {
			
			
			String unit = TryParseJsonNode(scaleNode.get("mvt_eenheid"), "");
			Map prodVals = new HashMap<>();
			scaleNode.get("mvt_varianten").forEach(varNode -> {
				prodVals.put(
					TryParseJsonNode(varNode.get("VariantProductID"), -1),
					TryParseJsonNode(varNode.get("VariantDimensie_1"), Double.NaN));
			});
			
			List tableProds = allProducts.stream()
					.filter(pc -> prodVals.keySet().contains(pc.getProductId()))
					.collect(Collectors.toList());
			
			// find quantities of the profile sets in the other products for the reference scaling dims
			prod.getProfileSets().forEach(ps -> {
				String psName = ps.getName();
				Map refDims = tableProds.parallelStream()
						.collect(Collectors.toMap(
								pc -> {
									return (Double)prodVals.get(pc.getProductId());
								},
								pc -> {
									Optional pcPs = pc.getProfileSetByName(psName);
									return pcPs.isPresent() ? pcPs.get().getQuantity() : Double.NaN; 
								}));
				
				Double[] cardRefDim = new Double[] {prodVals.get(prod.getProductId())};
				
				NmdScaler scaler = scalerFactory.createTableScaler(unit, refDims, cardRefDim);
				((NmdProfileSetImpl)ps).setScaler(scaler);
			});
		}		
	}

	private NmdScaler createLinearNMD23ScalerFromJson(JsonNode node) {
		
		NmdScaler scaler = null;
		try {
			JsonNode scaleNode = node.get("results").get(0);
			if (scaleNode != null) {
				Double scalerDim1 = TryParseJsonNode(scaleNode.get("Dimensie_1"), Double.NaN);
				Double scalerDim2 = TryParseJsonNode(scaleNode.get("Dimensie_2"), Double.NaN);
				String scaleUnit1 = TryParseJsonNode(scaleNode.get("rechtev_eenheid_dim1"), "");
				String scaleUnit2 = TryParseJsonNode(scaleNode.get("rechtev_eenheid_dim2"), "");
				String description = String.join("|",
					TryParseJsonNode(scaleNode.get("rechtev_grootheid_dim1"), ""),
					TryParseJsonNode(scaleNode.get("rechtev_grootheid_dim2"), ""));
				
				if (!(scalerDim1.isNaN() || scaleUnit1.isEmpty())) {
					scaler = scalerFactory.createLinScaler(scaleUnit1,
							new Double[] { 1d, 0d, 0d },
							new Double[] { Double.NaN, Double.NaN, Double.NaN, Double.NaN },
							new Double[] { scalerDim1, scalerDim2 });
					((NmdBaseScaler)scaler).setDescription(description);
				}
			}
		} catch (Exception e) {
			logger.warn("Could not create nmd23 scaler from json");
		}
		
		return scaler;
	}

	public HashMap> getQuantitiesOfProfileSetsForProduct(Integer productId) {
		if (!this.getIsConnected()) {
			login();
		}

		Map params = this.getBaseParamsForRequest();
		params.put("ProductID", productId.toString());
		params.put("includeNULLs", true);

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "ProfielsetsEnSchalingBijProduct",
				params, this.useCache);

		HashMap> res = new HashMap<>();
		// do something with the entity to get the token
		JsonNode resp_node = this.responseToJson(response);

		JsonNode psNodes = resp_node.get("results");
		if (psNodes != null) {
			psNodes.forEach(ps -> {
				NmdScaler scaler = getScalerFromJson(ps);
				Integer id = ps.get("ProfielSetID").asInt(-1);
				Double q = ps.get("Hoeveelheid").asDouble(1.0);
				q = q > 0.0 ? q : 1.0;
				res.put(id, new MutablePair(q, scaler));
			});
		}

		return res;
	}

	/*
	 * Add missing data to the NmdProfileSet
	 */
	@Override
	public Boolean getAdditionalProfileDataForCard(NmdProductCard c) {
		// check if data has already been loaded
		if (c.getProfileSets().size() > 0) {
			return true;
		}

		try {
			HashMap setData = getProfileSetsByIds(Arrays.asList(c.getProductId()));
			if (setData.size() > 0) {
				c.addProfileSets(setData.values());
				return true;
			}
		} catch (Exception e) {
			logger.warn("Could not find profile data for product with ID: " + c.getProductId().toString() + " and name: " + c.getDescription());
		}
		return false;
	}

	@Override
	public HashMap getProfileSetsByIds(List ids) {
		if (!this.getIsConnected()) {
			login();
		}
		HashMap profileSets = new HashMap<>();
		
		// First get the profileset data that is independent of the product card
		String idsString = String.join(",", ids.stream().map(id -> id.toString()).collect(Collectors.toList()));
		Map params = this.getBaseParamsForRequest();
		params.put("ProductIDs", idsString);
		params.put("includeNULLs", true);

		HttpResponse response = this.performGetRequestWithParams(this.apiPath + "ProductenProfielWaarden", params, this.useCache);

		if (response == null) return profileSets;
		// do something with the entity to get the token
		JsonNode resp_node = this.responseToJson(response);

		if (resp_node.get("results") != null) {
			if (!(resp_node.get("results") instanceof TextNode)) {
				JsonNode profielSetNodes = resp_node.get("results").get(0).get("ProfielSet");
				profielSetNodes.forEach(profielSetNode -> {

					Integer profielSetId = TryParseJsonNode(profielSetNode.get("ProfielSetID"), -1);

					// load set specific data
					NmdProfileSetImpl set = this.getDetailedProfielSetData(profielSetNode);

					profileSets.put(profielSetId, set);
				});
			}
		}

		// per product card it also needs the quantity and scaler information
		if (profileSets.size() > 0) {
			HashMap> ratios = this.getQuantitiesOfProfileSetsForProduct(ids.get(0));
			profileSets.values().forEach(ps -> {
				Integer id = ps.getProfielId();
				if (ratios.containsKey(id)) {
					((NmdProfileSetImpl) ps).setQuantity(ratios.get(id).getKey());
					if (ratios.get(id).getValue() != null) {
						((NmdProfileSetImpl) ps).setScaler(ratios.get(id).getValue());
					}
					((NmdProfileSetImpl) ps).setIsScalable(true);
				}
			});
		}
		return profileSets;
	}

	private void loadFaseProfielDataForSet(JsonNode profielSetNode, NmdProfileSetImpl set) {
		// laad faseprofiel specifieke data
		JsonNode profielen = profielSetNode.get("Profiel");
		if (profielen != null) {
			profielen.forEach(p -> {
				Integer fase = TryParseJsonNode(p.get("FaseID"), -1);
				String faseName = this.getResources().getFaseMapping().get(fase);

				NmdFaseProfielImpl profiel = new NmdFaseProfielImpl(faseName, this.getResources());

				p.get("ProfielMilieuEffecten").forEach(val -> {
					Integer catId = TryParseJsonNode(val.get("MilieuCategorieID"), -1);
					Double catVal = TryParseJsonNode(val.get("MilieuWaarde"), Double.NaN);

					profiel.setProfielCoefficient(
							this.getResources().getMilieuCategorieMapping().get(catId).getDescription(), catVal);
				});

				set.addFaseProfiel(faseName, profiel);
			});
		}
	}

	private NmdProfileSetImpl getDetailedProfielSetData(JsonNode psNode) {
		NmdProfileSetImpl set = new NmdProfileSetImpl();
		set.setProfileLifetime(TryParseJsonNode(psNode.get("Levensduur"), -1));
		set.setUnit(this.getResources().getUnitMapping().get(TryParseJsonNode(psNode.get("ProfielSetEenheidID"), -1)));

		set.setProfielId(TryParseJsonNode(psNode.get("ProfielSetID"), -1));
		set.setName(TryParseJsonNode(psNode.get("ProfielSetNaam"), ""));

		this.loadFaseProfielDataForSet(psNode, set);
		
		NmdScaler scaler = getScalerFromJson(psNode);
		if (scaler != null) {
			set.setScaler(scaler);
		}

		return set;
	}

	/**
	 * Try to get a set of profiel set data from the json node to populate the
	 * NmdProfielSetobject
	 * @param prodNode Json node to read product card infomration form
	 * @param nlsfbCode parent element nlsfb code
	 * @return a ProductCard object
	 */
	private NmdBaseProductCard getProductCardDataFromJson(JsonNode prodNode, NlsfbCode nlsfbCode) {
		NmdBaseProductCard prod;
		Boolean isNMD23Card = TryParseJsonNode(prodNode.get("IsConvertedFromNMD23"), false);
		if (isNMD23Card) {
			prod = new Nmd2ProductCard();
		} else {
			prod = new NmdBaseProductCard();
		}
		
		prod.setLifetime(TryParseJsonNode(prodNode.get("Levensduur"), -1));
		prod.setUnit(
				this.getResources().getUnitMapping().get(TryParseJsonNode(prodNode.get("FunctioneleEenheidID"), -1)));
		prod.setProductId(TryParseJsonNode(prodNode.get("ProductID"), -1));
		prod.setParentProductId(TryParseJsonNode(prodNode.get("OuderProductID"), -1));

		prod.setDescription(TryParseJsonNode(prodNode.get("ProductNaam"), ""));
		prod.setCategory(TryParseJsonNode(prodNode.get("CategorieID"), 3));

		prod.setNlsfbCode(nlsfbCode);
		return prod;
	}

	/*
	 * Get the essential product card info from an element JsonNode. With this info
	 * profielSet info can be retrieved
	 */
	private NmdElement getElementDataFromJson(JsonNode elementInfo) {
		NmdElementImpl newElement = new NmdElementImpl();
		newElement.setElementId(TryParseJsonNode(elementInfo.get("ElementID"), -1));

		// TODO: Omit any GWW elements?
		//	if (!TryParseJsonNode(elementInfo.get("ElementCode"), "").startsWith("gww")) { return null}
		if (elementInfo.has("PureElementCode")) {
			newElement.setNlsfbCode(new NlsfbCode(TryParseJsonNode(elementInfo.get("PureElementCode"), "")));
		} else {
			String elementCode = TryParseJsonNode(elementInfo.get("ElementCode"), "");
			String cleanedCode = elementCode.substring(0, elementCode.lastIndexOf("."));
			newElement.setNlsfbCode(new NlsfbCode(cleanedCode));
		}

		newElement.setElementName(TryParseJsonNode(elementInfo.get("ElementNaam"), ""));
		newElement.setParentId(TryParseJsonNode(elementInfo.get("OuderID"), -1));
		newElement.setIsMandatory(TryParseJsonNode(elementInfo.get("Verplicht"), false));

		return newElement;
	}
	
	private NmdScaler getScalerFromJson(JsonNode psNode) {
		int scalerType = TryParseJsonNode(psNode.get("SchalingsFormuleID"), -1);
		if (scalerType > 0) {
			String scalerDescription = TryParseJsonNode(psNode.get("OmschrijvingSchalingsmaat"), "");
			double scalerDim1 = TryParseJsonNode(psNode.get("SchalingsMaat_X1"), Double.NaN);
			double scalerDim2 = TryParseJsonNode(psNode.get("SchalingsMaat_X2"), Double.NaN);
			int scalerUnit = TryParseJsonNode(psNode.get("EenheidID_SchalingsMaat"), -1);

			double scalerMinDim1 = TryParseJsonNode(psNode.get("SchalingMinX1"), Double.NaN);
			double scalerMinDim2 = TryParseJsonNode(psNode.get("SchalingMinX2"), Double.NaN);
			double scalerMaxDim1 = TryParseJsonNode(psNode.get("SchalingMaxX1"), Double.NaN);
			double scalerMaxDim2 = TryParseJsonNode(psNode.get("SchalingMaxX2"), Double.NaN);

			JsonNode profNode = psNode.get("ProfielSetInfo").get(0);
			double scalerCoeffA = TryParseJsonNode(profNode.get("SchalingsFormuleA"), Double.NaN);
			double scalerCoeffB = TryParseJsonNode(profNode.get("SchalingsFormuleB"), Double.NaN);
			double scalerCoeffC = TryParseJsonNode(profNode.get("SchalingsFormuleC"), Double.NaN);

			String scalerTypeName = this.getResources().getScalingFormula().get(scalerType);
			String scalerUnitName = this.getResources().getUnitMapping().get(scalerUnit);
			NmdScaler scaler = null;
			try {
				scaler = scalerFactory.create(scalerTypeName, scalerDescription, scalerUnitName,
						new Double[] { scalerCoeffA, scalerCoeffB, scalerCoeffC },
						new Double[] { scalerMinDim1, scalerMaxDim1, scalerMinDim2, scalerMaxDim2 },
						new Double[] { scalerDim1, scalerDim2 });

			} catch (InputMismatchException e) {
				logger.error("encountered invalid input combinations in scaler creation");
			}
			return scaler;
		}
		return null;
	}
}