com.yahoo.bard.webservice.data.PreResponseDeserializer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of fili-core Show documentation
Show all versions of fili-core Show documentation
Fili web service library provides core capabilities for RESTful aggregation navigation, query planning and
metadata
// Copyright 2016 Yahoo Inc.
// Licensed under the terms of the Apache license. Please see LICENSE.md file distributed with this work for terms.
package com.yahoo.bard.webservice.data;
import static com.yahoo.bard.webservice.data.PreResponseSerializationProxy.RESPONSE_CONTEXT_KEY;
import static com.yahoo.bard.webservice.data.PreResponseSerializationProxy.RESULT_SET_KEY;
import static com.yahoo.bard.webservice.data.ResultSerializationProxy.DIMENSION_VALUES_KEY;
import static com.yahoo.bard.webservice.data.ResultSerializationProxy.METRIC_VALUES_KEY;
import static com.yahoo.bard.webservice.data.ResultSerializationProxy.TIMESTAMP_KEY;
import static com.yahoo.bard.webservice.data.ResultSetSerializationProxy.RESULTS_KEY;
import static com.yahoo.bard.webservice.data.ResultSetSerializationProxy.SCHEMA_DIM_COLUMNS;
import static com.yahoo.bard.webservice.data.ResultSetSerializationProxy.SCHEMA_GRANULARITY;
import static com.yahoo.bard.webservice.data.ResultSetSerializationProxy.SCHEMA_KEY;
import static com.yahoo.bard.webservice.data.ResultSetSerializationProxy.SCHEMA_METRIC_COLUMNS_TYPE;
import static com.yahoo.bard.webservice.data.ResultSetSerializationProxy.SCHEMA_TIMEZONE;
import com.yahoo.bard.webservice.config.SystemConfig;
import com.yahoo.bard.webservice.config.SystemConfigProvider;
import com.yahoo.bard.webservice.data.dimension.Dimension;
import com.yahoo.bard.webservice.data.dimension.DimensionColumn;
import com.yahoo.bard.webservice.data.dimension.DimensionDictionary;
import com.yahoo.bard.webservice.data.dimension.DimensionRow;
import com.yahoo.bard.webservice.data.metric.MetricColumn;
import com.yahoo.bard.webservice.data.metric.MetricColumnWithValueType;
import com.yahoo.bard.webservice.data.time.Granularity;
import com.yahoo.bard.webservice.data.time.GranularityParser;
import com.yahoo.bard.webservice.table.Column;
import com.yahoo.bard.webservice.util.GranularityParseException;
import com.yahoo.bard.webservice.web.ErrorMessageFormat;
import com.yahoo.bard.webservice.web.PreResponse;
import com.yahoo.bard.webservice.web.responseprocessors.ResponseContext;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Streams;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Class to de-serialize and prepare the PreResponse object from JSON. The advantages of custom deserialization are:
* a. We can reconstruct the dimension rows for each result with the help of dimension dictionary.
* b. Schema object de-serialization is more efficient then the default one.
* c. Provides the flexibility of optimizing serialization
*
* Sample serialized PreResponse string
*
*
* {
"responseContext": "[
"com.yahoo.bard.webservice.web.responseprocessors.ResponseContext",
{
"randomHeader": "someHeader",
"missingIntervals": ["java.util.ArrayList",
["a","b", "c",
["java.util.ArrayList",
[
[
"org.joda.time.Interval",
"2011-02-02T07:00:00.000Z/2011-02-03T10:15:00.000Z"
]
]
],
[
"java.math.BigDecimal",100
]
]
]
}
]",
"resultSet": {
"results": [
{
"dimensionRows": {
"ageBracket": "1",
"country": "US",
"gender": "m"
},
"metricsRows": {
"lookbackPageViews": 112,
"retentionPageViews": 113,
"simplePageViews": 111
},
"timeStamp": "2016-01-12T00:00:00.000Z"
},
{
"dimensionRows": {
"ageBracket": "4",
"country": "IN",
"gender": "f"
},
"metricsRows": {
"lookbackPageViews": 212,
"retentionPageViews": 213,
"simplePageViews": 211
},
"timeStamp": "2016-01-12T00:00:00.000Z"
}
],
"schema": {
"dimensionColumns": [
"ageBracket",
"country",
"gender"
],
"granularity": "day",
"metricColumns": [
"lookbackPageViews",
"simplePageViews",
"retentionPageViews"
],
"metricColumnsType": {
"lookbackPageViews": "java.math.BigDecimal",
"retentionPageViews": "java.math.BigDecimal",
"simplePageViews": "java.math.BigDecimal"
},
"timeZone": "UTC"
}
}
}
*
*
*/
public class PreResponseDeserializer {
public static final SystemConfig SYSTEM_CONFIG = SystemConfigProvider.getInstance();
private static final Logger LOG = LoggerFactory.getLogger(PreResponseDeserializer.class);
private final DimensionDictionary dimensionDictionary;
private final GranularityParser granularityParser;
private final ObjectMapper nonResponseContextMapper;
private final ObjectMapper responseContextMapper;
/**
* Class constructor.
*
* @param dimensionDictionary DimensionDictionary which contains all the details about dimensions
* @param nonResponseContextMapper Handles the deserialization of everything except the response context
* @param responseContextMapper Handles the deserialization of the response context of the PreResponse
* @param granularityParser Time grain provider
*/
public PreResponseDeserializer(
DimensionDictionary dimensionDictionary,
ObjectMapper nonResponseContextMapper,
ObjectMapper responseContextMapper,
GranularityParser granularityParser
) {
this.dimensionDictionary = dimensionDictionary;
this.nonResponseContextMapper = nonResponseContextMapper;
this.responseContextMapper = responseContextMapper;
this.granularityParser = granularityParser;
}
/**
* Deserialize the custom serialized PreResponse.
*
* @param preResponse Custom serialized PreResponse
*
* @return De-serialized PreResponse object
*
* @throws IOException in case of deserialization of ResponseContext fails
*/
public PreResponse deserialize(String preResponse) throws IOException {
JsonNode serializedPreResponse = nonResponseContextMapper.readTree(preResponse);
return new PreResponse(
getResultSet(serializedPreResponse.get(RESULT_SET_KEY)),
getResponseContext(serializedPreResponse.get(RESPONSE_CONTEXT_KEY))
);
}
/**
* Deserialize the serialized ResponseContext. Method throws an IOException when mapper fails to read the
* serialized ResponseContext.
*
* @param serializedResponseContext serialized ResponseContext as jsonNode
*
* @return Deserialized responseContext
*
* @throws IOException when there's a problem reading the response context from the JsonNode
*/
private ResponseContext getResponseContext(JsonNode serializedResponseContext) throws IOException {
return responseContextMapper.readValue(serializedResponseContext.asText(), ResponseContext.class);
}
/**
* Generates ResultSet object from the JsonNode which contains the serialized ResultSet.
*
* @param serializedResultSet JsonNode which contains the serialized ResultSet
*
* @return ResultSet object generated from JsonNode
*/
private ResultSet getResultSet(JsonNode serializedResultSet) {
ResultSetSchema resultSetSchema = getResultSetSchema(serializedResultSet.get(SCHEMA_KEY));
List results = Streams.stream(serializedResultSet.get(RESULTS_KEY))
.map(serializedResult -> getResult(serializedResult, resultSetSchema))
.collect(Collectors.toList());
return new ResultSet(resultSetSchema, results);
}
/**
* Generates ZonedSchema object from given JsonNode.
*
* @param schemaNode JsonNode which contains all the columns, timezone and granularity
*
* @return ResultSetSchema object generated from the JsonNode
*/
private ResultSetSchema getResultSetSchema(JsonNode schemaNode) {
DateTimeZone timezone = generateTimezone(
schemaNode.get(SCHEMA_TIMEZONE).asText(),
DateTimeZone.forID(
SYSTEM_CONFIG.getStringProperty(SYSTEM_CONFIG.getPackageVariableName("timezone"), "UTC")
)
);
//Recreate ResultSetSchema
LinkedHashSet columns = Stream.concat(
Streams.stream(schemaNode.get(SCHEMA_DIM_COLUMNS))
.map(JsonNode::asText)
.map(this::resolveDimensionName)
.map(DimensionColumn::new),
Streams.stream(() -> schemaNode.get(SCHEMA_METRIC_COLUMNS_TYPE).fields())
.map(entry -> new MetricColumnWithValueType(entry.getKey(), entry.getValue().asText()))
).collect(Collectors.toCollection(LinkedHashSet::new));
return new ResultSetSchema(generateGranularity(schemaNode.get(SCHEMA_GRANULARITY).asText(), timezone), columns);
}
/**
* Method to get dimension from DimensionDictionary for a given name.
*
* @param dimensionName To find Dimension from dimension dictionary
*
* @return The dimension with the given name
*/
private Dimension resolveDimensionName(String dimensionName) {
Dimension dimension = dimensionDictionary.findByApiName(dimensionName);
if (dimension == null) {
String msg = ErrorMessageFormat.UNABLE_TO_FIND_DIMENSION_FROM_DICTIONARY.format(dimensionName);
LOG.error(msg);
throw new DeserializationException(msg);
}
return dimension;
}
/**
* Creates new Result object from JsonNode.
*
* @param serializedResult JsonNode which contains all the serialized details to generate Result object
* @param resultSetSchema Schema of the result to generate the Result object
*
* @return Result object generated from given JsonNode
*/
private Result getResult(JsonNode serializedResult, ResultSetSchema resultSetSchema) {
return new Result(
extractDimensionValues(
serializedResult.get(DIMENSION_VALUES_KEY),
resultSetSchema.getColumns(DimensionColumn.class)
),
extractMetricValues(
serializedResult.get(METRIC_VALUES_KEY),
resultSetSchema.getColumns(MetricColumnWithValueType.class)
),
DateTime.parse(serializedResult.get(TIMESTAMP_KEY).asText())
);
}
/**
* Extracts dimension rows for the given dimension columns.
*
* @param dimensionRowsNode JsonNode which contains all the dimension rows which contains dimension names
* and its unique id as its value
* @param dimensionColumns DimensionColumns which needs to have dimension rows
*
* @return Map of all the dimensionRows associated with dimensionColumns
*/
private Map extractDimensionValues(
JsonNode dimensionRowsNode,
Set dimensionColumns
) {
return dimensionColumns.stream().collect(Collectors.toMap(
Function.identity(),
dimensionColumn -> dimensionColumn.getDimension().findDimensionRowByKeyValue(
dimensionRowsNode.get(dimensionColumn.getDimension().getApiName()).asText()
)
));
}
/**
* Extracts the metric values for all the metric columns from JsonNode which contains all the metrics names and
* respective values.
*
* @param metricsRows JsonNode which contains all the metric names and respective values as key-value pair
* @param metricColumns MetricColumns with value types
*
* @return metric columns with their values
*/
private Map extractMetricValues(
JsonNode metricsRows,
Set metricColumns
) {
Map metricColumnObjectMap = new HashMap<>();
for (MetricColumnWithValueType metricColumn : metricColumns) {
try {
metricColumnObjectMap.put(
metricColumn,
//deserialize the metric value based on its class type
nonResponseContextMapper.readValue(
metricsRows.get(metricColumn.getName()).toString(), metricColumn.getClassType()
)
);
} catch (JsonParseException | NullPointerException e) {
String msg = ErrorMessageFormat.METRIC_VALUE_PARSING_ERROR.format("parse");
LOG.error(msg, e);
throw new DeserializationException(msg, e);
} catch (JsonMappingException e) {
String msg = ErrorMessageFormat.METRIC_VALUE_PARSING_ERROR.format("map");
LOG.error(msg, e);
throw new DeserializationException(msg, e);
} catch (IOException e) {
String msg = ErrorMessageFormat.METRIC_VALUE_PARSING_ERROR.format("identify");
LOG.error(msg, e);
throw new DeserializationException(msg, e);
}
}
return metricColumnObjectMap;
}
/**
* Generate a Granularity instance based on given query granularity.
*
* @param granularity A string representation of the granularity
* @param dateTimeZone The time zone to use for this granularity
*
* @return A granularity instance with time zone information
*
* @throws DeserializationException if the string matches no meaningful granularity
*/
private Granularity generateGranularity(String granularity, DateTimeZone dateTimeZone)
throws DeserializationException {
try {
return granularityParser.parseGranularity(granularity, dateTimeZone);
} catch (GranularityParseException e) {
String msg = ErrorMessageFormat.GRANULARITY_PARSING_ERROR.format(granularity);
LOG.error(msg, e);
throw new DeserializationException(msg, e);
}
}
/**
* Get a time zone instance for the specified time zone id.
*
* @param timeZoneId the ID of the datetime zone, null means default
* @param systemTimeZone timeZone of the system
*
* @return the DateTimeZone object for the ID
*
* @throws DeserializationException When timeZoneId is unable to recognize
*/
private DateTimeZone generateTimezone(String timeZoneId, DateTimeZone systemTimeZone)
throws DeserializationException {
if (timeZoneId == null) {
return systemTimeZone;
}
try {
return DateTimeZone.forID(timeZoneId);
} catch (IllegalArgumentException e) {
String msg = ErrorMessageFormat.UNKNOWN_TIMEZONE_ID.format(timeZoneId);
LOG.error(msg, e);
throw new DeserializationException(msg , e);
}
}
@JsonIgnore
public DimensionDictionary getDimensionDictionary() {
return dimensionDictionary;
}
@JsonIgnore
public ObjectMapper getNonResponseContextMapper() {
return nonResponseContextMapper;
}
@JsonIgnore
public ObjectMapper getResponseContextMapper() {
return responseContextMapper;
}
@JsonIgnore
public GranularityParser getGranularityParser() {
return granularityParser;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy