io.hyperfoil.tools.horreum.svc.DatasetServiceImpl Maven / Gradle / Ivy
The newest version!
package io.hyperfoil.tools.horreum.svc;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import jakarta.annotation.security.PermitAll;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.persistence.EntityManager;
import jakarta.persistence.NoResultException;
import jakarta.persistence.PersistenceException;
import jakarta.persistence.Tuple;
import jakarta.transaction.TransactionManager;
import jakarta.transaction.Transactional;
import jakarta.ws.rs.DefaultValue;
import org.hibernate.Hibernate;
import org.hibernate.Session;
import org.hibernate.query.NativeQuery;
import org.hibernate.type.StandardBasicTypes;
import org.jboss.logging.Logger;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.hyperfoil.tools.horreum.api.SortDirection;
import io.hyperfoil.tools.horreum.api.data.*;
import io.hyperfoil.tools.horreum.api.services.DatasetService;
import io.hyperfoil.tools.horreum.api.services.SchemaService;
import io.hyperfoil.tools.horreum.bus.AsyncEventChannels;
import io.hyperfoil.tools.horreum.entity.FingerprintDAO;
import io.hyperfoil.tools.horreum.entity.PersistentLogDAO;
import io.hyperfoil.tools.horreum.entity.alerting.DatasetLogDAO;
import io.hyperfoil.tools.horreum.entity.data.*;
import io.hyperfoil.tools.horreum.hibernate.JsonBinaryType;
import io.hyperfoil.tools.horreum.mapper.DatasetMapper;
import io.hyperfoil.tools.horreum.server.WithRoles;
import io.quarkus.runtime.Startup;
import io.quarkus.security.identity.SecurityIdentity;
@ApplicationScoped
@Startup
public class DatasetServiceImpl implements DatasetService {
private static final Logger log = Logger.getLogger(DatasetServiceImpl.class);
//@formatter:off
private static final String LABEL_QUERY = """
WITH
used_labels AS (
SELECT label.id AS label_id, label.name, ds.schema_id, count(le) AS count
FROM dataset_schemas ds
JOIN label ON label.schema_id = ds.schema_id
LEFT JOIN label_extractors le ON le.label_id = label.id
WHERE ds.dataset_id = ?1 AND (?2 < 0 OR label.id = ?2) GROUP BY label.id, label.name, ds.schema_id
),
lvalues AS (
SELECT ul.label_id, le.name,
(CASE WHEN le.isarray THEN
jsonb_path_query_array(dataset.data -> ds.index, le.jsonpath::jsonpath)
ELSE
jsonb_path_query_first(dataset.data -> ds.index, le.jsonpath::jsonpath)
END) AS value
FROM dataset
JOIN dataset_schemas ds ON dataset.id = ds.dataset_id
JOIN used_labels ul ON ul.schema_id = ds.schema_id
LEFT JOIN label_extractors le ON ul.label_id = le.label_id
WHERE dataset.id = ?1
)
SELECT lvalues.label_id, ul.name, function,
(CASE
WHEN ul.count > 1 THEN jsonb_object_agg(COALESCE(lvalues.name, ''), lvalues.value)
WHEN ul.count = 1 THEN jsonb_agg(lvalues.value) -> 0
ELSE '{}'::jsonb END
) AS value
FROM label
JOIN lvalues ON lvalues.label_id = label.id
JOIN used_labels ul ON label.id = ul.label_id
GROUP BY lvalues.label_id, ul.name, function, ul.count
""";
protected static final String LABEL_PREVIEW = """
WITH
le AS (
SELECT * FROM jsonb_populate_recordset(NULL::extractor, (?1)::jsonb)
),
lvalues AS (
SELECT le.name,
(CASE WHEN le.isarray THEN
jsonb_path_query_array(dataset.data -> ds.index, le.jsonpath)
ELSE
jsonb_path_query_first(dataset.data -> ds.index, le.jsonpath)
END) AS value
FROM le, dataset
JOIN dataset_schemas ds ON dataset.id = ds.dataset_id
WHERE dataset.id = ?2 AND ds.schema_id = ?3
)
SELECT (CASE
WHEN jsonb_array_length((?1)::jsonb) > 1 THEN jsonb_object_agg(COALESCE(lvalues.name, ''), lvalues.value)
WHEN jsonb_array_length((?1)::jsonb) = 1 THEN jsonb_agg(lvalues.value) -> 0
ELSE '{}'::jsonb END
) AS value
FROM lvalues
""";
private static final String SCHEMAS_SELECT = """
SELECT dataset_id,
jsonb_agg(
jsonb_build_object('id', schema.id, 'uri', ds.uri, 'name', schema.name, 'source', 0, 'type', 2, 'key', ds.index::text, 'hasJsonSchema', schema.schema IS NOT NULL)
) AS schemas
FROM dataset_schemas ds
JOIN dataset ON dataset.id = ds.dataset_id
JOIN schema ON schema.id = ds.schema_id
""";
private static final String VALIDATION_SELECT = """
validation AS (
SELECT dataset_id, jsonb_agg(jsonb_build_object('schemaId', schema_id, 'error', error)) AS errors
FROM dataset_validationerrors GROUP BY dataset_id
)
""";
private static final String DATASET_SUMMARY_SELECT = """
SELECT ds.id, ds.runid AS runId,
ds.ordinal, ds.testid AS testId,
test.name AS testname, ds.description,
EXTRACT(EPOCH FROM ds.start) * 1000 AS start,
EXTRACT(EPOCH FROM ds.stop) * 1000 AS stop,
ds.owner, ds.access, dv.value AS view,
COALESCE(schema_agg.schemas, '[]') AS schemas,
COALESCE(validation.errors, '[]') AS validationErrors
FROM dataset ds
LEFT JOIN test ON test.id = ds.testid
LEFT JOIN schema_agg ON schema_agg.dataset_id = ds.id
LEFT JOIN validation ON validation.dataset_id = ds.id
LEFT JOIN dataset_view dv ON dv.dataset_id = ds.id AND dv.view_id =
""";
private static final String LIST_SCHEMA_DATASETS = """
WITH ids AS (
SELECT dataset_id AS id FROM dataset_schemas WHERE uri = ?1
),
schema_agg AS (
""" +
SCHEMAS_SELECT +
"""
WHERE dataset_id IN (SELECT id FROM ids) GROUP BY dataset_id
)
SELECT ds.id, ds.runid AS runId, ds.ordinal,
ds.testid AS testId, test.name AS testname, ds.description,
EXTRACT(EPOCH FROM ds.start) * 1000 AS start,
EXTRACT(EPOCH FROM ds.stop) * 1000 AS stop,
ds.owner, ds.access, dv.value AS view,
schema_agg.schemas AS schemas, '[]'::jsonb AS validationErrors
FROM dataset ds
LEFT JOIN test ON test.id = ds.testid
LEFT JOIN schema_agg ON schema_agg.dataset_id = ds.id
LEFT JOIN dataset_view dv ON dv.dataset_id = ds.id
WHERE ds.id IN (SELECT id FROM ids)
""";
private static final String ALL_LABELS_SELECT = """
SELECT dataset.id as dataset_id,
COALESCE(jsonb_object_agg(label.name, lv.value) FILTER (WHERE label.name IS NOT NULL), '{}'::jsonb) AS values
FROM dataset
LEFT JOIN label_values lv ON dataset.id = lv.dataset_id
LEFT JOIN label ON label.id = label_id
""";
//@formatter:on
@Inject
EntityManager em;
@Inject
ServiceMediator mediator;
@Inject
SecurityIdentity identity;
@Inject
TransactionManager tm;
@PermitAll
@WithRoles
@Override
public DatasetService.DatasetList listByTest(int testId, String filter, Integer limit, Integer page, String sort,
SortDirection direction, Integer viewId) {
StringBuilder sql = new StringBuilder("WITH schema_agg AS (")
.append(SCHEMAS_SELECT).append(" WHERE testid = :testId GROUP BY dataset_id")
.append("), ").append(VALIDATION_SELECT);
JsonNode jsonFilter = null;
if (filter != null && !filter.isBlank() && !filter.equals("{}")) {
sql.append(", all_labels AS (").append(ALL_LABELS_SELECT).append(" WHERE testid = :testId GROUP BY dataset.id)");
sql.append(DATASET_SUMMARY_SELECT);
addViewIdCondition(sql, viewId);
sql.append(
" JOIN all_labels ON all_labels.dataset_id = ds.id WHERE testid = :testId AND all_labels.values @> :jsonFilter");
jsonFilter = Util.parseFingerprint(filter);
} else {
sql.append(DATASET_SUMMARY_SELECT);
addViewIdCondition(sql, viewId);
sql.append(" WHERE testid = :testId");
}
addOrderAndPaging(limit, page, sort, direction, sql);
NativeQuery query = initTypes(sql.toString());
query.setParameter("testId", testId);
if (jsonFilter != null) {
query.setParameter("jsonFilter", jsonFilter, JsonBinaryType.INSTANCE);
}
if (viewId != null) {
query.setParameter("viewId", viewId);
}
DatasetService.DatasetList list = new DatasetService.DatasetList();
list.datasets = query.getResultList();
list.total = DatasetDAO.count("testid = ?1", testId);
return list;
}
private void addViewIdCondition(StringBuilder sql, Integer viewId) {
if (viewId == null) {
sql.append("(SELECT id FROM view WHERE test_id = :testId AND name = 'Default')");
} else {
sql.append(":viewId");
}
}
private NativeQuery initTypes(String sql) {
return em.unwrap(Session.class).createNativeQuery(sql.toString(), Tuple.class)
.addScalar("id", StandardBasicTypes.INTEGER)
.addScalar("runId", StandardBasicTypes.INTEGER)
.addScalar("ordinal", StandardBasicTypes.INTEGER)
.addScalar("testId", StandardBasicTypes.INTEGER)
.addScalar("testname", StandardBasicTypes.TEXT)
.addScalar("description", StandardBasicTypes.TEXT)
.addScalar("start", StandardBasicTypes.LONG)
.addScalar("stop", StandardBasicTypes.LONG)
.addScalar("owner", StandardBasicTypes.TEXT)
.addScalar("access", StandardBasicTypes.INTEGER)
.addScalar("view", JsonBinaryType.INSTANCE)
.addScalar("schemas", JsonBinaryType.INSTANCE)
.addScalar("validationErrors", JsonBinaryType.INSTANCE)
.setTupleTransformer((tuples, aliases) -> {
DatasetSummary summary = new DatasetSummary();
summary.id = (int) tuples[0];
summary.runId = (int) tuples[1];
summary.ordinal = (int) tuples[2];
summary.testId = (int) tuples[3];
summary.testname = (String) tuples[4];
summary.description = (String) tuples[5];
summary.start = Instant.ofEpochMilli((Long) tuples[6]);
summary.stop = Instant.ofEpochMilli((Long) tuples[7]);
summary.owner = (String) tuples[8];
summary.access = Access.fromInt((int) tuples[9]);
summary.view = IndexedLabelValueMap.fromObjectNode((ObjectNode) tuples[10]);
summary.schemas = Util.OBJECT_MAPPER.convertValue(tuples[11], new TypeReference<>() {
});
if (tuples[12] != null && !((ArrayNode) tuples[12]).isEmpty()) {
try {
summary.validationErrors = Arrays
.asList(Util.OBJECT_MAPPER.treeToValue((ArrayNode) tuples[12], ValidationError[].class));
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
;
}
return summary;
});
}
private void addOrderAndPaging(Integer limit, Integer page, String sort, SortDirection direction, StringBuilder sql) {
if (sort != null && sort.startsWith("view_data:")) {
String[] parts = sort.split(":", 3);
String vcid = parts[1];
String label = parts[2];
sql.append(" ORDER BY");
// prefer numeric sort
sql.append(" to_double(dv.value->'").append(vcid).append("'->>'").append(label).append("')");
Util.addDirection(sql, direction);
sql.append(", dv.value->'").append(vcid).append("'->>'").append(label).append("'");
Util.addDirection(sql, direction);
} else {
Util.addOrderBy(sql, sort, direction);
}
Util.addLimitOffset(sql, limit, page);
}
@WithRoles
@Override
public DatasetService.DatasetList listBySchema(String uri, Integer limit, Integer page, String sort,
@DefaultValue("Descending") SortDirection direction) {
StringBuilder sql = new StringBuilder(LIST_SCHEMA_DATASETS);
// TODO: filtering by fingerprint
addOrderAndPaging(limit, page, sort, direction, sql);
NativeQuery query = initTypes(sql.toString());
query.setParameter(1, uri);
DatasetService.DatasetList list = new DatasetService.DatasetList();
list.datasets = query.getResultList();
list.total = ((Number) em.createNativeQuery("SELECT COUNT(dataset_id) FROM dataset_schemas WHERE uri = ?1")
.setParameter(1, uri).getSingleResult()).longValue();
return list;
}
@Override
public List labelValues(int datasetId) {
Stream
© 2015 - 2025 Weber Informatics LLC | Privacy Policy