de.gematik.combine.execution.TableGenerator Maven / Gradle / Ivy
/*
* Copyright 2023 gematik GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package de.gematik.combine.execution;
import static com.google.common.collect.Lists.cartesianProduct;
import static de.gematik.combine.CombineMojo.ErrorType.MINIMAL_TABLE;
import static de.gematik.combine.CombineMojo.appendError;
import static de.gematik.combine.CombineMojo.getPluginLog;
import static de.gematik.combine.util.CurrentScenario.getCurrenScenarioName;
import static java.lang.String.format;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptySet;
import static java.util.Collections.shuffle;
import static java.util.Comparator.comparingInt;
import static java.util.Objects.nonNull;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static java.util.stream.Stream.concat;
import de.gematik.combine.filter.table.cell.CellFilter;
import de.gematik.combine.filter.table.row.RowFilter;
import de.gematik.combine.model.CombineItem;
import de.gematik.combine.model.TableCell;
import de.gematik.combine.tags.ConfiguredFilters;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import lombok.Data;
/**
* This TableGenerator combines given {@link CombineItem}s to tables. It knows two generation modes:
* 1. full table: applies cell filters to columns and calculates the cartesian product
* afterwards
*
2. minimal table: tries to use every item just once, but reuses items to fill otherwise
* incomplete rows. Table generation complies with cell and row filters
*/
public class TableGenerator {
private static final int ONE_MILLION = 1000000;
public List> generateTable(List combineItems,
ConfiguredFilters filters) {
if (filters.getActualConfig().isMinimalTable()) {
return generateMinimalTable(combineItems, filters);
}
return generateFullTable(combineItems, filters);
}
private List> generateFullTable(List combineItems,
ConfiguredFilters filters) {
List columns = filters.getColumns();
List> preparedColumns = preFilteredColumns(combineItems, filters);
getPluginLog().debug(
format("creating cartesianProduct table with %d columns", columns.size()));
if (columns.size() > 5) {
getPluginLog().warn(
format("creating cartesianProduct table with %d columns produces a huge amount "
+ "of entries and can cause out of memory errors", columns.size()));
}
List> table = new ArrayList<>(cartesianProduct(preparedColumns));
getPluginLog().debug(
format("created table with %d columns and %d rows", columns.size(), table.size()));
if (table.size() > ONE_MILLION) {
getPluginLog().warn(
"Such a big table will need chunking and a considerable amount of time "
+ "for filtering.\nPlease use '@Filter' tags with just one header reference to "
+ "filter columns before applying the cartesian product and therefore reduce "
+ "generated table size.");
}
return table;
}
/**
* applies cell filters to each column and returns a list with the possible values for each column
*/
private List> preFilteredColumns(List combineItems,
ConfiguredFilters filters) {
List headers = filters.getColumns();
getPluginLog().debug("Applied cell filters: " + filters.getCellFilters());
Map cellFilters = filters.combineCellFilters();
List> preparedColumns = new ArrayList<>();
for (String header : headers) {
List e = combineItems.stream()
.map(s -> new TableCell(header, s))
.filter(cellFilters.getOrDefault(header, x -> true))
.collect(toList());
if (filters.getActualConfig().isShuffleCombinations()) {
shuffle(e);
}
preparedColumns.add(e);
}
getPluginLog().debug(
format("prepared columns after applied cell filters: %s", preparedColumns));
return preparedColumns;
}
private List> generateMinimalTable(List combineItems,
ConfiguredFilters filters) {
List columns = filters.getColumns();
getPluginLog().debug(format("creating minimal table with %d columns", columns.size()));
final List> preparedColumns = preFilteredColumns(combineItems, filters);
boolean anyColumnEmpty = preparedColumns.stream().anyMatch(List::isEmpty);
if (anyColumnEmpty) {
getPluginLog().debug("could not generate any row for minimal table");
return emptyList();
}
getPluginLog().debug("Applied row filter: " + filters.getTableRowFilters());
getPluginLog().debug("Applied configuration: " + filters.getActualConfig());
StateInfo stateInfo = new StateInfo(filters, preparedColumns);
forEachUnusedValue(stateInfo,
unusedValue -> findValidRow(stateInfo, unusedValue).ifPresent(stateInfo::addNewRow));
if (stateInfo.getAllMissingValues().size() != 0) {
stateInfo.getAllMissingValues()
.forEach(e -> appendError(
format("Building minimal table failed for scenario: \"%s\". "
+ "No row could be build for -> value: %s%s",
getCurrenScenarioName(),
e.getValue(), nonNull(e.getUrl()) ? " url: " + e.getUrl() : ""),
MINIMAL_TABLE));
}
return stateInfo.getResult();
}
private void forEachUnusedValue(StateInfo stateInfo, Consumer unusedValueConsumer) {
for (List preparedColumn : stateInfo.getPreparedColumns()) {
for (TableCell tableCell : preparedColumn) {
if (stateInfo.getAllMissingValues().contains(tableCell.getCombineItem())) {
Optional.of(tableCell).ifPresent(unusedValueConsumer);
}
}
}
}
private Optional> findValidRow(StateInfo stateInfo, TableCell firstValue) {
List columns = stateInfo.getFilters().getColumns();
List> preparedColumns = stateInfo.getPreparedColumns();
Optional> completeRow = forEachColumnExtendRow(firstValue, columns,
preparedColumns, (row, possibleValues) -> addNewValueToRow(stateInfo, row, possibleValues));
return completeRow.map(row -> sortRowForColumns(columns, row));
}
private Optional> addNewValueToRow(StateInfo stateInfo, List row,
List possibleValues) {
List missingColumnValues = possibleValues.stream()
.filter(val -> stateInfo.getAllMissingValues().contains(val.getCombineItem()))
.collect(toList());
List rowFilters = new ArrayList<>(stateInfo.getFilters().getTableRowFilters());
Optional newColValue = findNewRowValue(stateInfo, possibleValues,
missingColumnValues, row, rowFilters);
return newColValue.map(value -> {
List extendedRow = new ArrayList<>(row);
extendedRow.add(value);
return extendedRow;
});
}
private Optional> forEachColumnExtendRow(TableCell firstValue,
List columns, List> preparedColumns,
BiFunction, List, Optional>> rowExtender) {
int startColumn = columns.indexOf(firstValue.getHeader());
List tmpRow = new ArrayList<>(columns.size());
tmpRow.add(firstValue);
Optional> row = Optional.of(tmpRow);
for (int currentColumn = (startColumn + 1) % columns.size(); currentColumn != startColumn;
currentColumn = (currentColumn + 1) % columns.size()) {
List possibleValues = preparedColumns.get(currentColumn);
row = rowExtender.apply(row.get(), possibleValues);
if (row.isEmpty()) {
getPluginLog().debug("could not create a valid row for: " + firstValue);
return row;
}
}
return row;
}
private Optional findNewRowValue(StateInfo stateInfo,
List preparedValues, List remainingValuesForThisColumn,
List row, List rowFilter) {
return findNewValueNotContainedIn(stateInfo, remainingValuesForThisColumn, row,
stateInfo.getAllUsedValues(), rowFilter)
.or(() -> findNewValueNotContainedIn(stateInfo, remainingValuesForThisColumn, row,
emptySet(),
rowFilter))
.or(() -> findNewValueMatchingFilter(stateInfo, preparedValues, row, rowFilter));
}
private Optional findNewValueMatchingFilter(StateInfo stateInfo,
List possibleValues,
List row, List rowFilters) {
for (TableCell possibleValue : possibleValues) {
ArrayList extendedRow = new ArrayList<>(row);
extendedRow.add(possibleValue);
if (checkRowFilter(extendedRow, rowFilters, stateInfo.getFilters().getColumns())) {
return Optional.of(possibleValue);
}
}
return Optional.empty();
}
private Optional findNewValueNotContainedIn(StateInfo stateInfo,
List possibleValues,
List usedRowValues, Set allUsedValues, List rowFilters) {
List allColumns = stateInfo.getFilters().getColumns();
Set usedCombineItems = concat(
allUsedValues.stream(),
usedRowValues.stream().map(TableCell::getCombineItem))
.collect(toSet());
for (TableCell possibleValue : possibleValues) {
ArrayList possibleRow = new ArrayList<>(usedRowValues);
possibleRow.add(possibleValue);
if (!usedCombineItems.contains(possibleValue.getCombineItem()) && checkRowFilter(possibleRow,
rowFilters, allColumns)) {
return Optional.of(possibleValue);
}
}
return Optional.empty();
}
private boolean checkRowFilter(List row, List rowFilters,
List allHeaders) {
return rowFilters.stream()
.filter(rf -> isApplicable(row, rf, allHeaders))
.allMatch(rf -> rf.test(row));
}
private boolean isApplicable(List row, RowFilter rowFilter, List allHeaders) {
List requiredColumns = rowFilter.getRequiredColumns(allHeaders);
Set filledColumns = row.stream()
.map(TableCell::getHeader)
.collect(toSet());
return filledColumns.containsAll(requiredColumns);
}
private List sortRowForColumns(List columns, List row) {
row.sort(comparingInt(a -> columns.indexOf(a.getHeader())));
return row;
}
@Data
public static class StateInfo {
private final ConfiguredFilters filters;
private final List> preparedColumns;
private final Set allMissingValues;
private final Set allUsedValues = new TreeSet<>();
private final List> result = new ArrayList<>();
public StateInfo(ConfiguredFilters filters, List> preparedColumns) {
this.filters = filters;
this.preparedColumns = preparedColumns;
this.allMissingValues = compute(preparedColumns);
}
private Set compute(List> preparedColumns) {
return preparedColumns.stream()
.flatMap(List::stream)
.map(TableCell::getCombineItem)
.collect(toSet());
}
public void addNewRow(List row) {
result.add(row);
Set rowValues = row.stream()
.map(TableCell::getCombineItem)
.collect(toSet());
allUsedValues.addAll(rowValues);
allMissingValues.removeAll(rowValues);
}
}
}