All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • dk.eobjects.metamodel.MetaModelHelper Maven / Gradle / Ivy

    /**
 *  This file is part of MetaModel.
 *
 *  MetaModel is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  MetaModel is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with MetaModel.  If not, see .
 */
package dk.eobjects.metamodel;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import dk.eobjects.metamodel.data.DataSet;
import dk.eobjects.metamodel.data.Row;
import dk.eobjects.metamodel.query.FilterItem;
import dk.eobjects.metamodel.query.FromItem;
import dk.eobjects.metamodel.query.GroupByItem;
import dk.eobjects.metamodel.query.OrderByItem;
import dk.eobjects.metamodel.query.Query;
import dk.eobjects.metamodel.query.SelectItem;
import dk.eobjects.metamodel.schema.Column;
import dk.eobjects.metamodel.schema.ColumnType;
import dk.eobjects.metamodel.schema.Table;
import dk.eobjects.metamodel.util.ObjectComparator;

/**
 * This class contains various helper functionality to common tasks involving
 * MetaModel, eg.:
 * 
      
 * 
    • Easy-access for traversing common schema items
      • 
 * 
    • Manipulate data in memory. These methods are primarily used to enable
 * queries for non-queryable data sources like CSV files and spreadsheets.
      • 
 * 
    • Query rewriting, traversing and manipulation.
      • 
 * 
    
 */
public class MetaModelHelper {

	private static final Log _log = LogFactory.getLog(MetaModelHelper.class);

	// Preven instantiation
	private MetaModelHelper() {
	}

	/**
	 * Creates an array of tables where all occurences of tables in the provided
	 * list of tables and columns are included
	 */
	public static Table[] getTables(List tableList,
			List columnList) {
		HashSet
     set = new HashSet
    ();
		set.addAll(tableList);
		for (Column column : columnList) {
			set.add(column.getTable());
		}
		return set.toArray(new Table[set.size()]);
	}

	/**
	 * Converts a list of columns to a corresponding array of tables
	 * 
	 * @param columns
	 *            the columns that the tables will be extracted from
	 * @return an array containing the tables of the provided columns.
	 */
	public static Table[] getTables(List columns) {
		ArrayList
     result = new ArrayList
    ();
		for (Column column : columns) {
			Table table = column.getTable();
			if (!result.contains(table)) {
				result.add(table);
			}
		}
		return result.toArray(new Table[result.size()]);
	}

	/**
	 * Creates a subset array of columns, where only columns that are contained
	 * within the specified table are included.
	 * 
	 * @param table
	 * @param columns
	 * @return an array containing the columns that exist in the table
	 */
	public static Column[] getTableColumns(Table table, List columns) {
		return getTableColumns(table, columns
				.toArray(new Column[columns.size()]));
	}

	/**
	 * Creates a subset array of columns, where only columns that are contained
	 * within the specified table are included.
	 * 
	 * @param table
	 * @param columns
	 * @return an array containing the columns that exist in the table
	 */
	public static Column[] getTableColumns(Table table, Column[] columns) {
		ArrayList result = new ArrayList();
		for (int i = 0; i < columns.length; i++) {
			Column column = columns[i];
			if (table == column.getTable()) {
				result.add(column);
			}
		}
		return result.toArray(new Column[result.size()]);
	}

	public static DataSet getCarthesianProduct(DataSet... fromDataSets) {
		return getCarthesianProduct(fromDataSets, new FilterItem[0]);
	}

	public static DataSet getCarthesianProduct(DataSet[] fromDataSets,
			List whereItems) {
		return getCarthesianProduct(fromDataSets, whereItems
				.toArray(new FilterItem[whereItems.size()]));
	}

	public static DataSet getCarthesianProduct(DataSet[] fromDataSets,
			FilterItem... filterItems) {
		List selectItems = new ArrayList();
		for (DataSet dataSet : fromDataSets) {
			for (int i = 0; i < dataSet.getSelectItems().length; i++) {
				SelectItem item = dataSet.getSelectItems()[i];
				selectItems.add(item);
			}
		}

		int selectItemOffset = 0;
		List data = new ArrayList();
		for (int fromDataSetIndex = 0; fromDataSetIndex < fromDataSets.length; fromDataSetIndex++) {
			DataSet fromDataSet = fromDataSets[fromDataSetIndex];
			SelectItem[] fromSelectItems = fromDataSet.getSelectItems();
			if (fromDataSetIndex == 0) {
				while (fromDataSet.next()) {
					Object[] values = fromDataSet.getRow().getValues();
					Object[] row = new Object[selectItems.size()];
					System.arraycopy(values, 0, row, selectItemOffset,
							values.length);
					data.add(row);
				}
				fromDataSet.close();
			} else {
				List fromDataRows = new ArrayList();
				while (fromDataSet.next()) {
					fromDataRows.add(fromDataSet.getRow().getValues());
				}
				fromDataSet.close();
				for (int i = 0; i < data.size(); i = i + fromDataRows.size()) {
					Object[] originalRow = data.get(i);
					data.remove(i);
					for (int j = 0; j < fromDataRows.size(); j++) {
						Object[] newRow = fromDataRows.get(j);
						System.arraycopy(newRow, 0, originalRow,
								selectItemOffset, newRow.length);
						data.add(i + j, originalRow.clone());
					}
				}
			}
			selectItemOffset += fromSelectItems.length;
		}

		SelectItem[] finalSelectItems = selectItems
				.toArray(new SelectItem[selectItems.size()]);
		for (Iterator it = data.iterator(); it.hasNext();) {
			Object[] objects = it.next();
			Row row = new Row(finalSelectItems, objects);
			for (FilterItem whereItem : filterItems) {
				if (!whereItem.evaluate(row)) {
					it.remove();
					break;
				}
			}
		}
		return new DataSet(finalSelectItems, data);
	}

	public static DataSet getFiltered(DataSet dataSet,
			List filterItems) {
		return getFiltered(dataSet, filterItems
				.toArray(new FilterItem[filterItems.size()]));
	}

	public static DataSet getFiltered(DataSet dataSet,
			FilterItem... filterItems) {
		List data = new ArrayList();
		while (dataSet.next()) {
			Row row = dataSet.getRow();
			boolean valid = true;
			for (FilterItem havingItem : filterItems) {
				if (!havingItem.evaluate(row)) {
					valid = false;
					break;
				}
			}
			if (valid) {
				data.add(row.getValues());
			}
		}
		dataSet.close();
		return new DataSet(dataSet.getSelectItems(), data);
	}

	public static DataSet getSelection(List selectItems,
			DataSet dataSet) {
		return getSelection(selectItems.toArray(new SelectItem[selectItems
				.size()]), dataSet);
	}

	public static DataSet getSelection(SelectItem[] selectItems, DataSet dataSet) {
		SelectItem[] dataSetSelectItems = dataSet.getSelectItems();
		if (!ArrayUtils.isEquals(dataSetSelectItems, selectItems)) {
			List resultData = new ArrayList();
			while (dataSet.next()) {
				Row row = dataSet.getRow();
				Object[] resultRow = new Object[selectItems.length];
				for (int i = 0; i < selectItems.length; i++) {
					SelectItem selectItem = selectItems[i];
					if (selectItem.getSubQuerySelectItem() != null) {
						resultRow[i] = row.getValue(selectItem
								.getSubQuerySelectItem());
					} else {
						resultRow[i] = row.getValue(selectItem);
					}
				}
				resultData.add(resultRow);
			}
			dataSet.close();
			dataSet = new DataSet(selectItems, resultData);
		}
		return dataSet;
	}

	public static DataSet getGrouped(List selectItems,
			DataSet dataSet, List groupByItems) {
		return getGrouped(selectItems
				.toArray(new SelectItem[selectItems.size()]), dataSet,
				groupByItems.toArray(new GroupByItem[groupByItems.size()]));
	}

	public static DataSet getGrouped(SelectItem[] selectItems, DataSet dataSet,
			GroupByItem[] groupByItems) {
		DataSet result = dataSet;
		if (groupByItems != null && groupByItems.length > 0) {
			Map>> uniqueRows = new HashMap>>();

			SelectItem[] groupBySelects = new SelectItem[groupByItems.length];
			for (int i = 0; i < groupBySelects.length; i++) {
				groupBySelects[i] = groupByItems[i].getSelectItem();
			}

			// Creates a list of SelectItems that have functions
			List functionItems = getFunctionSelectItems(Arrays
					.asList(selectItems));

			// Loop through the dataset and identify groups
			while (dataSet.next()) {
				Row row = dataSet.getRow();

				// Subselect a row prototype with only the unique values that
				// define the group
				Row uniqueRow = row.getSubSelection(groupBySelects);

				// function input is the values used for calculating aggregate
				// functions in the group
				Map> functionInput;
				if (!uniqueRows.containsKey(uniqueRow)) {
					// If this group already exist, use an existing function
					// input
					functionInput = new HashMap>();
					for (SelectItem item : functionItems) {
						functionInput.put(item, new ArrayList());
					}
					uniqueRows.put(uniqueRow, functionInput);
				} else {
					// If this is a new group, create a new function input
					functionInput = uniqueRows.get(uniqueRow);
				}

				// Loop through aggregate functions to check for validity
				for (SelectItem item : functionItems) {
					List objects = functionInput.get(item);
					Column column = item.getColumn();
					if (column != null) {
						Object value = row.getValue(new SelectItem(column));
						objects.add(value);
					} else if (SelectItem.isCountAllItem(item)) {
						// Just use the empty string, since COUNT(*) don't
						// evaluate values (but null values should be prevented)
						objects.add("");
					} else {
						throw new IllegalArgumentException(
								"Expression function not supported: " + item);
					}
				}
			}

			dataSet.close();
			List resultData = new ArrayList();

			// Loop through the groups to generate aggregates
			for (Entry>> entry : uniqueRows
					.entrySet()) {
				Row row = entry.getKey();
				Map> functionInput = entry.getValue();
				Object[] resultRow = new Object[selectItems.length];
				// Loop through select items to generate a row
				for (int i = 0; i < selectItems.length; i++) {
					SelectItem item = selectItems[i];
					int uniqueRowIndex = row.indexOf(item);
					if (uniqueRowIndex != -1) {
						// If there's already a value for the select item in the
						// row, keep it (it's one of the grouped by columns)
						resultRow[i] = row.getValue(uniqueRowIndex);
					} else {
						// Use the function input to calculate the aggregate
						// value
						List objects = functionInput.get(item);
						if (objects != null) {
							Object functionResult = item.getFunction()
									.evaluate(objects.toArray());
							resultRow[i] = functionResult;
						} else {
							if (item.getFunction() != null) {
								_log
										.error("No function input found for SelectItem: "
												+ item);
							}
						}
					}
				}
				resultData.add(resultRow);
			}
			result = new DataSet(selectItems, resultData);
		}
		result = getSelection(selectItems, result);
		return result;
	}

	public static DataSet getAggregated(List resultSelectItems,
			DataSet dataSet) {
		List functionItems = getFunctionSelectItems(resultSelectItems);
		if (functionItems.isEmpty()) {
			return dataSet;
		} else {
			SelectItem[] resultSelectItemsArray = resultSelectItems
					.toArray(new SelectItem[resultSelectItems.size()]);
			Map> functionInput = new HashMap>();
			for (SelectItem item : functionItems) {
				functionInput.put(item, new ArrayList());
			}

			boolean onlyAggregates = true;
			if (functionItems.size() != resultSelectItems.size()) {
				onlyAggregates = false;
			}

			List resultRows = new ArrayList();
			while (dataSet.next()) {
				Row inputRow = dataSet.getRow();
				for (SelectItem item : functionItems) {
					List objects = functionInput.get(item);
					Column column = item.getColumn();
					if (column != null) {
						Object value = inputRow
								.getValue(new SelectItem(column));
						objects.add(value);
					} else if (SelectItem.isCountAllItem(item)) {
						// Just use the empty string, since COUNT(*) don't
						// evaluate values (but null values should be prevented)
						objects.add("");
					} else {
						throw new IllegalArgumentException(
								"Expression function not supported: " + item);
					}
				}

				// If the result should also contain non-aggregated values, we
				// will keep those in the rows list
				if (!onlyAggregates) {
					Object[] values = new Object[resultSelectItemsArray.length];
					for (int i = 0; i < resultSelectItemsArray.length; i++) {
						Object value = inputRow
								.getValue(resultSelectItemsArray[i]);
						if (value != null) {
							values[i] = value;
						}
					}
					resultRows.add(new Row(resultSelectItemsArray, values));
				}
			}
			dataSet.close();

			// Create the aggregates
			Map functionResult = new HashMap();
			for (SelectItem item : functionItems) {
				List input = functionInput.get(item);
				Object result = item.getFunction().evaluate(input.toArray());
				functionResult.put(item, result);
			}
			functionInput = null;

			if (onlyAggregates) {
				// We will only create a single row with all the aggregates
				Object[] values = new Object[resultSelectItemsArray.length];
				for (int i = 0; i < resultSelectItemsArray.length; i++) {
					values[i] = functionResult.get(resultSelectItemsArray[i]);
				}
				Row row = new Row(resultSelectItemsArray, values);
				resultRows.add(row);
			} else {
				// We will create the aggregates as well as regular values
				for (int i = 0; i < resultRows.size(); i++) {
					Row row = resultRows.get(i);
					Object[] values = row.getValues();
					for (Entry entry : functionResult
							.entrySet()) {
						SelectItem item = entry.getKey();
						int itemIndex = row.indexOf(item);
						if (itemIndex != -1) {
							Object value = entry.getValue();
							values[itemIndex] = value;
						}
					}
					resultRows.set(i, new Row(resultSelectItemsArray, values));
				}
			}
			return new DataSet(resultRows);
		}
	}

	public static List getFunctionSelectItems(
			List selectItems) {
		List result = new ArrayList();
		for (SelectItem selectItem : selectItems) {
			if (selectItem.getFunction() != null) {
				result.add(selectItem);
			}
		}
		if (_log.isDebugEnabled()) {
			_log.debug("SelectItems with functions: " + result);
		}
		return result;
	}

	public static DataSet getOrdered(DataSet dataSet,
			List orderByItems) {
		return getOrdered(dataSet, orderByItems
				.toArray(new OrderByItem[orderByItems.size()]));
	}

	public static DataSet getOrdered(DataSet dataSet,
			final OrderByItem... orderByItems) {
		if (orderByItems != null && orderByItems.length != 0) {
			final int[] sortIndexes = new int[orderByItems.length];
			for (int i = 0; i < orderByItems.length; i++) {
				OrderByItem item = orderByItems[i];
				int indexOf = dataSet.indexOf(item.getSelectItem());
				sortIndexes[i] = indexOf;
			}
			List data = dataSet.toObjectArrays();

			final Comparator valueComparator = ObjectComparator
					.getComparator();

			Collections.sort(data, new Comparator() {
				public int compare(Object[] o1, Object[] o2) {
					for (int i = 0; i < sortIndexes.length; i++) {
						Object sortObj1 = o1[sortIndexes[i]];
						Object sortObj2 = o2[sortIndexes[i]];
						int compare = valueComparator.compare(sortObj1,
								sortObj2);
						if (compare != 0) {
							OrderByItem orderByItem = orderByItems[i];
							boolean ascending = orderByItem.isAscending();
							if (ascending) {
								return compare;
							} else {
								return compare * -1;
							}
						}
					}
					return 0;
				}
			});

			dataSet = new DataSet(dataSet.getSelectItems(), data);
		}
		return dataSet;
	}

	/**
	 * Examines a query and extracts an array of FromItem's that refer
	 * (directly) to tables (hence Joined FromItems and SubQuery FromItems are
	 * traversed but not included).
	 * 
	 * @param q
	 *            the query to examine
	 * @return an array of FromItem's that refer directly to tables
	 */
	public static FromItem[] getTableFromItems(Query q) {
		List result = new ArrayList();
		List items = q.getFromClause().getItems();
		for (FromItem item : items) {
			result.addAll(getTableFromItems(item));
		}
		return result.toArray(new FromItem[result.size()]);
	}

	public static List getTableFromItems(FromItem item) {
		List result = new ArrayList();
		if (item.getTable() != null) {
			result.add(item);
		} else if (item.getSubQuery() != null) {
			FromItem[] sqItems = getTableFromItems(item.getSubQuery());
			for (int i = 0; i < sqItems.length; i++) {
				result.add(sqItems[i]);
			}
		} else if (item.getJoin() != null) {
			FromItem leftSide = item.getLeftSide();
			result.addAll(getTableFromItems(leftSide));
			FromItem rightSide = item.getRightSide();
			result.addAll(getTableFromItems(rightSide));
		} else {
			throw new IllegalStateException(
					"FromItem was neither of Table type, SubQuery type or Join type: "
							+ item);
		}
		return result;
	}

	/**
	 * Executes a single row query, like "SELECT COUNT(*), MAX(SOME_COLUMN) FROM
	 * MY_TABLE" or similar.
	 * 
	 * @param dataContext
	 *            the DataContext object to use for executing the query
	 * @param query
	 *            the query to execute
	 * @return a row object representing the single row returned from the query
	 * @throws MetaModelException
	 *             if less or more than one Row is returned from the query
	 */
	public static Row executeSingleRowQuery(DataContext dataContext, Query query)
			throws MetaModelException {
		DataSet dataSet = dataContext.executeQuery(query);
		boolean next = dataSet.next();
		if (!next) {
			throw new MetaModelException("No rows returned from query: "
					+ query);
		}
		Row row = dataSet.getRow();
		next = dataSet.next();
		if (next) {
			throw new MetaModelException(
					"More than one row returned from query: " + query);
		}
		dataSet.close();
		return row;
	}

	/**
	 * Performs a left join (aka left outer join) operation on two datasets.
	 * 
	 * @param ds1
	 *            the left dataset
	 * @param ds2
	 *            the right dataset
	 * @param onConditions
	 *            the conditions to join by
	 * @return the left joined result dataset
	 */
	public static DataSet getLeftJoin(DataSet ds1, DataSet ds2,
			FilterItem[] onConditions) {
		if (ds1 == null) {
			throw new IllegalArgumentException("Left DataSet cannot be null");
		}
		if (ds2 == null) {
			throw new IllegalArgumentException("Right DataSet cannot be null");
		}
		SelectItem[] si1 = ds1.getSelectItems();
		SelectItem[] si2 = ds2.getSelectItems();
		SelectItem[] selectItems = new SelectItem[si1.length + si2.length];
		System.arraycopy(si1, 0, selectItems, 0, si1.length);
		System.arraycopy(si2, 0, selectItems, si1.length, si2.length);

		List resultRows = new ArrayList();
		List ds2data = ds2.toObjectArrays();
		while (ds1.next()) {

			// Construct a single-row dataset for making a carthesian product
			// against ds2
			Row ds1row = ds1.getRow();
			List ds1rows = new ArrayList();
			ds1rows.add(ds1row);

			List carthesianRows = getCarthesianProduct(
					new DataSet[] { new DataSet(ds1rows),
							new DataSet(si2, ds2data) }, onConditions)
					.toObjectArrays();
			if (carthesianRows.size() > 0) {
				resultRows.addAll(carthesianRows);
			} else {
				Object[] values = ds1row.getValues();
				Object[] row = new Object[selectItems.length];
				System.arraycopy(values, 0, row, 0, values.length);
				resultRows.add(row);
			}
		}
		ds1.close();

		return new DataSet(selectItems, resultRows);
	}

	/**
	 * Performs a right join (aka right outer join) operation on two datasets.
	 * 
	 * @param ds1
	 *            the left dataset
	 * @param ds2
	 *            the right dataset
	 * @param onConditions
	 *            the conditions to join by
	 * @return the right joined result dataset
	 */
	public static DataSet getRightJoin(DataSet ds1, DataSet ds2,
			FilterItem[] onConditions) {
		SelectItem[] ds1selects = ds1.getSelectItems();
		SelectItem[] ds2selects = ds2.getSelectItems();
		SelectItem[] leftOrderedSelects = new SelectItem[ds1selects.length
				+ ds2selects.length];
		System.arraycopy(ds1selects, 0, leftOrderedSelects, 0,
				ds1selects.length);
		System.arraycopy(ds2selects, 0, leftOrderedSelects, ds1selects.length,
				ds2selects.length);

		// We will reuse the left join algorithm (but switch the datasets
		// around)
		DataSet dataSet = getLeftJoin(ds2, ds1, onConditions);

		dataSet = getSelection(leftOrderedSelects, dataSet);
		return dataSet;
	}

	public static SelectItem[] createSelectItems(Column[] columns) {
		SelectItem[] items = new SelectItem[columns.length];
		for (int i = 0; i < items.length; i++) {
			items[i] = new SelectItem(columns[i]);
		}
		return items;
	}

	public static DataSet getDistinct(DataSet dataSet) {
		SelectItem[] selectItems = dataSet.getSelectItems();
		GroupByItem[] groupByItems = new GroupByItem[selectItems.length];
		for (int i = 0; i < groupByItems.length; i++) {
			groupByItems[i] = new GroupByItem(selectItems[i]);
		}
		return getGrouped(selectItems, dataSet, groupByItems);
	}

	public static Table[] getTables(Column[] columns) {
		return getTables(Arrays.asList(columns));
	}

	public static Column[] getColumnsByType(Column[] columns,
			ColumnType columnType) {
		ArrayList result = new ArrayList();
		for (Column column : columns) {
			if (columnType == column.getType()) {
				result.add(column);
			}
		}
		return result.toArray(new Column[result.size()]);
	}
}    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy