io.druid.segment.QueryableIndexIndexableAdapter Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of druid-processing Show documentation
Show all versions of druid-processing Show documentation
A module that is everything required to understands Druid Segments
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.segment;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.metamx.common.ISE;
import com.metamx.common.guava.CloseQuietly;
import com.metamx.common.logger.Logger;
import io.druid.segment.column.BitmapIndex;
import io.druid.segment.column.Column;
import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.column.ComplexColumn;
import io.druid.segment.column.DictionaryEncodedColumn;
import io.druid.segment.column.GenericColumn;
import io.druid.segment.column.IndexedFloatsGenericColumn;
import io.druid.segment.column.IndexedLongsGenericColumn;
import io.druid.segment.column.ValueType;
import io.druid.segment.data.ArrayBasedIndexedInts;
import io.druid.segment.data.BitmapCompressedIndexedInts;
import io.druid.segment.data.EmptyIndexedInts;
import io.druid.segment.data.Indexed;
import io.druid.segment.data.IndexedInts;
import io.druid.segment.data.IndexedIterable;
import io.druid.segment.data.ListIndexed;
import org.joda.time.Interval;
import java.io.Closeable;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
/**
*/
public class QueryableIndexIndexableAdapter implements IndexableAdapter
{
private static final Logger log = new Logger(QueryableIndexIndexableAdapter.class);
private final int numRows;
private final QueryableIndex input;
private final List availableDimensions;
private final Metadata metadata;
public QueryableIndexIndexableAdapter(QueryableIndex input)
{
this.input = input;
numRows = input.getNumRows();
// It appears possible that the dimensions have some columns listed which do not have a DictionaryEncodedColumn
// This breaks current logic, but should be fine going forward. This is a work-around to make things work
// in the current state. This code shouldn't be needed once github tracker issue #55 is finished.
this.availableDimensions = Lists.newArrayList();
for (String dim : input.getAvailableDimensions()) {
final Column col = input.getColumn(dim);
if (col == null) {
log.warn("Wtf!? column[%s] didn't exist!?!?!?", dim);
} else if (col.getDictionaryEncoding() != null) {
availableDimensions.add(dim);
} else {
log.info("No dictionary on dimension[%s]", dim);
}
}
this.metadata = input.getMetadata();
}
@Override
public Interval getDataInterval()
{
return input.getDataInterval();
}
@Override
public int getNumRows()
{
return numRows;
}
@Override
public Indexed getDimensionNames()
{
return new ListIndexed<>(availableDimensions, String.class);
}
@Override
public Indexed getMetricNames()
{
final Set columns = Sets.newLinkedHashSet(input.getColumnNames());
final HashSet dimensions = Sets.newHashSet(getDimensionNames());
return new ListIndexed<>(
Lists.newArrayList(Sets.difference(columns, dimensions)),
String.class
);
}
@Override
public Indexed getDimValueLookup(String dimension)
{
final Column column = input.getColumn(dimension);
if (column == null) {
return null;
}
final DictionaryEncodedColumn dict = column.getDictionaryEncoding();
if (dict == null) {
return null;
}
return new Indexed()
{
@Override
public Class getClazz()
{
return String.class;
}
@Override
public int size()
{
return dict.getCardinality();
}
@Override
public String get(int index)
{
return dict.lookupName(index);
}
@Override
public int indexOf(String value)
{
return dict.lookupId(value);
}
@Override
public Iterator iterator()
{
return IndexedIterable.create(this).iterator();
}
};
}
@Override
public Iterable getRows()
{
return new Iterable()
{
@Override
public Iterator iterator()
{
return new Iterator()
{
final GenericColumn timestamps = input.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();
final Object[] metrics;
final DictionaryEncodedColumn[] dictionaryEncodedColumns;
final int numMetrics = getMetricNames().size();
int currRow = 0;
boolean done = false;
{
this.dictionaryEncodedColumns = FluentIterable
.from(getDimensionNames())
.transform(
new Function()
{
@Override
public DictionaryEncodedColumn apply(String dimName)
{
return input.getColumn(dimName)
.getDictionaryEncoding();
}
}
).toArray(DictionaryEncodedColumn.class);
final Indexed availableMetrics = getMetricNames();
metrics = new Object[availableMetrics.size()];
for (int i = 0; i < metrics.length; ++i) {
final Column column = input.getColumn(availableMetrics.get(i));
final ValueType type = column.getCapabilities().getType();
switch (type) {
case FLOAT:
case LONG:
metrics[i] = column.getGenericColumn();
break;
case COMPLEX:
metrics[i] = column.getComplexColumn();
break;
default:
throw new ISE("Cannot handle type[%s]", type);
}
}
}
@Override
public boolean hasNext()
{
final boolean hasNext = currRow < numRows;
if (!hasNext && !done) {
CloseQuietly.close(timestamps);
for (Object metric : metrics) {
if (metric instanceof Closeable) {
CloseQuietly.close((Closeable) metric);
}
}
for (Object dimension : dictionaryEncodedColumns) {
if (dimension instanceof Closeable) {
CloseQuietly.close((Closeable) dimension);
}
}
done = true;
}
return hasNext;
}
@Override
public Rowboat next()
{
if (!hasNext()) {
throw new NoSuchElementException();
}
final int[][] dims = new int[dictionaryEncodedColumns.length][];
int dimIndex = 0;
for (final DictionaryEncodedColumn dict : dictionaryEncodedColumns) {
final IndexedInts dimVals;
if (dict.hasMultipleValues()) {
dimVals = dict.getMultiValueRow(currRow);
} else {
dimVals = new ArrayBasedIndexedInts(new int[]{dict.getSingleValueRow(currRow)});
}
int[] theVals = new int[dimVals.size()];
for (int j = 0; j < theVals.length; ++j) {
theVals[j] = dimVals.get(j);
}
dims[dimIndex++] = theVals;
}
Object[] metricArray = new Object[numMetrics];
for (int i = 0; i < metricArray.length; ++i) {
if (metrics[i] instanceof IndexedFloatsGenericColumn) {
metricArray[i] = ((GenericColumn) metrics[i]).getFloatSingleValueRow(currRow);
} else if (metrics[i] instanceof IndexedLongsGenericColumn) {
metricArray[i] = ((GenericColumn) metrics[i]).getLongSingleValueRow(currRow);
} else if (metrics[i] instanceof ComplexColumn) {
metricArray[i] = ((ComplexColumn) metrics[i]).getRowValue(currRow);
}
}
final Rowboat retVal = new Rowboat(
timestamps.getLongSingleValueRow(currRow), dims, metricArray, currRow
);
++currRow;
return retVal;
}
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
};
}
};
}
@VisibleForTesting
IndexedInts getBitmapIndex(String dimension, String value)
{
final Column column = input.getColumn(dimension);
if (column == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}
final BitmapIndex bitmaps = column.getBitmapIndex();
if (bitmaps == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}
return new BitmapCompressedIndexedInts(bitmaps.getBitmap(value));
}
@Override
public String getMetricType(String metric)
{
final Column column = input.getColumn(metric);
final ValueType type = column.getCapabilities().getType();
switch (type) {
case FLOAT:
return "float";
case LONG:
return "long";
case COMPLEX:
return column.getComplexColumn().getTypeName();
default:
throw new ISE("Unknown type[%s]", type);
}
}
@Override
public ColumnCapabilities getCapabilities(String column)
{
return input.getColumn(column).getCapabilities();
}
@Override
public IndexedInts getBitmapIndex(String dimension, int dictId)
{
final Column column = input.getColumn(dimension);
if (column == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}
final BitmapIndex bitmaps = column.getBitmapIndex();
if (bitmaps == null) {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}
if (dictId >= 0) {
return new BitmapCompressedIndexedInts(bitmaps.getBitmap(dictId));
} else {
return EmptyIndexedInts.EMPTY_INDEXED_INTS;
}
}
@Override
public Metadata getMetadata()
{
return metadata;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy