io.druid.query.IntervalChunkingQueryRunner Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of druid-processing Show documentation
A module that is everything required to understands Druid Segments
There is a newer version: 0.12.3
/*
 * Licensed to Metamarkets Group Inc. (Metamarkets) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. Metamarkets licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package io.druid.query;

import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.metamx.common.guava.FunctionalIterable;
import com.metamx.common.guava.Sequence;
import com.metamx.common.guava.Sequences;
import com.metamx.emitter.service.ServiceEmitter;
import com.metamx.emitter.service.ServiceMetricEvent;
import io.druid.granularity.PeriodGranularity;
import io.druid.query.spec.MultipleIntervalSegmentSpec;
import org.joda.time.Interval;
import org.joda.time.Period;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;

/**
 */
public class IntervalChunkingQueryRunner implements QueryRunner
{
  private final QueryRunner baseRunner;

  private final QueryToolChest> toolChest;
  private final ExecutorService executor;
  private final QueryWatcher queryWatcher;
  private final ServiceEmitter emitter;

  public IntervalChunkingQueryRunner(
      QueryRunner baseRunner, QueryToolChest> toolChest,
      ExecutorService executor, QueryWatcher queryWatcher, ServiceEmitter emitter
  )
  {
    this.baseRunner = baseRunner;
    this.toolChest = toolChest;
    this.executor = executor;
    this.queryWatcher = queryWatcher;
    this.emitter = emitter;
  }

  @Override
  public Sequence run(final Query query, final Map responseContext)
  {
    final Period chunkPeriod = getChunkPeriod(query);
    if (chunkPeriod.toStandardDuration().getMillis() == 0) {
      return baseRunner.run(query, responseContext);
    }

    List chunkIntervals = Lists.newArrayList(
        FunctionalIterable
            .create(query.getIntervals())
            .transformCat(
                new Function>()
                {
                  @Override
                  public Iterable apply(Interval input)
                  {
                    return splitInterval(input, chunkPeriod);
                  }
                }
            )
    );

    if (chunkIntervals.size() <= 1) {
      return baseRunner.run(query, responseContext);
    }

    return Sequences.concat(
        Lists.newArrayList(
            FunctionalIterable.create(chunkIntervals).transform(
                new Function>()
                {
                  @Override
                  public Sequence apply(Interval singleInterval)
                  {
                    return new AsyncQueryRunner(
                        //Note: it is assumed that toolChest.mergeResults(..) gives a query runner that is
                        //not lazy i.e. it does most of its work on call to run() method
                        toolChest.mergeResults(
                            new MetricsEmittingQueryRunner(
                                emitter,
                                new Function, ServiceMetricEvent.Builder>()
                                {
                                  @Override
                                  public ServiceMetricEvent.Builder apply(Query input)
                                  {
                                    return toolChest.makeMetricBuilder(input);
                                  }
                                },
                                baseRunner,
                                "query/intervalChunk/time",
                                ImmutableMap.of("chunkInterval", singleInterval.toString())
                            ).withWaitMeasuredFromNow()
                        ),
                        executor, queryWatcher
                    ).run(
                        query.withQuerySegmentSpec(new MultipleIntervalSegmentSpec(Arrays.asList(singleInterval))),
                        responseContext
                    );
                  }
                }
            )
        )
    );
  }

  private Iterable splitInterval(Interval interval, Period period)
  {
    if (interval.getEndMillis() == interval.getStartMillis()) {
      return Lists.newArrayList(interval);
    }

    List intervals = Lists.newArrayList();
    Iterator timestamps = new PeriodGranularity(period, null, null).iterable(
        interval.getStartMillis(),
        interval.getEndMillis()
    ).iterator();

    long start = Math.max(timestamps.next(), interval.getStartMillis());
    while (timestamps.hasNext()) {
      long end = timestamps.next();
      intervals.add(new Interval(start, end));
      start = end;
    }

    if (start < interval.getEndMillis()) {
      intervals.add(new Interval(start, interval.getEndMillis()));
    }

    return intervals;
  }

  private Period getChunkPeriod(Query query)
  {
    String p = query.getContextValue(QueryContextKeys.CHUNK_PERIOD, "P0D");
    return Period.parse(p);
  }
}