com.metamx.common.Granularity Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2011,2012 Metamarkets Group Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.metamx.common;
import org.joda.time.DateTime;
import org.joda.time.Days;
import org.joda.time.Hours;
import org.joda.time.Interval;
import org.joda.time.Minutes;
import org.joda.time.Months;
import org.joda.time.MutableDateTime;
import org.joda.time.ReadableInterval;
import org.joda.time.ReadablePeriod;
import org.joda.time.Seconds;
import org.joda.time.Weeks;
import org.joda.time.Years;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
*/
public enum Granularity
{
SECOND
{
final DateTimeFormatter defaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd/'H'=HH/'M'=mm/'S'=ss");
final DateTimeFormatter hiveFormat = DateTimeFormat.forPattern("'dt'=yyyy-MM-dd-HH-mm-ss");
final DateTimeFormatter lowerDefaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd/'h'=HH/'m'=mm/'s'=ss");
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
switch (type) {
case DEFAULT:
return defaultFormat;
case HIVE:
return hiveFormat;
case LOWER_DEFAULT:
return lowerDefaultFormat;
default:
throw new IAE("There is no format for type %s at granularity %s", type, this.name());
}
}
@Override
public ReadablePeriod getUnits(int count)
{
return Seconds.seconds(count);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Seconds.secondsIn(interval).getSeconds();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null
&& vals[2] != null
&& vals[3] != null
&& vals[4] != null
&& vals[5] != null
&& vals[6] != null) {
date = new DateTime(vals[1], vals[2], vals[3], vals[4], vals[5], vals[6], 0);
}
return date;
}
},
MINUTE
{
final DateTimeFormatter defaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd/'H'=HH/'M'=mm");
final DateTimeFormatter hiveFormat = DateTimeFormat.forPattern("'dt'=yyyy-MM-dd-HH-mm");
final DateTimeFormatter lowerDefaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd/'h'=HH/'m'=mm");
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
switch (type) {
case DEFAULT:
return defaultFormat;
case HIVE:
return hiveFormat;
case LOWER_DEFAULT:
return lowerDefaultFormat;
default:
throw new IAE("There is no format for type %s at granularity %s", type, this.name());
}
}
@Override
public ReadablePeriod getUnits(int count)
{
return Minutes.minutes(count);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
mutableDateTime.setSecondOfMinute(0);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Minutes.minutesIn(interval).getMinutes();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null && vals[2] != null && vals[3] != null && vals[4] != null && vals[5] != null) {
date = new DateTime(vals[1], vals[2], vals[3], vals[4], vals[5], 0, 0);
}
return date;
}
},
FIVE_MINUTE
{
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
return MINUTE.getFormatter(type);
}
@Override
public ReadablePeriod getUnits(int count)
{
return Minutes.minutes(count * 5);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
mutableDateTime.setSecondOfMinute(0);
mutableDateTime.setMinuteOfHour(mutableDateTime.getMinuteOfHour() - (mutableDateTime.getMinuteOfHour() % 5));
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Minutes.minutesIn(interval).getMinutes() / 5;
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
if (vals[1] != null && vals[2] != null && vals[3] != null && vals[4] != null && vals[5] != null) {
return truncate(new DateTime(vals[1], vals[2], vals[3], vals[4], vals[5], 0, 0));
}
return null;
}
},
TEN_MINUTE
{
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
return MINUTE.getFormatter(type);
}
@Override
public ReadablePeriod getUnits(int count)
{
return Minutes.minutes(count * 10);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
mutableDateTime.setSecondOfMinute(0);
mutableDateTime.setMinuteOfHour(mutableDateTime.getMinuteOfHour() - (mutableDateTime.getMinuteOfHour() % 10));
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Minutes.minutesIn(interval).getMinutes() / 10;
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
if (vals[1] != null && vals[2] != null && vals[3] != null && vals[4] != null && vals[5] != null) {
return truncate(new DateTime(vals[1], vals[2], vals[3], vals[4], vals[5], 0, 0));
}
return null;
}
},
FIFTEEN_MINUTE
{
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
return MINUTE.getFormatter(type);
}
@Override
public ReadablePeriod getUnits(int count)
{
return Minutes.minutes(count * 15);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
mutableDateTime.setSecondOfMinute(0);
mutableDateTime.setMinuteOfHour(mutableDateTime.getMinuteOfHour() - (mutableDateTime.getMinuteOfHour() % 15));
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Minutes.minutesIn(interval).getMinutes() / 15;
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
if (vals[1] != null && vals[2] != null && vals[3] != null && vals[4] != null && vals[5] != null) {
return truncate(new DateTime(vals[1], vals[2], vals[3], vals[4], vals[5], 0, 0));
}
return null;
}
},
HOUR
{
final DateTimeFormatter defaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd/'H'=HH");
final DateTimeFormatter hiveFormat = DateTimeFormat.forPattern("'dt'=yyyy-MM-dd-HH");
final DateTimeFormatter lowerDefaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd/'h'=HH");
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
switch (type) {
case DEFAULT:
return defaultFormat;
case HIVE:
return hiveFormat;
case LOWER_DEFAULT:
return lowerDefaultFormat;
default:
throw new IAE("There is no format for type %s at granularity %s", type, this.name());
}
}
@Override
public ReadablePeriod getUnits(int n)
{
return Hours.hours(n);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
mutableDateTime.setSecondOfMinute(0);
mutableDateTime.setMinuteOfHour(0);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Hours.hoursIn(interval).getHours();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null && vals[2] != null && vals[3] != null && vals[4] != null) {
date = new DateTime(vals[1], vals[2], vals[3], vals[4], 0, 0, 0);
}
return date;
}
},
SIX_HOUR
{
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
return HOUR.getFormatter(type);
}
@Override
public ReadablePeriod getUnits(int n)
{
return Hours.hours(n * 6);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfSecond(0);
mutableDateTime.setSecondOfMinute(0);
mutableDateTime.setMinuteOfHour(0);
mutableDateTime.setHourOfDay(mutableDateTime.getHourOfDay() - (mutableDateTime.getHourOfDay() % 6));
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Hours.hoursIn(interval).getHours() / 6;
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
if (vals[1] != null && vals[2] != null && vals[3] != null && vals[4] != null) {
return truncate(new DateTime(vals[1], vals[2], vals[3], vals[4], 0, 0, 0));
}
return null;
}
},
DAY
{
final DateTimeFormatter defaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd");
final DateTimeFormatter hiveFormat = DateTimeFormat.forPattern("'dt'=yyyy-MM-dd");
final DateTimeFormatter lowerDefaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM/'d'=dd");
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
switch (type) {
case DEFAULT:
return defaultFormat;
case HIVE:
return hiveFormat;
case LOWER_DEFAULT:
return lowerDefaultFormat;
default:
throw new IAE("There is no format for type %s at granularity %s", type, this.name());
}
}
@Override
public ReadablePeriod getUnits(int n)
{
return Days.days(n);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfDay(0);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Days.daysIn(interval).getDays();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null && vals[2] != null && vals[3] != null) {
date = new DateTime(vals[1], vals[2], vals[3], 0, 0, 0, 0);
}
return date;
}
},
WEEK
{
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
return DAY.getFormatter(type);
}
@Override
public ReadablePeriod getUnits(int n)
{
return Weeks.weeks(n);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfDay(0);
mutableDateTime.setDayOfWeek(1);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Weeks.weeksIn(interval).getWeeks();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null && vals[2] != null && vals[3] != null) {
date = truncate(new DateTime(vals[1], vals[2], vals[3], 0, 0, 0, 0));
}
return date;
}
},
MONTH
{
final DateTimeFormatter defaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM");
final DateTimeFormatter hiveFormat = DateTimeFormat.forPattern("'dt'=yyyy-MM");
final DateTimeFormatter lowerDefaultFormat = DateTimeFormat.forPattern("'y'=yyyy/'m'=MM");
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
switch (type) {
case DEFAULT:
return defaultFormat;
case HIVE:
return hiveFormat;
case LOWER_DEFAULT:
return lowerDefaultFormat;
default:
throw new IAE("There is no format for type %s at granularity %s", type, this.name());
}
}
@Override
public ReadablePeriod getUnits(int n)
{
return Months.months(n);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfDay(0);
mutableDateTime.setDayOfMonth(1);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Months.monthsIn(interval).getMonths();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null && vals[2] != null) {
date = new DateTime(vals[1], vals[2], 1, 0, 0, 0, 0);
}
return date;
}
},
YEAR
{
final DateTimeFormatter defaultFormat = DateTimeFormat.forPattern("'y'=yyyy");
final DateTimeFormatter hiveFormat = DateTimeFormat.forPattern("'dt'=yyyy");
final DateTimeFormatter lowerDefaultFormat = DateTimeFormat.forPattern("'y'=yyyy");
@Override
public DateTimeFormatter getFormatter(Formatter type)
{
switch (type) {
case DEFAULT:
return defaultFormat;
case HIVE:
return hiveFormat;
case LOWER_DEFAULT:
return lowerDefaultFormat;
default:
throw new IAE("There is no format for type %s at granularity %s", type, this.name());
}
}
@Override
public ReadablePeriod getUnits(int n)
{
return Years.years(n);
}
@Override
public DateTime truncate(DateTime time)
{
final MutableDateTime mutableDateTime = time.toMutableDateTime();
mutableDateTime.setMillisOfDay(0);
mutableDateTime.setDayOfMonth(1);
mutableDateTime.setMonthOfYear(1);
return mutableDateTime.toDateTime();
}
@Override
public int numIn(ReadableInterval interval)
{
return Years.yearsIn(interval).getYears();
}
@Override
public DateTime toDate(String filePath, Formatter formatter)
{
Integer[] vals = getDateValues(filePath, formatter);
DateTime date = null;
if (vals[1] != null) {
date = new DateTime(vals[1], 1, 1, 0, 0, 0, 0);
}
return date;
}
};
// Default patterns for parsing paths.
protected final Pattern defaultPathPattern =
Pattern.compile(
"^.*[Yy]=(\\d{4})/(?:[Mm]=(\\d{2})/(?:[Dd]=(\\d{2})/(?:[Hh]=(\\d{2})/(?:[Mm]=(\\d{2})/(?:[Ss]=(\\d{2})/)?)?)?)?)?.*$"
);
protected final Pattern hivePathPattern =
Pattern.compile("^.*dt=(\\d{4})(?:-(\\d{2})(?:-(\\d{2})(?:-(\\d{2})(?:-(\\d{2})(?:-(\\d{2})?)?)?)?)?)?/.*$");
// Abstract functions that individual enum's need to implement for the strategy.
public abstract DateTimeFormatter getFormatter(Formatter type);
public abstract ReadablePeriod getUnits(int n);
public abstract DateTime truncate(DateTime time);
public abstract int numIn(ReadableInterval interval);
public abstract DateTime toDate(String filePath, Formatter formatter);
public DateTime toDate(String filePath)
{
return toDate(filePath, Formatter.DEFAULT);
}
// Used by the toDate implementations.
protected final Integer[] getDateValues(String filePath, Formatter formatter)
{
Pattern pattern = defaultPathPattern;
switch (formatter) {
case DEFAULT:
case LOWER_DEFAULT:
break;
case HIVE:
pattern = hivePathPattern;
break;
default:
throw new IAE("Format %s not supported", formatter);
}
Matcher matcher = pattern.matcher(filePath);
Integer[] vals = new Integer[7];
if (matcher.matches()) {
for (int i = 1; i <= matcher.groupCount(); i++) {
vals[i] = (matcher.group(i) != null) ? Integer.parseInt(matcher.group(i)) : null;
}
}
return vals;
}
// Strategy Functions
public final DateTimeFormatter getFormatter(String type)
{
return getFormatter(Formatter.valueOf(type.toUpperCase()));
}
public final DateTime increment(DateTime time)
{
return time.plus(getUnits(1));
}
public final DateTime increment(DateTime time, int count)
{
return time.plus(getUnits(count));
}
public final DateTime decrement(DateTime time)
{
return time.minus(getUnits(1));
}
public final DateTime decrement(DateTime time, int count)
{
return time.minus(getUnits(count));
}
public final String toPath(DateTime time)
{
return toPath(time, "default");
}
public final String toPath(DateTime time, String type)
{
return toPath(time, Formatter.valueOf(type.toUpperCase()));
}
public final String toPath(DateTime time, Formatter type)
{
return getFormatter(type).print(time);
}
/**
* Return a granularity-sized Interval containing a particular DateTime.
*/
public final Interval bucket(DateTime t)
{
DateTime start = truncate(t);
return new Interval(start, increment(start));
}
/**
* Round out Interval such that it becomes granularity-aligned and nonempty.
*/
public final Interval widen(Interval interval)
{
final DateTime start = truncate(interval.getStart());
final DateTime end;
if (interval.getEnd().equals(start)) {
// Empty with aligned start/end; expand into a granularity-sized interval
end = increment(start);
} else if (truncate(interval.getEnd()).equals(interval.getEnd())) {
// Non-empty with aligned end; keep the same end
end = interval.getEnd();
} else {
// Non-empty with non-aligned end; push it out
end = increment(truncate(interval.getEnd()));
}
return new Interval(start, end);
}
// Iterable functions and classes.
public Iterable getIterable(final DateTime start, final DateTime end)
{
return getIterable(new Interval(start, end));
}
public Iterable getIterable(final Interval input)
{
return new IntervalIterable(input);
}
public Iterable getReverseIterable(final DateTime start, final DateTime end)
{
return getReverseIterable(new Interval(start, end));
}
public Iterable getReverseIterable(final Interval input)
{
return new ReverseIntervalIterable(input);
}
public class IntervalIterable implements Iterable
{
private final Interval inputInterval;
public IntervalIterable(Interval inputInterval)
{
this.inputInterval = inputInterval;
}
@Override
public Iterator iterator()
{
return new IntervalIterator(inputInterval);
}
}
public class ReverseIntervalIterable implements Iterable
{
private final Interval inputInterval;
public ReverseIntervalIterable(Interval inputInterval)
{
this.inputInterval = inputInterval;
}
@Override
public Iterator iterator()
{
return new ReverseIntervalIterator(inputInterval);
}
}
public class IntervalIterator implements Iterator
{
private final Interval inputInterval;
private DateTime currStart;
private DateTime currEnd;
public IntervalIterator(Interval inputInterval)
{
this.inputInterval = inputInterval;
currStart = truncate(inputInterval.getStart());
currEnd = increment(currStart);
}
@Override
public boolean hasNext()
{
return currStart.isBefore(inputInterval.getEnd());
}
@Override
public Interval next()
{
if (!hasNext()) {
throw new NoSuchElementException("There are no more intervals");
}
Interval retVal = new Interval(currStart, currEnd);
currStart = currEnd;
currEnd = increment(currStart);
return retVal;
}
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
}
public class ReverseIntervalIterator implements Iterator
{
private final Interval inputInterval;
private DateTime currStart;
private DateTime currEnd;
public ReverseIntervalIterator(Interval inputInterval)
{
this.inputInterval = inputInterval;
currEnd = inputInterval.getEnd();
currStart = decrement(currEnd);
}
@Override
public boolean hasNext()
{
return currEnd.isAfter(inputInterval.getStart());
}
@Override
public Interval next()
{
if (!hasNext()) {
throw new NoSuchElementException("There are no more intervals");
}
Interval retVal = new Interval(currStart, currEnd);
currEnd = currStart;
currStart = decrement(currEnd);
return retVal;
}
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
}
public enum Formatter
{
DEFAULT,
HIVE,
LOWER_DEFAULT
}
}