Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
io.druid.indexing.overlord.MetadataTaskStorage Maven / Gradle / Ivy
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.indexing.overlord;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.inject.Inject;
import io.druid.java.util.emitter.EmittingLogger;
import io.druid.indexing.common.TaskLock;
import io.druid.indexing.common.TaskStatus;
import io.druid.indexing.common.actions.TaskAction;
import io.druid.indexing.common.config.TaskStorageConfig;
import io.druid.indexing.common.task.Task;
import io.druid.java.util.common.DateTimes;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.Pair;
import io.druid.java.util.common.lifecycle.LifecycleStart;
import io.druid.java.util.common.lifecycle.LifecycleStop;
import io.druid.metadata.EntryExistsException;
import io.druid.metadata.MetadataStorageActionHandler;
import io.druid.metadata.MetadataStorageActionHandlerFactory;
import io.druid.metadata.MetadataStorageActionHandlerTypes;
import io.druid.metadata.MetadataStorageConnector;
import io.druid.metadata.MetadataStorageTablesConfig;
import org.joda.time.DateTime;
import javax.annotation.Nullable;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class MetadataTaskStorage implements TaskStorage
{
private static final MetadataStorageActionHandlerTypes TASK_TYPES = new MetadataStorageActionHandlerTypes()
{
@Override
public TypeReference getEntryType()
{
return new TypeReference()
{
};
}
@Override
public TypeReference getStatusType()
{
return new TypeReference()
{
};
}
@Override
public TypeReference getLogType()
{
return new TypeReference()
{
};
}
@Override
public TypeReference getLockType()
{
return new TypeReference()
{
};
}
};
private final MetadataStorageConnector metadataStorageConnector;
private final TaskStorageConfig config;
private final MetadataStorageActionHandler handler;
private static final EmittingLogger log = new EmittingLogger(MetadataTaskStorage.class);
@Inject
public MetadataTaskStorage(
final MetadataStorageConnector metadataStorageConnector,
final TaskStorageConfig config,
final MetadataStorageActionHandlerFactory factory
)
{
this.metadataStorageConnector = metadataStorageConnector;
this.config = config;
this.handler = factory.create(MetadataStorageTablesConfig.TASK_ENTRY_TYPE, TASK_TYPES);
}
@LifecycleStart
public void start()
{
metadataStorageConnector.createTaskTables();
}
@LifecycleStop
public void stop()
{
// do nothing
}
@Override
public void insert(final Task task, final TaskStatus status) throws EntryExistsException
{
Preconditions.checkNotNull(task, "task");
Preconditions.checkNotNull(status, "status");
Preconditions.checkArgument(
task.getId().equals(status.getId()),
"Task/Status ID mismatch[%s/%s]",
task.getId(),
status.getId()
);
log.info("Inserting task %s with status: %s", task.getId(), status);
try {
handler.insert(
task.getId(),
DateTimes.nowUtc(),
task.getDataSource(),
task,
status.isRunnable(),
status
);
}
catch (Exception e) {
if (e instanceof EntryExistsException) {
throw (EntryExistsException) e;
} else {
Throwables.propagate(e);
}
}
}
@Override
public void setStatus(final TaskStatus status)
{
Preconditions.checkNotNull(status, "status");
log.info("Updating task %s to status: %s", status.getId(), status);
final boolean set = handler.setStatus(
status.getId(),
status.isRunnable(),
status
);
if (!set) {
throw new ISE("Active task not found: %s", status.getId());
}
}
@Override
public Optional getTask(final String taskId)
{
return handler.getEntry(taskId);
}
@Override
public Optional getStatus(final String taskId)
{
return handler.getStatus(taskId);
}
@Override
public List getActiveTasks()
{
return ImmutableList.copyOf(
Iterables.transform(
Iterables.filter(
handler.getActiveEntriesWithStatus(),
new Predicate>()
{
@Override
public boolean apply(
@Nullable Pair input
)
{
return input.rhs.isRunnable();
}
}
),
new Function, Task>()
{
@Nullable
@Override
public Task apply(@Nullable Pair input)
{
return input.lhs;
}
}
)
);
}
@Override
public List getRecentlyFinishedTaskStatuses(@Nullable Integer maxTaskStatuses)
{
return ImmutableList.copyOf(
handler
.getInactiveStatusesSince(
DateTimes.nowUtc().minus(config.getRecentlyFinishedThreshold()),
maxTaskStatuses
)
.stream()
.filter(TaskStatus::isComplete)
.collect(Collectors.toList())
);
}
@Nullable
@Override
public Pair getCreatedDateTimeAndDataSource(String taskId)
{
return handler.getCreatedDateAndDataSource(taskId);
}
@Override
public void addLock(final String taskid, final TaskLock taskLock)
{
Preconditions.checkNotNull(taskid, "taskid");
Preconditions.checkNotNull(taskLock, "taskLock");
log.info(
"Adding lock on interval[%s] version[%s] for task: %s",
taskLock.getInterval(),
taskLock.getVersion(),
taskid
);
handler.addLock(taskid, taskLock);
}
@Override
public void replaceLock(String taskid, TaskLock oldLock, TaskLock newLock)
{
Preconditions.checkNotNull(taskid, "taskid");
Preconditions.checkNotNull(oldLock, "oldLock");
Preconditions.checkNotNull(newLock, "newLock");
log.info(
"Replacing lock on interval[%s] version[%s] for task: %s",
oldLock.getInterval(),
oldLock.getVersion(),
taskid
);
final Long oldLockId = handler.getLockId(taskid, oldLock);
if (oldLockId == null) {
throw new ISE("Cannot find lock[%s]", oldLock);
}
handler.replaceLock(taskid, oldLockId, newLock);
}
@Override
public void removeLock(String taskid, TaskLock taskLockToRemove)
{
Preconditions.checkNotNull(taskid, "taskid");
Preconditions.checkNotNull(taskLockToRemove, "taskLockToRemove");
final Long lockId = handler.getLockId(taskid, taskLockToRemove);
if (lockId == null) {
log.warn("Cannot find lock[%s]", taskLockToRemove);
} else {
log.info("Deleting TaskLock with id[%d]: %s", lockId, taskLockToRemove);
handler.removeLock(lockId);
}
}
@Override
public List getLocks(String taskid)
{
return ImmutableList.copyOf(
Iterables.transform(
getLocksWithIds(taskid).entrySet(), new Function, TaskLock>()
{
@Override
public TaskLock apply(Map.Entry e)
{
return e.getValue();
}
}
)
);
}
@Override
public void addAuditLog(final Task task, final TaskAction taskAction)
{
Preconditions.checkNotNull(taskAction, "taskAction");
log.info("Logging action for task[%s]: %s", task.getId(), taskAction);
handler.addLog(task.getId(), taskAction);
}
@Override
public List getAuditLogs(final String taskId)
{
return handler.getLogs(taskId);
}
private Map getLocksWithIds(final String taskid)
{
return handler.getLocks(taskid);
}
}
