com.baidu.hugegraph.task.HugeTask Maven / Gradle / Ivy
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.baidu.hugegraph.task;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.FutureTask;
import java.util.stream.Collector;
import java.util.stream.Collectors;
import org.apache.tinkerpop.gremlin.structure.Graph.Hidden;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.slf4j.Logger;
import com.baidu.hugegraph.HugeException;
import com.baidu.hugegraph.backend.id.Id;
import com.baidu.hugegraph.backend.id.IdGenerator;
import com.baidu.hugegraph.backend.serializer.BytesBuffer;
import com.baidu.hugegraph.exception.LimitExceedException;
import com.baidu.hugegraph.exception.NotFoundException;
import com.baidu.hugegraph.job.ComputerJob;
import com.baidu.hugegraph.job.EphemeralJob;
import com.baidu.hugegraph.type.define.SerialEnum;
import com.baidu.hugegraph.util.Blob;
import com.baidu.hugegraph.util.E;
import com.baidu.hugegraph.util.InsertionOrderUtil;
import com.baidu.hugegraph.util.JsonUtil;
import com.baidu.hugegraph.util.Log;
import com.baidu.hugegraph.util.StringEncoding;
public class HugeTask extends FutureTask {
private static final Logger LOG = Log.logger(HugeTask.class);
private static final float DECOMPRESS_RATIO = 10.0F;
private transient TaskScheduler scheduler = null;
private final TaskCallable callable;
private String type;
private String name;
private final Id id;
private final Id parent;
private Set dependencies;
private String description;
private String context;
private Date create;
private Id server;
private int load;
private volatile TaskStatus status;
private volatile int progress;
private volatile Date update;
private volatile int retries;
private volatile String input;
private volatile String result;
public HugeTask(Id id, Id parent, String callable, String input) {
this(id, parent, TaskCallable.fromClass(callable));
this.input(input);
}
public HugeTask(Id id, Id parent, TaskCallable callable) {
super(callable);
E.checkArgumentNotNull(id, "Task id can't be null");
E.checkArgument(id.number(), "Invalid task id type, it must be number");
assert callable != null;
this.callable = callable;
this.type = null;
this.name = null;
this.id = id;
this.parent = parent;
this.dependencies = null;
this.description = null;
this.context = null;
this.status = TaskStatus.NEW;
this.progress = 0;
this.create = new Date();
this.update = null;
this.retries = 0;
this.input = null;
this.result = null;
this.server = null;
this.load = 1;
}
public Id id() {
return this.id;
}
public Id parent() {
return this.parent;
}
public Set dependencies() {
return Collections.unmodifiableSet(this.dependencies);
}
public void depends(Id id) {
E.checkState(this.status == TaskStatus.NEW,
"Can't add dependency in status '%s'", this.status);
if (this.dependencies == null) {
this.dependencies = InsertionOrderUtil.newSet();
}
this.dependencies.add(id);
}
public TaskStatus status() {
return this.status;
}
public void type(String type) {
this.type = type;
}
public String type() {
return this.type;
}
public void name(String name) {
this.name = name;
}
public String name() {
return this.name;
}
public void description(String description) {
this.description = description;
}
public String description() {
return this.description;
}
public final void context(String context) {
E.checkArgument(this.context == null,
"Task context must be set once, but already set '%s'",
this.context);
E.checkArgument(this.status == TaskStatus.NEW,
"Task context must be set in state NEW instead of %s",
this.status);
this.context = context;
}
public final String context() {
return this.context;
}
public void progress(int progress) {
this.progress = progress;
}
public int progress() {
return this.progress;
}
public void createTime(Date create) {
this.create = create;
}
public Date createTime() {
return this.create;
}
public void updateTime(Date update) {
this.update = update;
}
public Date updateTime() {
return this.update;
}
public void retry() {
++this.retries;
}
public int retries() {
return this.retries;
}
public void input(String input) {
// checkPropertySize(input, P.INPUT);
this.input = input;
}
public String input() {
return this.input;
}
public String result() {
return this.result;
}
private void result(String result) {
checkPropertySize(result, P.RESULT);
this.result = result;
}
public void server(Id server) {
this.server = server;
}
public Id server() {
return this.server;
}
public void load(int load) {
this.load = load;
}
public int load() {
return this.load;
}
public boolean completed() {
return TaskStatus.COMPLETED_STATUSES.contains(this.status);
}
public boolean success() {
return this.status == TaskStatus.SUCCESS;
}
public boolean cancelled() {
return this.status == TaskStatus.CANCELLED || this.isCancelled();
}
public boolean cancelling() {
return this.status == TaskStatus.CANCELLING;
}
public boolean computer() {
return ComputerJob.COMPUTER.equals(this.type);
}
@Override
public String toString() {
return String.format("HugeTask(%s)%s", this.id, this.asMap());
}
@Override
public void run() {
if (this.cancelled()) {
// Scheduled task is running after cancelled
return;
}
TaskManager.setContext(this.context());
try {
assert this.status.code() < TaskStatus.RUNNING.code() : this.status;
if (this.checkDependenciesSuccess()) {
this.status(TaskStatus.RUNNING);
super.run();
}
} catch (Throwable e) {
this.setException(e);
} finally {
LOG.debug("Task is finished {}", this);
TaskManager.resetContext();
}
}
@Override
public boolean cancel(boolean mayInterruptIfRunning) {
// NOTE: Gremlin sleep() can't be interrupted by default
// https://mrhaki.blogspot.com/2016/10/groovy-goodness-interrupted-sleeping.html
boolean cancelled = super.cancel(mayInterruptIfRunning);
if (!cancelled) {
return cancelled;
}
try {
if (this.status(TaskStatus.CANCELLED)) {
// Callback for saving status to store
this.callable.cancelled();
} else {
// Maybe the worker is still running then set status SUCCESS
cancelled = false;
}
} catch (Throwable e) {
LOG.error("An exception occurred when calling cancelled()", e);
}
return cancelled;
}
public boolean fail(Throwable e) {
E.checkNotNull(e, "exception");
if (!(this.cancelled() && HugeException.isInterrupted(e))) {
LOG.warn("An exception occurred when running task: {}",
this.id(), e);
// Update status to FAILED if exception occurred(not interrupted)
if (this.status(TaskStatus.FAILED)) {
this.result(e.toString());
return true;
}
}
return false;
}
public void failSave(Throwable e) {
if (!this.fail(e)) {
// Can't update status, just set result to error message
this.result(e.toString());
}
}
@Override
protected void done() {
try {
// Callback for saving status to store
this.callable.done();
} catch (Throwable e) {
LOG.error("An exception occurred when calling done()", e);
} finally {
StandardTaskScheduler scheduler = (StandardTaskScheduler)
this.scheduler();
scheduler.taskDone(this);
}
}
@Override
protected void set(V v) {
String result = JsonUtil.toJson(v);
checkPropertySize(result, P.RESULT);
if (this.status(TaskStatus.SUCCESS)) {
this.result = result;
} else {
assert this.completed();
}
// Will call done() and may cause to save to store
super.set(v);
}
@Override
protected void setException(Throwable e) {
this.fail(e);
super.setException(e);
}
protected void scheduler(TaskScheduler scheduler) {
this.scheduler = scheduler;
}
protected TaskScheduler scheduler() {
E.checkState(this.scheduler != null,
"Can't call scheduler() before scheduling task");
return this.scheduler;
}
protected boolean checkDependenciesSuccess() {
if (this.dependencies == null || this.dependencies.isEmpty()) {
return true;
}
for (Id dependency : this.dependencies) {
HugeTask> task = this.scheduler().task(dependency);
if (!task.completed()) {
// Dependent task not completed, re-schedule self
this.scheduler().schedule(this);
return false;
} else if (task.status() == TaskStatus.CANCELLED) {
this.status(TaskStatus.CANCELLED);
this.result(String.format(
"Cancelled due to dependent task '%s' cancelled",
dependency));
this.done();
return false;
} else if (task.status() == TaskStatus.FAILED) {
this.status(TaskStatus.FAILED);
this.result(String.format(
"Failed due to dependent task '%s' failed",
dependency));
this.done();
return false;
}
}
return true;
}
protected TaskCallable callable() {
E.checkNotNull(this.callable, "callable");
return this.callable;
}
protected synchronized boolean status(TaskStatus status) {
E.checkNotNull(status, "status");
if (status.code() > TaskStatus.NEW.code()) {
E.checkState(this.type != null, "Task type can't be null");
E.checkState(this.name != null, "Task name can't be null");
}
if (!this.completed()) {
assert this.status.code() < status.code() ||
status == TaskStatus.RESTORING :
this.status + " => " + status + " (task " + this.id + ")";
this.status = status;
return true;
}
return false;
}
protected void property(String key, Object value) {
E.checkNotNull(key, "property key");
switch (key) {
case P.TYPE:
this.type = (String) value;
break;
case P.NAME:
this.name = (String) value;
break;
case P.CALLABLE:
// pass
break;
case P.STATUS:
this.status = SerialEnum.fromCode(TaskStatus.class,
(byte) value);
break;
case P.PROGRESS:
this.progress = (int) value;
break;
case P.CREATE:
this.create = (Date) value;
break;
case P.RETRIES:
this.retries = (int) value;
break;
case P.DESCRIPTION:
this.description = (String) value;
break;
case P.CONTEXT:
this.context = (String) value;
break;
case P.UPDATE:
this.update = (Date) value;
break;
case P.DEPENDENCIES:
@SuppressWarnings("unchecked")
Set values = (Set) value;
this.dependencies = values.stream().map(IdGenerator::of)
.collect(toOrderSet());
break;
case P.INPUT:
this.input = StringEncoding.decompress(((Blob) value).bytes(),
DECOMPRESS_RATIO);
break;
case P.RESULT:
this.result = StringEncoding.decompress(((Blob) value).bytes(),
DECOMPRESS_RATIO);
break;
case P.SERVER:
this.server = IdGenerator.of((String) value);
break;
default:
throw new AssertionError("Unsupported key: " + key);
}
}
protected Object[] asArray() {
E.checkState(this.type != null, "Task type can't be null");
E.checkState(this.name != null, "Task name can't be null");
List
© 2015 - 2025 Weber Informatics LLC | Privacy Policy