Please wait. This can take some minutes ...
Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
com.bigdata.resources.AsynchronousOverflowTask Maven / Gradle / Ivy
Go to download
Blazegraph(TM) DB Core Platform. It contains all Blazegraph DB dependencies other than Blueprints.
package com.bigdata.resources;
import java.nio.channels.AsynchronousCloseException;
import java.nio.channels.ClosedByInterruptException;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.Lock;
import org.apache.log4j.Logger;
import org.apache.log4j.MDC;
import com.bigdata.btree.BTree;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.IndexSegment;
import com.bigdata.btree.ScatterSplitConfiguration;
import com.bigdata.btree.proc.BatchLookup;
import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedure.ResultBuffer;
import com.bigdata.btree.proc.BatchLookup.BatchLookupConstructor;
import com.bigdata.btree.view.FusedView;
import com.bigdata.io.SerializerUtil;
import com.bigdata.journal.AbstractJournal;
import com.bigdata.journal.AbstractTask;
import com.bigdata.journal.ConcurrencyManager;
import com.bigdata.journal.IConcurrencyManager;
import com.bigdata.journal.ITx;
import com.bigdata.journal.TimestampUtility;
import com.bigdata.mdi.LocalPartitionMetadata;
import com.bigdata.mdi.MetadataIndex;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.resources.OverflowManager.ResourceScores;
import com.bigdata.service.DataService;
import com.bigdata.service.Event;
import com.bigdata.service.EventResource;
import com.bigdata.service.EventType;
import com.bigdata.service.IDataService;
import com.bigdata.service.ILoadBalancerService;
import com.bigdata.service.MetadataService;
import com.bigdata.util.Bytes;
import com.bigdata.util.DaemonThreadFactory;
import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.LatchedExecutor;
public class AsynchronousOverflowTask implements Callable {
protected static final Logger log = Logger.getLogger(AsynchronousOverflowTask.class );
private final ResourceManager resourceManager;
private final OverflowMetadata overflowMetadata;
private final long lastCommitTime;
private final Map used = new TreeMap();
private boolean isUsed(final String name) {
if (name == null )
throw new IllegalArgumentException();
return used.containsKey(name);
}
protected void putUsed(final String name, final String action) {
if (name == null )
throw new IllegalArgumentException();
if (action == null )
throw new IllegalArgumentException();
if (used.containsKey(name)) {
throw new IllegalStateException("Already used: " +name);
}
used.put(name,action);
}
public AsynchronousOverflowTask(final ResourceManager resourceManager,
final OverflowMetadata overflowMetadata) {
if (resourceManager == null )
throw new IllegalArgumentException();
if (overflowMetadata == null )
throw new IllegalArgumentException();
this .resourceManager = resourceManager;
this .overflowMetadata = overflowMetadata;
this .lastCommitTime = overflowMetadata.lastCommitTime;
}
private class AtomicCallable implements Callable {
private final OverflowActionEnum action;
private final ViewMetadata vmd;
private final boolean forceCompactingMerge;
private final AbstractTask task;
public AtomicCallable(final OverflowActionEnum action,
final ViewMetadata vmd,
final boolean forceCompactingMerge,
final AbstractTask task
) {
if (action == null )
throw new IllegalArgumentException();
if (vmd == null )
throw new IllegalArgumentException();
if (task == null )
throw new IllegalArgumentException();
this .action = action;
this .vmd = vmd;
this .forceCompactingMerge = forceCompactingMerge;
this .task = task;
}
public T call() throws Exception {
final Lock lock = vmd.lock;
lock.lock();
try {
if (vmd.getAction() != null ) {
if (log.isInfoEnabled())
log.info("Dropping task: runningAs=" + vmd.getAction()
+ ", plannedAction=" + action);
return null ;
}
vmd.setAction(action);
} finally {
lock.unlock();
}
if (action.equals(OverflowActionEnum.Merge)) {
if (((ConcurrencyManager) resourceManager
.getConcurrencyManager())
.getJournalOverextended() > resourceManager.overflowThreshold) {
return null ;
}
}
return resourceManager.getConcurrencyManager().submit(task).get();
}
}
private List> scheduleAndAwaitTasks(
final boolean forceCompactingMerges) throws InterruptedException {
final Iterator itr = overflowMetadata.views();
final Queue> buildList = new PriorityBlockingQueue>(
overflowMetadata.getIndexCount());
final Queue> mergeList = new PriorityBlockingQueue>(
overflowMetadata.getIndexCount());
while (itr.hasNext()) {
final ViewMetadata vmd = itr.next();
final String name = vmd.name;
if (overflowMetadata.isCopied(name)) {
if (log.isInfoEnabled())
log.info("was copied : " + vmd);
} else {
buildList.add(new Priority(vmd.buildPriority, vmd));
}
if (vmd.mergePriority > 0 d
|| (forceCompactingMerges && vmd.sourceCount > 1 )) {
mergeList
.add(new Priority(vmd.mergePriority, vmd));
}
}
if (log.isInfoEnabled()) {
log.info("Scheduling tasks: buildList=" +buildList.size()+", mergeList=" +mergeList.size());
}
final List> mergeFutures = new LinkedList>();
final List> buildFutures = new LinkedList>();
try {
final Executor buildService = new LatchedExecutor(
resourceManager.getFederation().getExecutorService(),
resourceManager.buildServiceCorePoolSize);
final Executor mergeService = new LatchedExecutor(
resourceManager.getFederation().getExecutorService(),
resourceManager.mergeServiceCorePoolSize);
for (Priority p : mergeList) {
final ViewMetadata vmd = p.v;
if (vmd.mergePriority > 0 || forceCompactingMerges) {
if (forceCompactingMerges && OverflowActionEnum.Copy.equals(vmd.getAction())) {
vmd.clearCopyAction();
}
final FutureTask ft = new FutureTask(
new AtomicCallable(OverflowActionEnum.Merge,
vmd, forceCompactingMerges,
new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
}
}
for (Priority p : buildList) {
final ViewMetadata vmd = p.v;
if (forceCompactingMerges && !vmd.compactView) {
final FutureTask ft = new FutureTask(new AtomicCallable(
OverflowActionEnum.Merge, vmd,
forceCompactingMerges,
new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
} else {
final FutureTask ft = new FutureTask(new AtomicCallable(
OverflowActionEnum.Build, vmd,
forceCompactingMerges,
new IncrementalBuildTask(vmd)));
buildFutures.add(ft);
buildService.execute(ft);
}
}
{
for (Future f : buildFutures) {
if (!f.isDone()) {
try {
f.get();
} catch (CancellationException ignore) {
} catch (ExecutionException ignore) {
}
}
}
}
{
for (Future f : mergeFutures) {
if (!f.isDone()) {
try {
f.get();
} catch (CancellationException ignore) {
} catch (ExecutionException ignore) {
}
}
}
}
final List> allFutures = new LinkedList>();
allFutures.addAll(buildFutures);
allFutures.addAll(mergeFutures);
return allFutures;
} finally {
for (Future f : buildFutures)
f.cancel(true );
for (Future f : mergeFutures)
f.cancel(true );
}
}
protected List chooseScatterSplits() {
final List tasks = new LinkedList();
final Iterator itr = overflowMetadata.views();
UUID[] moveTargets = null ;
while (itr.hasNext()) {
final ViewMetadata vmd = itr.next();
final String name = vmd.name;
if (isUsed(name)|| overflowMetadata.isCopied(name)) {
continue ;
}
final ScatterSplitConfiguration ssc = vmd.indexMetadata
.getScatterSplitConfiguration();
if (
(vmd.getIndexPartitionCount() == 1 L)
&& vmd.pmd.getSourcePartitionId() == -1
&& resourceManager.scatterSplitEnabled
&& ssc.isEnabled()
&& vmd.compactView
&& vmd.getPercentOfSplit() >= ssc.getPercentOfSplitThreshold()
) {
if (moveTargets == null ) {
final UUID[] a = resourceManager
.getFederation()
.getDataServiceUUIDs(
ssc.getDataServiceCount());
if (a == null || a.length == 1 ) {
if (log.isInfoEnabled())
log
.info("Will not scatter split - insufficient data services discovered." );
return tasks;
}
final Set tmp = new HashSet(Arrays.asList(a));
tmp.add(resourceManager.getDataServiceUUID());
moveTargets = tmp.toArray(new UUID[tmp.size()]);
}
final int nsplits = ssc.getIndexPartitionCount() == 0
? (2 * moveTargets.length) // two per data service.
: ssc.getIndexPartitionCount()
;
final AbstractTask task = new ScatterSplitTask(vmd, nsplits,
moveTargets);
tasks.add(task);
overflowMetadata.setAction(vmd.name, OverflowActionEnum.ScatterSplit);
putUsed(name, "willScatter(name=" + vmd + ")" );
if (log.isInfoEnabled())
log.info("will scatter: " + vmd);
continue ;
}
}
return tasks;
}
protected List chooseJoins() {
if (!resourceManager.joinsEnabled) {
if (log.isInfoEnabled())
log.info(OverflowManager.Options.JOINS_ENABLED + "="
+ resourceManager.joinsEnabled);
return EMPTY_LIST;
}
final List tasks = new LinkedList();
if (log.isInfoEnabled())
log.info("begin: lastCommitTime=" + lastCommitTime);
final Map undercapacityIndexPartitions = new HashMap();
{
int ndone = 0 ;
int nskip = 0 ;
int njoin = 0 ;
int nignored = 0 ;
assert used.isEmpty() : "There are " + used.size()
+ " used index partitions" ;
final Iterator itr = overflowMetadata.views();
while (itr.hasNext()) {
final ViewMetadata vmd = itr.next();
final String name = vmd.name;
final IIndex view = vmd.getView();
if (view == null ) {
throw new AssertionError(
"Index not found? : name=" + name
+ ", lastCommitTime=" + lastCommitTime);
}
final LocalPartitionMetadata pmd = vmd.pmd;
if (pmd.getSourcePartitionId() != -1 ) {
if (log.isInfoEnabled())
log.info("Skipping index: name=" + name
+ ", reason=moveInProgress" );
continue ;
}
if (log.isInfoEnabled())
log.info("Considering join: name=" + name + ", rangeCount="
+ vmd.getRangeCount() + ", pmd=" + pmd);
if (pmd.getRightSeparatorKey() != null
&& vmd.getPercentOfSplit() < .5 ) {
final String scaleOutIndexName = vmd.indexMetadata.getName();
BTree tmp = undercapacityIndexPartitions
.get(scaleOutIndexName);
if (tmp == null ) {
tmp = BTree.createTransient(new IndexMetadata(UUID
.randomUUID()));
undercapacityIndexPartitions
.put(scaleOutIndexName, tmp);
}
tmp.insert(pmd.getLeftSeparatorKey(), SerializerUtil
.serialize(pmd));
if (log.isInfoEnabled())
log.info("join candidate: " + name);
njoin++;
} else {
nignored++;
}
ndone++;
}
assert ndone == nskip + njoin + nignored : "ndone=" + ndone
+ ", nskip=" + nskip + ", njoin=" + njoin + ", nignored="
+ nignored;
}
{
final Iterator> itr = undercapacityIndexPartitions
.entrySet().iterator();
int ndone = 0 ;
int njoin = 0 ;
int nmove = 0 ;
assert used.isEmpty() : "There are " + used.size()
+ " used index partitions" ;
final UUID sourceDataService = resourceManager.getDataServiceUUID();
while (itr.hasNext()) {
final Map.Entry entry = itr.next();
final String scaleOutIndexName = entry.getKey();
if (log.isInfoEnabled())
log.info("Considering join candidates: "
+ scaleOutIndexName);
final BTree tmp = entry.getValue();
if (tmp.getEntryCount() > Integer.MAX_VALUE) {
log.error("Rediculous size for temp index." );
continue ;
}
final int ncandidates = (int ) tmp.getEntryCount();
assert ncandidates > 0 : "Expecting at least one candidate" ;
final ITupleIterator titr = tmp.rangeIterator();
if (log.isInfoEnabled())
log.info("Formulating rightSiblings query="
+ scaleOutIndexName + ", #underutilized="
+ ncandidates);
final byte [][] keys = new byte [ncandidates][];
final LocalPartitionMetadata[] underUtilizedPartitions = new LocalPartitionMetadata[ncandidates];
int i = 0 ;
while (titr.hasNext()) {
final ITuple tuple = titr.next();
final LocalPartitionMetadata pmd = (LocalPartitionMetadata) SerializerUtil
.deserialize(tuple.getValue());
underUtilizedPartitions[i] = pmd;
if (pmd.getRightSeparatorKey() == null ) {
throw new AssertionError(
"The last index partition may not be a join candidate: name="
+ scaleOutIndexName + ", " + pmd);
}
keys[i] = pmd.getRightSeparatorKey();
i++;
}
if (log.isInfoEnabled())
log.info("Looking for rightSiblings: name="
+ scaleOutIndexName + ", #underutilized="
+ ncandidates);
final BatchLookup op = BatchLookupConstructor.INSTANCE
.newInstance(0 , ncandidates,
keys, null );
final ResultBuffer resultBuffer;
try {
resultBuffer = (ResultBuffer) resourceManager
.getFederation().getMetadataService()
.submit(
TimestampUtility.asHistoricalRead(lastCommitTime),
MetadataService
.getMetadataIndexName(scaleOutIndexName),
op).get();
} catch (Exception e) {
log.error("Could not locate rightSiblings: index="
+ scaleOutIndexName, e);
continue ;
}
for (i = 0 ; i < ncandidates; i++) {
final LocalPartitionMetadata pmd = underUtilizedPartitions[i];
final ViewMetadata vmd = overflowMetadata
.getViewMetadata(DataService.getIndexPartitionName(
scaleOutIndexName, pmd.getPartitionId()));
final PartitionLocator rightSiblingLocator = (PartitionLocator) SerializerUtil
.deserialize(resultBuffer.getResult(i));
final UUID targetDataServiceUUID = rightSiblingLocator
.getDataServiceUUID();
final String[] resources = new String[2 ];
resources[0 ] = DataService.getIndexPartitionName(
scaleOutIndexName, pmd.getPartitionId());
resources[1 ] = DataService.getIndexPartitionName(
scaleOutIndexName, rightSiblingLocator.getPartitionId());
if (sourceDataService.equals(targetDataServiceUUID)) {
if (isUsed(resources[0 ])) continue ;
if (isUsed(resources[1 ])) continue ;
if (log.isInfoEnabled())
log.info("Will JOIN: " + Arrays.toString(resources));
final ViewMetadata vmd2 = overflowMetadata
.getViewMetadata(DataService
.getIndexPartitionName(
scaleOutIndexName,
rightSiblingLocator
.getPartitionId()));
final AbstractTask task = new JoinIndexPartitionTask(
resourceManager, lastCommitTime, resources,
new ViewMetadata[] { vmd, vmd2 });
tasks.add(task);
putUsed(resources[0 ], "willJoin(leftSibling="
+ resources[0 ] + ",rightSibling="
+ resources[1 ] + ")" );
putUsed(resources[1 ], "willJoin(leftSibling="
+ resources[0 ] + ",rightSibling="
+ resources[1 ] + ")" );
njoin++;
} else {
if (isUsed(resources[0 ])) continue ;
final String sourceIndexName = DataService
.getIndexPartitionName(scaleOutIndexName, pmd
.getPartitionId());
final AbstractTask task = new MoveTask(vmd,
targetDataServiceUUID);
String targetDataServiceName;
try {
targetDataServiceName = resourceManager.getFederation()
.getDataService(targetDataServiceUUID)
.getServiceName();
} catch (Throwable t) {
targetDataServiceName = targetDataServiceUUID.toString();
}
tasks.add(task);
putUsed(resources[0 ], "willMoveToJoinWithRightSibling"
+ "( " + sourceIndexName + " -> "
+ targetDataServiceName
+ ", leftSibling=" + resources[0 ]
+ ", rightSibling=" + resources[1 ]
+ ")" );
nmove++;
}
ndone++;
}
}
assert ndone == njoin + nmove;
}
return tasks;
}
protected ILoadBalancerService getLoadBalancerService() {
final ILoadBalancerService loadBalancerService;
try {
loadBalancerService = resourceManager.getFederation()
.getLoadBalancerService();
} catch (Exception ex) {
log.warn("Could not discover the load balancer service" , ex);
return null ;
}
if (loadBalancerService == null ) {
log.warn("Could not discover the load balancer service" );
return null ;
}
return loadBalancerService;
}
protected boolean shouldMove(final ILoadBalancerService loadBalancerService) {
if (loadBalancerService == null )
throw new IllegalArgumentException();
final boolean highlyUtilizedService;
try {
final UUID serviceUUID = resourceManager.getDataServiceUUID();
highlyUtilizedService = loadBalancerService
.isHighlyUtilizedDataService(serviceUUID);
} catch (Exception ex) {
log.warn("Could not determine if this data service is highly utilized" );
return false ;
}
if (!highlyUtilizedService) {
if (log.isInfoEnabled())
log.info("Service is not highly utilized." );
return false ;
}
final ResourceScores resourceScores = resourceManager.getResourceScores();
final boolean shouldMove =
(resourceScores.percentCPUTime >= resourceManager.movePercentCpuTimeThreshold) ||
(resourceScores.majorPageFaultsPerSec > 20 ) ||
(resourceScores.dataDirBytesFree < Bytes.gigabyte * 5 )||
(resourceScores.dataDirBytesFree < Bytes.gigabyte * .5 )
;
return shouldMove;
}
private List chooseMoves(
final ILoadBalancerService loadBalancerService) {
if (resourceManager.maximumMovesPerTarget == 0 ) {
return EMPTY_LIST;
}
final int minActiveIndexPartitions = resourceManager.minimumActiveIndexPartitions;
final int nactive = overflowMetadata.getActiveCount();
if (nactive <= minActiveIndexPartitions) {
if (log.isInfoEnabled())
log.info("Preconditions for move not satisified: nactive="
+ nactive + ", minActive=" + minActiveIndexPartitions);
return EMPTY_LIST;
}
final int maxMovesPerTarget = resourceManager.maximumMovesPerTarget;
final UUID sourceServiceUUID = resourceManager.getDataServiceUUID();
final UUID[] underUtilizedDataServiceUUIDs;
try {
underUtilizedDataServiceUUIDs = loadBalancerService
.getUnderUtilizedDataServices(
0 ,
0 ,
sourceServiceUUID
);
} catch (TimeoutException t) {
log.warn(t.getMessage());
return EMPTY_LIST;
} catch (InterruptedException t) {
log.warn(t.getMessage());
return EMPTY_LIST;
} catch (Throwable t) {
log.error("Could not obtain target service UUIDs: " , t);
return EMPTY_LIST;
}
if (underUtilizedDataServiceUUIDs == null
|| underUtilizedDataServiceUUIDs.length == 0 ) {
if (log.isInfoEnabled())
log.info("Load balancer does not report any underutilized services." );
return EMPTY_LIST;
}
final int maxMoves;
{
final int nactiveSurplus = nactive - minActiveIndexPartitions;
assert nactiveSurplus > 0 ;
assert underUtilizedDataServiceUUIDs != null ;
maxMoves = Math.min(resourceManager.maximumMoves,
Math.min(nactiveSurplus,
maxMovesPerTarget
* underUtilizedDataServiceUUIDs.length));
}
if (log.isInfoEnabled())
log.info("Considering index partition moves: #targetServices="
+ underUtilizedDataServiceUUIDs.length + ", maxMovesPerTarget="
+ maxMovesPerTarget + ", nactive=" + nactive + ", maxMoves="
+ maxMoves + ", sourceService=" +sourceServiceUUID+", targetServices="
+ Arrays.toString(underUtilizedDataServiceUUIDs));
long maxRangeCount = 0 L;
final List scores = new LinkedList();
for (Score score : overflowMetadata.getScores()) {
final String name = score.name;
if (isUsed(name)) continue ;
if (overflowMetadata.isCopied(name)) {
putUsed(name, "wasCopied(name=" + name + ")" );
continue ;
}
final StaleLocatorReason reason = resourceManager
.getIndexPartitionGone(score.name);
if (reason != null ) {
if (log.isInfoEnabled())
log.info("Skipping index: name=" + score.name + ", reason="
+ reason);
continue ;
}
final ViewMetadata vmd = overflowMetadata.getViewMetadata(name);
if (vmd == null ) {
if (log.isInfoEnabled())
log.info("Skipping index: name=" + name
+ ", reason=dropped" );
continue ;
}
if (vmd.pmd.getSourcePartitionId() != -1 ) {
if (log.isInfoEnabled())
log.info("Skipping index: name=" + name
+ ", reason=moveInProgress" );
continue ;
}
final long rangeCount = vmd.getRangeCount();
if (vmd.getPercentOfSplit() > resourceManager.maximumMovePercentOfSplit) {
if (log.isInfoEnabled())
log.info("Skipping index: name=" + name
+ ", reason=shouldSplit" );
continue ;
}
scores.add(score);
maxRangeCount = Math.max(maxRangeCount, rangeCount);
}
final PriorityQueue> moveQueue = new PriorityQueue>();
for (Score score : scores) {
final ViewMetadata vmd = overflowMetadata.getViewMetadata(score.name);
final double moveMinScore = .1 ;
final boolean moveCandidate =
score.drank >= moveMinScore
;
final double movePriority = vmd.isTailSplit()
? score.drank / .1//
: score.drank / vmd.getPercentOfSplit()/ /
;
if (log.isInfoEnabled())
log
.info(vmd.name + " : tailSplit=" + vmd.isTailSplit()
+ ", moveCandidate=" + moveCandidate
+ ", movePriority=" + movePriority + ", drank="
+ score.drank + ", percentOfSplit="
+ vmd.getPercentOfSplit() + " : " + vmd + " : "
+ score);
if (!moveCandidate) {
continue ;
}
moveQueue.add(new Priority(movePriority, vmd));
}
int nmove = 0 ;
final List tasks = new ArrayList(maxMoves);
while (nmove < maxMoves && !moveQueue.isEmpty()) {
final ViewMetadata vmd = moveQueue.poll().v;
if (log.isInfoEnabled())
log.info("Considering move candidate: " + vmd);
final UUID targetDataServiceUUID = underUtilizedDataServiceUUIDs[nmove
% underUtilizedDataServiceUUIDs.length];
if (sourceServiceUUID.equals(targetDataServiceUUID)) {
log
.error("LBS included the source data service in the set of possible targets: source="
+ sourceServiceUUID
+ ", targets="
+ Arrays
.toString(underUtilizedDataServiceUUIDs));
continue ;
}
String targetDataServiceName;
try {
targetDataServiceName = resourceManager.getFederation()
.getDataService(targetDataServiceUUID).getServiceName();
} catch (Throwable t) {
targetDataServiceName = targetDataServiceUUID.toString();
}
if (vmd.isTailSplit()) {
if (log.isInfoEnabled())
log.info("Will tailSplit " + vmd.name
+ " and move the rightSibling to dataService="
+ targetDataServiceName);
final AbstractTask task = new SplitTailTask(vmd,
targetDataServiceUUID);
tasks.add(task);
putUsed(vmd.name, "willTailSplit + moveRightSibling("
+ vmd.name + " -> " + targetDataServiceName + ") : "
+ vmd + " : " + overflowMetadata.getScore(vmd.name));
nmove++;
} else if (vmd.getPercentOfSplit() > .5 ) {
if (log.isInfoEnabled())
log
.info("Will split "
+ vmd.name
+ " and move the smallest post-split index partition to dataService="
+ targetDataServiceName);
final AbstractTask task = new SplitIndexPartitionTask(vmd,
targetDataServiceUUID);
tasks.add(task);
putUsed(vmd.name, "willSplit+Move(" + vmd.name + " -> "
+ targetDataServiceName + ") : " + vmd + " : "
+ overflowMetadata.getScore(vmd.name));
nmove++;
} else {
if (log.isInfoEnabled())
log.info("Will move " + vmd.name + " to dataService="
+ targetDataServiceName);
final AbstractTask task = new MoveTask(vmd,
targetDataServiceUUID);
tasks.add(task);
putUsed(vmd.name, "willMove(" + vmd.name + " -> "
+ targetDataServiceName + ") : " + vmd + " : "
+ overflowMetadata.getScore(vmd.name));
nmove++;
}
}
if (log.isInfoEnabled())
log.info("Will move " + nmove
+ " index partitions based on utilization." );
return tasks;
}
protected List chooseTasks(final boolean forceCompactingMerges)
throws Exception {
final AbstractJournal oldJournal = resourceManager
.getJournal(lastCommitTime);
final long oldJournalSize = oldJournal.size();
if (log.isInfoEnabled())
log.info("begin: lastCommitTime=" + lastCommitTime
+ ", compactingMerge=" + forceCompactingMerges
+ ", oldJournalSize=" + oldJournalSize);
final List tasks = new ArrayList(
(int ) oldJournal.getName2Addr().rangeCount());
if (!forceCompactingMerges) {
tasks.addAll(chooseScatterSplits());
tasks.addAll(chooseJoins());
final ILoadBalancerService lbs = getLoadBalancerService();
if (lbs != null && shouldMove(lbs)) {
tasks.addAll(chooseMoves(lbs));
}
}
tasks.addAll(chooseSplitBuildOrMerge(forceCompactingMerges));
{
final StringBuilder sb = new StringBuilder();
final Iterator> itrx = used.entrySet()
.iterator();
while (itrx.hasNext()) {
final Map.Entry entry = itrx.next();
sb.append("\n" + entry.getKey() + "\t = " + entry.getValue());
}
log.warn("\nlastCommitTime=" + lastCommitTime
+ ", compactingMerge=" + forceCompactingMerges
+ ", oldJournalSize=" + oldJournalSize + sb);
}
return tasks;
}
protected List chooseSplitBuildOrMerge(
final boolean compactingMerge) {
int ndone = 0 ;
int nskip = 0 ;
int nbuild = 0 ;
int nmerge = 0 ;
int nsplit = 0 ;
final List tasks = new LinkedList();
final Iterator itr = overflowMetadata.views();
final PriorityQueue> mergeQueue = new PriorityQueue>(
overflowMetadata.getIndexCount());
while (itr.hasNext()) {
final ViewMetadata vmd = itr.next();
final String name = vmd.name;
if (isUsed(name)) {
if (log.isInfoEnabled())
log.info("was handled: " + name);
nskip++;
ndone++;
continue ;
}
if (overflowMetadata.isCopied(name)) {
putUsed(name, "wasCopied(name=" + name + ")" );
if (log.isInfoEnabled())
log.info("was copied : " + vmd);
nskip++;
ndone++;
continue ;
}
if (compactingMerge || vmd.mandatoryMerge) {
final AbstractTask task = new CompactingMergeTask(vmd);
tasks.add(task);
overflowMetadata.setAction(vmd.name, OverflowActionEnum.Merge);
putUsed(name, "willManditoryMerge(" + vmd + ")" );
if (log.isInfoEnabled())
log.info("will merge : " + vmd);
nmerge++;
ndone++;
continue ;
}
if (!compactingMerge
&& vmd.pmd.getSourcePartitionId() == -1
&& vmd.isTailSplit()
) {
final AbstractTask task = new SplitTailTask(vmd, null );
tasks.add(task);
overflowMetadata.setAction(vmd.name,
OverflowActionEnum.TailSplit);
putUsed(name, "tailSplit(name=" + vmd + ")" );
if (log.isInfoEnabled())
log.info("will tailSpl: " + vmd);
nsplit++;
ndone++;
continue ;
}
if (
vmd.pmd.getSourcePartitionId() == -1
&& vmd.getPercentOfSplit() > 1.0
) {
final AbstractTask task = new SplitIndexPartitionTask(vmd,
(UUID) null );
tasks.add(task);
overflowMetadata.setAction(vmd.name, OverflowActionEnum.Split);
putUsed(name, "willSplit(name=" + vmd + ")" );
if (log.isInfoEnabled())
log.info("will split : " + vmd);
nsplit++;
ndone++;
continue ;
}
mergeQueue.add(new Priority(vmd.mergePriority, vmd));
}
while (!mergeQueue.isEmpty()) {
final Priority e = mergeQueue.poll();
final ViewMetadata vmd = e.v;
if (nmerge < resourceManager.maximumOptionalMergesPerOverflow) {
final AbstractTask task = new CompactingMergeTask(vmd);
tasks.add(task);
overflowMetadata.setAction(vmd.name, OverflowActionEnum.Merge);
putUsed(vmd.name, "willOptionalMerge(" + vmd + ")" );
if (log.isInfoEnabled())
log.info("will merge : " + vmd);
nmerge++;
ndone++;
} else {
final AbstractTask task = new IncrementalBuildTask(vmd);
tasks.add(task);
overflowMetadata.setAction(vmd.name, OverflowActionEnum.Build);
putUsed(vmd.name, "willBuild(" + vmd + ")" );
if (log.isInfoEnabled())
log.info("will build: " + vmd);
nbuild++;
ndone++;
}
}
if (ndone != nskip + nbuild + nmerge + nsplit) {
log.warn("ndone=" + ndone + ", but : nskip=" + nskip + ", nbuild="
+ nbuild + ", ncompact=" + nmerge + ", nsplit=" + nsplit);
}
if (ndone != used.size()) {
log.warn("ndone=" + ndone + ", but #used=" + used.size());
}
return tasks;
}
public Object call() throws Exception {
MDC.put("taskname" , "overflowService" );
if (resourceManager.overflowAllowed.get()) {
throw new AssertionError();
}
final long begin = System.currentTimeMillis();
final boolean forceCompactingMerges = resourceManager.compactingMerge
.getAndSet(false );
resourceManager.overflowCounters.asynchronousOverflowStartMillis.set(begin);
final Event e = new Event(resourceManager.getFederation(),
new EventResource(), EventType.AsynchronousOverflow).addDetail(
"asynchronousOverflowCounter" ,
resourceManager.overflowCounters.asynchronousOverflowCounter.get()).start();
try {
if (log.isInfoEnabled()) {
log.info("\npre-condition views: overflowCounter="
+ resourceManager.overflowCounters.asynchronousOverflowCounter.get()
+ "\n"
+ resourceManager.listIndexPartitions(TimestampUtility
.asHistoricalRead(lastCommitTime)));
}
if (resourceManager.compactingMergeWithAfterAction) {
final List> futures = scheduleAndAwaitTasks(forceCompactingMerges);
for (Future f : futures) {
try {
f.get();
} catch (CancellationException ex) {
log.error(ex, ex);
resourceManager.overflowCounters.asynchronousOverflowTaskCancelledCounter
.incrementAndGet();
} catch (ExecutionException ex) {
if (isNormalShutdown(ex)) {
log.warn("Normal shutdown? : " + ex, ex);
} else {
log.error(ex, ex);
}
resourceManager.overflowCounters.asynchronousOverflowTaskFailedCounter
.incrementAndGet();
}
}
} else {
final List tasks = chooseTasks(forceCompactingMerges);
runTasks((List) tasks);
}
final long overflowCounter = resourceManager.overflowCounters.asynchronousOverflowCounter
.incrementAndGet();
log.warn("done: overflowCounter=" + overflowCounter
+ ", lastCommitTime="
+ resourceManager.getLiveJournal().getLastCommitTime()
+ ", elapsed=" + (System.currentTimeMillis() - begin)
+ "ms" );
if (log.isInfoEnabled())
log.info("\npost-condition views: overflowCounter="
+ resourceManager.overflowCounters.asynchronousOverflowCounter.get()
+ "\n"
+ resourceManager.listIndexPartitions(ITx.UNISOLATED));
return null ;
} catch (Throwable t) {
resourceManager.overflowCounters.asynchronousOverflowFailedCounter.incrementAndGet();
if (isNormalShutdown(t)) {
log.warn("Normal shutdown? : " +t);
} else {
log.error(t, t);
}
throw new RuntimeException( t );
} finally {
e.end();
if (!resourceManager.overflowAllowed.compareAndSet(
false , true )) {
throw new AssertionError();
}
resourceManager.overflowCounters.asynchronousOverflowMillis
.addAndGet(e.getElapsed());
overflowMetadata.clearViews();
}
}
protected void runTasks(final List> tasks)
throws InterruptedException {
if (log.isInfoEnabled())
log.info("begin : will run " + tasks.size() + " update tasks" );
if (resourceManager.overflowTasksConcurrent == 1 ) {
runTasksInSingleThread(tasks);
} else {
runTasksConcurrent(tasks);
}
if (log.isInfoEnabled())
log.info("end" );
}
protected void runTasksInSingleThread(final List> tasks)
throws InterruptedException {
final ExecutorService executorService = Executors
.newSingleThreadExecutor(DaemonThreadFactory
.defaultThreadFactory());
try {
final long begin = System.nanoTime();
final long nanos = TimeUnit.MILLISECONDS
.toNanos(resourceManager.overflowTimeout);
long remaining = nanos;
final Iterator> titr = tasks.iterator();
int ndone = 0 ;
while (titr.hasNext() && remaining > 0 ) {
final boolean shouldOverflow = resourceManager
.isOverflowEnabled()
&& resourceManager.shouldOverflow();
if (shouldOverflow) {
if (resourceManager.overflowCancelledWhenJournalFull) {
break ;
} else {
final long elapsed = (System.nanoTime() - begin);
log.warn("Overflow still running: elapsed="
+ TimeUnit.NANOSECONDS.toMillis(elapsed));
}
}
final AbstractTask task = titr.next();
final Future f = resourceManager
.getConcurrencyManager().submit(task);
getFutureForTask(f, task, remaining, TimeUnit.NANOSECONDS);
remaining = nanos - (System.nanoTime() - begin);
ndone++;
}
log.warn("Completed " + ndone + " out of " + tasks.size()
+ " tasks" );
} finally {
executorService.shutdownNow();
}
}
protected void runTasksConcurrent(final List> tasks)
throws InterruptedException {
assert resourceManager.overflowTasksConcurrent >= 0 ;
try {
final List> futures = resourceManager
.getConcurrencyManager().invokeAll(tasks,
resourceManager.overflowTimeout,
TimeUnit.MILLISECONDS);
final Iterator> titr = tasks.iterator();
for (Future f : futures) {
final AbstractTask task = titr.next();
getFutureForTask(f, task, 0 L, TimeUnit.NANOSECONDS);
}
} finally {
}
}
private void getFutureForTask(final Future f,
final AbstractTask task, final long timeout, final TimeUnit unit) {
try {
f.get(timeout, unit);
final long elapsed = TimeUnit.NANOSECONDS
.toMillis(task.nanoTime_finishedWork
- task.nanoTime_beginWork);
if (log.isInfoEnabled())
log.info("Task complete: elapsed=" + elapsed + ", task=" + task);
} catch (Throwable t) {
final long elapsed = TimeUnit.NANOSECONDS
.toMillis(task.nanoTime_finishedWork
- task.nanoTime_beginWork);
if (t instanceof CancellationException) {
log.warn("Task cancelled: elapsed=" + elapsed + ", task="
+ task + " : " + t);
resourceManager.overflowCounters.asynchronousOverflowTaskCancelledCounter
.incrementAndGet();
} else if (isNormalShutdown(t)) {
log.warn("Normal shutdown? : elapsed=" + elapsed + ", task="
+ task + " : " + t);
} else {
resourceManager.overflowCounters.asynchronousOverflowTaskFailedCounter
.incrementAndGet();
log.error("Child task failed: elapsed=" + elapsed + ", task="
+ task + " : " + t, t);
}
}
}
private boolean isNormalShutdown(final Throwable t) {
return isNormalShutdown(resourceManager, t);
}
static protected boolean isNormalShutdown(
final ResourceManager resourceManager, final Throwable t) {
if (Thread.interrupted()) {
return true ;
}
if (!resourceManager.isRunning()
|| !resourceManager.getConcurrencyManager()
.isOpen()
|| InnerCause.isInnerCause(t,
InterruptedException.class )
|| InnerCause.isInnerCause(t,
ClosedByInterruptException.class )
|| InnerCause.isInnerCause(t,
ClosedChannelException.class )
|| InnerCause.isInnerCause(t,
AsynchronousCloseException.class )) {
return true ;
}
return false ;
}
@SuppressWarnings ("unchecked" )
static private final List EMPTY_LIST = Collections.EMPTY_LIST;
}