com.gemstone.gemfire.distributed.internal.locks.GrantorRequestProcessor Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of gemfire-core Show documentation
Show all versions of gemfire-core Show documentation
SnappyData store based off Pivotal GemFireXD
The newest version!
/*
* Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.gemstone.gemfire.distributed.internal.locks;
import com.gemstone.gemfire.CancelCriterion;
import com.gemstone.gemfire.DataSerializer;
import com.gemstone.gemfire.distributed.internal.*;
import com.gemstone.gemfire.i18n.LogWriterI18n;
import com.gemstone.gemfire.internal.Assert;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.util.concurrent.StoppableCondition;
import com.gemstone.gemfire.internal.util.concurrent.StoppableReentrantLock;
import java.io.*;
import java.util.Set;
import com.gemstone.gemfire.distributed.internal.membership.*;
/**
* A processor for sending a message to the elder asking it for the
* grantor of a dlock service.
*
* @since 4.0
* @author Darrel Schneider
*/
public class GrantorRequestProcessor extends ReplyProcessor21 {
private GrantorInfo result;
////////// Public static entry point /////////
/**
* The number of milliseconds to sleep for elder change if current elder
* is departing (and already sent shutdown msg) but is still in the View.
*/
public static final long ELDER_CHANGE_SLEEP =
Long.getLong("GrantorRequestProcessor.ELDER_CHANGE_SLEEP", 100).longValue();
private static final byte GET_OP = 0;
private static final byte BECOME_OP = 1;
private static final byte CLEAR_OP = 2;
private static final byte PEEK_OP = 3;
private static final byte CLEAR_WITH_LOCKS_OP = 4;
private static final GrantorInfo CLEAR_COMPLETE = new GrantorInfo(null, 0, 0, false);
/**
* Encapsulates the context necessary for processing a given grantor request
* for a given InternalDistributedSystem
*
* @author jpenney
*/
public static class GrantorRequestContext {
/**
* Locks access to elders
*/
final StoppableReentrantLock elderLock;
/**
* Subservient condition to {@link #elderLock}
*/
final StoppableCondition elderLockCondition;
/**
* Our notion of the current elder
*
* @guarded.By {@link #elderLock}
*/
InternalDistributedMember currentElder = null;
/**
* Count of the elder calls in-flight
*
* @guarded.By {@link #elderLock}
*/
int elderCallsInProgress = 0;
/**
* If true, we're cooling our heels waiting for the elders to pass
* the baton
*
* @guarded.By {@link #elderLock}
*/
boolean waitingToChangeElder = false;
public GrantorRequestContext(CancelCriterion cancelCriterion) {
elderLock = new StoppableReentrantLock(cancelCriterion);
elderLockCondition = elderLock.newCondition();
}
}
private final static boolean basicStartElderCall(
InternalDistributedSystem sys, ElderState es,
InternalDistributedMember elder, DLockService dls) {
GrantorRequestContext grc = sys.getGrantorRequestContext();
grc.elderLock.lock();
try {
if (es != null) {
// elder is in our vm
if (grc.elderCallsInProgress > 0) {
// wait until all the calls in progress to an old rmt elder complete.
// We know it is some other elder because we don't count the
// calls in progress to a local elder.
elderSyncWait(sys, elder, dls);
}
} else {
// elder is in remote vm
if (grc.elderCallsInProgress > 0) {
if (elder == grc.currentElder) {
grc.elderCallsInProgress += 1;
} else if (elder != null && elder.equals(grc.currentElder)) {
grc.elderCallsInProgress += 1;
} else {
elderSyncWait(sys, elder, dls);
return false;
}
} else {
grc.currentElder = elder;
grc.elderCallsInProgress = 1;
}
}
return true;
}
finally {
grc.elderLock.unlock();
}
}
/**
* Waits until elder recovery can proceed safely.
* Currently this is done by waiting until any in progress
* calls to an old elder are complete
* @param elderId the member id of the new elder; null if new elder is local
*/
static void readyForElderRecovery(
InternalDistributedSystem sys, InternalDistributedMember elderId,
DLockService dls) {
GrantorRequestContext grc = sys.getGrantorRequestContext();
if (elderId != null) {
grc.elderLock.lock();
try {
if (grc.elderCallsInProgress > 0) {
// make sure they are not going to the new elder
if (elderId != grc.currentElder && !elderId.equals(grc.currentElder)) {
elderSyncWait(sys, elderId, dls);
}
}
}
finally {
grc.elderLock.unlock();
}
} else {
grc.elderLock.lock();
try {
if (grc.elderCallsInProgress > 0) {
// wait until all the calls in progress to an old rmt elder complete.
// We know it is some other elder because we don't count the
// calls in progress to a local elder.
elderSyncWait(sys, /*elderId*/ null, dls);
}
}
finally {
grc.elderLock.unlock();
}
}
}
/**
* elderSyncWait
* @param newElder
* @param dls
*/
private static void elderSyncWait(
InternalDistributedSystem sys,
InternalDistributedMember newElder,
DLockService dls) {
GrantorRequestContext grc = sys.getGrantorRequestContext();
grc.waitingToChangeElder = true;
LogWriterI18n logger = sys.getLogWriterI18n();
while (grc.waitingToChangeElder) {
if (logger != null && logger.infoEnabled()) {
logger.info(
LocalizedStrings.GrantorRequestProcessor_GRANTORREQUESTPROCESSOR_ELDERSYNCWAIT_THE_CURRENT_ELDER_0_IS_WAITING_FOR_THE_NEW_ELDER_1,
new Object[] {grc.currentElder, newElder});
}
boolean interrupted = Thread.interrupted();
try {
grc.elderLockCondition.await();
break;
} catch (InterruptedException ignore) {
interrupted = true;
sys.getCancelCriterion().checkCancelInProgress(ignore);
}
finally {
if (interrupted) Thread.currentThread().interrupt();
}
}
}
/**
* Sets currentElder to the memberId of the current elder if elder is remote;
* null if elder is in our vm. TODO: collaboration lock was removed
*/
private final static ElderState startElderCall(
InternalDistributedSystem sys,
DLockService dls,
boolean usesElderCollaborationLock) {
InternalDistributedMember elder;
ElderState es = null;
final DM dm = sys.getDistributionManager();
boolean elderCallStarted = false;
while (!elderCallStarted) {
dm.throwIfDistributionStopped();
elder = dm.getElderId(); // call this before getElderState
Assert.assertTrue(elder != null,
"starting an elder call with no valid elder");
if (dm.getId().equals(elder)) {
if (usesElderCollaborationLock) {
try {
es = dm.getElderState(false, true);
}
catch (IllegalStateException e) {
// loop back around to reacquire Collaboration and try elder lock again
continue;
}
}
else {
es = dm.getElderState(false, false);
}
} else {
es = null;
}
elderCallStarted = basicStartElderCall(sys, es, elder, dls);
}
return es;
}
private final static void finishElderCall(GrantorRequestContext grc,
ElderState es) {
if (es == null) {
grc.elderLock.lock();
try {
Assert.assertTrue(grc.elderCallsInProgress > 0);
grc.elderCallsInProgress -= 1;
if (grc.elderCallsInProgress == 0) {
grc.currentElder = null;
if (grc.waitingToChangeElder) {
grc.waitingToChangeElder = false;
grc.elderLockCondition.signalAll();
}
}
}
finally {
grc.elderLock.unlock();
}
}
}
/**
* Asks the elder who the grantor is for the specified service.
* If no grantor exists then makes us the grantor.
* @param service the service we want to know the grantor of.
* @param sys the distributed system
* @return information describing the current grantor of this service
* and if it needs recovery.
*/
public static GrantorInfo getGrantor(DLockService service,
int dlsSerialNumber,
InternalDistributedSystem sys) {
return basicOp(-1, service, dlsSerialNumber, sys, null, GET_OP);
}
/**
* Asks the elder who the grantor is for the specified service.
* @param service the service we want to know the grantor of.
* @param sys th distributed system
* @return information describing the current grantor of this service
* and if recovery is needed
*/
public static GrantorInfo peekGrantor(DLockService service,
InternalDistributedSystem sys) {
return basicOp(-1, service, -1, sys, null, PEEK_OP);
}
public static GrantorInfo peekGrantor(String serviceName,
InternalDistributedSystem sys) {
return basicOp(-1, serviceName, null, -1, sys, null, PEEK_OP);
}
/**
* Tells the elder we want to become the grantor
* @param service the service we want to be the grantor of.
* @param oldTurk if non-null then only become grantor if it is currently oldTurk.
* @param sys the distributed system
* @return information describing the previous grantor, if any, and
* if we need to do a grantor recovery
*/
public static GrantorInfo becomeGrantor(DLockService service,
int dlsSerialNumber,
InternalDistributedMember oldTurk,
InternalDistributedSystem sys) {
return basicOp(-1, service, dlsSerialNumber, sys, oldTurk, BECOME_OP);
}
/**
* Tells the elder we are doing a clean destroy of our grantor
* @param service the service we are no longer the grantor of.
* @param sys the distributed system
*/
public static void clearGrantor(long grantorVersion,
DLockService service,
int dlsSerialNumber,
InternalDistributedSystem sys,
boolean withLocks) {
basicOp(grantorVersion, service, dlsSerialNumber, sys, null,
withLocks ? CLEAR_WITH_LOCKS_OP : CLEAR_OP);
}
/**
* @param opCode encodes what operation we are doing
*/
private static GrantorInfo basicOp(long grantorVersion,
DLockService service,
int dlsSerialNumber,
InternalDistributedSystem sys,
InternalDistributedMember oldTurk,
byte opCode) {
return basicOp(grantorVersion, service.getName(), service, dlsSerialNumber,
sys, oldTurk, opCode);
}
private static GrantorInfo basicOp(long grantorVersion,
String serviceName,
DLockService service,
int dlsSerialNumber,
InternalDistributedSystem system,
InternalDistributedMember oldTurk,
byte opCode) {
GrantorInfo result = null;
DM dm = system.getDistributionManager();
GrantorRequestContext grc = system.getGrantorRequestContext();
boolean tryNewElder;
boolean interrupted = false;
try {
do {
tryNewElder = false;
final boolean usesElderCollaborationLock =
opCode==GET_OP || opCode==BECOME_OP;
if (usesElderCollaborationLock) {
Assert.assertTrue(service != null,
"Attempting GrantorRequest without instance of DistributedLockService");
}
final ElderState es = startElderCall(
system, service, usesElderCollaborationLock);
dm.throwIfDistributionStopped();
try {
if (es != null) {
// local elder so do it without messaging
switch (opCode) {
case GET_OP:
result = es.getGrantor(serviceName, dm.getId(), dlsSerialNumber);
break;
case PEEK_OP:
result = es.peekGrantor(serviceName);
break;
case BECOME_OP:
result = es.becomeGrantor(serviceName, dm.getId(), dlsSerialNumber, oldTurk);
break;
case CLEAR_OP:
es.clearGrantor(grantorVersion, serviceName, dlsSerialNumber,
dm.getId(), false);
result = CLEAR_COMPLETE;
break;
case CLEAR_WITH_LOCKS_OP:
es.clearGrantor(grantorVersion, serviceName, dlsSerialNumber,
dm.getId(), true);
result = CLEAR_COMPLETE;
break;
default:
throw new IllegalStateException("Unknown opCode " + opCode);
}
}
else {
// remote elder so send message
GrantorRequestProcessor processor = new GrantorRequestProcessor(system,
grc.currentElder);
boolean sent = GrantorRequestMessage.send(grantorVersion,
dlsSerialNumber, serviceName,
grc.currentElder, dm, processor, oldTurk, opCode);
if (!sent) {
DLockLogWriter.fine(dm, "Unable to communicate with elder "
+ grc.currentElder);
}
try {
processor.waitForRepliesUninterruptibly();
} catch (ReplyException e) {
e.handleAsUnexpected();
}
if (processor.result != null) {
result = processor.result;
}
else {
// no result and no longer waiting...
// sleep if targeted elder still in view but not activeMembers
if (!dm.getDistributionManagerIds().contains(grc.currentElder) &&
dm.getViewMembers().contains(grc.currentElder)) {
// if true then elder no longer in DM activeMembers
// but elder is still in the View
// elder probably sent shutdown msg but may not yet left View
try {
Thread.sleep(ELDER_CHANGE_SLEEP);
}
catch (InterruptedException e) {
interrupted = true;
dm.getCancelCriterion().checkCancelInProgress(e);
}
}
// targetted elder either died or already sent us a shutdown msg
if (opCode != CLEAR_OP && opCode != CLEAR_WITH_LOCKS_OP) {
// Note we do not try a new elder if doing a clear because
// the new elder will not have anything for us to clear.
// He will have done an ElderInit.
tryNewElder = true;
}
}
}
}
finally {
finishElderCall(grc, es);
}
} while (tryNewElder);
}
finally {
if (interrupted) {
Thread.currentThread().interrupt();
}
}
return result;
}
//////////// Instance methods //////////////
/** Creates a new instance of GrantorRequestProcessor
*/
private GrantorRequestProcessor(InternalDistributedSystem system,
InternalDistributedMember elder) {
super(system, elder);
}
@Override
public void process(DistributionMessage msg) {
if (msg instanceof GrantorInfoReplyMessage) {
GrantorInfoReplyMessage giMsg = (GrantorInfoReplyMessage)msg;
this.result = giMsg.getGrantorInfo();
} else if (msg instanceof ReplyMessage) {
if (((ReplyMessage)msg).getException() == null) {
// must be a reply sent back from a CLEAR_OP
this.result = CLEAR_COMPLETE;
}
} else {
Assert.assertTrue(false,
"Expected instance of GrantorInfoReplyMessage or CReplyMessage but got "
+ msg.getClass());
}
super.process(msg);
}
/////////////// Inner message classes //////////////////
public static final class GrantorRequestMessage
extends PooledDistributionMessage implements MessageWithReply
{
private long grantorVersion;
private int dlsSerialNumber;
private String serviceName;
private int processorId;
private byte opCode;
private InternalDistributedMember oldTurk;
/**
*
* @param serviceName
* @param elder
* @param dm
* @param proc
* @param oldTurk
* @param opCode
* @return true if the message was sent
*/
protected static boolean send(long grantorVersion,
int dlsSerialNumber,
String serviceName,
InternalDistributedMember elder,
DM dm,
ReplyProcessor21 proc,
InternalDistributedMember oldTurk,
byte opCode)
{
// bug36361: the following assertion doesn't work, since the client that sent us
// the request might have a different notion of the elder (no view synchrony on the
// current notion of the elder).
// InternalDistributedMember moi = dm.getDistributionManagerId();
// Assert.assertTrue(!(
// // Sending a message to ourself is REALLY WEIRD, so
// // we make that the first test...
// moi.equals(dm.getElderId())
// && !moi.equals(elder)
// && dm.getDistributionManagerIds().contains(elder)
// ));
GrantorRequestMessage msg = new GrantorRequestMessage();
msg.grantorVersion = grantorVersion;
msg.dlsSerialNumber = dlsSerialNumber;
msg.serviceName = serviceName;
msg.oldTurk = oldTurk;
msg.opCode = opCode;
msg.processorId = proc.getProcessorId();
msg.setRecipient(elder);
if (DLockLogWriter.fineEnabled(dm)) {
DLockLogWriter.fine(dm, "GrantorRequestMessage sending " + msg
+ " to " + elder);
}
Set failures = dm.putOutgoing(msg);
return failures == null || failures.size() == 0;
}
@Override
public int getProcessorId() {
return this.processorId;
}
private void replyGrantorInfo(DM dm, GrantorInfo gi) {
GrantorInfoReplyMessage.send(this, dm, gi);
}
private void replyClear(DM dm) {
ReplyMessage.send(this.getSender(), this.getProcessorId(),
null, dm, null);
}
@Override
protected void process(DistributionManager dm) {
//executeBasicProcess(dm); // TODO change to this after things are stable
basicProcess(dm);
}
// private void executeBasicProcess(final DM dm) {
// final GrantorRequestMessage msg = this;
// try {
// dm.getWaitingThreadPool().execute(new Runnable() {
// public void run() {
// basicProcess(dm);
// }
// });
// }
// catch (RejectedExecutionException e) {
// if (DLockLogWriter.fineEnabled(dm)) {
// DLockLogWriter.fine(dm, "Rejected processing of <" + this + ">", e);
// }
// }
// }
protected void basicProcess(final DM dm) {
// we should be in the elder
ElderState es = dm.getElderState(true, false);
switch (this.opCode) {
case GET_OP:
replyGrantorInfo(dm, es.getGrantor(this.serviceName, getSender(),
this.dlsSerialNumber));
break;
case PEEK_OP:
replyGrantorInfo(dm, es.peekGrantor(this.serviceName));
break;
case BECOME_OP:
replyGrantorInfo(dm, es.becomeGrantor(this.serviceName, getSender(),
this.dlsSerialNumber, this.oldTurk));
break;
case CLEAR_OP:
es.clearGrantor(this.grantorVersion, this.serviceName,
this.dlsSerialNumber, getSender(), false);
replyClear(dm);
break;
case CLEAR_WITH_LOCKS_OP:
es.clearGrantor(this.grantorVersion, this.serviceName,
this.dlsSerialNumber, getSender(), true);
replyClear(dm);
break;
default:
throw new IllegalStateException("Unknown opCode " + this.opCode);
}
}
public int getDSFID() {
return GRANTOR_REQUEST_MESSAGE;
}
@Override
public void fromData(DataInput in)
throws IOException, ClassNotFoundException {
super.fromData(in);
this.grantorVersion = in.readLong();
this.dlsSerialNumber = in.readInt();
this.serviceName = DataSerializer.readString(in);
this.processorId = in.readInt();
this.opCode = in.readByte();
if (this.opCode == BECOME_OP) {
this.oldTurk = (InternalDistributedMember)DataSerializer.readObject(in);
}
}
@Override
public void toData(DataOutput out) throws IOException {
super.toData(out);
out.writeLong(this.grantorVersion);
out.writeInt(this.dlsSerialNumber);
DataSerializer.writeString(this.serviceName, out);
out.writeInt(this.processorId);
out.writeByte(this.opCode);
if (this.opCode == BECOME_OP) {
DataSerializer.writeObject(this.oldTurk, out);
}
}
public static String opCodeToString(int opCode) {
String string = null;
switch (opCode) {
case GET_OP: string = "GET_OP"; break;
case BECOME_OP: string = "BECOME_OP"; break;
case CLEAR_OP: string = "CLEAR_OP"; break;
case PEEK_OP: string = "PEEK_OP"; break;
case CLEAR_WITH_LOCKS_OP: string = "CLEAR_WITH_LOCKS_OP"; break;
default: string = "UNKNOWN:" + String.valueOf(opCode); break;
}
return string;
}
@Override
public String toString() {
String opCodeString = opCodeToString(this.opCode);
StringBuffer buff = new StringBuffer();
buff.append("GrantorRequestMessage (service='")
.append(this.serviceName)
.append("'; grantorVersion=")
.append(this.grantorVersion)
.append("'; dlsSerialNumber=")
.append(this.dlsSerialNumber)
.append("'; processorId=")
.append(this.processorId)
.append("'; opCode=")
.append(opCodeString)
.append("'; oldT=")
.append(this.oldTurk)
.append(")");
return buff.toString();
}
}
public static final class GrantorInfoReplyMessage extends ReplyMessage {
private InternalDistributedMember grantor;
private long elderVersionId;
private int grantorSerialNumber;
private boolean needsRecovery;
public static void send(MessageWithReply reqMsg, DM dm,
GrantorInfo gi)
{
GrantorInfoReplyMessage m = new GrantorInfoReplyMessage();
m.grantor = gi.getId();
m.needsRecovery = gi.needsRecovery();
m.elderVersionId = gi.getVersionId();
m.grantorSerialNumber = gi.getSerialNumber();
m.processorId = reqMsg.getProcessorId();
m.setRecipient(reqMsg.getSender());
dm.putOutgoing(m);
}
public GrantorInfo getGrantorInfo() {
return new GrantorInfo(this.grantor, this.elderVersionId,
this.grantorSerialNumber, this.needsRecovery);
}
@Override
public int getDSFID() {
return GRANTOR_INFO_REPLY_MESSAGE;
}
@Override
public void fromData(DataInput in)
throws IOException, ClassNotFoundException {
super.fromData(in);
this.grantor = (InternalDistributedMember)DataSerializer.readObject(in);
this.elderVersionId = in.readLong();
this.grantorSerialNumber = in.readInt();
this.needsRecovery = in.readBoolean();
}
@Override
public void toData(DataOutput out) throws IOException {
super.toData(out);
DataSerializer.writeObject(this.grantor, out);
out.writeLong(this.elderVersionId);
out.writeInt(this.grantorSerialNumber);
out.writeBoolean(this.needsRecovery);
}
@Override
public String toString() {
StringBuffer buff = new StringBuffer();
buff.append("GrantorInfoReplyMessage")
.append("; sender=")
.append(getSender())
.append("; processorId=")
.append(super.processorId)
.append("; grantor=")
.append(this.grantor)
.append("; elderVersionId=")
.append(this.elderVersionId)
.append("; grantorSerialNumber=")
.append(this.grantorSerialNumber)
.append("; needsRecovery=")
.append(this.needsRecovery)
.append(")");
return buff.toString();
}
}
}