com.ibm.etcd.client.watch.EtcdWatchClient Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of etcd-java Show documentation
Show all versions of etcd-java Show documentation
etcd3 java client and utilities
/*
* Copyright 2017, 2018 IBM Corp. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy
* of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.ibm.etcd.client.watch;
import java.io.Closeable;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.ForkJoinPool;
import javax.annotation.concurrent.GuardedBy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.ibm.etcd.client.GrpcClient;
import com.ibm.etcd.client.GrpcClient.ResilientResponseObserver;
import com.ibm.etcd.client.kv.WatchUpdate;
import com.ibm.etcd.client.kv.KvClient.Watch;
import com.ibm.etcd.client.kv.KvClient.WatchIterator;
import com.ibm.etcd.api.ResponseHeader;
import com.ibm.etcd.api.WatchCancelRequest;
import com.ibm.etcd.api.WatchCreateRequest;
import com.ibm.etcd.api.WatchGrpc;
import com.ibm.etcd.api.WatchRequest;
import com.ibm.etcd.api.WatchResponse;
import io.grpc.MethodDescriptor;
import io.grpc.Status.Code;
import io.grpc.stub.StreamObserver;
/**
*
*/
public class EtcdWatchClient implements Closeable {
private static final Logger logger = LoggerFactory.getLogger(EtcdWatchClient.class);
private static final Exception CANCEL_EXCEPTION = new CancellationException();
private static final String UNAUTH_REASON_PREFIX = "rpc error: code = PermissionDenied";
private static final MethodDescriptor METHOD_WATCH =
WatchGrpc.getWatchMethod();
/* Watcher states:
* - in pendingCreate only, watchId < 0, finished == false
* - in activeWatchers only, watchId >= 0, finished == false
* (stream completion event published in this transition)
* - in neither, watchId < 0, finished == true
*/
private final GrpcClient client;
private final Executor observerExecutor; // "parent" executor
private final Executor eventLoop; // serialized
@GuardedBy("this")
private StreamObserver requestStream;
// additions done only in "this" lock, removals done only in eventLoop
private final Queue pendingCreate = new ConcurrentLinkedQueue<>();
@GuardedBy("eventLoop")
private final Map activeWatchers = new HashMap<>();
public EtcdWatchClient(GrpcClient client) {
this(client, client.getResponseExecutor());
}
public EtcdWatchClient(GrpcClient client, Executor executor) {
this.client = client;
this.observerExecutor = executor != null ? executor : ForkJoinPool.commonPool();
this.eventLoop = GrpcClient.serialized(client.getInternalExecutor(), 128); // bounded for back-pressure
}
/**
* Internal per-watch state
*/
class WatcherRecord {
private final StreamObserver observer;
private final WatchCreateRequest request;
private final Executor watcherExecutor;
private WatchHandle creationFuture;
long upToRevision;
long watchId = -2L; // -2 only pre-creation, >= -1 after
boolean userCancelled, finished;
volatile boolean vUserCancelled;
public WatcherRecord(WatchCreateRequest request,
StreamObserver observer,
Executor parentExecutor) {
this.observer = observer;
this.request = request;
long rev = request.getStartRevision();
this.upToRevision = rev - 1;
// bounded for back-pressure
this.watcherExecutor = GrpcClient.serialized(parentExecutor, 64);
}
// null => cancelled (non-error)
public void publishCompletionEvent(final Exception err) {
watcherExecutor.execute(() -> {
completeCreateFuture(false, err);
try {
if(err == null || vUserCancelled) observer.onCompleted();
else observer.onError(err);
} catch(RuntimeException e) {
logger.warn("Watch "+watchId
+" observer onCompleted/onError threw", e);
}
});
}
@GuardedBy("eventLoop")
public void processWatchEvents(final WatchResponse wr) {
if(userCancelled) return; // suppress events if cancelled
int eventsCount = wr.getEventsCount();
final long newRevision = eventsCount <= 0 ? wr.getHeader().getRevision() - 1
: wr.getEvents(eventsCount-1).getKv().getModRevision();
if(newRevision <= upToRevision) return;
watcherExecutor.execute(() -> {
try {
// if(first) observer.onNext(new WatchUpdate(wr.getHeader(),
// null, null, EventType.ESTABLISHED));
if(vUserCancelled) return;
observer.onNext(new EtcdWatchUpdate(wr));
} catch(RuntimeException e) {
logger.warn("Watch observer onNext() threw", e);
// must cancel the watch here per StreamObserver contract
cancel();
//TODO this will result in the watcher receiving
// a final onComplete, but it should really be onError in this case
}
});
upToRevision = newRevision;
}
// returns true if addition to activeWatchers was made
@GuardedBy("eventLoop")
public boolean processCreatedResponse(WatchResponse wr, boolean cancelled) {
long newWatchId = wr.getWatchId();
if(cancelled || newWatchId == -1L) {
String reason = wr.getCancelReason();
if(reason != null && reason.startsWith(UNAUTH_REASON_PREFIX)) {
// If watch creation fails due to an authentication issue (likely
// expired), we trigger a reauth+refresh of the stream by sending
// an UNAUTHENTICATED error. This watch must first be re-created
// so that it will be retried properly in our onRefresh method.
synchronized(EtcdWatchClient.this) {
boolean notClosed = createNewWatch(this);
if(notClosed) {
StreamObserver reqStream = getRequestStream();
if(reqStream != null) reqStream.onError(Code.UNAUTHENTICATED
.toStatus().withDescription(reason).asException());
}
}
}
else processCancelledResponse(wr);
} else {
boolean first = this.watchId < 0, veryFirst = this.watchId == -2;;
this.watchId = newWatchId;
if(activeWatchers.putIfAbsent(newWatchId, this) == null) {
if(userCancelled) sendCancel(watchId);
else {
if(veryFirst) {
watcherExecutor.execute(() -> completeCreateFuture(true, null));
}
if(!first || wr.getEventsCount() > 0) {
processWatchEvents(wr);
}
return true;
}
} else {
logger.error("State error: watchId conflict: "+watchId);
//TODO cancel existing watch here?
}
}
return false;
}
@GuardedBy("eventLoop")
public void processCancelledResponse(WatchResponse wr) {
watchId = -1L;
if(finished) {
logger.warn("Ignoring unexpected cancel response for watch "
+wr.getWatchId()+", reason="+wr.getCancelReason());
return;
}
finished = true;
Exception error;
if(userCancelled) error = null;
else {
ResponseHeader header = wr.getHeader();
long cRev = wr.getCompactRevision();
String reason = wr.getCancelReason();
if(cRev != 0) error = new RevisionCompactedException(header, reason, cRev);
else if(wr.getCreated()) error = new WatchCreateException(header, reason);
else error = new WatchCancelledException(header, reason);
}
publishCompletionEvent(error);
}
@GuardedBy("eventLoop")
public WatchRequest newCreateWatchRequest() {
return WatchRequest.newBuilder()
.setCreateRequest(request.toBuilder()
.setStartRevision(upToRevision +1)).build();
}
public WatchRequest firstCreateWatchRequest() {
return WatchRequest.newBuilder()
.setCreateRequest(request).build();
}
// NOT guarded - called by user or watcherExecutor
public void cancel() {
if(closed || finished || userCancelled) return;
eventLoop.execute(() -> {
if(closed || userCancelled || finished) return;
sendCancel(watchId);
vUserCancelled = userCancelled = true;
});
}
private void completeCreateFuture(boolean created, Exception error) {
WatchHandle wh = creationFuture;
if(wh == null) return;
wh.complete(created, error);
creationFuture = null;
}
}
// @Override
public Watch watch(WatchCreateRequest createReq, StreamObserver observer) {
return watch(createReq, observer, null);
}
public Watch watch(WatchCreateRequest createReq,
StreamObserver observer, Executor executor) {
if(closed) throw new IllegalStateException("closed");
final WatcherRecord wrec = new WatcherRecord(createReq,
observer, executor != null ? executor : observerExecutor);
WatchHandle handle = new WatchHandle(wrec);
boolean succ = createNewWatch(wrec);
if(!succ) throw new IllegalStateException("closed");
return handle;
}
/**
* Blocking watch
*/
public WatchIterator watch(WatchCreateRequest createReq) {
EtcdWatchIterator watchIt = new EtcdWatchIterator();
Watch handle = watch(createReq, watchIt, MoreExecutors.directExecutor());
return watchIt.setWatch(handle);
}
//TODO probably change to have a request thread instead
private boolean createNewWatch(WatcherRecord wrec) {
WatchRequest createReq = wrec.firstCreateWatchRequest();
synchronized(this) {
StreamObserver reqStream = getRequestStream();
if(reqStream == null) return false;
pendingCreate.add(wrec);
reqStream.onNext(createReq);
}
return true;
}
static class WatchHandle extends AbstractFuture implements Watch {
private final WeakReference wrecRef;
public WatchHandle(WatcherRecord wrec) {
wrecRef = new WeakReference<>(wrec);
wrec.creationFuture = this;
}
@Override
public void close() {
WatcherRecord wrec = wrecRef.get();
if(wrec != null) wrec.cancel();
}
@Override
protected void interruptTask() {
close();
}
void complete(boolean created, Exception error) {
if(error != null) setException(error);
else set(created);
}
}
@GuardedBy("eventLoop")
protected void sendCancel(long watchId) {
if(watchId < 0) return; // not created yet
// send cancel request
WatchRequest cancelReq = WatchRequest.newBuilder()
.setCancelRequest(WatchCancelRequest.newBuilder()
.setWatchId(watchId).build()).build();
synchronized(this) {
// don't need to re-initialize reqstream if null (watch can't exist)
StreamObserver reqStream = closed ? null : requestStream;
if(reqStream != null) {
reqStream.onNext(cancelReq);
}
}
}
/**
* get current watch request stream if exists, otherwise establish new one
*/
@GuardedBy("this")
protected StreamObserver getRequestStream() {
if(closed) return null;
if(requestStream == null) {
logger.debug("watch stream starting");
requestStream = client.callStream(METHOD_WATCH, responseObserver, eventLoop);
}
return requestStream;
}
@GuardedBy("eventLoop")
protected void closeRequestStreamIfNoWatches() {
synchronized(this) {
if(requestStream == null || !activeWatchers.isEmpty() || !pendingCreate.isEmpty()) return;
requestStream.onError(CANCEL_EXCEPTION);
logger.debug("watch stream cancelled due to there being no active watches");
requestStream = null;
}
}
protected final ResilientResponseObserver responseObserver
= new ResilientResponseObserver() {
// all methods @GuardedBy("eventLoop")
@Override
public void onEstablished() {
// nothing to do here
logger.debug("onEstablished called for watch request stream");
}
@Override
public void onNext(WatchResponse wr) {
processResponse(wr);
}
@Override
public void onReplaced(StreamObserver newStreamRequestObserver) {
if(!closed) logger.info("onReplaced called for watch request stream"
+(newStreamRequestObserver==null?" with newReqStream == null":""));
onReplacedOrFailed(newStreamRequestObserver, null);
}
@Override
public void onCompleted() {
logger.debug("onCompleted called for watch request stream");
// alldone
}
@Override
public void onError(Throwable t) {
logger.debug("onError called for watch request stream", t);
if(closed || GrpcClient.causedBy(t, CancellationException.class)) return;
synchronized(EtcdWatchClient.this) {
if(closed) return;
}
logger.warn("Unexpected fatal watch stream error", t);
// this will cancel/complete all open user watches -
// complete their futures exceptionally if not started,
// and send a final onError to their stream observers
onReplacedOrFailed(null, t instanceof Exception
? (Exception)t : new RuntimeException(t));
}
void onReplacedOrFailed(StreamObserver newRequestStream, Exception err) {
List pending = null;
synchronized(EtcdWatchClient.this) {
requestStream = newRequestStream;
if(!activeWatchers.isEmpty() || !pendingCreate.isEmpty()) {
pending = new ArrayList(pendingCreate);
pending.addAll(activeWatchers.values());
pendingCreate.clear();
activeWatchers.clear();
}
}
boolean watchesExist = false;
// recreate all the non-cancelled watches
if(pending != null) for(WatcherRecord wrec : pending) {
if(wrec.finished) continue;
if(wrec.watchId >= 0) wrec.watchId = -1L;
boolean cancelled = wrec.userCancelled || closed || newRequestStream == null;
if(!cancelled) {
WatchRequest createReq = wrec.newCreateWatchRequest();
synchronized(EtcdWatchClient.this) {
if(closed) cancelled = true; // (client closed)
else {
pendingCreate.add(wrec);
newRequestStream.onNext(createReq);
watchesExist = true;
}
}
}
if(cancelled) {
wrec.vUserCancelled = wrec.userCancelled = true;
wrec.finished = true;
// err is null here in normal cancellation case
wrec.publishCompletionEvent(err);
}
}
if(!watchesExist) closeRequestStreamIfNoWatches();
}
};
@GuardedBy("eventLoop")
protected void processResponse(WatchResponse wr) {
boolean cancelled = wr.getCanceled() || wr.getCompactRevision() != 0;
long watchId = wr.getWatchId();
boolean watchCountReduced = false;
WatcherRecord wrec;
if(wr.getCreated()) {
if(logger.isDebugEnabled()) {
logger.debug("watch create response received for id "+watchId);
}
wrec = pendingCreate.poll();
if(wrec == null) {
logger.error("State error: received unexpected watch create response: "+wr);
sendCancel(wr.getWatchId()); // or maybe close/refresh stream
//throw new IllegalStateException("unexpected watch create response");
return;
}
watchCountReduced = !wrec.processCreatedResponse(wr, cancelled);
} else if(cancelled) {
wrec = activeWatchers.remove(watchId);
watchCountReduced = true;
if(wrec != null) {
//TODO try to resume on "unexpected" cancellations?
wrec.processCancelledResponse(wr);
}
} else {
wrec = activeWatchers.get(watchId);
if(wrec != null) {
wrec.processWatchEvents(wr);
} else {
logger.warn("State error: received response for unrecognized watcher: "+watchId);
sendCancel(watchId); // or maybe close/refresh stream
}
}
if(watchCountReduced && activeWatchers.isEmpty() && pendingCreate.isEmpty()) {
closeRequestStreamIfNoWatches();
}
}
@GuardedBy("this") // but lazy-read from other contexts
protected boolean closed;
@Override
public void close() {
if(closed) return;
eventLoop.execute(() -> {
if(!closed) synchronized(EtcdWatchClient.this) {
if(closed) return;
closed = true;
if(requestStream != null) {
requestStream.onError(CANCEL_EXCEPTION);
requestStream = null;
}
responseObserver.onReplaced(null); // this will close any individual watches
}
});
}
}