io.mantisrx.server.worker.client.MetricsClientImpl Maven / Gradle / Ivy
The newest version!
/*
* Copyright 2019 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.mantisrx.server.worker.client;
import io.mantisrx.common.metrics.Gauge;
import io.mantisrx.common.metrics.Metrics;
import io.mantisrx.common.metrics.MetricsRegistry;
import io.mantisrx.common.network.WorkerEndpoint;
import io.mantisrx.server.master.client.MasterClientWrapper;
import io.reactivex.mantis.remote.observable.EndpointChange;
import io.reactivx.mantis.operators.DropOperator;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Observer;
import rx.Subscriber;
import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
import rx.subscriptions.Subscriptions;
class MetricsClientImpl implements MetricsClient {
private static final Logger logger = LoggerFactory.getLogger(MetricsClientImpl.class);
final String jobId;
final WorkerConnectionFunc workerConnectionFunc;
final JobWorkerMetricsLocator jobWorkerMetricsLocator;
final private AtomicBoolean nowClosed = new AtomicBoolean(false);
final private WorkerConnections workerConnections = new WorkerConnections();
private final String workersGuageName = "MetricsConnections";
private final String expectedWorkersGaugeName = "ExpectedMetricsConnections";
private final String workerConnReceivingDataGaugeName = "metricsRecvngData";
private final Gauge workersGauge;
private final Gauge expectedWorkersGauge;
private final Gauge workerConnReceivingDataGauge;
private final AtomicInteger numWorkers = new AtomicInteger();
private final Observer workerConnectionsStatusObserver;
private final long dataRecvTimeoutSecs;
MetricsClientImpl(String jobId, WorkerConnectionFunc workerConnectionFunc, JobWorkerMetricsLocator jobWorkerMetricsLocator,
Observable numWorkersObservable,
Observer workerConnectionsStatusObserver, long dataRecvTimeoutSecs) {
this.jobId = jobId;
this.workerConnectionFunc = workerConnectionFunc;
this.jobWorkerMetricsLocator = jobWorkerMetricsLocator;
Metrics metrics = new Metrics.Builder()
.name(MetricsClientImpl.class.getCanonicalName() + "-" + jobId)
.addGauge(workersGuageName)
.addGauge(expectedWorkersGaugeName)
.addGauge(workerConnReceivingDataGaugeName)
.build();
metrics = MetricsRegistry.getInstance().registerAndGet(metrics);
workersGauge = metrics.getGauge(workersGuageName);
expectedWorkersGauge = metrics.getGauge(expectedWorkersGaugeName);
workerConnReceivingDataGauge = metrics.getGauge(workerConnReceivingDataGaugeName);
numWorkersObservable
.doOnNext(new Action1() {
@Override
public void call(Integer integer) {
numWorkers.set(integer);
}
})
.takeWhile(new Func1() {
@Override
public Boolean call(Integer integer) {
return !nowClosed.get();
}
})
.subscribe();
this.workerConnectionsStatusObserver = workerConnectionsStatusObserver;
this.dataRecvTimeoutSecs = dataRecvTimeoutSecs;
}
private String toWorkerConnName(String host, int port) {
return host + "-" + port;
}
@Override
public boolean hasError() {
return false;
}
@Override
public String getError() {
return null;
}
@Override
public Observable> getResults() {
return Observable
.create(new Observable.OnSubscribe>() {
@Override
public void call(final Subscriber subscriber) {
internalGetResults().subscribe(subscriber);
}
})
.subscribeOn(Schedulers.io());
}
private Observable> internalGetResults() {
return jobWorkerMetricsLocator
.locateWorkerMetricsForJob(jobId)
.map(new Func1>() {
@Override
public Observable call(EndpointChange endpointChange) {
if (nowClosed.get())
return Observable.empty();
if (endpointChange.getType() == EndpointChange.Type.complete) {
return handleEndpointClose(endpointChange);
} else {
return handleEndpointConnect(endpointChange);
}
}
})
.lift(new Observable.Operator, Observable>() {
@Override
public Subscriber> call(Subscriber> subscriber) {
subscriber.add(Subscriptions.create(new Action0() {
@Override
public void call() {
try {
logger.warn("Closing metrics connections to workers of job " + jobId);
closeAllConnections();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}));
return subscriber;
}
})
.share()
.lift(new DropOperator>("client_metrics_share"))
;
}
private Observable handleEndpointConnect(EndpointChange ec) {
logger.info("Opening connection to metrics sink at " + ec.toString());
final String unwrappedHost = MasterClientWrapper.getUnwrappedHost(ec.getEndpoint().getHost());
final int metricsPort;
if (ec.getEndpoint() instanceof WorkerEndpoint) {
metricsPort = ((WorkerEndpoint) ec.getEndpoint()).getMetricPort();
} else {
logger.error("endpoint received on Endpoint connect is not a WorkerEndpoint {}, no metrics port to connect to", ec.getEndpoint());
return Observable.empty();
}
WorkerConnection workerConnection = workerConnectionFunc.call(unwrappedHost, metricsPort,
new Action1() {
@Override
public void call(Boolean flag) {
updateWorkerConx(flag);
}
},
new Action1() {
@Override
public void call(Boolean flag) {
updateWorkerDataReceivingStatus(flag);
}
},
dataRecvTimeoutSecs
);
if (nowClosed.get()) {// check if closed before adding
try {
workerConnection.close();
} catch (Exception e) {
logger.warn("Error closing worker metrics connection " + workerConnection.getName() + " - " + e.getMessage(), e);
}
return Observable.empty();
}
workerConnections.put(toWorkerConnName(unwrappedHost, metricsPort), workerConnection);
if (nowClosed.get()) {
try {
workerConnection.close();
workerConnections.remove(toWorkerConnName(unwrappedHost, metricsPort));
return Observable.empty();
} catch (Exception e) {
logger.warn("Error closing worker metrics connection - " + e.getMessage());
}
}
return workerConnection.call()
// .flatMap(new Func1, Observable>() {
// @Override
// public Observable call(Observable tObservable) {
// return tObservable;
// }
// })
;
}
private void updateWorkerDataReceivingStatus(Boolean flag) {
if (flag)
workerConnReceivingDataGauge.increment();
else
workerConnReceivingDataGauge.decrement();
expectedWorkersGauge.set(numWorkers.get());
if (workerConnectionsStatusObserver != null) {
synchronized (workerConnectionsStatusObserver) {
workerConnectionsStatusObserver.onNext(new WorkerConnectionsStatus(workerConnReceivingDataGauge.value(), workersGauge.value(), numWorkers.get()));
}
}
}
private void updateWorkerConx(Boolean flag) {
if (flag)
workersGauge.increment();
else
workersGauge.decrement();
expectedWorkersGauge.set(numWorkers.get());
if (workerConnectionsStatusObserver != null) {
synchronized (workerConnectionsStatusObserver) {
workerConnectionsStatusObserver.onNext(new WorkerConnectionsStatus(workerConnReceivingDataGauge.value(), workersGauge.value(), numWorkers.get()));
}
}
}
private Observable handleEndpointClose(EndpointChange ec) {
logger.info("Closed connection to metrics sink at " + ec.toString());
final String unwrappedHost = MasterClientWrapper.getUnwrappedHost(ec.getEndpoint().getHost());
final int metricsPort;
if (ec.getEndpoint() instanceof WorkerEndpoint) {
metricsPort = ((WorkerEndpoint) ec.getEndpoint()).getMetricPort();
} else {
logger.warn("endpoint received on Endpoint close is not a WorkerEndpoint {}, worker endpoint required for metrics port", ec.getEndpoint());
return Observable.empty();
}
final WorkerConnection removed = workerConnections.remove(toWorkerConnName(unwrappedHost, metricsPort));
if (removed != null) {
try {
removed.close();
} catch (Exception e) {
// shouldn't happen
logger.error("Unexpected exception on closing worker metrics connection: " + e.getMessage(), e);
}
}
return Observable.empty();
}
private void closeAllConnections() throws Exception {
nowClosed.set(true);
workerConnections.closeOut(new Action1>() {
@Override
public void call(WorkerConnection tWorkerConnection) {
try {
tWorkerConnection.close();
} catch (Exception e) {
logger.warn("Error closing worker metrics connection " + tWorkerConnection.getName() +
" - " + e.getMessage(), e);
}
}
});
}
class WorkerConnections {
final private Map> workerConnections = new HashMap<>();
private boolean isClosed = false;
private void put(String key, WorkerConnection val) {
synchronized (workerConnections) {
if (isClosed)
return;
workerConnections.put(key, val);
}
}
private WorkerConnection remove(String key) {
synchronized (workerConnections) {
return workerConnections.remove(key);
}
}
private void closeOut(Action1> onClose) {
synchronized (workerConnections) {
isClosed = true;
}
for (WorkerConnection workerConnection : workerConnections.values()) {
logger.info("Closing " + workerConnection.getName());
onClose.call(workerConnection);
}
}
}
}