io.reactivex.mantis.remote.observable.ServeGroupedObservable Maven / Gradle / Ivy
/*
* Copyright 2019 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.reactivex.mantis.remote.observable;
import io.mantisrx.common.codec.Encoder;
import io.mantisrx.common.metrics.Counter;
import io.mantisrx.common.metrics.Metrics;
import io.mantisrx.common.metrics.MetricsRegistry;
import io.mantisrx.common.network.HashFunctions;
import io.mantisrx.server.core.ServiceRegistry;
import io.reactivex.mantis.remote.observable.filter.ServerSideFilters;
import io.reactivex.mantis.remote.observable.slotting.ConsistentHashing;
import io.reactivex.mantis.remote.observable.slotting.SlottingStrategy;
import io.reactivx.mantis.operators.DisableBackPressureOperator;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Notification;
import rx.Observable;
import rx.Observer;
import rx.Subscription;
import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.observables.GroupedObservable;
public class ServeGroupedObservable extends ServeConfig> {
private static final Logger logger = LoggerFactory.getLogger(ServeGroupedObservable.class);
private Encoder keyEncoder;
private Encoder valueEncoder;
private int groupBufferTimeMSec = 250;
private long expiryInSecs = Long.MAX_VALUE;
private Counter groupsExpiredCounter;
public ServeGroupedObservable(Builder builder) {
super(builder.name, builder.slottingStrategy, builder.filterFunction,
builder.maxWriteAttempts);
// TODO this should be pushed into builder, default is 0 buffer
String groupBufferTimeMSecStr =
ServiceRegistry.INSTANCE.getPropertiesService().getStringValue("mantis.remoteObservable.groupBufferMSec", "250");
if (groupBufferTimeMSecStr != null && !groupBufferTimeMSecStr.equals("250")) {
groupBufferTimeMSec = Integer.parseInt(groupBufferTimeMSecStr);
}
this.keyEncoder = builder.keyEncoder;
this.valueEncoder = builder.valueEncoder;
this.expiryInSecs = builder.expiryTimeInSecs;
Metrics m = new Metrics.Builder()
.name("ServeGroupedObservable")
.addCounter("groupsExpiredCounter")
.build();
m = MetricsRegistry.getInstance().registerAndGet(m);
groupsExpiredCounter = m.getCounter("groupsExpiredCounter");
applySlottingSideEffectToObservable(builder.observable, builder.minConnectionsToSubscribe);
}
private void applySlottingSideEffectToObservable(
Observable>> o,
final Observable minConnectionsToSubscribe) {
final AtomicInteger currentMinConnectionsToSubscribe = new AtomicInteger();
minConnectionsToSubscribe
.subscribe(new Action1() {
@Override
public void call(Integer t1) {
currentMinConnectionsToSubscribe.set(t1);
}
});
Observable>>> listOfGroups = o
.map(new Func1>, Observable>>>() {
@Override
public Observable>> call(
Observable> og) {
return
og
.flatMap(new Func1, Observable>>>() {
@Override
public Observable>> call(
final GroupedObservable group) {
final byte[] keyBytes = keyEncoder.encode(group.getKey());
final String keyValue = group.getKey();
return
group
// comment out as this causes a NPE to happen in merge. supposedly fixed in rxjava 1.0
.doOnUnsubscribe(new Action0() {
@Override
public void call() {
//logger.info("Expiring group stage in serveGroupedObservable " + group.getKey());
groupsExpiredCounter.increment();
}
})
.timeout(expiryInSecs, TimeUnit.SECONDS, (Observable) Observable.empty())
.materialize()
.lift(new DisableBackPressureOperator>())
.buffer(groupBufferTimeMSec, TimeUnit.MILLISECONDS)
.filter(new Func1>, Boolean>() {
@Override
public Boolean call(List> t1) {
return t1 != null && !t1.isEmpty();
}
})
.map(new Func1>, List>>() {
@Override
public List> call(List> notifications) {
List> groups = new ArrayList<>(notifications.size());
for (Notification notification : notifications) {
groups.add(new Group(keyValue, keyBytes, notification));
}
return groups;
}
});
}
});
}
});
final Observable>> withSideEffects =
Observable
.merge(
listOfGroups
)
.doOnEach(new Observer>>() {
@Override
public void onCompleted() {
slottingStrategy.completeAllConnections();
}
@Override
public void onError(Throwable e) {
e.printStackTrace();
slottingStrategy.errorAllConnections(e);
}
@Override
public void onNext(List> listOfGroups) {
for (Group group : listOfGroups) {
slottingStrategy.writeOnSlot(group.getKeyBytes(), group);
}
}
});
final MutableReference subscriptionRef = new MutableReference<>();
final AtomicInteger connectionCount = new AtomicInteger(0);
final AtomicBoolean isSubscribed = new AtomicBoolean();
slottingStrategy.registerDoOnEachConnectionAdded(new Action0() {
@Override
public void call() {
Integer minNeeded = currentMinConnectionsToSubscribe.get();
Integer current = connectionCount.incrementAndGet();
if (current >= minNeeded) {
if (isSubscribed.compareAndSet(false, true)) {
logger.info("MinConnectionsToSubscribe: " + minNeeded + ", has been met, subscribing to observable, current connection count: " + current);
subscriptionRef.setValue(withSideEffects.subscribe());
}
} else {
logger.info("MinConnectionsToSubscribe: " + minNeeded + ", has NOT been met, current connection count: " + current);
}
}
});
slottingStrategy.registerDoAfterLastConnectionRemoved(new Action0() {
@Override
public void call() {
subscriptionRef.getValue().unsubscribe();
logger.info("All connections deregistered, unsubscribed to observable, resetting current connection count: 0");
connectionCount.set(0);
isSubscribed.set(false);
}
});
}
public Encoder getKeyEncoder() {
return keyEncoder;
}
public Encoder getValueEncoder() {
return valueEncoder;
}
public static class Builder {
private String name;
private Observable>> observable;
private SlottingStrategy> slottingStrategy =
new ConsistentHashing<>(name, HashFunctions.ketama());
private Encoder keyEncoder;
private Encoder valueEncoder;
private Func1, Func1> filterFunction = ServerSideFilters.noFiltering();
private int maxWriteAttempts = 3;
private long expiryTimeInSecs = Long.MAX_VALUE;
private Observable minConnectionsToSubscribe = Observable.just(1);
public Builder name(String name) {
if (name != null && name.length() > 127) {
throw new IllegalArgumentException("Observable name must be less than 127 characters");
}
this.name = name;
return this;
}
public Builder observable(Observable>> observable) {
this.observable = observable;
return this;
}
public Builder maxWriteAttempts(int maxWriteAttempts) {
this.maxWriteAttempts = maxWriteAttempts;
return this;
}
public Builder withExpirySecs(long expiryInSecs) {
this.expiryTimeInSecs = expiryInSecs;
return this;
}
public Builder minConnectionsToSubscribe(Observable minConnectionsToSubscribe) {
this.minConnectionsToSubscribe = minConnectionsToSubscribe;
return this;
}
public Builder slottingStrategy(SlottingStrategy> slottingStrategy) {
this.slottingStrategy = slottingStrategy;
return this;
}
public Builder keyEncoder(Encoder keyEncoder) {
this.keyEncoder = keyEncoder;
return this;
}
public Builder valueEncoder(Encoder valueEncoder) {
this.valueEncoder = valueEncoder;
return this;
}
public Builder serverSideFilter(
Func1, Func1> filterFunc) {
this.filterFunction = filterFunc;
return this;
}
public ServeGroupedObservable build() {
return new ServeGroupedObservable(this);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy