org.jgroups.protocols.PerDestinationBundler Maven / Gradle / Ivy
Go to download
This artifact provides a single jar that contains all classes required to use remote EJB and JMS, including
all dependencies. It is intended for use by those not using maven, maven users should just import the EJB and
JMS BOM's instead (shaded JAR's cause lots of problems with maven, as it is very easy to inadvertently end up
with different versions on classes on the class path).
package org.jgroups.protocols;
import org.jgroups.Address;
import org.jgroups.Message;
import org.jgroups.NullAddress;
import org.jgroups.View;
import org.jgroups.annotations.ManagedAttribute;
import org.jgroups.annotations.Property;
import org.jgroups.conf.AttributeType;
import org.jgroups.logging.Log;
import org.jgroups.util.ByteArrayDataOutputStream;
import org.jgroups.util.FastArray;
import org.jgroups.util.Profiler;
import org.jgroups.util.Util;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Stream;
import static org.jgroups.protocols.TP.MSG_OVERHEAD;
/**
* Queues messages per destination ('null' is a special destination), sending when the last sender thread to the same
* destination returns or max_size has been reached. This uses 1 thread per destination, so it won't scale to many
* cluster members (unless virtual threads are used).
*
* See https://issues.redhat.com/browse/JGRP-2639 for details.
* @author Bela Ban
* @since 5.2.7
*/
public class PerDestinationBundler implements Bundler {
/**
* Maximum number of bytes for messages to be queued until they are sent.
* This value needs to be smaller than the largest datagram packet size in case of UDP
*/
@Property(name="max_size", type= AttributeType.BYTES,
description="Maximum number of bytes for messages to be queued (per destination) until they are sent")
protected int max_size=64000;
@ManagedAttribute(description="Total number of messages sent (single and batches)",type=AttributeType.SCALAR)
protected final LongAdder total_msgs_sent=new LongAdder();
@ManagedAttribute(description="Number of single messages sent",type=AttributeType.SCALAR)
protected final LongAdder num_single_msgs_sent=new LongAdder();
@ManagedAttribute(description="Number of batches sent",type=AttributeType.SCALAR)
protected final LongAdder num_batches_sent=new LongAdder();
@ManagedAttribute(description="Number of batches sent because no more messages were available",type=AttributeType.SCALAR)
protected final LongAdder num_send_due_to_no_msgs=new LongAdder();
@ManagedAttribute(description="Number of batches sent because the queue was full",type=AttributeType.SCALAR)
protected final LongAdder num_sends_due_to_max_size=new LongAdder();
protected TP transport;
protected Log log;
protected Address local_addr;
protected final Map dests=Util.createConcurrentMap();
protected static final Address NULL=new NullAddress();
public int size() {
return dests.values().stream().map(SendBuffer::size).reduce(0, Integer::sum);
}
public int getQueueSize() {return -1;}
public int getMaxSize() {return max_size;}
public Bundler setMaxSize(int s) {this.max_size=s; return this;}
@ManagedAttribute(description="Average number of messages in an BatchMessage")
public double avgBatchSize() {
long num_batches=num_batches_sent.sum(), total_msgs=total_msgs_sent.sum(), single_msgs=num_single_msgs_sent.sum();
if(num_batches == 0 || total_msgs == 0) return 0.0;
long batched_msgs=total_msgs - single_msgs;
return batched_msgs / (double)num_batches;
}
@Override public void resetStats() {
Stream.of(total_msgs_sent, num_batches_sent, num_single_msgs_sent, num_sends_due_to_max_size)
.forEach(LongAdder::reset);
p_send_msg_list.reset();
p_send_single_msg.reset();
p_send_msgs.reset();
}
public void init(TP transport) {
this.transport=Objects.requireNonNull(transport);
this.log=transport.getLog();
}
public void start() {
local_addr=Objects.requireNonNull(transport.getAddress());
dests.values().forEach(SendBuffer::start);
}
public void stop() {
dests.values().forEach(SendBuffer::stop);
}
public void send(Message msg) throws Exception {
if(msg.getSrc() == null)
msg.setSrc(local_addr);
Address dest=msg.dest() == null ? NULL : msg.dest();
SendBuffer buf=dests.computeIfAbsent(dest, k -> new SendBuffer(dest).start());
buf.send(msg);
}
public void viewChange(View view) {
List mbrs=view.getMembers();
if(mbrs == null) return;
mbrs.stream().filter(dest -> !dests.containsKey(dest))
.forEach(dest -> dests.putIfAbsent(dest, new SendBuffer(dest).start()));
// remove left members
dests.keySet().stream().filter(dest -> !mbrs.contains(dest) && !(dest == NULL))
.forEach(dests::remove);
}
@ManagedAttribute final Profiler p_send_msg_list=new Profiler(),
p_send_single_msg=new Profiler(),
p_send_msgs=new Profiler();
protected class SendBuffer implements Runnable {
private final Address dest;
protected final FastArray msgs=new FastArray(16).increment(10);
private final Lock lock=new ReentrantLock(false);
private final BlockingQueue queue=new ArrayBlockingQueue<>(8192);
private final List remove_queue=new ArrayList<>(1024);
private final ByteArrayDataOutputStream output=new ByteArrayDataOutputStream(max_size + MSG_OVERHEAD);
private volatile Thread bundler_thread;
private volatile boolean running=true;
private long count;
protected SendBuffer(Address dest) {
this.dest=dest;
}
public void run() {
while(running) {
Message msg=null;
try {
if((msg=queue.take()) == null)
continue;
addAndSendIfSizeExceeded(msg);
while(true) {
remove_queue.clear();
int num_msgs=queue.drainTo(remove_queue);
if(num_msgs <= 0)
break;
for(int i=0; i < remove_queue.size(); i++) {
msg=remove_queue.get(i);
addAndSendIfSizeExceeded(msg);
}
}
if(count > 0) {
sendBundledMessages();
num_send_due_to_no_msgs.increment();
}
}
catch(Throwable t) {
}
}
}
protected void addAndSendIfSizeExceeded(Message msg) {
int size=msg.size();
if(count + size >= max_size) {
sendBundledMessages();
num_sends_due_to_max_size.increment();
}
addMessage(msg, size);
}
protected void addMessage(Message msg, int size) {
msgs.add(msg);
count+=size;
}
protected void sendBundledMessages() {
sendBatch(dest, msgs);
msgs.clear(false);
count=0;
}
public SendBuffer start() {
if(running)
stop();
bundler_thread=transport.getThreadFactory().newThread(this, "pd-bundler"); // new Thread(this, "pd-bundler");
running=true;
bundler_thread.start();
return this;
}
public void stop() {
running=false;
Thread tmp=bundler_thread;
if(tmp != null)
tmp.interrupt();
}
protected void send(Message msg) throws Exception {
queue.put(msg);
}
protected void sendBatch(Address destination, FastArray list) {
if(list.isEmpty()) // should never happen!
return;
Address dst=destination == NULL? null : destination;
sendMessages(dst, local_addr, list);
}
protected void sendSingleMessage(final Address dest, final Message msg) {
p_send_single_msg.start();
try {
output.position(0);
Util.writeMessage(msg, output, dest == null);
transport.doSend(output.buffer(), 0, output.position(), dest);
if(transport.statsEnabled())
transport.getMessageStats().incrNumSingleMsgsSent(1);
num_single_msgs_sent.increment();
}
catch(Throwable e) {
log.error("%s: failed sending message to %s: %s", local_addr, dest, e);
}
finally {
p_send_single_msg.stop();
}
}
protected void sendMessageList(final Address dest, final Address src, final FastArray list) {
p_send_msg_list.start();
output.position(0);
try {
Util.writeMessageList(dest, src, transport.cluster_name.chars(), list,
output, dest == null, transport.getId());
transport.doSend(output.buffer(), 0, output.position(), dest);
if(transport.statsEnabled())
transport.getMessageStats().incrNumBatchesSent(1);
num_batches_sent.increment();
}
catch(Throwable e) {
log.trace(Util.getMessage("FailureSendingMsgBundle"), transport.getAddress(), e);
}
finally {
p_send_msg_list.stop();
}
}
protected void sendMessages(final Address dest, final Address src, final FastArray list) {
p_send_msgs.start();
try {
int size=list.size();
if(size == 0)
return;
if(size == 1)
sendSingleMessage(dest, list.get(0));
else
sendMessageList(dest, src, list);
total_msgs_sent.add(size);
}
catch(Throwable e) {
log.trace(Util.getMessage("FailureSendingMsgBundle"), transport.getAddress(), e);
}
finally {
p_send_msgs.stop();
}
}
public String toString() {
return String.format("%d msgs", size());
}
protected int size() {
lock.lock();
try {
return msgs.size();
}
finally {
lock.unlock();
}
}
}
}