io.nextop.client.node.http.HttpNode Maven / Gradle / Ivy
package io.nextop.client.node.http;
import io.nextop.*;
import io.nextop.client.MessageControl;
import io.nextop.client.MessageControlState;
import io.nextop.client.node.AbstractMessageControlNode;
import io.nextop.client.retry.SendStrategy;
import io.nextop.org.apache.http.*;
import io.nextop.org.apache.http.client.HttpRequestRetryHandler;
import io.nextop.org.apache.http.client.config.RequestConfig;
import io.nextop.org.apache.http.client.methods.*;
import io.nextop.org.apache.http.client.protocol.HttpClientContext;
import io.nextop.org.apache.http.client.protocol.RequestClientConnControl;
import io.nextop.org.apache.http.client.utils.URIUtils;
import io.nextop.org.apache.http.config.ConnectionConfig;
import io.nextop.org.apache.http.config.MessageConstraints;
import io.nextop.org.apache.http.conn.*;
import io.nextop.org.apache.http.conn.HttpConnectionFactory;
import io.nextop.org.apache.http.conn.routing.HttpRoute;
import io.nextop.org.apache.http.entity.ContentLengthStrategy;
import io.nextop.org.apache.http.impl.DefaultConnectionReuseStrategy;
import io.nextop.org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import io.nextop.org.apache.http.impl.conn.ConnectionShutdownException;
import io.nextop.org.apache.http.impl.conn.DefaultHttpResponseParserFactory;
import io.nextop.org.apache.http.impl.conn.DefaultManagedHttpClientConnection;
import io.nextop.org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import io.nextop.org.apache.http.impl.entity.LaxContentLengthStrategy;
import io.nextop.org.apache.http.impl.entity.StrictContentLengthStrategy;
import io.nextop.org.apache.http.impl.execchain.*;
import io.nextop.org.apache.http.impl.io.DefaultHttpRequestWriterFactory;
import io.nextop.org.apache.http.io.HttpMessageParserFactory;
import io.nextop.org.apache.http.io.HttpMessageWriterFactory;
import io.nextop.org.apache.http.io.SessionInputBuffer;
import io.nextop.org.apache.http.io.SessionOutputBuffer;
import io.nextop.org.apache.http.protocol.*;
import rx.functions.Func1;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CodingErrorAction;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public final class HttpNode extends AbstractMessageControlNode {
public static final class Config {
public final String userAgent;
public final int maxConcurrentConnections;
public Config(String userAgent, int maxConcurrentConnections) {
this.userAgent = userAgent;
this.maxConcurrentConnections = maxConcurrentConnections;
}
}
public static final Config DEFAULT_CONFIG = new Config(
/** FIXME set a user agent with the version, e.g. Nextop/0.1.4 */ "Nextop",
2
);
/** retry once immediately, then defer to the retake strategy.
* this is a heuristic in case the request got redirected to a bad box. */
public static final SendStrategy DEFAULT_SEND_STRATEGY = new SendStrategy.Builder().init(0, TimeUnit.MILLISECONDS
).repeat(1
).build();
public static final SendStrategy DEFAULT_RETAKE_STRATEGY = new SendStrategy.Builder()
.withUniformRandom(2000, TimeUnit.MILLISECONDS)
.repeatIndefinitely()
.build();
/** guaranteed to repeat indefinitely, in case a custom retake strategy expires */
static final SendStrategy FALLBACK_RETAKE_STRATEGY = DEFAULT_RETAKE_STRATEGY;
/** yield at this many of bytes to emit progress,
* transfer request, etc. */
static final int DEFAULT_YIELD_Q_BYTES = 4 * 1024;
final Config config;
final PoolingHttpClientConnectionManager clientConnectionManager =
new PoolingHttpClientConnectionManager(new NextopHttpClientConnectionFactory());
// volatile for reads
volatile boolean active = false;
@Nullable
List looperThreads = null;
// configuration
/* retry in the http node is controlled by two strategies: send and retake.
* send = [send attempt, [send delay, [send attempt, ...]?]?]
* retake (for yieldable messages) = [[send], [move to back, other sends, delay?, [send], [move to back, other sends, delay?, [send], ...]?]?]
* retake (for un-yieldable messages) = [[send], [delay?, [send], [delay?, [send], ...]?]?]
*
* the delay in the send sequence is controlled by #sendStrategy
* the delay in the retake sequences is controlled by #retakeStrategy
*/
// this is the strategy for one entry before possibly yielding
// each time the entry is taken, the strategy is run from the beginning
// this is a really aggressive strategy that relies on CONNECTIVITY STATUS
// to stop retrying on a bad connection
volatile SendStrategy sendStrategy = DEFAULT_SEND_STRATEGY;
// if an entry yields or is otherwise taken after a failed take (either a protocol error or sendStrategy expired)
// this strategy runs. this strategy should repeat indefinitely, since an entry can circulate indefinitely
volatile SendStrategy retakeStrategy = DEFAULT_RETAKE_STRATEGY;
@Nullable
volatile Wire.Adapter wireAdapter = null;
public HttpNode() {
this(DEFAULT_CONFIG);
}
public HttpNode(Config config) {
this.config = config;
}
/////// CONFIG ///////
public void setSendStrategy(SendStrategy sendStrategy) {
this.sendStrategy = sendStrategy;
// loopers pick this up eventually
}
public void setWireAdapter(Wire.Adapter wireAdapter) {
this.wireAdapter = wireAdapter;
// loopers pick this up eventually
}
/////// NODE ///////
@Override
protected void initSelf(@Nullable Bundle savedState) {
// ready to receive
upstream.onActive(true);
}
@Override
public void onActive(boolean active) {
if (this.active != active) {
this.active = active;
if (active) {
assert null == looperThreads;
MessageControlState mcs = getMessageControlState();
SharedLooperState sls = new SharedLooperState();
// note that the message control state coordinates between multiple loopers
// (and between multiple nodes)
int n = config.maxConcurrentConnections;
Thread[] threads = new Thread[n];
for (int i = 0; i < n; ++i) {
threads[i] = new RequestLooper(mcs, sls);
}
looperThreads = Arrays.asList(threads);
for (int i = 0; i < n; ++i) {
threads[i].start();
}
} else {
assert null != looperThreads;
for (Thread t : looperThreads) {
t.interrupt();
}
looperThreads = null;
}
}
}
@Override
public void onMessageControl(MessageControl mc) {
assert MessageControl.Direction.SEND.equals(mc.dir);
assert active;
if (active) {
MessageControlState mcs = getMessageControlState();
if (!mcs.onActiveMessageControl(mc, upstream)) {
mcs.add(mc);
}
}
// TODO else send back upstream?
}
private static final class SharedLooperState {
final Map mostRecentSends = new ConcurrentHashMap(8);
/** this is here to handle a one bad case: an entry is retaken that can yield,
* but it has been less than this number of ms since the last yield.
* To avoid spinning the CPU yielding the same id(s), wait this number of ms before the next eval. */
final int retakeYieldQMs = 50;
SharedLooperState() {
}
static final class MostRecentSend {
final long nanos;
final SendStrategy activeStrategy;
MostRecentSend(long nanos, SendStrategy activeStrategy) {
this.nanos = nanos;
this.activeStrategy = activeStrategy;
}
}
}
static final Func1 IS_SENDABLE = new Func1() {
@Override
public Boolean call(MessageControlState.Entry entry) {
// HTTP can't send to the Nextop local route
return !Message.isLocal(entry.message.route);
}
};
final class RequestLooper extends Thread {
final MessageControlState mcs;
final SharedLooperState sls;
// set at the beginning of a request; reset at the end of request
@Nullable
ProgressCallback progressCallback = null;
/** local cache of HttpNode#wireAdapter */
@Nullable
Wire.Adapter wireAdapter = null;
RequestLooper(MessageControlState mcs, SharedLooperState sls) {
this.mcs = mcs;
this.sls = sls;
}
@Override
public void run() {
top:
while (active) {
@Nullable MessageControlState.Entry entry;
try {
entry = mcs.takeFirstAvailable(IS_SENDABLE, HttpNode.this,
Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
continue top;
}
if (null != entry) {
@Nullable SharedLooperState.MostRecentSend mostRecentSend = sls.mostRecentSends.get(entry.id);
if (null != mostRecentSend) {
int delayMs = (int) mostRecentSend.activeStrategy.getDelay(TimeUnit.MILLISECONDS);
int elapsedMs = (int) TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - mostRecentSend.nanos);
if (elapsedMs < delayMs) {
// - if entry is not yieldable, wait delayMs - elapsedMs
// - else (yieldable)
// - if elapsesMs < retakeYieldQMs
// - if delayMs - elapsesMs < retakeYieldQMs, wait delayMs - elapsesMs then exec
// - else wait retakeYieldQMs then yield
// - else yield
int remainingMs = delayMs - elapsedMs;
if (!Message.isYieldable(entry.message)) {
try {
Thread.sleep(remainingMs);
} catch (InterruptedException e) {
mcs.release(entry.id, HttpNode.this);
continue;
}
} else {
// message is yieldable
if (elapsedMs < sls.retakeYieldQMs) {
if (remainingMs < sls.retakeYieldQMs) {
try {
Thread.sleep(remainingMs);
} catch (InterruptedException e) {
mcs.release(entry.id, HttpNode.this);
continue;
}
} else {
try {
Thread.sleep(sls.retakeYieldQMs);
} catch (InterruptedException e) {
mcs.release(entry.id, HttpNode.this);
continue;
}
mcs.yield(entry.id);
mcs.release(entry.id, HttpNode.this);
continue;
}
} else {
mcs.yield(entry.id);
mcs.release(entry.id, HttpNode.this);
continue;
}
}
}
}
this.wireAdapter = HttpNode.this.wireAdapter;
assert null == entry.end;
try {
end(entry, execute(entry));
} catch (IOException e) {
retake(entry);
} catch (HttpException e) {
retake(entry);
} catch (Throwable t) {
// an internal issue
// can never recover from this (assume the system is deterministic)
end(entry, MessageControlState.End.ERROR);
}
}
}
}
private void retake(MessageControlState.Entry entry) {
assert null == entry.end;
SendStrategy nextStrategy;
@Nullable SharedLooperState.MostRecentSend mostRecentSend = sls.mostRecentSends.get(entry.id);
if (null != mostRecentSend) {
nextStrategy = mostRecentSend.activeStrategy.retry();
} else {
nextStrategy = retakeStrategy.retry();
}
if (!nextStrategy.isSend()) {
// this case indicates a bug in a custom retake strategy, where the strategy does not repeat indefinitely
nextStrategy = FALLBACK_RETAKE_STRATEGY.retry();
}
assert nextStrategy.isSend();
sls.mostRecentSends.put(entry.id, new SharedLooperState.MostRecentSend(System.nanoTime(), nextStrategy));
// at this point the entry was elected to yield
// in this case, check whether the message has indicated it can be moved to the end of the line
if (Message.isYieldable(entry.message)) {
mcs.yield(entry.id);
}
mcs.release(entry.id, HttpNode.this);
}
private void end(final MessageControlState.Entry entry, MessageControlState.End end) {
assert null == entry.end;
sls.mostRecentSends.remove(entry.id);
mcs.remove(entry.id, end);
final Route route = entry.message.inboxRoute();
switch (end) {
case COMPLETED:
post(new Runnable() {
@Override
public void run() {
upstream.onMessageControl(MessageControl.receive(MessageControl.Type.COMPLETE, route));
}
});
break;
case ERROR:
post(new Runnable() {
@Override
public void run() {
upstream.onMessageControl(MessageControl.receive(MessageControl.Type.ERROR, route));
}
});
break;
default:
throw new IllegalStateException();
}
}
private MessageControlState.End execute(final MessageControlState.Entry entry) throws IOException, HttpException {
final HttpRequest request;
try {
request = Message.toHttpRequest(entry.message);
} catch (URISyntaxException e) {
// can never send this
return MessageControlState.End.ERROR;
}
final HttpHost target;
try {
target = Message.toHttpHost(entry.message);
} catch (URISyntaxException e) {
// can never send this
return MessageControlState.End.ERROR;
}
final Message responseMessage;
progressCallback = new ProgressAdapter(entry);
try {
HttpResponse response = doExecute(createExecChain(entry),
target, request, null);
responseMessage = Message.fromHttpResponse(response).setRoute(entry.message.inboxRoute()).build();
} finally {
progressCallback = null;
}
post(new Runnable() {
@Override
public void run() {
upstream.onMessageControl(MessageControl.receive(responseMessage));
}
});
return MessageControlState.End.COMPLETED;
}
/** lifted version of {@link io.nextop.org.apache.http.impl.client.CloseableHttpClient#doExecute} */
private CloseableHttpResponse doExecute(
ClientExecChain execChain,
HttpHost target,
HttpRequest request,
@Nullable HttpContext context) throws IOException, HttpException {
HttpExecutionAware execAware = null;
if (request instanceof HttpExecutionAware) {
execAware = (HttpExecutionAware) request;
}
final HttpRequestWrapper wrapper = HttpRequestWrapper.wrap(request);
final HttpClientContext localcontext = HttpClientContext.adapt(
null != context ? context : new BasicHttpContext());
final HttpRoute route = new HttpRoute(target);
RequestConfig config = null;
if (request instanceof Configurable) {
config = ((Configurable) request).getConfig();
}
if (config != null) {
localcontext.setRequestConfig(config);
}
return execChain.execute(route, wrapper, localcontext, execAware);
}
private ClientExecChain createExecChain(MessageControlState.Entry entry) {
NextopClientExec nextopExec = new NextopClientExec(
new NextopHttpRequestExecutor(progressCallback),
clientConnectionManager,
DefaultConnectionReuseStrategy.INSTANCE,
DefaultConnectionKeepAliveStrategy.INSTANCE,
config.userAgent
);
return new RetryExec(nextopExec, new NextopHttpRequestRetryHandler(sendStrategy, entry, mcs));
}
}
final class ProgressAdapter implements ProgressCallback {
final MessageControlState.Entry entry;
MessageControlState mcs = getMessageControlState();
ProgressAdapter(MessageControlState.Entry entry) {
this.entry = entry;
}
@Override
public void onSendStarted(int tryCount) {
post(new Runnable() {
@Override
public void run() {
mcs.setOutboxTransferProgress(entry.id,
MessageControlState.TransferProgress.none(entry.id));
}
});
}
@Override
public void onSendProgress(final long sentBytes, final long sendTotalBytes) {
post(new Runnable() {
@Override
public void run() {
mcs.setOutboxTransferProgress(entry.id,
MessageControlState.TransferProgress.create(entry.id, sentBytes, sendTotalBytes));
}
});
}
@Override
public void onSendCompleted(final long sentBytes, final long sendTotalBytes) {
post(new Runnable() {
@Override
public void run() {
mcs.setOutboxTransferProgress(entry.id,
MessageControlState.TransferProgress.create(entry.id, sentBytes, sendTotalBytes));
}
});
}
@Override
public void onReceiveStarted(int tryCount) {
post(new Runnable() {
@Override
public void run() {
mcs.setInboxTransferProgress(entry.id,
MessageControlState.TransferProgress.none(entry.id));
}
});
}
@Override
public void onReceiveProgress(final long receivedBytes, final long receiveTotalBytes) {
post(new Runnable() {
@Override
public void run() {
mcs.setInboxTransferProgress(entry.id,
MessageControlState.TransferProgress.create(entry.id, receivedBytes, receiveTotalBytes));
}
});
}
@Override
public void onReceiveCompleted(final long receivedBytes, final long receiveTotalBytes) {
post(new Runnable() {
@Override
public void run() {
mcs.setInboxTransferProgress(entry.id,
MessageControlState.TransferProgress.create(entry.id, receivedBytes, receiveTotalBytes));
}
});
}
}
/** can be called from any thread. Expect the IO thread to call. */
static interface ProgressCallback {
void onSendStarted(int tryCount);
void onSendProgress(long sentBytes, long sendTotalBytes);
void onSendCompleted(long sentBytes, long sendTotalBytes);
void onReceiveStarted(int tryCount);
void onReceiveProgress(long receivedBytes, long receiveTotalBytes);
void onReceiveCompleted(long receivedBytes, long receiveTotalBytes);
}
// PoolingHttpClientConnectionManager
// -- uses ManagedHttpClientConnectionFactory
// -- uses LoggingManagedHttpClientConnection #getOutputStream(socket) #getInputStream(Socket)
// -- implements ManagedHttpClientConnection #bind(Socket)
// -- extends DefaultManagedHttpClientConnection WANT TO USE THIS
// DefaultConnectionReuseStrategy
// FIXME wrap in a retry exec with NextopHttpRequestRetryHandler
// FIXME use the message property idempotent to influence retry also
// DefaultHttpRequestRetryHandler
// FIXME create Nextop exec chain per request
// FIXME
// NextopRetryExec:
// check that request is still the head before retry (this is sort of the solution to head of line blocking)
static final class NextopHttpRequestRetryHandler implements HttpRequestRetryHandler {
private SendStrategy sendStrategy;
private final MessageControlState.Entry entry;
private final MessageControlState mcs;
NextopHttpRequestRetryHandler(SendStrategy sendStrategy,
MessageControlState.Entry entry, MessageControlState mcs) {
this.sendStrategy = sendStrategy;
this.entry = entry;
this.mcs = mcs;
}
@Override
public boolean retryRequest(final IOException exception,
final int executionCount,
final HttpContext context) {
sendStrategy = sendStrategy.retry();
if (!sendStrategy.isSend()) {
return false;
}
// check ended - don't retry an ended entry
if (null != entry.end) {
return false;
}
// retry if not fully sent (server starts processing on fully received message)
// or if the request is idempotent (either because it is nullipotent or marked as idempotent)
if (HttpClientContext.adapt(context).isRequestSent() && Message.isIdempotent(entry.message)) {
return false;
}
// fail if there is a higher priority request
int timeoutMs = (int) sendStrategy.getDelay(TimeUnit.MILLISECONDS);
try {
return !mcs.hasFirstAvailable(entry.id, timeoutMs, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
return false;
}
}
}
// implement a subclass of HttpRequestExector that surfaces SendIOException(final chunk), ReceiveIOException
// implement a custom RetryHandler that always retries if send failed on not final chunk,
/** based on org.apache.http.impl.execchain.MinimalClientExec */
static final class NextopClientExec implements ClientExecChain {
// ProgressCallback progressCallback;
private final HttpRequestExecutor requestExecutor;
private final HttpClientConnectionManager connManager;
private final ConnectionReuseStrategy reuseStrategy;
private final ConnectionKeepAliveStrategy keepAliveStrategy;
private final HttpProcessor httpProcessor;
public NextopClientExec(
final HttpRequestExecutor requestExecutor,
final HttpClientConnectionManager connManager,
final ConnectionReuseStrategy reuseStrategy,
final ConnectionKeepAliveStrategy keepAliveStrategy,
String userAgent) {
this.httpProcessor = new ImmutableHttpProcessor(
new RequestContent(),
new RequestTargetHost(),
new RequestClientConnControl(),
new RequestUserAgent(userAgent));
this.requestExecutor = requestExecutor;
this.connManager = connManager;
this.reuseStrategy = reuseStrategy;
this.keepAliveStrategy = keepAliveStrategy;
// this.progressCallback = progressCallback;
}
static void rewriteRequestURI(
final HttpRequestWrapper request,
final HttpRoute route) throws ProtocolException {
try {
URI uri = request.getURI();
if (uri != null) {
// Make sure the request URI is relative
if (uri.isAbsolute()) {
uri = URIUtils.rewriteURI(uri, null, true);
} else {
uri = URIUtils.rewriteURI(uri);
}
request.setURI(uri);
}
} catch (final URISyntaxException ex) {
throw new ProtocolException("Invalid URI: " + request.getRequestLine().getUri(), ex);
}
}
@Override
public CloseableHttpResponse execute(
final HttpRoute route,
final HttpRequestWrapper request,
final HttpClientContext context,
final HttpExecutionAware execAware) throws IOException, HttpException {
rewriteRequestURI(request, route);
final ConnectionRequest connRequest = connManager.requestConnection(route, null);
if (execAware != null) {
if (execAware.isAborted()) {
connRequest.cancel();
throw new RequestAbortedException("Request aborted");
} else {
execAware.setCancellable(connRequest);
}
}
final RequestConfig config = context.getRequestConfig();
final HttpClientConnection managedConn;
try {
final int timeout = config.getConnectionRequestTimeout();
managedConn = connRequest.get(timeout > 0 ? timeout : 0, TimeUnit.MILLISECONDS);
} catch(final InterruptedException interrupted) {
Thread.currentThread().interrupt();
throw new RequestAbortedException("Request aborted", interrupted);
} catch(final ExecutionException ex) {
Throwable cause = ex.getCause();
if (cause == null) {
cause = ex;
}
throw new RequestAbortedException("Request execution failed", cause);
}
final NextopConnectionHolder releaseTrigger = new NextopConnectionHolder(connManager, managedConn);
try {
if (execAware != null) {
if (execAware.isAborted()) {
releaseTrigger.close();
throw new RequestAbortedException("Request aborted");
} else {
execAware.setCancellable(releaseTrigger);
}
}
if (!managedConn.isOpen()) {
final int timeout = config.getConnectTimeout();
this.connManager.connect(
managedConn,
route,
timeout > 0 ? timeout : 0,
context);
this.connManager.routeComplete(managedConn, route, context);
}
final int timeout = config.getSocketTimeout();
if (timeout >= 0) {
managedConn.setSocketTimeout(timeout);
}
HttpHost target = null;
final HttpRequest original = request.getOriginal();
if (original instanceof HttpUriRequest) {
final URI uri = ((HttpUriRequest) original).getURI();
if (uri.isAbsolute()) {
target = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());
}
}
if (target == null) {
target = route.getTargetHost();
}
context.setAttribute(HttpCoreContext.HTTP_TARGET_HOST, target);
context.setAttribute(HttpCoreContext.HTTP_REQUEST, request);
context.setAttribute(HttpCoreContext.HTTP_CONNECTION, managedConn);
context.setAttribute(HttpClientContext.HTTP_ROUTE, route);
// TODO managedConn is an instance of CPoolProxy
// TODO an easy way to call getConnection or get the connection out of it without reflection?
httpProcessor.process(request, context);
final HttpResponse response = requestExecutor.execute(request, managedConn, context);
httpProcessor.process(response, context);
// The connection is in or can be brought to a re-usable state.
if (reuseStrategy.keepAlive(response, context)) {
// Set the idle duration of this connection
final long duration = keepAliveStrategy.getKeepAliveDuration(response, context);
releaseTrigger.setValidFor(duration, TimeUnit.MILLISECONDS);
releaseTrigger.markReusable();
} else {
releaseTrigger.markNonReusable();
}
// check for entity, release connection if possible
final HttpEntity entity = response.getEntity();
if (entity == null || !entity.isStreaming()) {
// connection not needed and (assumed to be) in re-usable state
releaseTrigger.releaseConnection();
return new NextopHttpResponseProxy(response, null);
} else {
return new NextopHttpResponseProxy(response, releaseTrigger);
}
} catch (final ConnectionShutdownException ex) {
final InterruptedIOException ioex = new InterruptedIOException(
"Connection has been shut down");
ioex.initCause(ex);
throw ioex;
} catch (final HttpException ex) {
releaseTrigger.abortConnection();
throw ex;
} catch (final IOException ex) {
releaseTrigger.abortConnection();
throw ex;
} catch (final RuntimeException ex) {
releaseTrigger.abortConnection();
throw ex;
}
}
}
static final class NextopHttpClientConnectionFactory
implements HttpConnectionFactory {
private final HttpMessageWriterFactory requestWriterFactory;
private final HttpMessageParserFactory responseParserFactory;
private final ContentLengthStrategy incomingContentStrategy;
private final ContentLengthStrategy outgoingContentStrategy;
private final AtomicInteger connectionCounter = new AtomicInteger(0);
public NextopHttpClientConnectionFactory(
@Nullable HttpMessageWriterFactory requestWriterFactory,
@Nullable HttpMessageParserFactory responseParserFactory,
@Nullable ContentLengthStrategy incomingContentStrategy,
@Nullable ContentLengthStrategy outgoingContentStrategy) {
super();
this.requestWriterFactory = requestWriterFactory != null ? requestWriterFactory :
DefaultHttpRequestWriterFactory.INSTANCE;
this.responseParserFactory = responseParserFactory != null ? responseParserFactory :
DefaultHttpResponseParserFactory.INSTANCE;
this.incomingContentStrategy = incomingContentStrategy != null ? incomingContentStrategy :
LaxContentLengthStrategy.INSTANCE;
this.outgoingContentStrategy = outgoingContentStrategy != null ? outgoingContentStrategy :
StrictContentLengthStrategy.INSTANCE;
}
public NextopHttpClientConnectionFactory(
@Nullable HttpMessageWriterFactory requestWriterFactory,
@Nullable HttpMessageParserFactory responseParserFactory) {
this(requestWriterFactory, responseParserFactory, null, null);
}
public NextopHttpClientConnectionFactory(
@Nullable HttpMessageParserFactory responseParserFactory) {
this(null, responseParserFactory);
}
public NextopHttpClientConnectionFactory() {
this(null, null);
}
@Override
public NextopHttpClientConnection create(final HttpRoute route, final ConnectionConfig config) {
final ConnectionConfig cconfig = config != null ? config : ConnectionConfig.DEFAULT;
CharsetDecoder chardecoder = null;
CharsetEncoder charencoder = null;
final Charset charset = cconfig.getCharset();
final CodingErrorAction malformedInputAction = cconfig.getMalformedInputAction() != null ?
cconfig.getMalformedInputAction() : CodingErrorAction.REPORT;
final CodingErrorAction unmappableInputAction = cconfig.getUnmappableInputAction() != null ?
cconfig.getUnmappableInputAction() : CodingErrorAction.REPORT;
if (charset != null) {
chardecoder = charset.newDecoder();
chardecoder.onMalformedInput(malformedInputAction);
chardecoder.onUnmappableCharacter(unmappableInputAction);
charencoder = charset.newEncoder();
charencoder.onMalformedInput(malformedInputAction);
charencoder.onUnmappableCharacter(unmappableInputAction);
}
final String id = String.format("nextop-http-%d", connectionCounter.getAndIncrement());
return new NextopHttpClientConnection(
id,
cconfig.getBufferSize(),
cconfig.getFragmentSizeHint(),
chardecoder,
charencoder,
cconfig.getMessageConstraints(),
incomingContentStrategy,
outgoingContentStrategy,
requestWriterFactory,
responseParserFactory);
}
}
// be able to reset progress
// be able to attach callback that gets called after A bytes of upload, B bytes of download indiviudally
static final class NextopHttpClientConnection extends DefaultManagedHttpClientConnection {
final int yieldQBytes = DEFAULT_YIELD_Q_BYTES;
private boolean wireSet = false;
@Nullable
private Wire wire = null;
public NextopHttpClientConnection(
final String id,
final int buffersize,
final int fragmentSizeHint,
final CharsetDecoder chardecoder,
final CharsetEncoder charencoder,
final MessageConstraints constraints,
final ContentLengthStrategy incomingContentStrategy,
final ContentLengthStrategy outgoingContentStrategy,
final HttpMessageWriterFactory requestWriterFactory,
final HttpMessageParserFactory responseParserFactory) {
super(id, buffersize, fragmentSizeHint,
chardecoder, charencoder,
constraints,
incomingContentStrategy, outgoingContentStrategy,
requestWriterFactory, responseParserFactory);
}
@Nullable
private ProgressCallback getProgressCallback() {
// TODO passing this via the thread is nasty, but is there a good way for the ExecChain to inject into this
// TODO (through the pool adapter)
RequestLooper t = (RequestLooper) Thread.currentThread();
return t.progressCallback;
}
@Nullable
private Wire.Adapter getAdapter() {
// TODO (see notes in #getProgressCallback)
RequestLooper t = (RequestLooper) Thread.currentThread();
return t.wireAdapter;
}
/** sets {@link #wire} from the socket input/output,
* if there is an adapter (which is most commonly used to condition the wire).
* After this call, {@link #wire} may be null. */
private void setWire(Socket socket) throws IOException {
if (!wireSet) {
wireSet = true;
@Nullable Wire.Adapter adapter = getAdapter();
if (null != adapter) {
InputStream is = super.getSocketInputStream(socket);
OutputStream os = super.getSocketOutputStream(socket);
try {
wire = adapter.adapt(Wires.io(is, os));
} catch (InterruptedException e) {
throw new IOException(e);
}
}
}
}
@Override
protected InputStream getSocketInputStream(Socket socket) throws IOException {
setWire(socket);
if (null != wire) {
return Wires.inputStream(wire);
} else {
return super.getSocketInputStream(socket);
}
}
@Override
protected OutputStream getSocketOutputStream(Socket socket) throws IOException {
setWire(socket);
if (null != wire) {
return Wires.outputStream(wire);
} else {
return super.getSocketOutputStream(socket);
}
}
// FIXME if TCP error on close, throw SendIOException
// FIXME this means all packets sent up to the tcp window size,
// FIXME but failed to ack the end
// FIXME otherwise, up to the end of the entity was not sent, so the server knows it has a hanging request
@Override
protected OutputStream createOutputStream(final long len, SessionOutputBuffer outbuffer) {
@Nullable final ProgressCallback progressCallback = getProgressCallback();
final long sendTotalBytes = 0 < len ? len : 0;
final OutputStream os = super.createOutputStream(len, outbuffer);
return new OutputStream() {
long sentBytes = 0L;
long lastNotificationIndex = -1L;
/** scales the total in the case the actual transfer is exceeding the total (bug in the size calc) */
private long scaledSendTotalBytes(long b) {
long t = sendTotalBytes;
while (0 < t && t <= b) {
long u = 161 * t / 100;
if (t < u) {
t = u;
} else {
t *= 2;
}
}
return t;
}
private void onSendProgress(long bytes) {
sentBytes += bytes;
if (null != progressCallback) {
long notificationIndex = sentBytes / yieldQBytes;
if (lastNotificationIndex != notificationIndex) {
lastNotificationIndex = notificationIndex;
progressCallback.onSendProgress(sentBytes, scaledSendTotalBytes(sentBytes));
}
}
}
private void onSendCompleted() {
if (null != progressCallback) {
progressCallback.onSendCompleted(sentBytes, sentBytes);
}
}
@Override
public void write(int b) throws IOException {
os.write(b);
onSendProgress(1);
}
@Override
public void write(byte[] b) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
for (int i = 0; i < len; i += yieldQBytes) {
int c = Math.min(yieldQBytes, len - i);
os.write(b, off + i, c);
onSendProgress(c);
// FIXME
// try {
// Thread.sleep(200);
// } catch (InterruptedException e) {
// // ignore
// }
}
}
@Override
public void flush() throws IOException {
os.flush();
}
@Override
public void close() throws IOException {
os.close();
onSendCompleted();
}
};
}
@Override
public void sendRequestEntity(HttpEntityEnclosingRequest request) throws HttpException, IOException {
super.sendRequestEntity(request);
}
@Override
protected InputStream createInputStream(final long len, SessionInputBuffer inbuffer) {
@Nullable final ProgressCallback progressCallback = getProgressCallback();
final long receiveTotalBytes = 0 < len ? len : 0;
final InputStream is = super.createInputStream(len, inbuffer);
return new InputStream() {
long receivedBytes = 0L;
long lastNotificationIndex = -1L;
/** scales the total in the case the actual transfer is exceeding the total (bug in the size calc) */
private long scaledReceiveTotalBytes(long b) {
long t = receiveTotalBytes;
while (0 < t && t <= b) {
long u = 161 * t / 100;
if (t < u) {
t = u;
} else {
t *= 2;
}
}
return t;
}
private void onReceiveProgress(long bytes) {
receivedBytes += bytes;
if (null != progressCallback) {
long notificationIndex = receivedBytes / yieldQBytes;
if (lastNotificationIndex != notificationIndex) {
lastNotificationIndex = notificationIndex;
progressCallback.onReceiveProgress(receivedBytes, scaledReceiveTotalBytes(receivedBytes));
}
}
}
private void onReceiveCompleted() {
if (null != progressCallback) {
progressCallback.onReceiveCompleted(receivedBytes, receivedBytes);
}
}
@Override
public int read() throws IOException {
int b = is.read();
onReceiveProgress(1);
return b;
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
for (int i = 0; i < len; i += yieldQBytes) {
int c = Math.min(yieldQBytes, len - i);
int r = is.read(b, off + i, c);
if (0 < r) {
onReceiveProgress(r);
}
if (r < c) {
return i + r;
}
}
return len;
}
@Override
public void close() throws IOException {
super.close();
onReceiveCompleted();
}
@Override
public long skip(long n) throws IOException {
return is.skip(n);
}
@Override
public int available() throws IOException {
return is.available();
}
@Override
public boolean markSupported() {
return is.markSupported();
}
@Override
public void mark(int readlimit) {
is.mark(readlimit);
}
@Override
public void reset() throws IOException {
is.reset();
}
};
}
// OVERRIDE sendRequestEntity
// throw a SendIO
}
// not to be shared. one per exec chain/request
static class NextopHttpRequestExecutor extends HttpRequestExecutor {
ProgressCallback progressCallback;
int sendTryCount = 0;
int receiveTryCount = 0;
NextopHttpRequestExecutor(ProgressCallback progressCallback) {
this.progressCallback = progressCallback;
}
@Override
protected HttpResponse doSendRequest(
final HttpRequest request,
final HttpClientConnection conn,
final HttpContext context) throws IOException, HttpException {
++sendTryCount;
if (null != progressCallback) {
progressCallback.onSendStarted(sendTryCount);
}
return super.doSendRequest(request, conn, context);
}
@Override
protected HttpResponse doReceiveResponse(
final HttpRequest request,
final HttpClientConnection conn,
final HttpContext context) throws HttpException, IOException {
++receiveTryCount;
if (null != progressCallback) {
progressCallback.onReceiveStarted(receiveTryCount);
}
return super.doReceiveResponse(request, conn, context);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy