nstream.adapter.common.ingress.IngestorMetricsAgent Maven / Gradle / Ivy
Show all versions of nstream-adapter-common Show documentation
// Copyright 2015-2024 Nstream, inc.
//
// Licensed under the Redis Source Available License 2.0 (RSALv2) Agreement;
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://redis.com/legal/rsalv2-agreement/
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nstream.adapter.common.ingress;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import nstream.adapter.common.AdapterSettings;
import nstream.adapter.common.schedule.DeferrableException;
import swim.api.SwimLane;
import swim.api.lane.DemandLane;
import swim.concurrent.TimerRef;
import swim.structure.Record;
import swim.structure.Value;
public abstract class IngestorMetricsAgent extends IngestorAgent {
private static final long MESSAGE_RATE_INTERVAL = 1000;
private TimerRef messageRateTimer;
private volatile long startTime;
private volatile long bucketStartTime;
private final AtomicLong messageCount = new AtomicLong();
private final AtomicLong errorCount = new AtomicLong();
private final AtomicLong recentMessageCount = new AtomicLong();
private volatile double currentMessageRate;
private final AtomicLong recentErrorCount = new AtomicLong();
private volatile double currentErrorRate;
protected IngestorMetricsAgent() {
}
@SwimLane("pulse")
DemandLane pulse = this.demandLane()
.onCue(uplink -> pulse());
private Value pulse() {
final long now = System.currentTimeMillis();
if (now <= this.startTime || this.startTime == 0L) {
return Value.absent();
}
final double eventRate = (double) (this.messageCount.get() * 1000L) / (now - this.startTime);
return Record.create(6)
.slot("startTimestamp", this.startTime)
.slot("messageCount", this.messageCount.get())
.slot("errorCount", this.errorCount.get())
.slot("messageRate", eventRate)
.slot("currentMessageRate", this.currentMessageRate)
.slot("currentErrorRate", this.currentErrorRate);
}
private void updateRecentBucket() {
final long now = System.currentTimeMillis();
final long elapsed = now - this.bucketStartTime;
this.currentMessageRate = (double) (this.recentMessageCount.getAndSet(0L) * 1000L) / elapsed;
this.currentErrorRate = (double) (this.recentErrorCount.getAndSet(0L) * 1000L) / elapsed;
this.bucketStartTime = now;
}
private void incrementMessageCount() {
this.messageCount.incrementAndGet();
this.recentMessageCount.incrementAndGet();
}
private void incrementErrorCount() {
this.errorCount.incrementAndGet();
this.recentErrorCount.incrementAndGet();
}
protected void didFailIngest(final V ingestee, final Exception exception) {
didFail(exception);
}
/**
* Per-message ingestion strategy that treats any exception incurred as
* nonfatal except for {@link IngestPanicException IngestFaultExceptions}.
*
* This is the default ingestion strategy.
*
* @param ingestee the ingested object
*/
protected final void ingestOrContinue(final V ingestee) {
try {
incrementMessageCount();
ingest(ingestee);
} catch (Exception e) {
if (e instanceof IngestPanicException) {
didFailIngest(ingestee, e);
cancel();
return;
}
incrementErrorCount();
handleDeferrableException(e);
}
}
protected final void ingestOrCancel(final V ingestee) {
ingestOrCancel(ingestee, false, v -> false);
}
/**
* Per-message ingestion strategy that aggressively cleans up resources on
* incurred exceptions.
*
*
Any exception that is not a {@code DeferrableException}, or any {@code
* ingestee} that yields {@code true} when applied to {@code shouldTerminate},
* will trigger termination and cleanup.
*
* @param ingestee the ingested object
* @param shouldIngestTerminator whether to apply the ingestion logic to a
* termination-causing message
* @param shouldTerminate whether {@code ingestee} should trigger termination
* and cleanup of ingestor resources
*/
protected final void ingestOrCancel(final V ingestee, boolean shouldIngestTerminator,
Function shouldTerminate) {
try {
incrementMessageCount();
if (shouldTerminate.apply(ingestee)) {
if (shouldIngestTerminator) {
ingest(ingestee);
}
cancel();
} else {
ingest(ingestee);
}
} catch (final DeferrableException deferrableException) {
incrementErrorCount();
handleDeferrableException(deferrableException);
} catch (final Exception exception) {
didFailIngest(ingestee, exception);
cancel();
}
}
/**
* Callback after successful staging.
* If overridden should always make call to super first to initialise
* metrics.
*/
@Override
protected void didStageReception() {
this.startTime = System.currentTimeMillis();
this.bucketStartTime = this.startTime;
this.messageRateTimer = setTimer(MESSAGE_RATE_INTERVAL, () -> {
updateRecentBucket();
this.pulse.cue();
this.messageRateTimer.reschedule(MESSAGE_RATE_INTERVAL);
});
}
}