Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka.network
import java.net.InetAddress
import java.nio.ByteBuffer
import java.util.concurrent._
import com.typesafe.scalalogging.Logger
import com.yammer.metrics.core.{Gauge, Meter}
import kafka.metrics.KafkaMetricsGroup
import kafka.utils.{Logging, NotNothing, Pool}
import org.apache.kafka.common.memory.MemoryPool
import org.apache.kafka.common.network.Send
import org.apache.kafka.common.protocol.{ApiKeys, Errors}
import org.apache.kafka.common.requests._
import org.apache.kafka.common.security.auth.KafkaPrincipal
import org.apache.kafka.common.utils.{Sanitizer, Time}
import scala.collection.mutable
import scala.collection.JavaConverters._
import scala.reflect.ClassTag
object RequestChannel extends Logging {
private val requestLogger = Logger("kafka.request.logger")
val RequestQueueSizeMetric = "RequestQueueSize"
val ResponseQueueSizeMetric = "ResponseQueueSize"
val ProcessorMetricTag = "processor"
def isRequestLoggingEnabled: Boolean = requestLogger.underlying.isDebugEnabled
sealed trait BaseRequest
case object ShutdownRequest extends BaseRequest
case class Session(principal: KafkaPrincipal, clientAddress: InetAddress) {
val sanitizedUser = Sanitizer.sanitize(principal.getName)
}
class Metrics {
private val metricsMap = mutable.Map[String, RequestMetrics]()
(ApiKeys.values.toSeq.map(_.name) ++
Seq(RequestMetrics.consumerFetchMetricName, RequestMetrics.followFetchMetricName)).foreach { name =>
metricsMap.put(name, new RequestMetrics(name))
}
def apply(metricName: String) = metricsMap(metricName)
def close(): Unit = {
metricsMap.values.foreach(_.removeMetrics())
}
}
class Request(val processor: Int,
val context: RequestContext,
val startTimeNanos: Long,
memoryPool: MemoryPool,
@volatile private var buffer: ByteBuffer,
metrics: RequestChannel.Metrics) extends BaseRequest {
// These need to be volatile because the readers are in the network thread and the writers are in the request
// handler threads or the purgatory threads
@volatile var requestDequeueTimeNanos = -1L
@volatile var apiLocalCompleteTimeNanos = -1L
@volatile var responseCompleteTimeNanos = -1L
@volatile var responseDequeueTimeNanos = -1L
@volatile var apiRemoteCompleteTimeNanos = -1L
@volatile var messageConversionsTimeNanos = 0L
@volatile var temporaryMemoryBytes = 0L
@volatile var recordNetworkThreadTimeCallback: Option[Long => Unit] = None
val session = Session(context.principal, context.clientAddress)
private val bodyAndSize: RequestAndSize = context.parseRequest(buffer)
def header: RequestHeader = context.header
def sizeOfBodyInBytes: Int = bodyAndSize.size
//most request types are parsed entirely into objects at this point. for those we can release the underlying buffer.
//some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference
//to the buffer. for those requests we cannot release the buffer early, but only when request processing is done.
if (!header.apiKey.requiresDelayedAllocation) {
releaseBuffer()
}
def requestDesc(details: Boolean): String = s"$header -- ${body[AbstractRequest].toString(details)}"
def body[T <: AbstractRequest](implicit classTag: ClassTag[T], nn: NotNothing[T]): T = {
bodyAndSize.request match {
case r: T => r
case r =>
throw new ClassCastException(s"Expected request with type ${classTag.runtimeClass}, but found ${r.getClass}")
}
}
trace(s"Processor $processor received request: ${requestDesc(true)}")
def requestThreadTimeNanos = {
if (apiLocalCompleteTimeNanos == -1L) apiLocalCompleteTimeNanos = Time.SYSTEM.nanoseconds
math.max(apiLocalCompleteTimeNanos - requestDequeueTimeNanos, 0L)
}
def updateRequestMetrics(networkThreadTimeNanos: Long, response: Response) {
val endTimeNanos = Time.SYSTEM.nanoseconds
// In some corner cases, apiLocalCompleteTimeNanos may not be set when the request completes if the remote
// processing time is really small. This value is set in KafkaApis from a request handling thread.
// This may be read in a network thread before the actual update happens in KafkaApis which will cause us to
// see a negative value here. In that case, use responseCompleteTimeNanos as apiLocalCompleteTimeNanos.
if (apiLocalCompleteTimeNanos < 0)
apiLocalCompleteTimeNanos = responseCompleteTimeNanos
// If the apiRemoteCompleteTimeNanos is not set (i.e., for requests that do not go through a purgatory), then it is
// the same as responseCompleteTimeNanos.
if (apiRemoteCompleteTimeNanos < 0)
apiRemoteCompleteTimeNanos = responseCompleteTimeNanos
/**
* Converts nanos to millis with micros precision as additional decimal places in the request log have low
* signal to noise ratio. When it comes to metrics, there is little difference either way as we round the value
* to the nearest long.
*/
def nanosToMs(nanos: Long): Double = {
val positiveNanos = math.max(nanos, 0)
TimeUnit.NANOSECONDS.toMicros(positiveNanos).toDouble / TimeUnit.MILLISECONDS.toMicros(1)
}
val requestQueueTimeMs = nanosToMs(requestDequeueTimeNanos - startTimeNanos)
val apiLocalTimeMs = nanosToMs(apiLocalCompleteTimeNanos - requestDequeueTimeNanos)
val apiRemoteTimeMs = nanosToMs(apiRemoteCompleteTimeNanos - apiLocalCompleteTimeNanos)
val apiThrottleTimeMs = nanosToMs(responseCompleteTimeNanos - apiRemoteCompleteTimeNanos)
val responseQueueTimeMs = nanosToMs(responseDequeueTimeNanos - responseCompleteTimeNanos)
val responseSendTimeMs = nanosToMs(endTimeNanos - responseDequeueTimeNanos)
val messageConversionsTimeMs = nanosToMs(messageConversionsTimeNanos)
val totalTimeMs = nanosToMs(endTimeNanos - startTimeNanos)
val fetchMetricNames =
if (header.apiKey == ApiKeys.FETCH) {
val isFromFollower = body[FetchRequest].isFromFollower
Seq(
if (isFromFollower) RequestMetrics.followFetchMetricName
else RequestMetrics.consumerFetchMetricName
)
}
else Seq.empty
val metricNames = fetchMetricNames :+ header.apiKey.name
metricNames.foreach { metricName =>
val m = metrics(metricName)
m.requestRate(header.apiVersion).mark()
m.requestQueueTimeHist.update(Math.round(requestQueueTimeMs))
m.localTimeHist.update(Math.round(apiLocalTimeMs))
m.remoteTimeHist.update(Math.round(apiRemoteTimeMs))
m.throttleTimeHist.update(Math.round(apiThrottleTimeMs))
m.responseQueueTimeHist.update(Math.round(responseQueueTimeMs))
m.responseSendTimeHist.update(Math.round(responseSendTimeMs))
m.totalTimeHist.update(Math.round(totalTimeMs))
m.requestBytesHist.update(sizeOfBodyInBytes)
m.messageConversionsTimeHist.foreach(_.update(Math.round(messageConversionsTimeMs)))
m.tempMemoryBytesHist.foreach(_.update(temporaryMemoryBytes))
}
// Records network handler thread usage. This is included towards the request quota for the
// user/client. Throttling is only performed when request handler thread usage
// is recorded, just before responses are queued for delivery.
// The time recorded here is the time spent on the network thread for receiving this request
// and sending the response. Note that for the first request on a connection, the time includes
// the total time spent on authentication, which may be significant for SASL/SSL.
recordNetworkThreadTimeCallback.foreach(record => record(networkThreadTimeNanos))
if (isRequestLoggingEnabled) {
val detailsEnabled = requestLogger.underlying.isTraceEnabled
val responseString = response.responseString.getOrElse(
throw new IllegalStateException("responseAsString should always be defined if request logging is enabled"))
val builder = new StringBuilder(256)
builder.append("Completed request:").append(requestDesc(detailsEnabled))
.append(",response:").append(responseString)
.append(" from connection ").append(context.connectionId)
.append(";totalTime:").append(totalTimeMs)
.append(",requestQueueTime:").append(requestQueueTimeMs)
.append(",localTime:").append(apiLocalTimeMs)
.append(",remoteTime:").append(apiRemoteTimeMs)
.append(",throttleTime:").append(apiThrottleTimeMs)
.append(",responseQueueTime:").append(responseQueueTimeMs)
.append(",sendTime:").append(responseSendTimeMs)
.append(",securityProtocol:").append(context.securityProtocol)
.append(",principal:").append(session.principal)
.append(",listener:").append(context.listenerName.value)
if (temporaryMemoryBytes > 0)
builder.append(",temporaryMemoryBytes:").append(temporaryMemoryBytes)
if (messageConversionsTimeMs > 0)
builder.append(",messageConversionsTime:").append(messageConversionsTimeMs)
requestLogger.debug(builder.toString)
}
}
def releaseBuffer(): Unit = {
if (buffer != null) {
memoryPool.release(buffer)
buffer = null
}
}
override def toString = s"Request(processor=$processor, " +
s"connectionId=${context.connectionId}, " +
s"session=$session, " +
s"listenerName=${context.listenerName}, " +
s"securityProtocol=${context.securityProtocol}, " +
s"buffer=$buffer)"
}
abstract class Response(val request: Request) {
locally {
val nowNs = Time.SYSTEM.nanoseconds
request.responseCompleteTimeNanos = nowNs
if (request.apiLocalCompleteTimeNanos == -1L)
request.apiLocalCompleteTimeNanos = nowNs
}
def processor: Int = request.processor
def responseString: Option[String] = Some("")
def onComplete: Option[Send => Unit] = None
override def toString: String
}
/** responseAsString should only be defined if request logging is enabled */
class SendResponse(request: Request,
val responseSend: Send,
val responseAsString: Option[String],
val onCompleteCallback: Option[Send => Unit]) extends Response(request) {
override def responseString: Option[String] = responseAsString
override def onComplete: Option[Send => Unit] = onCompleteCallback
override def toString: String =
s"Response(type=Send, request=$request, send=$responseSend, asString=$responseAsString)"
}
class NoOpResponse(request: Request) extends Response(request) {
override def toString: String =
s"Response(type=NoOp, request=$request)"
}
class CloseConnectionResponse(request: Request) extends Response(request) {
override def toString: String =
s"Response(type=CloseConnection, request=$request)"
}
class StartThrottlingResponse(request: Request) extends Response(request) {
override def toString: String =
s"Response(type=StartThrottling, request=$request)"
}
class EndThrottlingResponse(request: Request) extends Response(request) {
override def toString: String =
s"Response(type=EndThrottling, request=$request)"
}
}
class RequestChannel(val queueSize: Int, val metricNamePrefix : String) extends KafkaMetricsGroup {
import RequestChannel._
val metrics = new RequestChannel.Metrics
private val requestQueue = new ArrayBlockingQueue[BaseRequest](queueSize)
private val processors = new ConcurrentHashMap[Int, Processor]()
val requestQueueSizeMetricName = metricNamePrefix.concat(RequestQueueSizeMetric)
val responseQueueSizeMetricName = metricNamePrefix.concat(ResponseQueueSizeMetric)
newGauge(requestQueueSizeMetricName, new Gauge[Int] {
def value = requestQueue.size
})
newGauge(responseQueueSizeMetricName, new Gauge[Int]{
def value = processors.values.asScala.foldLeft(0) {(total, processor) =>
total + processor.responseQueueSize
}
})
def addProcessor(processor: Processor): Unit = {
if (processors.putIfAbsent(processor.id, processor) != null)
warn(s"Unexpected processor with processorId ${processor.id}")
newGauge(responseQueueSizeMetricName,
new Gauge[Int] {
def value = processor.responseQueueSize
},
Map(ProcessorMetricTag -> processor.id.toString)
)
}
def removeProcessor(processorId: Int): Unit = {
processors.remove(processorId)
removeMetric(responseQueueSizeMetricName, Map(ProcessorMetricTag -> processorId.toString))
}
/** Send a request to be handled, potentially blocking until there is room in the queue for the request */
def sendRequest(request: RequestChannel.Request) {
requestQueue.put(request)
}
/** Send a response back to the socket server to be sent over the network */
def sendResponse(response: RequestChannel.Response) {
if (isTraceEnabled) {
val requestHeader = response.request.header
val message = response match {
case sendResponse: SendResponse =>
s"Sending ${requestHeader.apiKey} response to client ${requestHeader.clientId} of ${sendResponse.responseSend.size} bytes."
case _: NoOpResponse =>
s"Not sending ${requestHeader.apiKey} response to client ${requestHeader.clientId} as it's not required."
case _: CloseConnectionResponse =>
s"Closing connection for client ${requestHeader.clientId} due to error during ${requestHeader.apiKey}."
case _: StartThrottlingResponse =>
s"Notifying channel throttling has started for client ${requestHeader.clientId} for ${requestHeader.apiKey}"
case _: EndThrottlingResponse =>
s"Notifying channel throttling has ended for client ${requestHeader.clientId} for ${requestHeader.apiKey}"
}
trace(message)
}
val processor = processors.get(response.processor)
// The processor may be null if it was shutdown. In this case, the connections
// are closed, so the response is dropped.
if (processor != null) {
processor.enqueueResponse(response)
}
}
/** Get the next request or block until specified time has elapsed */
def receiveRequest(timeout: Long): RequestChannel.BaseRequest =
requestQueue.poll(timeout, TimeUnit.MILLISECONDS)
/** Get the next request or block until there is one */
def receiveRequest(): RequestChannel.BaseRequest =
requestQueue.take()
def updateErrorMetrics(apiKey: ApiKeys, errors: collection.Map[Errors, Integer]) {
errors.foreach { case (error, count) =>
metrics(apiKey.name).markErrorMeter(error, count)
}
}
def clear() {
requestQueue.clear()
}
def shutdown() {
clear()
metrics.close()
}
def sendShutdownRequest(): Unit = requestQueue.put(ShutdownRequest)
}
object RequestMetrics {
val consumerFetchMetricName = ApiKeys.FETCH.name + "Consumer"
val followFetchMetricName = ApiKeys.FETCH.name + "Follower"
val RequestsPerSec = "RequestsPerSec"
val RequestQueueTimeMs = "RequestQueueTimeMs"
val LocalTimeMs = "LocalTimeMs"
val RemoteTimeMs = "RemoteTimeMs"
val ThrottleTimeMs = "ThrottleTimeMs"
val ResponseQueueTimeMs = "ResponseQueueTimeMs"
val ResponseSendTimeMs = "ResponseSendTimeMs"
val TotalTimeMs = "TotalTimeMs"
val RequestBytes = "RequestBytes"
val MessageConversionsTimeMs = "MessageConversionsTimeMs"
val TemporaryMemoryBytes = "TemporaryMemoryBytes"
val ErrorsPerSec = "ErrorsPerSec"
}
class RequestMetrics(name: String) extends KafkaMetricsGroup {
import RequestMetrics._
val tags = Map("request" -> name)
val requestRateInternal = new Pool[Short, Meter]()
// time a request spent in a request queue
val requestQueueTimeHist = newHistogram(RequestQueueTimeMs, biased = true, tags)
// time a request takes to be processed at the local broker
val localTimeHist = newHistogram(LocalTimeMs, biased = true, tags)
// time a request takes to wait on remote brokers (currently only relevant to fetch and produce requests)
val remoteTimeHist = newHistogram(RemoteTimeMs, biased = true, tags)
// time a request is throttled
val throttleTimeHist = newHistogram(ThrottleTimeMs, biased = true, tags)
// time a response spent in a response queue
val responseQueueTimeHist = newHistogram(ResponseQueueTimeMs, biased = true, tags)
// time to send the response to the requester
val responseSendTimeHist = newHistogram(ResponseSendTimeMs, biased = true, tags)
val totalTimeHist = newHistogram(TotalTimeMs, biased = true, tags)
// request size in bytes
val requestBytesHist = newHistogram(RequestBytes, biased = true, tags)
// time for message conversions (only relevant to fetch and produce requests)
val messageConversionsTimeHist =
if (name == ApiKeys.FETCH.name || name == ApiKeys.PRODUCE.name)
Some(newHistogram(MessageConversionsTimeMs, biased = true, tags))
else
None
// Temporary memory allocated for processing request (only populated for fetch and produce requests)
// This shows the memory allocated for compression/conversions excluding the actual request size
val tempMemoryBytesHist =
if (name == ApiKeys.FETCH.name || name == ApiKeys.PRODUCE.name)
Some(newHistogram(TemporaryMemoryBytes, biased = true, tags))
else
None
private val errorMeters = mutable.Map[Errors, ErrorMeter]()
Errors.values.foreach(error => errorMeters.put(error, new ErrorMeter(name, error)))
def requestRate(version: Short): Meter = {
requestRateInternal.getAndMaybePut(version, newMeter("RequestsPerSec", "requests", TimeUnit.SECONDS, tags + ("version" -> version.toString)))
}
class ErrorMeter(name: String, error: Errors) {
private val tags = Map("request" -> name, "error" -> error.name)
@volatile private var meter: Meter = null
def getOrCreateMeter(): Meter = {
if (meter != null)
meter
else {
synchronized {
if (meter == null)
meter = newMeter(ErrorsPerSec, "requests", TimeUnit.SECONDS, tags)
meter
}
}
}
def removeMeter(): Unit = {
synchronized {
if (meter != null) {
removeMetric(ErrorsPerSec, tags)
meter = null
}
}
}
}
def markErrorMeter(error: Errors, count: Int) {
errorMeters(error).getOrCreateMeter().mark(count.toLong)
}
def removeMetrics(): Unit = {
for (version <- requestRateInternal.keys) removeMetric(RequestsPerSec, tags + ("version" -> version.toString))
removeMetric(RequestQueueTimeMs, tags)
removeMetric(LocalTimeMs, tags)
removeMetric(RemoteTimeMs, tags)
removeMetric(RequestsPerSec, tags)
removeMetric(ThrottleTimeMs, tags)
removeMetric(ResponseQueueTimeMs, tags)
removeMetric(TotalTimeMs, tags)
removeMetric(ResponseSendTimeMs, tags)
removeMetric(RequestBytes, tags)
removeMetric(ResponseSendTimeMs, tags)
if (name == ApiKeys.FETCH.name || name == ApiKeys.PRODUCE.name) {
removeMetric(MessageConversionsTimeMs, tags)
removeMetric(TemporaryMemoryBytes, tags)
}
errorMeters.values.foreach(_.removeMeter())
errorMeters.clear()
}
}
