Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package kafka.tools
import java.io._
import java.nio.ByteBuffer
import joptsimple.OptionParser
import kafka.coordinator.{GroupMetadataKey, GroupMetadataManager, OffsetKey}
import kafka.log._
import kafka.message._
import kafka.serializer.Decoder
import kafka.utils.{VerifiableProperties, _}
import org.apache.kafka.clients.consumer.internals.ConsumerProtocol
import org.apache.kafka.common.KafkaException
import org.apache.kafka.common.utils.Utils
import scala.collection.mutable
object DumpLogSegments {
def main(args: Array[String]) {
val parser = new OptionParser
val printOpt = parser.accepts("print-data-log", "if set, printing the messages content when dumping data logs")
val verifyOpt = parser.accepts("verify-index-only", "if set, just verify the index log without printing its content")
val filesOpt = parser.accepts("files", "REQUIRED: The comma separated list of data and index log files to be dumped")
.withRequiredArg
.describedAs("file1, file2, ...")
.ofType(classOf[String])
val maxMessageSizeOpt = parser.accepts("max-message-size", "Size of largest message.")
.withRequiredArg
.describedAs("size")
.ofType(classOf[java.lang.Integer])
.defaultsTo(5 * 1024 * 1024)
val deepIterationOpt = parser.accepts("deep-iteration", "if set, uses deep instead of shallow iteration")
val valueDecoderOpt = parser.accepts("value-decoder-class", "if set, used to deserialize the messages. This class should implement kafka.serializer.Decoder trait. Custom jar should be available in kafka/libs directory.")
.withOptionalArg()
.ofType(classOf[java.lang.String])
.defaultsTo("kafka.serializer.StringDecoder")
val keyDecoderOpt = parser.accepts("key-decoder-class", "if set, used to deserialize the keys. This class should implement kafka.serializer.Decoder trait. Custom jar should be available in kafka/libs directory.")
.withOptionalArg()
.ofType(classOf[java.lang.String])
.defaultsTo("kafka.serializer.StringDecoder")
val offsetsOpt = parser.accepts("offsets-decoder", "if set, log data will be parsed as offset data from __consumer_offsets topic")
if(args.length == 0)
CommandLineUtils.printUsageAndDie(parser, "Parse a log file and dump its contents to the console, useful for debugging a seemingly corrupt log segment.")
val options = parser.parse(args : _*)
CommandLineUtils.checkRequiredArgs(parser, options, filesOpt)
val print = if(options.has(printOpt)) true else false
val verifyOnly = if(options.has(verifyOpt)) true else false
val files = options.valueOf(filesOpt).split(",")
val maxMessageSize = options.valueOf(maxMessageSizeOpt).intValue()
val isDeepIteration = if(options.has(deepIterationOpt)) true else false
val messageParser = if (options.has(offsetsOpt)) {
new OffsetsMessageParser
} else {
val valueDecoder: Decoder[_] = CoreUtils.createObject[Decoder[_]](options.valueOf(valueDecoderOpt), new VerifiableProperties)
val keyDecoder: Decoder[_] = CoreUtils.createObject[Decoder[_]](options.valueOf(keyDecoderOpt), new VerifiableProperties)
new DecoderMessageParser(keyDecoder, valueDecoder)
}
val misMatchesForIndexFilesMap = new mutable.HashMap[String, List[(Long, Long)]]
val nonConsecutivePairsForLogFilesMap = new mutable.HashMap[String, List[(Long, Long)]]
for(arg <- files) {
val file = new File(arg)
if(file.getName.endsWith(Log.LogFileSuffix)) {
println("Dumping " + file)
dumpLog(file, print, nonConsecutivePairsForLogFilesMap, isDeepIteration, maxMessageSize , messageParser)
} else if(file.getName.endsWith(Log.IndexFileSuffix)) {
println("Dumping " + file)
dumpIndex(file, verifyOnly, misMatchesForIndexFilesMap, maxMessageSize)
}
}
misMatchesForIndexFilesMap.foreach {
case (fileName, listOfMismatches) => {
System.err.println("Mismatches in :" + fileName)
listOfMismatches.foreach(m => {
System.err.println(" Index offset: %d, log offset: %d".format(m._1, m._2))
})
}
}
nonConsecutivePairsForLogFilesMap.foreach {
case (fileName, listOfNonConsecutivePairs) => {
System.err.println("Non-secutive offsets in :" + fileName)
listOfNonConsecutivePairs.foreach(m => {
System.err.println(" %d is followed by %d".format(m._1, m._2))
})
}
}
}
/* print out the contents of the index */
private def dumpIndex(file: File,
verifyOnly: Boolean,
misMatchesForIndexFilesMap: mutable.HashMap[String, List[(Long, Long)]],
maxMessageSize: Int) {
val startOffset = file.getName().split("\\.")(0).toLong
val logFile = new File(file.getAbsoluteFile.getParent, file.getName.split("\\.")(0) + Log.LogFileSuffix)
val messageSet = new FileMessageSet(logFile, false)
val index = new OffsetIndex(file = file, baseOffset = startOffset)
for(i <- 0 until index.entries) {
val entry = index.entry(i)
val partialFileMessageSet: FileMessageSet = messageSet.read(entry.position, maxMessageSize)
val messageAndOffset = getIterator(partialFileMessageSet.head, isDeepIteration = true).next()
if(messageAndOffset.offset != entry.offset + index.baseOffset) {
var misMatchesSeq = misMatchesForIndexFilesMap.getOrElse(file.getAbsolutePath, List[(Long, Long)]())
misMatchesSeq ::=(entry.offset + index.baseOffset, messageAndOffset.offset)
misMatchesForIndexFilesMap.put(file.getAbsolutePath, misMatchesSeq)
}
// since it is a sparse file, in the event of a crash there may be many zero entries, stop if we see one
if(entry.offset == 0 && i > 0)
return
if (!verifyOnly)
println("offset: %d position: %d".format(entry.offset + index.baseOffset, entry.position))
}
}
private trait MessageParser[K, V] {
def parse(message: Message): (Option[K], Option[V])
}
private class DecoderMessageParser[K, V](keyDecoder: Decoder[K], valueDecoder: Decoder[V]) extends MessageParser[K, V] {
override def parse(message: Message): (Option[K], Option[V]) = {
if (message.isNull) {
(None, None)
} else {
val key = if (message.hasKey)
Some(keyDecoder.fromBytes(Utils.readBytes(message.key)))
else
None
val payload = Some(valueDecoder.fromBytes(Utils.readBytes(message.payload)))
(key, payload)
}
}
}
private class OffsetsMessageParser extends MessageParser[String, String] {
private def hex(bytes: Array[Byte]): String = {
if (bytes.isEmpty)
""
else
String.format("%X", BigInt(1, bytes))
}
private def parseOffsets(offsetKey: OffsetKey, payload: ByteBuffer) = {
val group = offsetKey.key.group
val (topic, partition) = offsetKey.key.topicPartition.asTuple
val offset = GroupMetadataManager.readOffsetMessageValue(payload)
val keyString = s"offset::${group}:${topic}:${partition}"
val valueString = if (offset.metadata.isEmpty)
String.valueOf(offset.offset)
else
s"${offset.offset}:${offset.metadata}"
(Some(keyString), Some(valueString))
}
private def parseGroupMetadata(groupMetadataKey: GroupMetadataKey, payload: ByteBuffer) = {
val groupId = groupMetadataKey.key
val group = GroupMetadataManager.readGroupMessageValue(groupId, payload)
val protocolType = group.protocolType
val assignment = group.allMemberMetadata.map { member =>
if (protocolType == ConsumerProtocol.PROTOCOL_TYPE) {
val partitionAssignment = ConsumerProtocol.deserializeAssignment(ByteBuffer.wrap(member.assignment))
val userData = hex(Utils.toArray(partitionAssignment.userData()))
if (userData.isEmpty)
s"${member.memberId}=${partitionAssignment.partitions()}"
else
s"${member.memberId}=${partitionAssignment.partitions()}:${userData}"
} else {
s"${member.memberId}=${hex(member.assignment)}"
}
}.mkString("{", ",", "}")
val keyString = s"metadata::${groupId}"
val valueString = s"${protocolType}:${group.protocol}:${group.generationId}:${assignment}"
(Some(keyString), Some(valueString))
}
override def parse(message: Message): (Option[String], Option[String]) = {
if (message.isNull)
(None, None)
else if (!message.hasKey) {
throw new KafkaException("Failed to decode message using offset topic decoder (message had a missing key)")
} else {
GroupMetadataManager.readMessageKey(message.key) match {
case offsetKey: OffsetKey => parseOffsets(offsetKey, message.payload)
case groupMetadataKey: GroupMetadataKey => parseGroupMetadata(groupMetadataKey, message.payload)
case _ => throw new KafkaException("Failed to decode message using offset topic decoder (message had an invalid key)")
}
}
}
}
/* print out the contents of the log */
private def dumpLog(file: File,
printContents: Boolean,
nonConsecutivePairsForLogFilesMap: mutable.HashMap[String, List[(Long, Long)]],
isDeepIteration: Boolean,
maxMessageSize: Int,
parser: MessageParser[_, _]) {
val startOffset = file.getName().split("\\.")(0).toLong
println("Starting offset: " + startOffset)
val messageSet = new FileMessageSet(file, false)
var validBytes = 0L
var lastOffset = -1l
val shallowIterator = messageSet.iterator(maxMessageSize)
for(shallowMessageAndOffset <- shallowIterator) { // this only does shallow iteration
val itr = getIterator(shallowMessageAndOffset, isDeepIteration)
for (messageAndOffset <- itr) {
val msg = messageAndOffset.message
if(lastOffset == -1)
lastOffset = messageAndOffset.offset
// If we are iterating uncompressed messages, offsets must be consecutive
else if (msg.compressionCodec == NoCompressionCodec && messageAndOffset.offset != lastOffset +1) {
var nonConsecutivePairsSeq = nonConsecutivePairsForLogFilesMap.getOrElse(file.getAbsolutePath, List[(Long, Long)]())
nonConsecutivePairsSeq ::=(lastOffset, messageAndOffset.offset)
nonConsecutivePairsForLogFilesMap.put(file.getAbsolutePath, nonConsecutivePairsSeq)
}
lastOffset = messageAndOffset.offset
print("offset: " + messageAndOffset.offset + " position: " + validBytes + " isvalid: " + msg.isValid +
" payloadsize: " + msg.payloadSize + " magic: " + msg.magic +
" compresscodec: " + msg.compressionCodec + " crc: " + msg.checksum)
if(msg.hasKey)
print(" keysize: " + msg.keySize)
if(printContents) {
val (key, payload) = parser.parse(msg)
key.map(key => print(s" key: ${key}"))
payload.map(payload => print(s" payload: ${payload}"))
}
println()
}
validBytes += MessageSet.entrySize(shallowMessageAndOffset.message)
}
val trailingBytes = messageSet.sizeInBytes - validBytes
if(trailingBytes > 0)
println("Found %d invalid bytes at the end of %s".format(trailingBytes, file.getName))
}
private def getIterator(messageAndOffset: MessageAndOffset, isDeepIteration: Boolean) = {
if (isDeepIteration) {
val message = messageAndOffset.message
message.compressionCodec match {
case NoCompressionCodec =>
getSingleMessageIterator(messageAndOffset)
case _ =>
ByteBufferMessageSet.deepIterator(message)
}
} else
getSingleMessageIterator(messageAndOffset)
}
private def getSingleMessageIterator(messageAndOffset: MessageAndOffset) = {
new IteratorTemplate[MessageAndOffset] {
var messageIterated = false
override def makeNext(): MessageAndOffset = {
if (!messageIterated) {
messageIterated = true
messageAndOffset
} else
allDone()
}
}
}
}
