com.gemstone.gemfire.internal.MergeLogFiles Maven / Gradle / Ivy
Show all versions of gemfire-core Show documentation
/*
* Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package com.gemstone.gemfire.internal;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import com.gemstone.gemfire.SystemFailure;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
/**
* This program merges entries from multiple GemFire log files (those
* written using a {@link com.gemstone.gemfire.i18n.LogWriterI18n} together,
* sorting them by their timestamp. Note that this program assumes
* that the entries in the individual log files are themselves sorted
* by timestamp.
*
* MergeLogFiles has several command line options:
*
* -pids tells the program to look for hydra-style naming and pick out
* process identifiers, then use these to distinguish log entries instead of
* full log file names.
*
* -dirCount x when -pids is not being used, or it is being used
* and a process ID can't be found, the -dirCount option instructs the
* program as to how many parent directory names should be included in the name
* of each log file.
*
* -noblanks tells the program to suppress blank lines found in log files
*
* -align tells the program to align log content not having timestamps with
* log content that does have timestamps.
*
* -threads tells the program to attempt to make use of multiple CPUs
*
* The -pids option will cause a file nickname table to be built and
* emitted at the beginning of the merged log files. Nicknames are of the
* form pid-x, where pid is the process ID that emitted the log entry, and
* x is the position of this log file in the ordered set of log files
* created by that process.
*
* Normally, one log file reader is created per log file, and they are iterated
* over repeatedly to find and write the next earliest timestamped line to
* the output stream. Text without a timestamp is associated with the prior
* timestamped line in the file.
*
* The -threads option will cause the program to also create threads
* for each reader that are backed by bounded {@link BlockingQueue}s,
* as outlined in the diagram below. This can consume more memory, so it is
* wise to increase the Xmx of the java virtual machine if you are going to
* use this option.
*
*
* 
*
*
* @see SortLogFile
* @see LogFileParser
*
* @author David Whitlock
* @author Bruce Schuchardt
*
*
* @since 2.0 (-pids, -threads, -align, and -noblanks added in 5.1)
*/
public class MergeLogFiles {
private static PrintStream out = System.out;
private static PrintStream err = System.err;
/**
* Merges the log files from a given set of
* InputStreams into a PrinWriter.
*
* @param logFiles
* The log files to be merged
* @param logFileNames
* The names of the log files to be printed in the merged log
* @param mergedFile
* Where the merged logs are printed to
*
* @return Whether or not problems occurred while merging the log
* files.
*
* @throws IllegalArgumentException
* If the length of logFiles is not the same
* as the length of logFileNames
*
* @see Sorter#mergeLogFiles
*/
public static boolean mergeLogFiles(InputStream[] logFiles,
String[] logFileNames, PrintWriter mergedFile) {
return mergeLogFiles(logFiles, logFileNames, mergedFile, false, false, false, new LinkedList());
}
/**
* Merges the log files from a given set of
* InputStreams into a PrinWriter.
*
* @param logFiles
* The log files to be merged
* @param logFileNames
* The names of the log files to be printed in the merged log
* @param mergedFile
* Where the merged logs are printed to
* @param tabOut
* Whether to align non-timestamped lines with timestamped lines
* @param suppressBlanks
* Whether to omit blank lines
* @return Whether or not problems occurred while merging the log
* files.
*
* @throws IllegalArgumentException
* If the length of logFiles is not the same
* as the length of logFileNames
*
* @see Sorter#mergeLogFiles
*/
public static boolean mergeLogFiles(InputStream[] logFiles,
String[] logFileNames, PrintWriter mergedFile,
boolean tabOut, boolean suppressBlanks, boolean multithreaded) {
return mergeLogFiles(logFiles, logFileNames, mergedFile, tabOut, suppressBlanks, multithreaded, new LinkedList());
}
/**
* Merges the log files from a given set of
* InputStreams into a PrinWriter.
*
* @param logFiles
* The log files to be merged
* @param logFileNames
* The names of the log files to be printed in the merged log
* @param mergedFile
* Where the merged logs are printed to
* @param tabOut
* Whether to align non-timestamped lines with timestamped lines
* @param suppressBlanks
* Whether to omit blank lines
* @param patterns
* Regular expression patterns that lines must match to be included
* @return Whether or not problems occurred while merging the log
* files.
*
* @throws IllegalArgumentException
* If the length of logFiles is not the same
* as the length of logFileNames
*
* @see Sorter#mergeLogFiles
*/
public static boolean mergeLogFiles(InputStream[] logFiles,
String[] logFileNames, PrintWriter mergedFile,
boolean tabOut, boolean suppressBlanks, boolean multithreaded, List patterns) {
return Sorter.mergeLogFiles(logFiles, logFileNames, mergedFile,
tabOut, suppressBlanks, multithreaded, patterns);
}
// ////////////////// Main Program ////////////////////
/**
* Prints usage information about this program
*/
private static void usage(String s) {
// note that we don't document the -pids switch because it is tailored
// to how hydra works and would not be useful for customers
err.println("\n** " + s + "\n");
err.println(LocalizedStrings.MergeLogFiles_USAGE.toLocalizedString() + ": java MergeLogFiles [(directory | logFile)]+");
err.println("-dirCount n " + LocalizedStrings.MergeLogFiles_NUMBER_OF_PARENT_DIRS_TO_PRINT.toLocalizedString());
err.println("-mergeFile file " + LocalizedStrings.MergeLogFiles_FILE_IN_WHICH_TO_PUT_MERGED_LOGS.toLocalizedString());
err.println("-pids " + LocalizedStrings.MergeLogFiles_SEARCH_FOR_PIDS_IN_FILE_NAMES_AND_USE_THEM_TO_IDENTIFY_FILES.toLocalizedString());
err.println("-align " + LocalizedStrings.MergeLogFiles_ALIGN_NONTIMESTAMPED_LINES_WITH_OTHERS.toLocalizedString());
err.println("-noblanks " + LocalizedStrings.MergeLogFiles_SUPPRESS_OUTPUT_OF_BLANK_LINES.toLocalizedString());
err.println("-threaded " + LocalizedStrings.MergeLogFiles_USE_MULTITHREADING_TO_TAKE_ADVANTAGE_OF_MULTIPLE_CPUS.toLocalizedString());
// err.println("-regex pattern Case-insensitive search for a regular expression.");
// err.println(" May be used multiple times. Use Java regular ");
// err.println(" expression syntax (see java.util.regex.Pattern).");
err.println("");
err.println(LocalizedStrings.MergeLogFiles_MERGES_MULTIPLE_GEMFIRE_LOG_FILES_AND_SORTS_THEM_BY_TIMESTAMP.toLocalizedString());
err.println(LocalizedStrings.MergeLogFiles_THE_MERGED_LOG_FILE_IS_WRITTEN_TO_SYSTEM_OUT_OR_A_FILE.toLocalizedString());
err.println("");
err.println(LocalizedStrings.MergeLogFiles_IF_A_DIRECTORY_IS_SPECIFIED_ALL_LOG_FILES_IN_THAT_DIRECTORY_ARE_MERGED.toLocalizedString());
err.println("");
System.exit(1);
}
/**
* Find all of the .log files in the given directory
* @param dirName directory to search
* @return all of the .log files found (Files)
*/
static ArrayList getLogFiles(String dirName) {
ArrayList result = new ArrayList();
File dir = new File(dirName);
File names[] = FileUtil.listFiles(dir);
for (int i = 0; i < names.length; i ++) {
String n = names[i].getAbsolutePath();
if (n.endsWith(".log") || n.endsWith(".log.gz")) {
result.add(names[i]);
}
} // for
return result;
}
public static void main(String[] args) throws IOException {
File mergeFile = null;
ArrayList files = new ArrayList();
List nickNames = null;
int dirCount = 0;
boolean findPIDs = false;
boolean tabOut = false;
boolean suppressBlanks = false;
boolean multithreaded = false;
List patterns = new LinkedList();
// Parse command line
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-align")) {
tabOut = true;
}
else if (args[i].equals("-noblanks")) {
suppressBlanks = true;
}
else if (args[i].equals("-pids")) {
findPIDs = true;
}
else if (args[i].equals("-threaded")) {
multithreaded = true;
}
else if (args[i].equals("-regex")) {
if (i+1 >= args.length) {
usage("missing pattern for -regex option");
}
patterns.add(args[i+1]);
i++;
}
else if (args[i].equals("-dirCount")) {
if (++i >= args.length) {
usage(LocalizedStrings.MergeLogFiles_MISSING_NUMBER_OF_PARENT_DIRECTORIES.toLocalizedString());
}
try {
dirCount = Integer.parseInt(args[i]);
} catch (NumberFormatException ex) {
usage(LocalizedStrings.MergeLogFiles_NOT_A_NUMBER_0.toLocalizedString(args[i]));
}
} else if (args[i].equals("-mergeFile")) {
if (++i >= args.length) {
usage(LocalizedStrings.MergeLogFiles_MISSING_MERGE_FILE_NAME.toLocalizedString());
}
mergeFile = new File(args[i]);
} else {
File file = new File(args[i]);
if (!file.exists()) {
usage(LocalizedStrings.MergeLogFiles_FILE_0_DOES_NOT_EXIST.toLocalizedString(file));
}
files.add(file.getAbsoluteFile());
}
} // for
if ( files.isEmpty() ) {
usage(LocalizedStrings.MergeLogFiles_MISSING_FILENAME.toLocalizedString());
}
// Expand directory names found in list
ArrayList expandedFiles = new ArrayList();
for (int i = 0; i < files.size(); i ++) {
File f = (File)files.get(i);
String n = f.getAbsolutePath();
if (!f.exists()) {
usage(LocalizedStrings.MergeLogFiles_FILE_0_DOES_NOT_EXIST.toLocalizedString(n));
}
if (f.isFile()) {
expandedFiles.add(f);
continue;
}
if (f.isDirectory()) {
ArrayList moreFiles = getLogFiles(n);
expandedFiles.addAll(moreFiles);
continue;
}
usage(LocalizedStrings.MergeLogFiles_FILE_0_IS_NEITHER_A_FILE_NOR_A_DIRECTORY.toLocalizedString(n));
}
Collections.sort(expandedFiles);
files = expandedFiles;
// Create output stream
PrintStream ps;
if (mergeFile != null) {
ps = new PrintStream(new FileOutputStream(mergeFile), true);
} else {
ps = out;
}
PrintWriter mergedFile = new PrintWriter(ps, true);
ps.println("Merged files (count = " + expandedFiles.size() + ") input list:");
for (int i = 0; i < expandedFiles.size(); i ++) {
ps.println(" " + expandedFiles.get(i));
}
ps.println("");
if (findPIDs) {
nickNames = findPIDs(files, mergedFile);
}
InputStream[] logFiles = new InputStream[files.size()];
String[] logFileNames = new String[files.size()];
for (int i = 0; i < files.size(); i++) {
File file = (File) files.get(i);
logFiles[i] = new FileInputStream(file);
if (findPIDs && (nickNames.get(i) != null)) {
if (file.getCanonicalPath().toLowerCase().endsWith("gz")) {
logFileNames[i] = (String)nickNames.get(i)+".gz";
} else {
logFileNames[i] = (String)nickNames.get(i);
}
}
else {
StringBuffer sb = new StringBuffer();
File parent = file.getParentFile();
for (int j = 0; j < dirCount && parent != null; j++) {
String parentName = parent.getName() + "/";
// don't add dot-slash
if (parentName.equals("./")) {
parent = null;
} else {
sb.insert(0, parentName);
parent = parent.getParentFile();
}
}
sb.append(file.getName());
logFileNames[i] = sb.toString();
}
}
mergeLogFiles(logFiles, logFileNames, mergedFile, tabOut, suppressBlanks,
multithreaded, patterns);
System.exit(0); // In case we're running JProbe
}
////////////////////// Inner Classes //////////////////////
/**
* hydra log files usually have the process's PID in their path name.
* This method extracts the PID number and assigns the corresponding File
* a nickname using the PID and the position of the File in the list of those
* also having this PID.
* e.g., bgexec32414_1043.log --> 1043-1
* gemfire_1043/system.log --> 1043-2
* gemfire_1043/system_01_00.log --> 1043-3
*/
private static ArrayList findPIDs(ArrayList files, PrintWriter output) {
int pidTable[] = new int[files.size()];
int pidTableCounter[] = new int[pidTable.length];
ArrayList nickNames = new ArrayList();
char sep = File.separatorChar;
// System.out.println("findPids() invoked");
for (Iterator it=files.iterator(); it.hasNext(); ) {
File f = (File)it.next();
String name = f.getPath();
// System.out.println("considering " + name);
String slashdotslash = "" + sep + "." + sep;
int startIdx = name.lastIndexOf(slashdotslash);
// get rid of the parent directories and any /./ in the path
if (startIdx > 0) {
name = name.substring(startIdx + slashdotslash.length());
}
startIdx = name.lastIndexOf(sep);
// first see if there's a number at the end of the file's directory name
if (startIdx > 0) {
startIdx--;
char c = name.charAt(startIdx);
if ( ! ('0' <= c && c <= '9') ) {
// System.out.println("no number found in directory name");
startIdx = 0;
}
else {
// see if this is a hydra-generated test directory name, like
// parReg-0504-161349
int testIdx = startIdx-1;
while (testIdx > 0 && '0' <= name.charAt(testIdx) && name.charAt(testIdx) <= '9') {
testIdx--;
}
if (testIdx < 1 || name.charAt(testIdx) == '-') {
startIdx = 0;
}
// System.out.println("using directory name: '" + name.substring(0, startIdx+1) + "'");
}
}
// if there's no number in the directory name, use the file name
if (startIdx <= 0) {
startIdx = name.length() - 1;
if (startIdx > 3
&& name.charAt(startIdx) == 'g'
&& name.charAt(startIdx-1) == 'o'
&& name.charAt(startIdx-2) == 'l'
&& name.charAt(startIdx-3) == '.') {
startIdx -= 4;
// System.out.println("using file name: '" + name.substring(0,startIdx+1) + "'");
}
// else {
// System.out.println("could not find a PID");
// }
}
// find the string of numbers at the end of the test area and use it
// as a PID
String PID = null;
for (int i=startIdx; i>=0; i--) {
char c = name.charAt(i);
// System.out.println("charAt("+i+")="+c);
if ( ! ('0' <= c && c <= '9') ) {
if (i < (name.length()-1)) { // have a number
// there's a number - assume it's a PID if it's not zero
PID = name.substring(i+1, startIdx+1);
// System.out.println("parsing '" + PID + "'");
try {
int iPID = Integer.valueOf(PID).intValue();
if (iPID > 0) {
// System.out.println("Found PID " + iPID);
int p = 0;
// find the PID in the table of those seen so far, or assign it
// a new slot. increment the number of files for this PID and
// assign a nickname for the file
for ( ; p patterns;
//////////////////// Constructors ////////////////////
/**
* Creates a new Reader that reads from the given log
* file with the given name. Invoking this constructor will start
* this reader thread.
* @param patterns java regular expressions that an entry must match one or more of
*/
public NonThreadedReader(InputStream logFile, String logFileName,
ThreadGroup group, boolean tabOut, boolean suppressBlanks, List patterns) {
// super(group, "Reader for " + ((logFileName != null) ? logFileName : logFile.toString()));
if (logFileName.endsWith(".gz")) {
try {
this.logFile = new BufferedReader(new InputStreamReader(new GZIPInputStream(logFile)));
} catch (IOException e) {
System.err.println(logFileName + " does not appear to be in gzip format");
this.logFile = new BufferedReader(new InputStreamReader(logFile));
}
} else {
this.logFile =
new BufferedReader(new InputStreamReader(logFile));
}
this.logFileName = logFileName;
this.patterns = patterns;
// this.suppressBlanks = suppressBlanks;
// this.tabOut = tabOut;
this.parser = new LogFileParser(this.logFileName, this.logFile, tabOut, suppressBlanks);
}
/** returns the file name being read */
public String getFileName() {
return this.logFileName;
}
/** unique identifier, used for sorting instead of file name */
private int uniqueId;
/** set the unique identifier for this reader */
public void setUniqueId(int id) {
uniqueId = id;
}
/** retrieve the unique identifier for this reader */
public int getUniqueId() {
return uniqueId;
}
/**
* Peeks at the oldest log entry read from the log file, waits for
* a log entry to become available.
*
* @return null if interrupted while waiting
*/
public synchronized LogFileParser.LogEntry peek() {
while (this.nextEntry == null) {
try {
nextEntry = parser.getNextEntry();
if (nextEntry == null) {
return null;
}
if (!nextEntry.isLast() && !patternMatch(nextEntry)) {
continue;
}
}
catch (IOException ioe) {
ioe.printStackTrace(System.err);
}
}
return nextEntry;
}
/** return true if the entry matches one or more regex patterns */
private boolean patternMatch(LogFileParser.LogEntry entry) {
if (this.patterns == null || this.patterns.isEmpty()) {
return true;
}
for (Pattern p: patterns) {
if (p.matcher(entry.getContents()).matches()) {
return true;
}
}
return false;
}
/**
* Removes the old log entry read from the log file
*/
public LogFileParser.LogEntry poll() {
LogFileParser.LogEntry returnValue = null;
if (this.nextEntry != null) {
returnValue = this.nextEntry;
nextEntry = null;
}
else {
while (returnValue == null) {
try {
returnValue = parser.getNextEntry();
if (!returnValue.isLast() && !patternMatch(returnValue)) {
returnValue = null;
continue;
}
}
catch (IOException ioe) {
ioe.printStackTrace(System.err);
break;
}
}
}
return returnValue;
}
}
/**
* Thread that reads an entry from a GemFire log file and adds it a
* bounded queue. The entries are consumed by a {@link
* MergeLogFiles.Sorter}.
*/
static class ThreadedReader extends Thread implements Reader {
/** The maximum size of the entry queue */
private static int QUEUE_CAPACITY = 1000;
//////////////////// Instance Methods ////////////////////
/** The log file */
private BufferedReader logFile;
/** The name of the log file */
private String logFileName;
/** The queue containing log entries */
private BlockingQueue queue;
/** whether to suppress blank lines */
private boolean suppressBlanks;
/** whether to align non-timestamped lines with timestamped lines */
private boolean tabOut;
private List patterns;
//////////////////// Constructors ////////////////////
/**
* Creates a new Reader that reads from the given log
* file with the given name. Invoking this constructor will start
* this reader thread.
* @param patterns TODO
*
* @see #run
*/
public ThreadedReader(InputStream logFile, String logFileName,
ThreadGroup group, boolean tabOut, boolean suppressBlanks, List patterns) {
// super(group, "Reader for " + ((logFileName != null) ? logFileName : logFile.toString()));
super(group, LocalizedStrings.MergeLogFiles_LOG_FILE_READER.toLocalizedString());
this.logFile =
new BufferedReader(new InputStreamReader(logFile));
this.logFileName = logFileName;
this.queue = new LinkedBlockingQueue(QUEUE_CAPACITY);
// new UnsharedMessageQueue(QUEUE_CAPACITY,
// (75 * QUEUE_CAPACITY) / 100);
this.suppressBlanks = suppressBlanks;
this.tabOut = tabOut;
this.patterns = patterns;
this.start();
}
/** returns the file name being read */
public String getFileName() {
return this.logFileName;
}
/** unique identifier, used for sorting instead of file name */
private int uniqueId;
/** set the unique identifier for this reader */
public void setUniqueId(int id) {
uniqueId = id;
}
/** retrieve the unique identifier for this reader */
public int getUniqueId() {
return uniqueId;
}
/**
* Reads the log file and places {@link LogFileParser.LogEntry}
* objects into the queue. When it is finished, it places a
* LogEntry that whose {@link
* LogFileParser.LogEntry#isLast isLast} method will return
* true.
*/
@Override
public void run() {
LogFileParser parser =
new LogFileParser(this.logFileName, this.logFile, tabOut, suppressBlanks);
try {
while (true) {
SystemFailure.checkFailure();
LogFileParser.LogEntry entry = parser.getNextEntry();
if (entry.isLast() || patternMatch(entry)) {
this.queue.put(entry);
synchronized (this) {
this.notify();
}
}
if (entry.isLast()) {
break;
}
}
} catch (IOException ex) {
ex.printStackTrace(System.err);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
/** return true if the entry matches one or more regex patterns */
private boolean patternMatch(LogFileParser.LogEntry entry) {
if (this.patterns == null || this.patterns.isEmpty()) {
return true;
}
for (Pattern p: patterns) {
if (p.matcher(entry.getContents()).matches()) {
return true;
}
}
return false;
}
/**
* Peeks at the oldest log entry read from the log file, waits for
* a log entry to become available.
*
* @return null if interrupted while waiting
*/
public LogFileParser.LogEntry peek() {
// out.println(this.getName() + " size " + this.queue.size());
LogFileParser.LogEntry entry =
(LogFileParser.LogEntry) this.queue.peek();
if (entry == null) {
synchronized (this) {
entry = (LogFileParser.LogEntry) this.queue.peek();
while (entry == null) {
boolean interrupted = Thread.interrupted();
try {
this.wait();
entry = (LogFileParser.LogEntry) this.queue.peek();
}
catch (InterruptedException e) {
interrupted = true;
}
finally {
if (interrupted) {
Thread.currentThread().interrupt();
}
}
} // while
} // synchronized
}
return entry;
}
/**
* Removes the old log entry read from the log file
*/
public LogFileParser.LogEntry poll() {
return (LogFileParser.LogEntry) this.queue.poll();
}
}
/**
* A thread group that contains the reader threads and logs uncaught
* exceptions to standard error.
*/
static class ReaderGroup extends ThreadGroup {
/** Did an uncaught exception occur? */
private boolean exceptionOccurred;
ReaderGroup(String groupName) {
super(groupName);
this.exceptionOccurred = false;
}
@Override
public void uncaughtException(Thread t, Throwable e) {
if (e instanceof Error && SystemFailure.isJVMFailureError((Error)e)) {
SystemFailure.setFailure((Error)e); // don't throw
}
this.exceptionOccurred = true;
System.err.println(LocalizedStrings.MergeLogFiles_EXCEPTION_IN_0.toLocalizedString(t));
e.printStackTrace(System.err);
}
/**
* Returns whether or not an uncaught exception occurred in one of
* the threads in this group.
*/
public boolean exceptionOccurred() {
return this.exceptionOccurred;
}
}
/**
* Examines the {@link MergeLogFiles.Reader#peek oldest entry} in
* each log file and writes it to a merged log file.
*/
static class Sorter {
/**
* Merges the log files from a given set of
* InputStreams into a PrintWriter.
*
* @param logFiles
* The log files to be merged
* @param logFileNames
* The names of the log files to be printed in the merged log
* @param mergedFile
* Where the merged logs are printed to
* @param tabOut
* Whether to align non-timestamped lines with others
* @param suppressBlanks
* Whether to suppress output of blank lines
* @param patterns TODO
* @return Whether or not problems occurred while merging the log
* files.
*
* @throws IllegalArgumentException
* If the length of logFiles is not the same
* as the length of logFileNames
*/
public static boolean mergeLogFiles(InputStream[] logFiles,
String[] logFileNames, PrintWriter mergedFile,
boolean tabOut, boolean suppressBlanks, boolean multithreaded, List patterns) {
if (logFiles.length != logFileNames.length) {
throw new IllegalArgumentException(LocalizedStrings.MergeLogFiles_NUMBER_OF_LOG_FILES_0_IS_NOT_THE_SAME_AS_THE_NUMBER_OF_LOG_FILE_NAMES_1.toLocalizedString(new Object[] {Integer.valueOf(logFiles.length), Integer.valueOf(logFileNames.length)}));
}
List compiledPatterns = new LinkedList();
for (String pattern: patterns) {
compiledPatterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE));
}
// First start the Reader threads
ReaderGroup group = new ReaderGroup(LocalizedStrings.MergeLogFiles_READER_THREADS.toLocalizedString());
Collection readers = new ArrayList(logFiles.length);
for (int i = 0; i < logFiles.length; i++) {
if (multithreaded) {
readers.add(new ThreadedReader(logFiles[i], logFileNames[i], group,
tabOut, suppressBlanks, compiledPatterns));
}
else {
readers.add(new NonThreadedReader(logFiles[i], logFileNames[i], group,
tabOut, suppressBlanks, compiledPatterns));
}
}
// Merge the log files together
Reader lastOldest = null;
// sort readers by their next time-stamp
Set sorted = sortReaders(readers);
while (!readers.isEmpty()) {
Reader oldest = null;
Iterator sortedIt = sorted.iterator();
if (!sortedIt.hasNext()) {
break;
}
oldest = (Reader)sortedIt.next();
sortedIt.remove();
String nextReaderTimestamp = null;
Reader nextInLine = null;
if (sortedIt.hasNext()) {
nextInLine = (Reader)sortedIt.next();
nextReaderTimestamp = nextInLine.peek().getTimestamp();
}
// if we've switched to a different reader, emit a blank line
// for readability
if (oldest != lastOldest) {
mergedFile.println();
lastOldest = oldest;
}
LogFileParser.LogEntry entry = null;
// write until we hit the next file's time-stamp
do {
entry = oldest.peek();
String timestamp = entry.getTimestamp();
if (nextReaderTimestamp != null) {
if (nextReaderTimestamp.compareTo(timestamp) < 0) {
sorted.add(oldest);
entry = null;
break;
}
}
entry = oldest.poll();
entry.writeTo(mergedFile);
} while (!entry.isLast());
if (entry != null && entry.isLast()) {
readers.remove(oldest);
}
}
return group.exceptionOccurred();
}
private static Set sortReaders(Collection readers) {
Set sorted = new TreeSet(new ReaderComparator());
int uniqueId = 1;
for (Iterator iter = readers.iterator(); iter.hasNext(); ) {
Reader reader = (Reader)iter.next();
if (reader == null) {
continue;
}
reader.setUniqueId(uniqueId++);
sorted.add(reader);
}
return sorted;
}
}
protected static class ReaderComparator implements Comparator {
public int compare(Object o1, Object o2) {
Reader reader1 = (Reader)o1;
int id1 = reader1.getUniqueId();
Reader reader2 = (Reader)o2;
int id2 = reader2.getUniqueId();
LogFileParser.LogEntry entry1 = reader1.peek();
LogFileParser.LogEntry entry2 = reader2.peek();
if (entry1 == null) {
if (entry2 == null) {
if (id1 < id2) {
return -1;
}
// IDs are unique, so no need for == test
return 1;
}
// sort readers with no entries before readers with entries so they'll
// be removed quickly
return -1;
}
if (entry2 == null) {
return 1;
}
String timestamp1 = entry1.getTimestamp();
String timestamp2 = entry2.getTimestamp();
int compare = timestamp1.compareTo(timestamp2);
if (compare == 0) {
if (id1 < id2) {
return -1;
}
return 1;
}
return compare;
}
}
}