com.gemstone.gemfire.internal.sequencelog.io.GemfireLogConverter Maven / Gradle / Ivy

/*
 * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */
package com.gemstone.gemfire.internal.sequencelog.io;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.gemstone.gemfire.internal.LogWriterImpl;
import com.gemstone.gemfire.internal.sequencelog.GraphType;
import com.gemstone.gemfire.internal.sequencelog.Transition;

/**
 * A utility to convert existing gemfire logs
 * into a graph logger file. This will help
 * me debug hydra runs more quickly. 
 * @author dsmith
 *
 */
public class GemfireLogConverter {

  /**
   * 
   */
  private static final Pattern DATE_PATTERN = Pattern.compile("(\\d\\d\\d\\d)/(\\d\\d)/(\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d).(\\d\\d\\d)");
  private static final Pattern ALL = Pattern.compile(".*");
  private static ArrayList tests = buildTests();
  
  
  public static void convertFiles(OutputStream output, File[] files) throws IOException {
    Context context = new Context();
    context.appender = new OutputStreamAppender(output);
    for(File file : files) {
      context.currentMember = null;
      BufferedReader reader = new BufferedReader(new FileReader(file));
      String line;
      int linecount = 1;
      while((line = reader.readLine()) != null) {
        linecount++;
        for(Test test: tests) {
          if(test.apply(context, line)) {
            //go to the next line
            break;
          }
        }
      }
    }
  }
  
  /**
   * @param args
   * @throws IOException 
   */
  public static void main(String[] args) throws IOException {

    if(args.length == 0) {
      usage();
      System.exit(1);
    }

    File outputFile = new File(args[0]);
    
    File[] files = new File[args.length - 1];
    for(int i = 1; i < args.length; i++) {
      String fileName  = args[i];
      File file = new File(fileName);
      files[i-1] = file;
    }
    
    convertFiles(new BufferedOutputStream(new FileOutputStream(outputFile)), files);
  }
  
  private static ArrayList buildTests()  {
    ArrayList tests = new ArrayList();
    
    //Membership level events
    //[info 2011/04/27 00:43:31.020 PDT dataStoregemfire5_hs20e_16643  tid=0x17] Starting DistributionManager hs20e(16643):6872/49575.
    tests.add(new Test("Starting DistributionManager (.*)\\.") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
        String member = matcher.group(1);
        context.currentMember = member;
        context.appender.write(new Transition(timestamp, GraphType.MEMBER, "member", "start", "running", member, member));
      }
    });
    
  //[info 2011/04/27 00:49:39.254 PDT dataStoregemfire5_hs20e_16643  tid=0x71] Member at hs20e(16640):50554/49570 unexpectedly left the distributed cache: departed JGroups view
    tests.add(new Test("Member at (.*) unexpectedly left the distributed cache") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
        String member = matcher.group(1);
        context.appender.write(new Transition(timestamp, GraphType.MEMBER, "member", "crashed", "destroyed", member, member));
        context.appender.write(new Transition(timestamp, GraphType.REGION, ALL, "crash", "destroyed", member, member));
        context.appender.write(new Transition(timestamp, GraphType.KEY, ALL, "crash", "destroyed", member, member));
        context.appender.write(new Transition(timestamp, GraphType.MESSAGE, ALL, "crash", "destroyed", member, member));
      }});
    
    //[info 2011/04/27 00:49:33.154 PDT dataStoregemfire5_hs20e_16643  tid=0x18] VM is exiting - shutting down distributed system
    tests.add(new Test("VM is exiting - shutting down distributed system") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
      String member = context.currentMember;
      context.appender.write(new Transition(timestamp, GraphType.MEMBER, "member", "stop", "destroyed", member, member));
      context.appender.write(new Transition(timestamp, GraphType.REGION, ALL, "destroy", "destroyed", member, member));
      context.appender.write(new Transition(timestamp, GraphType.KEY, ALL, "destroy", "destroyed", member, member));
      context.appender.write(new Transition(timestamp, GraphType.MESSAGE, ALL, "destroy", "destroyed", member, member));
    }});
    //[info 2011/04/27 00:49:33.104 PDT dataStoregemfire3_hs20e_16635  tid=0xaf] GemFireCache[id = 746885; isClosing = true; created = Wed Apr 27 00:43:41 PDT 2011; server = false; copyOnRead = false; lockLease = 120; lockTimeout = 60]: Now closing.
      tests.add(new Test("GemFireCache .*Now closing") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
        String member = context.currentMember;
        context.appender.write(new Transition(timestamp, GraphType.MEMBER, "member", "stop", "destroyed", member, member));
        context.appender.write(new Transition(timestamp, GraphType.REGION, ALL, "destroy", "destroyed", member, member));
        context.appender.write(new Transition(timestamp, GraphType.KEY, ALL, "destroy", "destroyed", member, member));
        context.appender.write(new Transition(timestamp, GraphType.MESSAGE, ALL, "destroy", "destroyed", member, member));
      }
    });
    //[info 2011/04/27 00:49:45.152 PDT dataStoregemfire3_hs20e_16635  tid=0xaf] Shutting down DistributionManager hs20e(16635):54340/49574. 
      tests.add(new Test("Shutting down DistributionManager") {
        public void process(Context context, long timestamp, Matcher matcher) throws IOException {
          String member = context.currentMember;
          context.appender.write(new Transition(timestamp, GraphType.MEMBER, "member", "stop", "destroyed", member, member));
          context.appender.write(new Transition(timestamp, GraphType.REGION, ALL, "destroy", "destroyed", member, member));
          context.appender.write(new Transition(timestamp, GraphType.KEY, ALL, "destroy", "destroyed", member, member));
          context.appender.write(new Transition(timestamp, GraphType.MESSAGE, ALL, "destroy", "destroyed", member, member));
        }
      });
      


    
    //Region level events
    //[info 2011/04/27 00:44:39.811 PDT dataStoregemfire5_hs20e_16643  tid=0x17] initializing region __PR
    tests.add(new Test("initializing region (.*)") {
    public void process(Context context, long timestamp, Matcher matcher) throws IOException {
      String region = matcher.group(1);
      context.appender.write(new Transition(timestamp, GraphType.REGION, region, "create", "created", context.currentMember, context.currentMember));
    }
    });
    //[info 2011/05/09 00:12:19.789 GMT+05:30 peergemfire_2_1_pcc40_11040  tid=0x18] Partitioned Region /Region_GlobalVillage is created with prId=1
    tests.add(new Test("Partitioned Region /(.*) is created with") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
        String region = matcher.group(1);
        context.appender.write(new Transition(timestamp, GraphType.REGION, region, "create", "created", context.currentMember, context.currentMember));
      }
      });
    //[info 2011/04/27 00:48:31.077 PDT dataStoregemfire5_hs20e_16643  tid=0x8b] Region _B__partitionedRegion_1 requesting initial image from hs20e(16650):13093/49571
    tests.add(new Test(" Region (.*) requesting initial image from (.*)") {
    public void process(Context context, long timestamp, Matcher matcher) throws IOException {
      String region = matcher.group(1);
      String source = matcher.group(2);
      context.appender.write(new Transition(timestamp, GraphType.REGION, region, "GII", "created", source, context.currentMember));
    }
    });
    //[info 2011/04/27 00:48:31.088 PDT dataStoregemfire5_hs20e_16643  tid=0x8b] Region _B__partitionedRegion_1 initialized persistent id: hs20e.gemstone.com/10.138.45.5:/export/hs20e2/users/lynn/class_versioning_dev_Mar11_5/parRegPdxInstance/2011-04-26-14-46-03/serialParRegHAPersistPdx-0427-003656/vm_5_dataStore5_disk_1 created at timestamp 1303890276172 version 0 diskStoreId af159596-d932-4f69-8f78-b8914f88fab1 with data from hs20e(16650):13093/49571.
    tests.add(new Test(" Region (.*) initialized persistent id: .* diskStoreId (.*) with data from ") {
    public void process(Context context, long timestamp, Matcher matcher) throws IOException {
      String region = matcher.group(1);
      String store = matcher.group(2);
      context.appender.write(new Transition(timestamp, GraphType.REGION, region, "persist", "persisted", context.currentMember, store));
    }
    });
    
    //[info 2011/04/27 00:48:31.243 PDT dataStoregemfire5_hs20e_16643  tid=0x8b] Region /__PR/_B__partitionedRegion_2 was created on this member with the persistent id hs20e.gemstone.com/10.138.45.5:/export/hs20e2/users/lynn/class_versioning_dev_Mar11_5/parRegPdxInstance/2011-04-26-14-46-03/serialParRegHAPersistPdx-0427-003656/vm_5_dataStore5_disk_1 created at timestamp 1303890276172 version 0 diskStoreId af159596-d932-4f69-8f78-b8914f88fab1.
    //TODO, need to clean this up. We're logging full paths with this message, but only region
    //name with other messages.
    tests.add(new Test(" Region /__PR/(.*) was created on this member with the persistent id .* diskStoreId (.*)\\.") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
        String region = matcher.group(1);
        String store = matcher.group(2);
        context.appender.write(new Transition(timestamp, GraphType.REGION, region, "persist", "persisted", context.currentMember, store));
      }
    });
    
    //[info 2011/04/27 00:54:04.413 PDT dataStoregemfire3_hs20e_19190  tid=0x7b] Region /__PR/_B__partitionedRegion_25 recovered from the local disk. Old persistent ID: /10.138.45.5:/export/hs20e2/users/lynn/class_versioning_dev_Mar11_5/parRegPdxInstance/2011-04-26-14-46-03/serialParRegHAPersistPdx-0427-003656/vm_3_dataStore3_disk_1 created at timestamp 1303890276172 version 0 diskStoreId 0b21e233-e1a9-4b2e-9190-13a191219760, new persistent ID hs20e.gemstone.com/10.138.45.5:/export/hs20e2/users/lynn/class_versioning_dev_Mar11_5/parRegPdxInstance/2011-04-26-14-46-03/serialParRegHAPersistPdx-0427-003656/vm_3_dataStore3_disk_1 created at timestamp 1303890815750 version 0 diskStoreId 0b21e233-e1a9-4b2e-9190-13a191219760
    tests.add(new Test(" Region /__PR/(.*) recovered from the local disk. .* new persistent ID.*diskStoreId (.*)") {
      public void process(Context context, long timestamp, Matcher matcher) throws IOException {
        String region = matcher.group(1);
        String store = matcher.group(2);
        context.appender.write(new Transition(timestamp, GraphType.REGION, region, "recover", "created", store, context.currentMember));
      }
    });

    return tests;
  }
  
  private static long extractDate(String line) {
    Matcher matcher = DATE_PATTERN.matcher(line);
    if(!matcher.find()) {
      throw new IllegalStateException("Could not parse timestamp from line " + line);
    }
    matcher.start();
    SimpleDateFormat formatter = new SimpleDateFormat(LogWriterImpl.FORMAT);
    Date date = formatter.parse(line, new ParsePosition(matcher.start()));
    return date.getTime();
  }

  private static void usage() {
    System.err.println("Usage java com.gemstone.sequence.logconversion.ConvertGemfireLogs outputfile [log_file]+");
    
  }
  
  private static abstract class Test {
    private Pattern pattern;
    
    public Test(String pattern) {
      this.pattern = Pattern.compile(pattern);
    }
    
    public boolean apply(Context context, String line) throws IOException {
      Matcher matcher = pattern.matcher(line);
      if(matcher.find()) {
        long timestamp = extractDate(line);
        process(context, timestamp, matcher);
        return true;
      }
      return false;
    }
    
    public abstract void process(Context context, long timestamp, Matcher matcher) throws IOException;
  }
  
  private static class Context {
    private OutputStreamAppender appender;
    private String currentMember;
  }
}