com.github.hackerwin7.mysql.tracker.mysql.dbsync.DirectLogFetcher Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of mysql-tracker Show documentation
Show all versions of mysql-tracker Show documentation
mysql-tracker is a mysql binlog dumper that fetch and parse the binlog event
The newest version!
package com.github.hackerwin7.mysql.tracker.mysql.dbsync;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.SocketTimeoutException;
import java.sql.Connection;
import java.sql.SQLException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* TODO: Document It!!
*
*
* DirectLogFetcher fetcher = new DirectLogFetcher();
* fetcher.open(conn, file, 0, 13);
*
* while (fetcher.fetch())
* {
* LogEvent mysql.dbsync.event;
* do
* {
* mysql.dbsync.event = decoder.decode(fetcher, context);
*
* // process log mysql.dbsync.event.
* }
* while (mysql.dbsync.event != null);
* }
* // connection closed.
*
*
* @author Changyuan.lh
* @version 1.0
*/
public final class DirectLogFetcher extends LogFetcher
{
protected static final Log logger = LogFactory.getLog(DirectLogFetcher.class);
/** Command to dump binlog */
public static final byte COM_BINLOG_DUMP = 18;
/** Packet header sizes */
public static final int NET_HEADER_SIZE = 4;
public static final int SQLSTATE_LENGTH = 5;
/** Packet offsets */
public static final int PACKET_LEN_OFFSET = 0;
public static final int PACKET_SEQ_OFFSET = 3;
/** Maximum packet length */
public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
/** BINLOG_DUMP options */
public static final int BINLOG_DUMP_NON_BLOCK = 1;
public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
private Connection conn;
private OutputStream mysqlOutput;
private InputStream mysqlInput;
public DirectLogFetcher()
{
super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
}
public DirectLogFetcher(final int initialCapacity)
{
super(initialCapacity, DEFAULT_GROWTH_FACTOR);
}
public DirectLogFetcher(final int initialCapacity, final float growthFactor)
{
super(initialCapacity, growthFactor);
}
private static final Object unwrapConnection(Object conn, Class connClazz)
throws IOException
{
while (!connClazz.isInstance(conn))
{
try
{
Class connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
if (connProxy.isInstance(conn))
{
conn = invokeMethod(conn, connProxy, "getTargetConnection");
continue;
}
}
catch (ClassNotFoundException e)
{
// org.springframework.jdbc.datasource.ConnectionProxy not found.
}
try
{
Class connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
if (connProxy.isInstance(conn))
{
conn = getDeclaredField(conn, connProxy, "_conn");
continue;
}
}
catch (ClassNotFoundException e)
{
// org.apache.commons.dbcp.DelegatingConnection not found.
}
try
{
if (conn instanceof java.sql.Wrapper)
{
Class connIface = Class.forName("com.mysql.jdbc.Connection");
conn = ((java.sql.Wrapper) conn).unwrap(connIface);
continue;
}
}
catch (ClassNotFoundException e)
{
// com.mysql.jdbc.Connection not found.
}
catch (SQLException e)
{
logger.warn("Unwrap " + conn.getClass().getName() + " to "
+ connClazz.getName() + " failed: " + e.getMessage(), e);
}
return null;
}
return conn;
}
private static final Object invokeMethod(Object obj, Class objClazz,
String name)
{
try
{
Method method = objClazz.getMethod(name, (Class[]) null);
return method.invoke(obj, (Object[]) null);
}
catch (NoSuchMethodException e)
{
throw new IllegalArgumentException("No such method: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
catch (IllegalAccessException e)
{
throw new IllegalArgumentException("Cannot invoke method: \'"
+ name + "\' @ " + objClazz.getName(), e);
}
catch (InvocationTargetException e)
{
throw new IllegalArgumentException("Invoke method failed: \'"
+ name + "\' @ " + objClazz.getName(),
e.getTargetException());
}
}
private static final Object getDeclaredField(Object obj, Class objClazz,
String name)
{
try
{
Field field = objClazz.getDeclaredField(name);
field.setAccessible(true);
return field.get(obj);
}
catch (NoSuchFieldException e)
{
throw new IllegalArgumentException("No such field: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
catch (IllegalAccessException e)
{
throw new IllegalArgumentException("Cannot get field: \'" + name
+ "\' @ " + objClazz.getName(), e);
}
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final int serverId)
throws IOException
{
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final int serverId,
boolean nonBlocking) throws IOException
{
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, final long filePosition,
final int serverId) throws IOException
{
open(conn, fileName, filePosition, serverId, false);
}
/**
* Connect MySQL master to fetch binlog.
*/
public void open(Connection conn, String fileName, long filePosition,
final int serverId, boolean nonBlocking) throws IOException
{
try
{
this.conn = conn;
Class connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
Object unwrapConn = unwrapConnection(conn, connClazz);
if (unwrapConn == null)
{
throw new IOException("Unable to unwrap "
+ conn.getClass().getName()
+ " to com.mysql.jdbc.ConnectionImpl");
}
// Get underlying IO streams for network communications.
Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
if (connIo == null)
{
throw new IOException("Get null field:"
+ conn.getClass().getName() + "#io");
}
mysqlOutput = (OutputStream) getDeclaredField(connIo,
connIo.getClass(), "mysqlOutput");
mysqlInput = (InputStream) getDeclaredField(connIo,
connIo.getClass(), "mysqlInput");
if (filePosition == 0)
filePosition = BIN_LOG_HEADER_SIZE;
sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
position = 0;
}
catch (IOException e)
{
close(); /* Do cleanup */
logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
+ ", position = " + filePosition);
throw e;
}
catch (ClassNotFoundException e)
{
close(); /* Do cleanup */
throw new IOException(
"Unable to load com.mysql.jdbc.ConnectionImpl", e);
}
}
/**
* Put a byte in the buffer.
*
* @param b the byte to put in the buffer
*/
protected final void putByte(byte b)
{
ensureCapacity(position + 1);
buffer[position++] = b;
}
/**
* Put 16-bit integer in the buffer.
*
* @param i16 the integer to put in the buffer
*/
protected final void putInt16(int i16)
{
ensureCapacity(position + 2);
byte[] buf = buffer;
buf[position++] = (byte) (i16 & 0xff);
buf[position++] = (byte) (i16 >>> 8);
}
/**
* Put 32-bit integer in the buffer.
*
* @param i32 the integer to put in the buffer
*/
protected final void putInt32(long i32)
{
ensureCapacity(position + 4);
byte[] buf = buffer;
buf[position++] = (byte) (i32 & 0xff);
buf[position++] = (byte) (i32 >>> 8);
buf[position++] = (byte) (i32 >>> 16);
buf[position++] = (byte) (i32 >>> 24);
}
/**
* Put a string in the buffer.
*
* @param s the value to put in the buffer
*/
protected final void putString(String s)
{
ensureCapacity(position + (s.length() * 2) + 1);
System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
position += s.length();
buffer[position++] = 0;
}
protected final void sendBinlogDump(String fileName,
final long filePosition, final int serverId, boolean nonBlocking)
throws IOException
{
position = NET_HEADER_SIZE;
putByte(COM_BINLOG_DUMP);
putInt32(filePosition);
int binlog_flags = nonBlocking ? BINLOG_DUMP_NON_BLOCK : 0;
binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
putInt16(binlog_flags); // binlog_flags
putInt32(serverId); // slave's server-id
putString(fileName);
final byte[] buf = buffer;
final int len = position - NET_HEADER_SIZE;
buf[0] = (byte) (len & 0xff);
buf[1] = (byte) (len >>> 8);
buf[2] = (byte) (len >>> 16);
mysqlOutput.write(buffer, 0, position);
mysqlOutput.flush();
}
/**
* {@inheritDoc}
*
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
*/
public boolean fetch() throws IOException
{
try
{
// Fetching packet header from input.
if (!fetch0(0, NET_HEADER_SIZE))
{
logger.warn("Reached end of input stream while fetching header");
return false;
}
// Fetching the first packet(may a multi-packet).
int netlen = getUint24(PACKET_LEN_OFFSET);
int netnum = getUint8(PACKET_SEQ_OFFSET);
if (!fetch0(NET_HEADER_SIZE, netlen))
{
logger.warn("Reached end of input stream: packet #" + netnum
+ ", len = " + netlen);
return false;
}
// Detecting error code.
final int mark = getUint8(NET_HEADER_SIZE);
if (mark != 0)
{
if (mark == 255) // error from master
{
// Indicates an error, for example trying to fetch from wrong
// binlog position.
position = NET_HEADER_SIZE + 1;
final int errno = getInt16();
String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
String errmsg = getFixString(limit - position);
throw new IOException("Received error packet:"
+ " errno = " + errno + ", sqlstate = " + sqlstate
+ " errmsg = " + errmsg);
}
else if (mark == 254)
{
// Indicates end of stream. It's not clear when this would
// be sent.
logger.warn("Received EOF packet from server, apparent"
+ " master disconnected.");
return false;
}
else
{
// Should not happen.
throw new IOException("Unexpected response " + mark
+ " while fetching binlog: packet #" + netnum
+ ", len = " + netlen);
}
}
// The first packet is a multi-packet, concatenate the packets.
while (netlen == MAX_PACKET_LENGTH)
{
if (!fetch0(0, NET_HEADER_SIZE))
{
logger.warn("Reached end of input stream while fetching header");
return false;
}
netlen = getUint24(PACKET_LEN_OFFSET);
netnum = getUint8(PACKET_SEQ_OFFSET);
if (!fetch0(limit, netlen))
{
logger.warn("Reached end of input stream: packet #"
+ netnum + ", len = " + netlen);
return false;
}
}
// Preparing buffer variables to decoding.
origin = NET_HEADER_SIZE + 1;
position = origin;
limit -= origin;
return true;
}
catch (SocketTimeoutException e)
{
close(); /* Do cleanup */
logger.error("Socket timeout expired, closing connection", e);
throw e;
}
catch (InterruptedIOException e)
{
close(); /* Do cleanup */
logger.warn("I/O interrupted while reading from client socket", e);
throw e;
}
catch (IOException e)
{
close(); /* Do cleanup */
logger.error("I/O error while reading from client socket", e);
throw e;
}
}
private final boolean fetch0(final int off, final int len)
throws IOException
{
ensureCapacity(off + len);
for (int count, n = 0; n < len; n += count)
{
if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
{
// Reached end of input stream
return false;
}
}
if (limit < off + len)
limit = off + len;
return true;
}
/**
* {@inheritDoc}
*
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
*/
public void close() throws IOException
{
try
{
if (conn != null)
conn.close();
conn = null;
mysqlInput = null;
mysqlOutput = null;
}
catch (SQLException e)
{
logger.warn("Unable to close connection", e);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy