org.tentackle.dbms.MpxConnectionManager Maven / Gradle / Ivy
Show all versions of tentackle-database Show documentation
/**
* Tentackle - http://www.tentackle.org
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package org.tentackle.dbms;
import org.tentackle.log.Logger;
import org.tentackle.log.LoggerFactory;
import org.tentackle.misc.FormatHelper;
import org.tentackle.session.PersistenceException;
import org.tentackle.sql.BackendInfo;
import java.util.Date;
/**
* Multiplexing connection manager.
*
* A connection manager for applications with a large number of session instances,
* e.g. application servers. The manager will multiplex N session instances against
* M connections, allowing N > M. This is not to be mixed up with session-pooling, as connection
* multiplexing is completely transparent to the application whereas pooling requires
* some handling to request and release a session.
*
* @author harald
*/
public class MpxConnectionManager extends DefaultConnectionManager {
private static final Logger LOGGER = LoggerFactory.getLogger(MpxConnectionManager.class);
private static final int MAX_LOOP = 100; // max. number of sleep-with-retry if no connection available
private static final int BASE_MS = 50; // base milliseconds to sleep before next try
private static final long MUTEX_MS = 1000; // mutex wait timeout
// args
/** the backend info. */
final protected BackendInfo backendInfo;
/** increment size. */
final protected int incSize;
/** minimum size. */
final protected int minSize;
// local
/** free list for unattached connections. */
protected int[] unConList;
/** number of entries in unConList. */
protected int unConCount;
/** true if initialized. */
protected boolean initialized;
// connect thread
private boolean shutdownRequested; // true if shutdown procedure initiated
private Thread connectThread; // thread to bring up connections
private final Object connectGoMutex; // mutex to trigger connect thread to go for more connections
private final Object connectDoneMutex; // mutex to signal waiting threads that connections were brought up
private int conRequestCount; // number of new connections requested
private int warnLogSize; // number of connections when to start WARNING logging
private int infoLogSize; // number of connections when to start INFO logging
/**
* Creates a new connection manager.
*
* @param name the name of the connection manager
* @param backendInfo the backend info to use for creating connections
* @param maxSessions the maximum number of Db instances, 0 = no limit
* @param idOffset the offset for connection ids (> 0)
* @param iniSize the initial size of the connection pool
* @param incSize the number of connections to add if all in use
* @param minSize the minimum number of connections
* @param maxSize the maximum number of connections
* @param minMinutes minimum minutes a connection should be used
* @param maxMinutes maximum minutes a connection should be used
*/
public MpxConnectionManager(String name, BackendInfo backendInfo, int maxSessions, int idOffset, int iniSize, int incSize,
int minSize, int maxSize, int minMinutes, int maxMinutes) {
super(name, iniSize, maxSize, idOffset, minMinutes, maxMinutes);
if (minSize > maxSize) {
throw new IllegalArgumentException("maxSize must be >= minSize");
}
if (maxSessions > 0 && maxSessions < iniSize) {
throw new IllegalArgumentException("maxSessions must be 0 or >= iniSize");
}
this.backendInfo = backendInfo;
this.maxDbSize = maxSessions;
this.incSize = incSize;
this.minSize = minSize;
infoLogSize = maxConSize / 4;
warnLogSize = maxConSize / 2;
// setup initial connections
unConList = new int[iniSize];
// create and start the connect thread
connectGoMutex = new Object();
connectDoneMutex = new Object();
// connections are brought up in an extra thread
connectThread = new ConnectionThread(getName() + " Connection Management Thread");
}
/**
* Creates a connection manager with reasonable values for most servers.
*
* Maximum of 1000 Db instances (500 clients).
* Start with 8 connections.
* Add 2 connections at a time if all connections are in use.
* Don't drop below 4 connections.
* Maximum of 40 connections, i.e. concurrent db operations (i.e. info above 10, warning above 20)
* Within 12 to 36h (approx. once a day), close and reopen connections
* (allows updates of the database servers's QEPs, prepared statements cache, etc...).
*
* @param backendInfo backend info to use for creating connections (may be open or closed)
* @param idOffset the offset for connection ids (> 0)
*/
public MpxConnectionManager(BackendInfo backendInfo, int idOffset) {
this("mpx-mgr", backendInfo, 1000, idOffset, 8, 2, 4, 40, 720, 2160);
}
/**
* Shuts down this connection manager.
* All connections are closed and the threads stopped.
* Application servers should invoke this method when shut down.
*/
@Override
public synchronized void shutdown() {
shutdownRequested = true;
if (connectThread.isAlive()) {
connectThread.interrupt();
try {
connectThread.join(); // wait until connect thread terminates
}
catch (InterruptedException ex) {
LOGGER.warning(this + ": stopping the connect thread failed", ex);
}
}
// close all connections
super.shutdown();
unConCount = 0;
}
/**
* Gets the number of connections when to start info logging.
*
* The default is 1/4th of the maximum number of connections.
*
* @return the info level
*/
public int getInfoLogSize() {
return infoLogSize;
}
/**
* Sets the number of connections when to start info logging.
*
* @param infoLogSize the info level
*/
public void setInfoLogSize(int infoLogSize) {
this.infoLogSize = infoLogSize;
}
/**
* Gets the number of connections when to start warning logging.
*
* The default is half of the maximum number of connections.
*
* @return the warning level
*/
public int getWarnLogSize() {
return warnLogSize;
}
/**
* Sets the number of connections when to start warning logging.
*
* @param warnLogSize the warning level
*/
public void setWarnLogSize(int warnLogSize) {
this.warnLogSize = warnLogSize;
}
@Override
public synchronized int login(Db session) {
if (!initialized) {
createConnections(iniSize);
connectThread.start();
initialized = true;
}
int id = addDb(session) + idOffset;
LOGGER.fine("{0} logged into {1}, id={2}", session, this, id);
return id;
}
@Override
public synchronized void logout(Db session) {
assertSessionBelongsToMe(session);
int index = getIndexFromSessionId(session);
removeDb(index);
LOGGER.fine("{0} logged out from {1}, id={2}", session, this, session.getSessionId());
try {
// check that connection is not attached, if so, detach it!
ManagedConnection con = session.getConnection();
if (con != null) {
con.forceDetached();
if (con.isDead()) {
// remove connection if dead and check to reopen
cleanupDeadConnection(con);
/**
* Client crashed due to a dead server connection.
* This is a strong indicator that the database is facing some severe
* problems. Because we don't know how many unattached connections are down
* we will probe them here. This is a little time consuming but better
* than waiting for other clients to crash.
*/
int[] newUnConList = new int[unConList.length];
int newUnConCount = 0;
for (int i = 0; i < unConCount; i++) {
ManagedConnection c = conList[unConList[i]];
c.verifyConnection();
if (c.isDead()) {
cleanupDeadConnection(c);
}
else {
newUnConList[newUnConCount++] = c.getIndex();
}
}
unConList = newUnConList;
unConCount = newUnConCount;
// open any missing connections
reopenConnections();
}
else {
// add connection to freelist
pushUnattached(con.getIndex());
}
}
}
finally {
session.clearSessionId();
}
}
@Override
public void attach(Db session) {
assertSessionBelongsToMe(session);
int loopCount = 0; // number of retries so far
for (;;) { // until attached
synchronized(this) {
ManagedConnection con = session.getConnection();
if (con != null) {
// already attached (e.g. within transaction)
con.attachSession(session); // just increment the attach count
break;
}
// find an unattached connection
int conIndex = popUnattached();
if (conIndex >= 0) {
con = conList[conIndex];
if (con.getIndex() != conIndex) {
throw new PersistenceException(session,
"connection " + con + " has wrong index " + con.getIndex() + ", expected " + conIndex);
}
/**
* If connection is detached for a long time (minMinutes/2), verify the connection first.
* This is mainly for databases like MySQL that close connections after a certain time
* of inactivity. For those databases it is recommended to set the idle-timeout to minTime.
*/
if (System.currentTimeMillis() - con.getDetachedSince() > minMinutes * 30000L) {
/**
* Although this will block all threads from attaching/detaching a connection,
* this is the preferred solution compared to an extra "verification thread",
* because that thread would need some synchronisation on the connection object
* to prevent attachment while running the verification statement.
* However, verification happens very rarely, so the performance would suffer
* more from permanent and mostly unnecessary syncs than rare blocking verifications --
* hopefully... ;)
*/
con.verifyConnection();
}
if (con.isDead()) {
cleanupDeadConnection(con);
continue;
}
// found: attach!
con.attachSession(session);
break;
}
}
/**
* continue unsyncd (allow other threads to use the connection manager).
* In order not to stop all clients for the duration of establishing new connections
* this is done in the connecThread. We will just trigger the thread and wait.
*/
boolean request = false;
boolean runningOutOfConnections = false;
synchronized(this) {
if (conRequestCount == 0) {
// no request running: request more connections
conRequestCount = incSize;
request = true;
}
else if (conRequestCount < 0) {
// max connections exhausted
conRequestCount = 0;
loopCount++;
if (loopCount > MAX_LOOP) { // we tried hard, but it took too long, sorry :-(
// this will probably close the client unfriendly, but what else can we do?
throw new PersistenceException(this + ": max. number of concurrent connections in use: " + maxConSize);
}
runningOutOfConnections = true;
}
// else: some request is already running
}
// unsyncd again...
if (runningOutOfConnections) {
try {
// sleep a randomly with increasing time according to loopcount
int baseMs = loopCount * BASE_MS;
long ms = baseMs + random.nextInt(baseMs);
LOGGER.warning("{0}: Running out of connections! Putting {1} to sleep for {2} ms, attempt {3}",
this, session, ms, loopCount);
Thread.sleep(ms);
}
catch (InterruptedException ex) {
// just start over and try again
}
}
else {
if (request) {
// start a new request for bringing up connections
synchronized(connectGoMutex) {
connectGoMutex.notifyAll();
}
}
// wait for connections to be brought up
synchronized(connectDoneMutex) {
try {
connectDoneMutex.wait(MUTEX_MS); // wait max 1 second
}
catch (InterruptedException ex) {
// just start over and try again
}
}
}
}
}
@Override
public void detach(Db session) {
assertSessionBelongsToMe(session);
boolean closeIt = false;
ManagedConnection con;
synchronized (this) {
con = session.getConnection();
if (con == null) {
throw new PersistenceException(this + ": no connection attached to " + session);
}
con.detachSession(session);
if (!con.isAttached()) {
if (con.isClosed()) {
LOGGER.warning("detached session {0}: connection {1} already closed!", session, con);
int ndx = con.getIndex();
if (ndx >= 0) {
removeConnection(ndx);
}
}
else {
if (!con.getAutoCommit()) {
/**
* The connection has been detached even though a transaction is still running!
* This is bad and indicates some MT problem or that some thread keeps on using
* a Db even if there has been an exception causing a rollback.
* Anyway: mark the connection dead so that it is closed and cleaned up.
* The Db is also closed to prevent any further use.
*/
LOGGER.severe("connection {0} detached while still in transaction: *** MARKED DEAD ***", con);
con.setDead(true);
}
if (con.isDead()) {
cleanupDeadConnection(con);
try {
// close the db to make it unusable anymore
session.close();
}
catch (RuntimeException rex) {
LOGGER.warning("closing Db failed", rex);
}
}
else if (con.isExpired()) {
// connection time elapsed: close it
removeConnection(con.getIndex());
closeIt = true; // will close below
}
else {
// add unattached connection
pushUnattached(con.getIndex());
}
}
}
}
// unsyncd...
if (closeIt) {
LOGGER.info("{0}: closing connection {1}, open since {2}",
this, con, FormatHelper.formatTimestamp(new Date(con.getEstablishedSince())));
try {
con.close();
}
catch (RuntimeException rex) {
LOGGER.warning("closing old connection failed", rex);
}
reopenConnections();
}
}
@Override
public synchronized void forceDetach(Db session) {
assertSessionBelongsToMe(session);
ManagedConnection con = session.getConnection();
if (con != null) {
con.forceDetached();
pushUnattached(con.getIndex());
}
}
/**
* Adds the index of an unused connection to the freelist.
*
* @param index the index of the connection in the connections list
*/
protected void pushUnattached(int index) {
if (unConCount >= unConList.length) {
// list is full, enlarge it
int[] nUnConList = new int[unConList.length << 1];
System.arraycopy(unConList, 0, nUnConList, 0, unConList.length);
for (int j=unConList.length; j < nUnConList.length; j++) {
nUnConList[j] = -1;
}
unConList = nUnConList;
}
unConList[unConCount++] = index; // add to freelist
}
/**
* Gets a connection from the unattached freelist.
*
* @return the index to the connections list, -1 if no more unattached found
*/
protected int popUnattached() {
return unConCount > 0 ? unConList[--unConCount] : -1;
}
/**
* Create spare connections.
*
* @param count the number of connections to create
* @return the number of connections created
*/
protected int createConnections(int count) {
int conSize;
synchronized(this) {
conSize = getNumConnections();
}
if (conSize + count < minSize) {
// at least to minsize
count = minSize - conSize;
}
// align count before we run into DbRuntimeException in addConnection()
if (maxConSize > 0) {
if (conSize + count > maxConSize) {
count = maxConSize - conSize;
}
if (count == 0) {
LOGGER.severe(this + ": *** maximum number of connections reached: " + maxConSize + " ***");
}
else {
if (conSize + count > getInfoLogSize()) {
String msg = this + ": increasing number of connections by " + count + " to " + (conSize + count) + " of " +
maxConSize + " (unattached=" + unConCount + ")";
if (conSize + count > getWarnLogSize()) {
// Half of the connections concurrently in use in most cases tells us that the db-server
// is reaching its limits, i.e. the operations take too long. Or maxConSize is simply too low.
// Log attached connections.
StringBuilder buf = new StringBuilder(msg).append("\n\n attached connections:");
for (ManagedConnection con: ManagedConnection.getManagedConnections()) {
if (con.getManager() == this && con.isAttached()) {
buf.append("\n\n ").append(con.toDiagnosticString());
}
}
LOGGER.warning(buf.toString());
}
else {
// pre warning
LOGGER.info(msg);
}
}
}
}
for (int i=0; i < count; i++) {
ManagedConnection con = createConnection(backendInfo); // may take some time dep. on the db backend...
synchronized(this) {
pushUnattached(addConnection(con)); // add to established connections and unattached freelist
}
}
return count;
}
/**
* closes a dead connection and removes it from the
* connection list.
*/
private void cleanupDeadConnection(ManagedConnection con) {
// marked dead: close it (hard close) and remove it from connection list
removeConnection(con.getIndex());
try {
LOGGER.warning(this + ": closing **DEAD** connection " + con);
con.close();
}
catch (PersistenceException ex) {
LOGGER.warning("closing **DEAD** connection failed -> ignored", ex);
}
}
/**
* checks whether the number of connections dropped below minSize.
* If so, reopen missing connections.
*/
private void reopenConnections() {
boolean reopen = false;
synchronized(this) {
if (conRequestCount == 0) {
// no connect running
int num = minSize - getNumConnections();
if (num > 0) {
// we dropped below minSize: open connection(s)
reopen = true;
conRequestCount = num;
}
}
}
if (reopen) {
synchronized(connectGoMutex) {
connectGoMutex.notifyAll(); // fire connect, but don't wait for completion
}
}
}
/**
* The connection management thread.
*/
private class ConnectionThread extends Thread {
public ConnectionThread(String name) {
super(name);
setDaemon(true);
setPriority(MAX_PRIORITY);
}
@Override
public void run() {
LOGGER.info("{0} started", getName());
while (!shutdownRequested) {
synchronized (connectGoMutex) { // syncs conRequestCount as well
try {
connectGoMutex.wait(MUTEX_MS);
}
catch (InterruptedException ex) {
}
if (!shutdownRequested && conRequestCount > 0) {
/**
* Bring up connections.
* This will probably throw exceptions that we catch here and log them.
* There's not much more we can do.
*/
try {
// create missing connections
int count = createConnections(conRequestCount);
conRequestCount = count == 0 ? -1 : 0; // -1 = max. connections exhausted
}
catch (Exception e) {
LOGGER.warning(MpxConnectionManager.this + ": creating connections failed", e);
}
synchronized (connectDoneMutex) { // syncs conRequestCount as well
connectDoneMutex.notifyAll();
}
}
}
}
LOGGER.info("{0} stopped", getName());
}
}
}