All Downloads are FREE. Search and download functionalities are using the official Maven repository.

diffcases.919148.ReplicationRun.919148_ReplicationRun_0_t Maven / Gradle / Ivy

The newest version!
 /*
  
 Derby - Class org.apache.derbyTesting.functionTests.tests.replicationTests.ReplicationRun
  
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at
  
    http://www.apache.org/licenses/LICENSE-2.0
  
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
  
  */
 package org.apache.derbyTesting.functionTests.tests.replicationTests;
 
 
 import org.apache.derbyTesting.functionTests.util.PrivilegedFileOpsForTests;
 import org.apache.derbyTesting.junit.TestConfiguration;
 import org.apache.derby.drda.NetworkServerControl;
 import java.net.InetAddress;
 import java.util.Properties;
 
 import java.sql.*;
 import java.io.*;
 import org.apache.derby.jdbc.ClientDataSource;
 
 import org.apache.derbyTesting.junit.BaseJDBCTestCase;
 import org.apache.derbyTesting.junit.BaseTestCase;
 
 /**
  * Framework to run replication tests.
  * Subclass to create specific tests as 
  * in ReplicationRun_Local and ReplicationRun_Distributed.
  */
 
 public class ReplicationRun extends BaseTestCase
 {
     
     /**
      * Name of properties file defining the test environment
      * and replication tests to be run.
      * Located in ${user.dir}
      */
     final static String REPLICATIONTEST_PROPFILE = "replicationtest.properties";
     
     final static String REPLICATION_MASTER_TIMED_OUT           = "XRE06";
     final static String REPLICATION_SLAVE_STARTED_OK           = "XRE08";
     final static String REPLICATION_DB_NOT_BOOTED              = "XRE11";
     final static String SLAVE_OPERATION_DENIED_WHILE_CONNECTED = "XRE41";
     final static String REPLICATION_SLAVE_SHUTDOWN_OK          = "XRE42";
 
     static String testUser = null;
     
     static String userDir = null;
     
     static String dataEncryption = null;
       // Set to a legal encryption string to
       // create or connect to an encrypted db.
     
     static String masterServerHost = "localhost"; 
     static int masterServerPort = 1527; // .. default..
     static String slaveServerHost = "localhost";
     static int slaveServerPort = 3527; // .. ..
     static String testClientHost = "localhost";
     static int slaveReplPort = 6666;
     
     static String masterDatabasePath = null;
     static String slaveDatabasePath = null;
     static String replicatedDb = "test";
     
     static String bootLoad = ""; // The "test" to run when booting the master database.
     
     static String freezeDB = ""; // Preliminary: need to "manually" freeze db as part of initialization.
     static String unFreezeDB = ""; // Preliminary: need to "manually" unfreeze db as part of initialization.
     
     static boolean runUnReplicated = false;
     
     static boolean simpleLoad = true;
     static int simpleLoadTuples = 1000;
     
     static int tuplesToInsertPerf = 10000;
     static int commitFreq = 0; // autocommit
     
     static String masterDbSubPath = "db_master";
     static String slaveDbSubPath = "db_slave";
     static final String DB_UID = "";
     static final String DB_PASSWD = "";
     
     
     static String replicationTest = "";
     static String replicationVerify = "";
     
     static String sqlLoadInit = "";
     
     final static String networkServerControl = "org.apache.derby.drda.NetworkServerControl";
     static String specialTestingJar = null;
     // None null if using e.g. your own modified tests.
     static String jvmVersion = null;
     static String masterJvmVersion = null;
     static String slaveJvmVersion = null;
     static String derbyVersion = null;
     static String derbyMasterVersion = null; // Needed for PoC. Remove when committed.
     static String derbySlaveVersion = null;  // Needed for PoC. Remove when committed.
     
     static String junit_jar = null; // Path for JUnit jar
     static String test_jars = null; // Path for derbyTesting.jar:junit_jar
     
     final static String FS = File.separator;
     final static String PS = File.pathSeparator;
     final static String JVMloc = BaseTestCase.getJavaExecutableName();
     
     static boolean showSysinfo = false;
     static long PINGSERVER_SLEEP_TIME_MILLIS = 500L;
     
     static long sleepTime = 5000L; // millisecs.
     
     static final String DRIVER_CLASS_NAME = "org.apache.derby.jdbc.ClientDriver";
     static final String DB_PROTOCOL="jdbc:derby";
     
     static final String ALL_INTERFACES = "0.0.0.0";
     
     static String LF = null;
     
     static final String remoteShell = "/usr/bin/ssh -x"; // or /usr/bin/ssh ?
     
    private static final long DEFAULT_SERVER_START_TIMEOUT = 75000;
    
     Utils util = new Utils();
     
     State state = new State();
 
     static boolean localEnv = false; // true if all hosts have access to
                                              // same filesystem (NFS...)
     
     static String derbyProperties = null;
 
     NetworkServerControl masterServer;
 
     NetworkServerControl slaveServer;
 
     String classPath = null; // Used in "localhost" testing.
     
     /** A Connection to the master database*/
     private Connection masterConn = null;
     /** A Connection to the slave database*/
     private Connection slaveConn = null;
     /** The exception thrown as a result of a startSlave connection attempt  */
     private volatile Exception startSlaveException = null;
 
     /**
      * Creates a new instance of ReplicationRun
      * @param testcaseName Identifying the test.
      */
     public ReplicationRun(String testcaseName)
     {
         super(testcaseName);
         
         LF = System.getProperties().getProperty("line.separator");
     }
     
     /**
      * Parent super()
      * @throws java.lang.Exception .
      */
     protected void setUp() throws Exception
     {
         super.setUp();
     }
     
     /**
      * Parent super()
      * @throws java.lang.Exception .
      */
     protected void tearDown() throws Exception
     {
         stopServer(jvmVersion, derbyVersion,
                 slaveServerHost, slaveServerPort);
         
         stopServer(jvmVersion, derbyVersion,
                 masterServerHost, masterServerPort);
 
         super.tearDown();
     }
     
     /**
      * Run the test. Extra logic in addition to BaseTestCase's similar logic,
      * to save derby.log and database files for replication directories if a
      * failure happens.
      */
     public void runBare() throws Throwable {
 
         try {
 
             super.runBare();
 
         } catch (Throwable running) {
 
             // Copy the master and slave's derby.log file and databases
             //
             PrintWriter stackOut = null;
 
             try {
                 String failPath = PrivilegedFileOpsForTests.
                     getAbsolutePath(getFailureFolder());
 
                 stackOut = new PrintWriter(
                         PrivilegedFileOpsForTests.getFileOutputStream(
                             new File(failPath, ERRORSTACKTRACEFILE), true));
 
                 String[] replPaths = new String[]{masterDbSubPath,
                                                   slaveDbSubPath};
 
                 for (int i=0; i < 2; i++) {
                     // Copy the derby.log file.
                     //
                     File origLog = new File(replPaths[i], DERBY_LOG);
                     File newLog = new File(failPath,
                                            replPaths[i] + "-" + DERBY_LOG);
                     PrivilegedFileOpsForTests.copy(origLog, newLog);
 
                     // Copy the database.
                     //
                     String dbName = TestConfiguration.getCurrent().
                         getDefaultDatabaseName();
                     File dbDir = new File(replPaths[i], dbName );
                     File newDbDir = new File(failPath,
                                              replPaths[i] + "-" + dbName);
                     PrivilegedFileOpsForTests.copy(dbDir,newDbDir);
                 }
             } catch (IOException ioe) {
                 // We need to throw the original exception so if there
                 // is an exception saving the db or derby.log we will print it
                 // and additionally try to log it to file.
                 BaseTestCase.printStackTrace(ioe);
                 if (stackOut != null) {
                     stackOut.println("Copying db_slave/db_master's " +
                                      DERBY_LOG + " or database failed:");
                     ioe.printStackTrace(stackOut);
                     stackOut.println();
                 }
             } finally {
                 if (stackOut != null) {
                     stackOut.close();
                 }
 
                 // Let JUnit take over
                 throw running;
             }
         }
     }
 
 
     String useEncryption(boolean create)
     {
         String encryptionString = "";
         if ( dataEncryption != null)
         {
             if ( create ) encryptionString = ";dataEncryption=true";
             encryptionString = encryptionString+";"+dataEncryption;
         }
         return encryptionString;
     }
     
     //////////////////////////////////////////////////////////////
     ////
     //// The replication test framework (testReplication()):
     //// a) "clean" replication run starting master and slave servers,
     ////     preparing master and slave databases,
     ////     starting and stopping replication and doing
     ////     failover for a "normal"/"failure free" replication
     ////     test run.
     //// b)  Running (positive and negative) tests at the various states 
     ////     of replication to test what is and is not accepted compared to
     ////     the functional specification.
     //// c)  Adding additional load on master and slave servers in 
     ////     different states of replication.
     ////
     //////////////////////////////////////////////////////////////
     
     /* Template
     public void testReplication()
     throws Exception
     {
         util.DEBUG("WARNING: Define in subclass of ReplicationRun. "
                 + "See ReplicationRun_Local for an example.");
     }
      */
 
     void connectPing(String fullDbPath, 
             String serverHost, int serverPort, 
             String testClientHost)
         throws Exception
     {
         
         String serverURL = "jdbc:derby:"
                 +"//"+serverHost+":"+serverPort+"/";
         String dbURL = serverURL
                 +fullDbPath
                 +useEncryption(false);
         Connection conn = null;
         String lastmsg = null;
         long sleeptime = 200L;
         boolean done = false;
         int count = 0;
         while ( !done )
         {
             try
             {
                 Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
                 conn = DriverManager.getConnection(dbURL);
                 done = true;
                 util.DEBUG("Got connection after " 
                         + count +" * "+ sleeptime + " ms.");
                 conn.close();
             }
             catch ( SQLException se )
             {
                 int errCode = se.getErrorCode();
                 lastmsg = se.getMessage();
                 String sState = se.getSQLState();
                 String expectedState = "08004";
                 lastmsg = errCode + " " + sState + " " + lastmsg 
                         + ". Expected: "+ expectedState;
                 util.DEBUG("Got SQLException: " + lastmsg);
                 if ( (errCode == -1)
                 && (sState.equalsIgnoreCase(expectedState) ) )
                 {
                     util.DEBUG("Failover not complete.");
                     Thread.sleep(sleeptime); // ms.
                 }
                 else
                 {
                     se.printStackTrace(); // FIXME!
                     assertTrue("Connect failed. " + lastmsg, false);
                     return;
                 }
             }
             
             assertTrue("Failover did not succeed.", count++ < 600); // 600*200ms = 120s = 2mins.
         }
     }
     
     String showCurrentState(String ID, long waitTime,
             String fullDbPath, 
             String serverHost, int serverPort)
         throws Exception
     {
         int errCode = 0;
         String sState = "CONNECTED";
         String msg = null;
         Thread.sleep(waitTime); // .... until stable...
         try
         {
             ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
             ds.setDatabaseName(fullDbPath);
             ds.setServerName(serverHost);
             ds.setPortNumber(serverPort);
             ds.setConnectionAttributes(useEncryption(false));
             Connection conn = ds.getConnection();
             conn.close();
         }
         catch ( SQLException se )
         {
             errCode = se.getErrorCode();
             msg = se.getMessage();
             sState = se.getSQLState();
         }
         util.DEBUG(ID+": ["+serverHost+":"+serverPort+"/"+fullDbPath+"] "
                 + errCode + " " + sState + " " + msg);
         return sState;
     }
     void waitForConnect(long sleepTime, int tries,
             String fullDbPath, 
             String serverHost, int serverPort)
         throws Exception
     {
         int count = 0;
         String msg = null;
         while ( count++ <= tries )
         {
             try
             {
                 ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
                 ds.setDatabaseName(fullDbPath);
                 ds.setServerName(serverHost);
                 ds.setPortNumber(serverPort);
                 ds.setConnectionAttributes(useEncryption(false));
                 Connection conn = ds.getConnection();
                 util.DEBUG("Got connection after " 
                         + (count-1) +" * "+ sleepTime + " ms.");
                 conn.close();
                 return;
             }
             catch ( SQLException se )
             {
                 msg = se.getErrorCode() + "' '" + se.getSQLState()
                         + "' '" + se.getMessage();
                 util.DEBUG(count  + " got '" + msg +"'.");
                 Thread.sleep(sleepTime); // ms. Sleep and try again...
             }
         }        
         assertTrue(msg + ": Could NOT connect in "
                 + tries+"*"+sleepTime + "ms.",false);
     }
     void waitForSQLState(String expectedState, 
             long sleepTime, int tries,
             String fullDbPath, 
             String serverHost, int serverPort)
         throws Exception
     {
         int count = 0;
         String msg = null;
         while ( count++ <= tries )
         {
             try
             {
                 ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
                 ds.setDatabaseName(fullDbPath);
                 ds.setServerName(serverHost);
                 ds.setPortNumber(serverPort);
                 ds.setConnectionAttributes(useEncryption(false));
                 Connection conn = ds.getConnection();
                 // Should never get here!
                 conn.close();
                 assertTrue("Expected SQLState'"+expectedState
                             + "', but got connection!",
                         false);
             }
             catch ( SQLException se )
             {
                 int errCode = se.getErrorCode();
                 msg = se.getMessage();
                 String sState = se.getSQLState();
                 msg = "'" + errCode + "' '" + sState + "' '" + msg +"'";
                 util.DEBUG(count 
                         + ": SQLState expected '"+expectedState+"'," +
                         " got " + msg);
                 if ( sState.equals(expectedState) )
                 {
                     util.DEBUG("Reached SQLState '" + expectedState +"' in "
                             + (count-1)+"*"+sleepTime + "ms.");
                     return; // Got desired SQLState.
                 }
                 else
                 {
                     Thread.sleep(sleepTime); // ms. Sleep and try again...
                 }
             }
             
         }
         assertTrue(msg + ": SQLState '"+expectedState+"' was not reached in "
                 + tries+"*"+sleepTime + "ms.",false);
         
     }
     void shutdownDb(String jvmVersion, // Not yet used
             String serverHost, int serverPort, 
             String dbPath, String replicatedDb,
             String clientHost) // Not yet used
         throws Exception
     {
         String serverURL = "jdbc:derby:"
                 +"//"+serverHost+":"+serverPort+"/";
         String dbURL = serverURL
                 +dbPath
                 +FS+replicatedDb
                 +useEncryption(false);
         util.DEBUG("**** DriverManager.getConnection(\"" + dbURL+";shutdown=true\");");
 
         try{
             Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
             DriverManager.getConnection(dbURL+";shutdown=true");
         }
         catch (SQLException se)
         {
             
         }
         
     }
     
     ////////////////////////////////////////////////////////////////
     /* Utilities.... */
     void startServerMonitor(String slaveHost)
     {
         util.DEBUG("startServerMonitor(" + slaveHost + ") NOT YET IMPLEMENTED.");
     }
     
     void runTest(String replicationTest,
             String clientVM,
             String testClientHost,
             String serverHost, int serverPort,
             String dbName)
             throws Exception
     {
         util.DEBUG("runTest(" + replicationTest
                 + ", " + clientVM
                 + ", " + testClientHost
                 + ", " + serverHost
                 + ", " + serverPort
                 + ", " + dbName
                 + ") "
                 );
         
         if ( replicationTest == null ) 
         {
             util.DEBUG("No replicationTest specified. Exitting.");
             return;
         } 
         
         if ( simpleLoad )
         {
             _testInsertUpdateDeleteOnMaster(serverHost, serverPort,
                 dbName, simpleLoadTuples);
             return;
         }
         
         String URL = masterURL(dbName);
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbyTesting.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         String testingClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbynet.jar" // WHY IS THIS NEEDED?
                 // See TestConfiguration: startNetworkServer and stopNetworkServer
                 + PS + test_jars;
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = null;
         
         util.DEBUG("replicationTest: " + replicationTest);
         if ( replicationTest.indexOf(".sql") >= 0 )
         {
             command = clientJvm
                     + " -Dij.driver=" + DRIVER_CLASS_NAME
                     + " -Dij.connection.startTestClient=" + URL
                     + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                     + " " + replicationTest
                     ;
         }
         else
         { // JUnit
             if ( testClientHost.equals("localhost") )
             {
                 testingClassPath = classPath; // Using the complete classpath
             }
             command = clientJvm
                     + " -Dderby.tests.trace=true"
                     // + " -Djava.security.policy=\"\"" // Now using noSecurityManager decorator
                     + " -Dtest.serverHost=" + serverHost  // Tell the test what server
                     + " -Dtest.serverPort=" + serverPort  // and port to connect to.
                     + " -Dtest.inserts=" + tuplesToInsertPerf // for SimplePerfTest
                     + " -Dtest.commitFreq=" +  commitFreq // for SimplePerfTest
                     + " -classpath " + testingClassPath
                     + " junit.textui.TestRunner"
                     + " " + replicationTest
                     ;
         }
         
         long startTime = System.currentTimeMillis();
         String results = null;
         if ( testClientHost.equalsIgnoreCase("localhost") )
         {
             runUserCommandLocally(command, userDir+FS+masterDbSubPath, "runTest ");
         }
         else
         {
             command = "cd "+ userDir +";" // Must be positioned where the properties file is located.
                     + command;
             results = runUserCommandRemotely(command, testClientHost, testUser,
                     "runTest ");
         }
         util.DEBUG("Time: " + (System.currentTimeMillis() - startTime) / 1000.0);
         
     }
     void runTestOnSlave(String replicationTest,
             String clientVM,
             String testClientHost,
             String serverHost, int serverPort,
             String dbName)
             throws Exception
     {
         util.DEBUG("runTestOnSlave(" + replicationTest
                 + ", " + clientVM
                 + ", " + testClientHost
                 + ", " + serverHost
                 + ", " + serverPort
                 + ", " + dbName
                 + ") "
                 );
         
         
         String URL = slaveURL(dbName);
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbyTesting.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         String testingClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbynet.jar" // WHY IS THIS NEEDED?
                 // See TestConfiguration: startNetworkServer and stopNetworkServer
                 + PS + test_jars;
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = null;
         
         if ( replicationTest == null ) 
         {
             util.DEBUG("No replicationTest specified. Exitting.");
             return;
         } 
         
         if ( serverHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
             testingClassPath = classPath;
         }
         util.DEBUG("replicationTest: " + replicationTest);
         if ( replicationTest.indexOf(".sql") >= 0 )
         {
             command = clientJvm
                     + " -Dij.driver=" + DRIVER_CLASS_NAME
                     + " -Dij.connection.startTestClient=" + URL
                     + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                     + " " + replicationTest
                     ;
         }
         else
         { // JUnit
             if ( testClientHost.equals("localhost") ) 
             {
                 testingClassPath = classPath; // Using the complete classpath
             }
             command = clientJvm
                     + " -Dderby.tests.trace=true"
                     // + " -Djava.security.policy=\"\""  // Now using noSecurityManager decorator
                     + " -Dtest.serverHost=" + serverHost  // Tell the test what server
                     + " -Dtest.serverPort=" + serverPort  // and port to connect to.
                     + " -Dtest.inserts=" + tuplesToInsertPerf // for SimplePerfTest
                     + " -Dtest.commitFreq=" +  commitFreq // for SimplePerfTest
                     + " -classpath " + testingClassPath
                     + " junit.textui.TestRunner"
                     + " " + replicationTest
                     ;
         }
         
         long startTime = System.currentTimeMillis();
         String results = null;
         if ( testClientHost.equalsIgnoreCase("localhost") )
         {
             runUserCommandLocally(command, userDir+FS+slaveDbSubPath, "runTestOnSlave ");
         }
         else
         {
             command = "cd "+ userDir +";" // Must be positioned where the properties file is located.
                     + command;
             results = runUserCommandRemotely(command, testClientHost, testUser,
                     "runTestOnSlave ");
         }
         util.DEBUG("Time: " + (System.currentTimeMillis() - startTime) / 1000.0);
         
     }
     
     /*
      *
      * Should allow:
      * - Run load in separate thread.
      *
      */
     private void runLoad(String load,
             String clientVM,
             String testClientHost,
             String masterHost, int masterPort,
             String dbSubPath) // FIXME? Should we allow extra URL options?
             throws Exception
     {
         util.DEBUG("runLoad(" + load
                 + ", " + clientVM
                 + ", " + testClientHost
                 + ", " + masterHost
                 + ", " + masterPort
                 + ", " + dbSubPath
                 + ") "
                 );
         
         
         String URL = masterLoadURL(dbSubPath);
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbyTesting.jar"
                 // Needed for 'run resource 'createTestProcedures.subsql';' cases?
                 // Nope? what is 'resource'?
                 + PS + derbyVersion +FS+ "derbytools.jar";
         String testingClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbynet.jar" // WHY IS THIS NEEDED?
                 // See TestConfiguration: startNetworkServer and stopNetworkServer
                 + PS + test_jars;
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = null;
         
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
             testingClassPath = classPath;
         }
         util.DEBUG("load: " + load);
         if ( load.indexOf(".sql") >= 0 )
         {
             command = clientJvm // "java"
                     + " -Dij.driver=" + DRIVER_CLASS_NAME
                     + " -Dij.connection.startTestClient=" + URL
                     + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                     + " " + load
                     ;
         }
         else
         {
             /* BEGIN For junit: */
             command = clientJvm // "java"
                     + " -Dderby.tests.trace=true"
                     // + " -Djava.security.policy=\"\""  // Now using noSecurityManager decorator
                     + " -classpath " + testingClassPath
                     + " junit.textui.TestRunner"
                     + " " + load //
                     // + " org.apache.derbyTesting.functionTests.tests.replicationTests.ReplicationTestRun"
                     ;
             /* END */
         }
         
         /* String results = */
         if ( testClientHost.equalsIgnoreCase("localhost") )
         {
             runUserCommandInThread(command,  testUser, dbSubPath,
                     "runLoad["+dbSubPath+"] ");
         }
         else
         {
             runUserCommandInThreadRemotely(command,
                     testClientHost, testUser, dbSubPath,
                     "runLoad["+dbSubPath+"] ");
         }
         
     }
     
     private void runStateTest(String stateTest,
             String clientVM,
             String testClientHost,
             String masterHost, int masterPort, // serverHost?, serverPort?
             String dbSubPath) // FIXME? Should we allow extra URL options?
             throws Exception
     {
         util.DEBUG("runStateTest(" + stateTest
                 + ", " + clientVM
                 + ", " + testClientHost
                 + ", " + masterHost
                 + ", " + masterPort
                 + ", " + dbSubPath
                 + ") "
                 );
         
         
         String URL = masterLoadURL(dbSubPath);
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbyTesting.jar"
                 // Needed for 'run resource 'createTestProcedures.subsql';' cases?
                 // Nope? what is 'resource'?
                 + PS + derbyVersion +FS+ "derbytools.jar";
         String testingClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbynet.jar" // WHY IS THIS NEEDED?
                 // See TestConfiguration: startNetworkServer and stopNetworkServer
                 + PS + test_jars;
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = null;
         
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
             testingClassPath = classPath;
         }
         util.DEBUG("stateTest: " + stateTest);
         if ( stateTest.indexOf(".sql") >= 0 )
         {
             command = clientJvm 
                     + " -Dij.driver=" + DRIVER_CLASS_NAME
                     + " -Dij.connection.startTestClient=" + URL
                     + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                     + " " + stateTest
                     ;
         }
         else
         {
             /* BEGIN For junit: */
             command = "cd "+ userDir +";" // Must be positioned where the properties file is located.
                     + clientJvm 
                     + " -Dderby.tests.trace=true"
                     // + " -Djava.security.policy=\"\""  // Now using noSecurityManager decorator
                     + " -classpath " + testingClassPath
                     + " junit.textui.TestRunner"
                     + " " + stateTest
                     ;
             /* END */
         }
         
         /* String results = */
         runUserCommandRemotely(command,
                 testClientHost, // masterHost,
                 testUser,
                 // dbSubPath,
                 "runStateTest "); // ["+dbSubPath+"]
         
     }
     
     void bootMasterDatabase(String clientVM, 
             String dbSubPath,
             String dbName,
             String masterHost,  // Where the command is to be executed.
             int masterServerPort, // master server interface accepting client requests
             String load)
         throws Exception
     {
         // Should just do a "connect....;create=true" here, instead of copying in initMaster.
         
         String URL = masterURL(dbName)
                 +";create=true"
                 +useEncryption(true);
 
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
         }
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String results = null;
         if ( masterHost.equalsIgnoreCase("localhost") || localEnv )
         {
             /* results = 
                 runUserCommand(command,
                 testUser,
                 "bootMasterDatabase "); */
             util.DEBUG("bootMasterDatabase getConnection("+URL+")");
             Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
             Connection conn = DriverManager.getConnection(URL);
             conn.close();
         }
         else
         {
         // Execute the ij command on the testClientHost as testUser
             String command = clientJvm 
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.createMaster=\"" + URL + "\""
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + "exit.sql" // FIXME!!
                 ;
         
             results =
                 runUserCommandRemotely(command,
                 testClientHost,
                 testUser,
                 "bootMasterDatabase ");
         }
         util.DEBUG(results);
         
         
         // NB! should be done by startMaster. Preliminary needs to freeze db before copying to slave and setting replication mode.
         util.DEBUG("************************** DERBY-???? Preliminary needs to freeze db before copying to slave and setting replication mode.");
         if ( masterServerHost.equalsIgnoreCase("localhost") || localEnv )
         {
              URL = masterURL(dbName);
             Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
             util.DEBUG("bootMasterDatabase getConnection("+URL+")");
             Connection conn = DriverManager.getConnection(URL);
             Statement s = conn.createStatement();
             s.execute("call syscs_util.syscs_freeze_database()");
             conn.close();
         }
         else
         {
         runTest(freezeDB,
                 clientVM,
                 testClientHost,
                 masterServerHost, masterServerPort,
                 dbName
                 );
         }
         
         if ( load != null )
         {
             runLoad(load,
                     clientVM, // jvmVersion,
                     testClientHost,
                     masterServerHost, masterServerPort,
                     dbSubPath+FS+dbName);
         }
         util.DEBUG("bootMasterDatabase done.");
     }
 
     /**
      * Set master db in replication master mode.
      */
     void startMaster(String clientVM,
             String dbName,
             String masterHost,  // Where the command is to be executed.
             int masterServerPort, // master server interface accepting client requests
             String slaveClientInterface, // Will be = slaveReplInterface = slaveHost if only one interface card used.
             int slaveServerPort, // masterPort, // Not used since slave don't accept client requests
             String slaveReplInterface, // slaveHost,
             int slaveReplPort) // slavePort)
             throws Exception
     {
         if ( masterHost.equalsIgnoreCase("localhost") )
         {
             startMaster_direct(dbName,
                     masterHost,  masterServerPort,
                     slaveReplInterface, slaveReplPort);
         }
         else
         {
             startMaster_ij(dbName,
                     masterHost,
                     masterServerPort, slaveReplInterface,
                     slaveReplPort, testClientHost);
         }
         /* else if ...
         {
             startMaster_CLI(clientVM,
                     dbName,
                     masterHost,masterServerPort,
                     slaveClientInterface,slaveServerPort,
                     slaveReplInterface,slaveReplPort);
         } */
     }
     /* CLI not available for 10.4 */
     private void startMaster_CLI(String clientVM,
             String dbName,
             String masterHost,  // Where the command is to be executed.
             int masterServerPort, // master server interface accepting client requests
             String slaveClientInterface, // Will be = slaveReplInterface = slaveHost if only one interface card used.
             int slaveServerPort, // masterPort, // Not used since slave don't accept client requests
             String slaveReplInterface, // slaveHost,
             int slaveReplPort) // slavePort)
             throws Exception
     {
         
         String masterClassPath = derbyMasterVersion +FS+ "derbynet.jar";
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             masterClassPath = classPath;
         }
         /* java -classpath ${MASTER_LIB}/derbynet.jar \
          *       org.apache.derby.drda.NetworkServerControl startreplication test \
          *       -slavehost ${SLAVEREPLINTERFACE} -slaveport ${SLAVEREPLPORT} \
          *       -h ${SLAVECLIENTINTERFACE} -p ${SLAVESERVERPORT}?? \
          *       -noSecurityManager
          */
         String command = clientJvm
                 + " -classpath " + masterClassPath
                 + " " + networkServerControl
                 + " startreplication" // startmaster!
                 + " " + dbName
                 + " -slavehost " + /*slaveHost*/ slaveReplInterface + " -slaveport " + /*slavePort*/ slaveReplPort
                 + " -h " + /*masterHost*/ slaveClientInterface + " -p " + masterServerPort /*masterPort*/ /* see comment above slaveServerPort */
                 + " -noSecurityManager"
                 ;
         
         util.DEBUG("Executing '" + command + "' on " + masterHost);
         
         // Do rsh/ssh to masterHost and execute the command there.
         
         // runUserCommandRemotely(command, // FIXME?! Should NOT be in sep. thread? Wait for it to complete!?
         runUserCommandInThreadRemotely(command, //
                 masterHost,
                 testUser,
                 masterDbSubPath+FS+dbName,
                 "startMaster_CLI ");
         
     }
     private void startMaster_ij(String dbName,
             String masterHost,
             int masterServerPort,  // Where the master db is run.
             String slaveReplInterface, // master server interface accepting client requests
             
             int slaveReplPort, // slaveHost,
             String testClientHost)
             throws Exception
     {
         
         String masterClassPath = derbyMasterVersion +FS+ "derbynet.jar";
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             masterClassPath = classPath;
         }
                 
         String URL = masterURL(dbName)
                 +";startMaster=true;slaveHost="+slaveReplInterface
                 +";slavePort="+slaveReplPort;
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
         }
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = clientJvm // "java"
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.startMaster=\"" + URL + "\""
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + "/home/os136789/Replication/testing/exit.sql"
                 ;
         
         // Execute the ij command on the testClientHost as testUser
         String results =
                 runUserCommandRemotely(command,
                 testClientHost,
                 testUser,
                 "startMaster_ij ");
         util.DEBUG(results);
     }
     private void startMaster_direct(String dbName,
             String masterHost,  // Where the master db is run.
             int masterServerPort, // master server interface accepting client requests
             
             String slaveReplInterface, // slaveHost,
             int slaveReplPort)
             throws Exception
     {
         
         String URL = masterURL(dbName)
                 +";startMaster=true;slaveHost="+slaveReplInterface
                 +";slavePort="+slaveReplPort;
                 
             util.DEBUG("startMaster_direct getConnection("+URL+")");
             Connection conn = null;
             boolean done = false;
             int count = 0;
             while ( !done )
             {
                 try
                 {
                     /* On 1.5 locking of Drivermanager.class prevents
                      * using DriverManager.getConnection() concurrently
                      * in startMaster and startSlave!
                     Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
                     conn = DriverManager.getConnection(URL);
                      */
                     ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
                     ds.setDatabaseName(masterDatabasePath+FS+masterDbSubPath+FS+dbName);
                     ds.setServerName(masterHost);
                     ds.setPortNumber(masterServerPort);
                     ds.setConnectionAttributes("startMaster=true"
                             +";slaveHost="+slaveReplInterface
                             +";slavePort="+slaveReplPort
                             +useEncryption(false));
                     conn = ds.getConnection();
                     
                     done = true;
                     conn.close();
                     util.DEBUG("startMaster_direct connected in " + count + " * 100ms.");
                 }
                 catch ( SQLException se )
                 {
                     int errCode = se.getErrorCode();
                     String msg = se.getMessage();
                     String sState = se.getSQLState();
                     String expectedState = "XRE04";
                     util.DEBUG("startMaster Got SQLException: " 
                             + errCode + " " + sState + " " + msg + ". Expected " + expectedState);
                     if ( (errCode == -1)
                     && (sState.equalsIgnoreCase(expectedState) ) )
                     {
                         util.DEBUG("Not ready to startMaster. "
                                 +"Beware: Will also report "
                                 + "'... got a fatal error for database '...../'"
                                 + " in master derby.log.");
                         Thread.sleep(100L); // ms.
                     }
                     else
                     {
                         if (REPLICATION_MASTER_TIMED_OUT.equals(sState)) // FIXME! CANNOT_START_MASTER_ALREADY_BOOTED
                         {
                             util.DEBUG("Master already started?");
                         }
                         util.DEBUG("startMaster_direct Got: "
                                 +state+" Expected "+expectedState);
                         throw se;
                     }
                 }
 
                 assertTrue("startMaster did not succeed.", count++ < 1200); // 1200*100ms = 120s = 2mins.
             }
             util.DEBUG("startMaster_direct exit.");
     }
     
     /**
      * Get a connection to the master database.
      * @return A connection to the master database
      */
     protected Connection getMasterConnection() {
         if (masterConn == null) {
             String url = masterURL(replicatedDb);
             try {
                 masterConn = DriverManager.getConnection(url);
             } catch (SQLException sqle) {
                 fail("Could not connect to master database");
             }
         }
         return masterConn;
     }
     
     /**
      * Get a connection to the slave database.
      * @return A connection to the slave database
      */
     protected Connection getSlaveConnection() {
         if (slaveConn == null) {
             String url = slaveURL(replicatedDb);
             try {
                 slaveConn = DriverManager.getConnection(url);
             } catch (SQLException sqle) {
                 fail("Could not connect to slave database");
             }
         }
         return slaveConn;
     }
 
 
     /**
      * Execute SQL on the master database through a Statement
      * @param sql The sql that should be executed on the master database
      * @throws java.sql.SQLException thrown if an error occured while
      * executing the sql
      */
     protected void executeOnMaster(String sql) throws SQLException {
          Statement s = getMasterConnection().createStatement();
          s.execute(sql);
          s.close();
     }
 
     /**
      * Execute SQL on the slave database through a Statement
      * @param sql The sql that should be executed on the slave database
      * @throws java.sql.SQLException thrown if an error occured while
      * executing the sql
      */
     protected void executeOnSlave(String sql) throws SQLException {
          Statement s = getSlaveConnection().createStatement();
          s.execute(sql);
          s.close();
     }
 
     /**
      * Set slave db in replication slave mode
      */
     void startSlave(String clientVM,
             String dbName,
             String slaveClientInterface, // slaveHost, // Where the command is to be executed.
             int slaveServerPort,
             String slaveReplInterface,
             int slaveReplPort,
             String testClientHost)
     throws Exception
     {
         if ( testClientHost.equalsIgnoreCase("localhost") )
         {
             startSlave_direct(dbName,
                     slaveClientInterface, slaveServerPort,
                     slaveReplInterface,slaveReplPort);
         }
         else
         {
             startSlave_ij(jvmVersion,
                     dbName,
                     slaveClientInterface, slaveServerPort,
                     slaveReplInterface, slaveReplPort,
                     testClientHost);
         }
         /* else if ... 
         {
             startSlave_CLI(clientVM,
                     dbName,
                     slaveClientInterface,slaveServerPort,
                     slaveReplInterface,slaveReplPort);
         } */
         
     }
     /* CLI Not available in 10.4 */
     private void startSlave_CLI(String clientVM,
             String dbName,
             String slaveClientInterface, // slaveHost, // Where the command is to be executed.
             int slaveServerPort,
             String slaveReplInterface,
             int slaveReplPort)
             throws InterruptedException
     {
         
         String slaveClassPath = derbySlaveVersion +FS+ "derbynet.jar";
         if ( slaveClientInterface.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             slaveClassPath = classPath;
         }
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         /*
          * java -classpath ${SLAVE_LIB}/derbynet.jar org.apache.derby.drda.NetworkServerControl \
          *       startslave test -slavehost ${SLAVEREPLINTERFACE}  -slaveport ${SLAVEREPLPORT} \
          *       -h ${SLAVECLIENTINTERFACE} -p ${SLAVESERVERPORT}??  \
          *       -noSecurityManager
          */
         String command = clientJvm
                 + " -classpath " + slaveClassPath
                 + " " + networkServerControl
                 + " startslave"
                 + " " + dbName
                 + " -slavehost " + slaveReplInterface + " -slaveport " + slaveReplPort
                 + " -h " + slaveClientInterface + " -p " + slaveServerPort
                 + " -noSecurityManager"
                 ;
         
         util.DEBUG("Executing  '" + command + "' on " + slaveClientInterface); // slaveHost
         
         runUserCommandInThreadRemotely(command,
                 slaveClientInterface, // slaveHost,
                 testUser,
                 slaveDbSubPath+FS+dbName,
                 "startSlave_CLI ");
         
     }
     private void startSlave_ij(String jvmVersion,
             String dbName,
             String slaveHost,  // Where the slave db is run.
             int slaveServerPort, // slave server interface accepting client requests
             
             String slaveReplInterface, // slaveHost,
             int slaveReplPort, // slavePort)
             
             String testClientHost)
             throws Exception
     {
         
         String slaveClassPath = derbySlaveVersion +FS+ "derbynet.jar";
         if ( slaveHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             slaveClassPath = classPath;
         }
         String URL = slaveURL(dbName)
                 +";startSlave=true;slaveHost="+slaveReplInterface
                 +";slavePort="+slaveReplPort;
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         if ( slaveHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
         }
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = clientJvm // "java"
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.startSlave=\"" + URL + "\""
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + "/home/os136789/Replication/testing/exit.sql"
                 ;
         
         // Execute the ij command on the testClientHost as testUser
         /* String results =
                 runUserCommandRemotely(command,
                 testClientHost,
                 testUser,
                 "startSlave_ij ");
         util.DEBUG(results); */
         // 618220 + failover_impl_3205_v3.diff + derby-3361-1a.diff :
         runUserCommandInThreadRemotely(command,
                 testClientHost,
                 testUser,
                 slaveDbSubPath+FS+dbName,
                 "startSlave_ij ");
         
     }
     private void startSlave_direct(String dbName,
             String slaveHost,  // Where the slave db is run.
             int slaveServerPort, // slave server interface accepting client requests
             String slaveReplInterface,
             int slaveReplPort)
             throws Exception
     {
         final String URL = slaveURL(dbName)
                 +";startSlave=true;slaveHost="+slaveReplInterface
                 +";slavePort="+slaveReplPort;
         
             util.DEBUG("startSlave_direct getConnection("+URL+")");
             
             final String fDbPath = slaveDatabasePath+FS+slaveDbSubPath+FS+dbName;
             final String fSlaveHost = slaveHost;
             final int fSlaveServerPort = slaveServerPort;
             final String fConnAttrs = "startSlave=true"
                                 +";slaveHost="+slaveReplInterface
                                 +";slavePort="+slaveReplPort
                                 +useEncryption(false);
             Thread connThread = new Thread(
                     new Runnable()
             {
                 public void run()
                 {
                     startSlaveException = null;
                     Connection conn = null;
                     String expectedState = "XRE08";
                     try {
                         // NB! WIll hang here until startMaster is executed!
                         /*On 1.5 locking of Drivermanager.class prevents
                          * using DriverManager.getConnection() concurrently
                          * in startMaster and startSlave!
                         Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
                         conn = DriverManager.getConnection(URL);
                          */
                         ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
                         ds.setDatabaseName(fDbPath);
                         ds.setServerName(fSlaveHost);
                         ds.setPortNumber(fSlaveServerPort);
                         ds.setConnectionAttributes(fConnAttrs);
                         conn = ds.getConnection();
                         conn.close();
                     }
                     catch (SQLException se)
                     {
                         startSlaveException = se;
                     }
                     catch (Exception ex)
                     {
                         startSlaveException = ex;
                     }
                 }
             }
             );
             connThread.start();
             util.DEBUG("startSlave_direct exit.");
     }
     
 
     void failOver(String jvmVersion,
             String dbPath, String dbSubPath, String dbName,
             String host,  // Where the db is run.
             int serverPort,
             
             String testClientHost)
             throws Exception
     {
         if ( host.equalsIgnoreCase("localhost") )
         {
             failOver_direct(dbPath, dbSubPath, dbName, host, serverPort);
         }
         else
         {
             failOver_ij(jvmVersion, dbPath, dbSubPath, dbName, host, serverPort,
                     testClientHost);
         }
 
     }
     private void failOver_ij(String jvmVersion,
             String dbPath, String dbSubPath, String dbName,
             String host,  // Where the db is run.
             int serverPort,
             
             String testClientHost)
             throws Exception
     {
         
         String masterClassPath = derbyMasterVersion +FS+ "derbynet.jar";
                 
         String URL = masterURL(dbName)
                 +";failover=true";
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         if ( host.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
         }
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = clientJvm // "java"
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.failover=\"" + URL + "\""
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + "/home/os136789/Replication/testing/exit.sql"
                 ;
         
         // Execute the ij command on the testClientHost as testUser
         String results =
                 runUserCommandRemotely(command,
                 testClientHost,
                 testUser,
                 "failOver_ij ");
         util.DEBUG(results);
     }
     private void failOver_direct(String dbPath, String dbSubPath, String dbName,
             String host,  // Where the db is run.
             int serverPort)
             throws Exception
     {
         String URL = masterURL(dbName)
                 +";failover=true";
                
             util.DEBUG("failOver_direct getConnection("+URL+")");
 
             Connection conn = null;
             try
             {
                 Class.forName(DRIVER_CLASS_NAME); // Needed when running from classes!
                 conn = DriverManager.getConnection(URL);
                 // conn.close();
             }
             catch (SQLException se)
             {
                 int errCode = se.getErrorCode();
                 String msg = se.getMessage();
                 String sState = se.getSQLState();
                 String expectedState = "XRE20";
                 msg = "failOver_direct Got SQLException: " 
                         + errCode + " " + sState + " " + msg 
                         + ". Expected: " + expectedState;
                 util.DEBUG(msg);
                 BaseJDBCTestCase.assertSQLState(expectedState, se);
             }
    }
     
     private void stopMaster(String dbName)
     {
         util.DEBUG("Simulating '... stopreplication/stopmaster -db "+dbName
                 + " NB! Doing nothing now!");
     }
     private void stopMaster_ij(String jvmVersion,
             String dbName,
             String masterHost,  // Where the master db is run.
             int masterServerPort,
             
             String testClientHost)
             throws Exception
     {
         
         String masterClassPath = derbyMasterVersion +FS+ "derbynet.jar";
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             masterClassPath = classPath;
         }
                 
         String URL = masterURL(dbName)
                 +";stopMaster=true"; 
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         if ( masterHost.equals("localhost") )
         { // Use full classpath when running locally. Can not vary server versions!
             ijClassPath = classPath;
         }
         
         String clientJvm = BaseTestCase.getJavaExecutableName();
         
         String command = clientJvm // "java"
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.stopMaster=\"" + URL + "\""
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + "/home/os136789/Replication/testing/exit.sql"
                 ;
         
         // Execute the ij command on the testClientHost as testUser
         String results =
                 runUserCommandRemotely(command,
                 testClientHost,
                 testUser,
                 "stopMaster_ij ");
         util.DEBUG(results);
     }
     
     int xFindServerPID(String serverHost, int serverPort)
     throws InterruptedException
     {
         if ( masterServer != null ) // If master (and assuming then also slave)
             // is started via new NetworkServerContol() use 0 for "PID".
         {
             return 0;
         }
         if ( serverHost.equalsIgnoreCase("localhost") ) 
         { // Assuming we do not need the PID.
             return 0;
         }
         int pid = -1;
         /* String command = "ps auxwww "
                 + " | grep " + serverPort
                 + " | grep '.NetworkServerControl start -h '"
                 // + " | grep '/trunk_slave/jars/'" or trunk_master...
                 + " | grep -v grep"
                 + " | grep -v ssh"
                 + " | gawk '{ print $2 }'"; */
         String p1 = "ps auxwww"; // "/bin/ps auxwww";
         String p2 = " | grep " + serverPort; // /bin/grep
         String p3 = " | grep '.NetworkServerControl start -h '"; // /bin/grep
         String p4 = ""; // | /bin/grep '/trunk_slave/jars/'"; // Also used for master...
         String p5 = " | grep -v grep"; // /bin/grep
         String p6 = " | grep -v ssh"; // /bin/grep
         String p7 = " | grep -v tcsh";  // /bin/grep// Assuming always doing remote command (ssh)
         String p8 = " | gawk '{ print $2 }'"; // /bin/gawk
         String p9 = " | head -1"; // For cases where we also get some error...
         
         String command = p1 + p2 + p3 + p4 + p5 + p6 + p7 /* + p8 */ + ";";
         String result = runUserCommandRemotely(command,serverHost, testUser);
         util.DEBUG("xFindServerPID: '" + result + "'");
         // result = result.split(" ")[1]; // Without ' + p8 ' should show full line. But fails with PID less than 10000!
         command = p1 + p2 + p3 + p4 + p5 + p6 + p7 + p8 + p9 + ";";
         result = runUserCommandRemotely(command, serverHost, testUser);
         if ( result == null )
         {util.DEBUG("xFindServerPID: Server process not found");return -1;} // Avoid error on parseInt below
         util.DEBUG("xFindServerPID: '" + result + "'");
         pid = Integer.parseInt(result.trim());
         util.DEBUG("xFindServerPID: " + pid);
         return pid;
     }
     void xStopServer(String serverHost, int serverPID)
     throws InterruptedException
     {
         if ( serverPID == -1 || serverPID == 0 )
         {util.DEBUG("Illegal PID");return;}
         String command = "kill " + serverPID;
         runUserCommandRemotely(command,
                 serverHost,
                 testUser,
                 "xStopServer");
     }
     
     void verifySlave()
     throws Exception
     {
         util.DEBUG("BEGIN verifySlave "+slaveServerHost+":"
                 +slaveServerPort+"/"+slaveDatabasePath+FS+slaveDbSubPath+FS+replicatedDb);
         
         if ( (replicationTest != null) // If 'replicationTest==null' no table was created/filled
                 && simpleLoad )
         {
             _verifyDatabase(slaveServerHost, slaveServerPort, 
                     slaveDatabasePath+FS+slaveDbSubPath+FS+replicatedDb,
                     simpleLoadTuples);
             // return;
         }
 
         ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
         ds.setDatabaseName(slaveDatabasePath + FS + slaveDbSubPath + FS +
                            replicatedDb);
         ds.setServerName(slaveServerHost);
         ds.setPortNumber(slaveServerPort);
         ds.setConnectionAttributes(useEncryption(false));
         Connection conn = ds.getConnection();
             
         simpleVerify(conn);
         conn.close();
         /*
         runSlaveVerificationCLient(jvmVersion,
                 testClientHost,
                 replicatedDb,
                 slaveServerHost, slaveServerPort);
         */
         util.DEBUG("END   verifySlave");
     }
     void verifyMaster()
     throws Exception
     {
         util.DEBUG("BEGIN verifyMaster " + masterServerHost + ":"
                 +masterServerPort+"/"+masterDatabasePath+FS+masterDbSubPath+FS+replicatedDb);
         
         if ( (replicationTest != null)  // If 'replicationTest==null' no table was created/filled
                 && simpleLoad )
         {
             _verifyDatabase(masterServerHost, masterServerPort, 
                     masterDatabasePath+FS+masterDbSubPath+FS+replicatedDb,
                     simpleLoadTuples);
             // return;
         }
 
         ClientDataSource ds = new org.apache.derby.jdbc.ClientDataSource();
         ds.setDatabaseName(masterDatabasePath + FS + masterDbSubPath + FS +
                            replicatedDb);
         ds.setServerName(masterServerHost);
         ds.setPortNumber(masterServerPort);
         ds.setConnectionAttributes(useEncryption(false));
         Connection conn = ds.getConnection();
             
         simpleVerify(conn);
         conn.close();
         /*
         runMasterVerificationCLient(jvmVersion,
                 testClientHost,
                 replicatedDb,
                 masterServerHost, masterServerPort);
         */
         util.DEBUG("END   verifyMaster");
     }
     private void simpleVerify(Connection conn) // Verification code..
     throws SQLException
     {
         Statement s = conn.createStatement();
         ResultSet rs = s.executeQuery("select SCHEMAID, TABLENAME from sys.systables");
         while (rs.next())
         {
             util.DEBUG(rs.getString(1) + " " + rs.getString(2));
         }
     }
     
     private void runSlaveVerificationCLient(String jvmVersion,
             String testClientHost,
             String dbName,
             String serverHost,
             int serverPort)
             throws Exception
     {
         util.DEBUG("runSlaveVerificationCLient");
         
         runTestOnSlave(replicationVerify,
                 jvmVersion,
                 testClientHost,
                 serverHost,serverPort,
                 dbName);
         /*
         String URL = slaveURL(dbName);
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         
         String clientJvm = jvmVersion+JVMloc;
         
         String command = clientJvm // "java"
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.slave=" + URL
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + replicationVerify // FIXME! handle junit verificationclients too!
                 ;
         
         String results = null;
         if ( serverHost.equalsIgnoreCase("localhost") )
         {
             runUserCommandLocally(command,
                     userDir,
                     "runSlaveVerificationCLient ");
         }
         else
         {
             results = runUserCommandRemotely(command,
                     testClientHost,
                     testUser,
                     // dbName,
                     "runSlaveVerificationCLient ");
         }
         util.DEBUG(results);
        */
     }
     
     private void runMasterVerificationCLient(String jvmVersion,
             String testClientHost,
             String dbName,
             String serverHost,
             int serverPort)
             throws Exception
     {
         util.DEBUG("runMasterVerificationCLient");
         
         runTest(replicationVerify,
                 jvmVersion,
                 testClientHost,
                 serverHost,serverPort,
                 dbName);
         /*
         String URL = masterURL(dbName);
         String ijClassPath = derbyVersion +FS+ "derbyclient.jar"
                 + PS + derbyVersion +FS+ "derbytools.jar";
         
         String clientJvm = jvmVersion+JVMloc;
         
         String command = clientJvm // "java"
                 + " -Dij.driver=" + DRIVER_CLASS_NAME
                 + " -Dij.connection.master=" + URL
                 + " -classpath " + ijClassPath + " org.apache.derby.tools.ij"
                 + " " + replicationVerify // FIXME! handle junit verificationclients too!
                 ;
         
         String results = null;
         if ( serverHost.equalsIgnoreCase("localhost") )
         {
             runUserCommandLocally(command,
                     userDir,
                     "runMasterVerificationCLient ");            
         }
         else
         {
             results =
                     runUserCommandRemotely(command,
                     testClientHost,
                     testUser,
                     // dbName,
                     "runMasterVerificationCLient ");
         }
         util.DEBUG(results);
        */
     }
     
     /* UNUSED!
     private static Connection getConnection(String driverClassName,String url,String dbUid,String dbPasswd)
     {
         DEBUG("getConnection: '"+driverClassName+"' '"+url+"' '"+dbUid+"' '"+dbPasswd+"'");
         Connection con = null;
         try
         {
             Class.forName(driverClassName).newInstance();
             con = DriverManager.getConnection(url); //,dbUid,dbPasswd);
         }
         catch (Exception e)
         {
             util.DEBUG("Failed to create connection: " + url + ", " + e.getMessage());
             e.printStackTrace();
         }
         return con;
     }
      */
     
     
     /* private */ String runUserCommand(String command,
             String testUser)
     {
         util.DEBUG("Execute '"+ command +"'");
         String output = "";
         
         try
         {
             Runtime rt = Runtime.getRuntime();
             Process proc = rt.exec(command);
             
             output = processOutput(proc);
             
             int exitVal = proc.waitFor();
             util.DEBUG("ExitValue: " + exitVal);
         }
         catch (Throwable t)
         {
             t.printStackTrace();
         }
         
         return output;
     }
     private String runUserCommand(String command,
             String testUser,
             // String dbDir,
             String id)
     {
         final String ID= "runUserCommand "+id+" ";
         util.DEBUG("Execute '"+ command +"'");
         String output = "";
         
         try
         {
             Runtime rt = Runtime.getRuntime();
             Process proc = rt.exec(command);
             
             output = processOutput(ID, proc);
             
             int exitVal = proc.waitFor();
             util.DEBUG("ExitValue: " + exitVal);
         }
         catch (Throwable t)
         {
             t.printStackTrace();
         }
         
         return output;
     }
     private void runUserCommandLocally(String command, String user_dir, String ID)
     { // Running on localhost.
         util.DEBUG("");
         final String debugId = "runUserCommandLocally " + ID + " ";
         util.DEBUG("+++ runUserCommandLocally " + command + " / " + user_dir);
                         
         String tmp ="";
         util.DEBUG(debugId+command);
         
         final String fullCmd = command;
         
         String[] envElements = null; // rt.exec() will inherit..
         /*
         tmp ="";
         for ( int i=0;i 0) {
                 gotEx = null;
 
                 try {
                     DriverManager.getConnection(connectionURL);
                     fail("Unexpectedly connected");
                 } catch (SQLException se) {
                     if (se.getSQLState().
                             equals(SLAVE_OPERATION_DENIED_WHILE_CONNECTED)) {
                         // Try again, shutdown did not complete yet..
                         gotEx = se;
                         util.DEBUG
                             ("got SLAVE_OPERATION_DENIED_WHILE_CONNECTED, " +
                              "sleep");
                         Thread.sleep(1000L);
                         continue;
 
                     } else if (se.getSQLState().
                                    equals(REPLICATION_SLAVE_SHUTDOWN_OK)) {
                         // Try again, shutdown started but did not complete yet.
                         gotEx = se;
                         util.DEBUG("got REPLICATION_SLAVE_SHUTDOWN_OK, " +
                                    "sleep..");
                         Thread.sleep(1000L);
                         continue;
 
                     } else if (se.getSQLState().
                                    equals(REPLICATION_DB_NOT_BOOTED)) {
                         // All is fine, so proceed
                         util.DEBUG("Got REPLICATION_DB_NOT_BOOTED as expected");
                         break;
 
                     } else {
                         // Something else, so report.
                         gotEx = se;
                         break;
                     }
                 }
             }
 
             if (gotEx != null) {
                 // We did not get what we expected as the final state
                 // (REPLICATION_DB_NOT_BOOTED) in reasonable time, or we saw
                 // something that is not a legal intermediate state, so we fail
                 // now:
                 throw gotEx;
             }
 
             return null;
         }
     }
 }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy