org.pentaho.di.www.SlaveServerConfig Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of kettle-engine Show documentation
Show all versions of kettle-engine Show documentation
Container pom for Pentaho Data Integration modules
The newest version!
/*! ******************************************************************************
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/
package org.pentaho.di.www;
import org.pentaho.di.cluster.SlaveServer;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.database.Database;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.encryption.Encr;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.logging.LogChannel;
import org.pentaho.di.core.logging.LogChannelInterface;
import org.pentaho.di.core.logging.LoggingObjectInterface;
import org.pentaho.di.core.logging.LoggingObjectType;
import org.pentaho.di.core.logging.SimpleLoggingObject;
import org.pentaho.di.core.plugins.PluginRegistry;
import org.pentaho.di.core.plugins.RepositoryPluginType;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.metastore.MetaStoreConst;
import org.pentaho.di.repository.RepositoriesMeta;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.repository.RepositoryMeta;
import org.pentaho.metastore.api.exceptions.MetaStoreException;
import org.pentaho.metastore.stores.delegate.DelegatingMetaStore;
import org.pentaho.metastore.stores.memory.MemoryMetaStore;
import org.pentaho.metastore.stores.xml.XmlMetaStore;
import org.w3c.dom.Node;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
public class SlaveServerConfig {
public static final String XML_TAG = "slave_config";
public static final String XML_TAG_MASTERS = "masters";
public static final String XML_TAG_REPOSITORY = "repository";
public static final String XML_TAG_SEQUENCES = "sequences";
public static final String XML_TAG_AUTOSEQUENCE = "autosequence";
public static final String XML_TAG_AUTO_CREATE = "autocreate";
public static final String XML_TAG_JETTY_OPTIONS = "jetty_options";
public static final String XML_TAG_ACCEPTORS = "acceptors";
public static final String XML_TAG_ACCEPT_QUEUE_SIZE = "acceptQueueSize";
public static final String XML_TAG_LOW_RES_MAX_IDLE_TIME = "lowResourcesMaxIdleTime";
private List masters;
private SlaveServer slaveServer;
private boolean reportingToMasters;
private boolean joining;
private int maxLogLines;
private int maxLogTimeoutMinutes;
private int objectTimeoutMinutes;
private String filename;
private List databases;
private List slaveSequences;
private SlaveSequence autoSequence;
private boolean automaticCreationAllowed;
private Repository repository;
private RepositoryMeta repositoryMeta;
private String repositoryId;
private String repositoryUsername;
private String repositoryPassword;
private DelegatingMetaStore metaStore;
private String passwordFile;
public SlaveServerConfig() {
masters = new ArrayList();
databases = new ArrayList();
slaveSequences = new ArrayList();
automaticCreationAllowed = false;
metaStore = new DelegatingMetaStore();
// Add the local Hitachi Vantara MetaStore to the delegation.
// This sets it as the active one.
//
try {
XmlMetaStore localStore = new XmlMetaStore( MetaStoreConst.getDefaultPentahoMetaStoreLocation() );
metaStore.addMetaStore( localStore );
metaStore.setActiveMetaStoreName( localStore.getName() );
} catch ( MetaStoreException e ) {
LogChannel.GENERAL.logError( "Unable to open local Pentaho meta store from [" + MetaStoreConst.getDefaultPentahoMetaStoreLocation() + "]", e );
// now replace this with an in memory metastore.
//
try {
MemoryMetaStore memoryStore = new MemoryMetaStore();
memoryStore.setName( "Memory metastore" );
metaStore.addMetaStore( memoryStore );
metaStore.setActiveMetaStoreName( memoryStore.getName() );
} catch ( MetaStoreException e2 ) {
throw new RuntimeException( "Unable to add a default memory metastore to the delegating store", e );
}
}
passwordFile = null; // force lookup by server in ~/.kettle or local folder
}
public SlaveServerConfig( SlaveServer slaveServer ) {
this();
this.slaveServer = slaveServer;
}
public SlaveServerConfig( List masters, boolean reportingToMasters, SlaveServer slaveServer ) {
this.masters = masters;
this.reportingToMasters = reportingToMasters;
this.slaveServer = slaveServer;
}
public String getXML() {
StringBuilder xml = new StringBuilder();
xml.append( XMLHandler.openTag( XML_TAG ) );
for ( SlaveServer slaveServer : masters ) {
xml.append( slaveServer.getXML() );
}
XMLHandler.addTagValue( "report_to_masters", reportingToMasters );
if ( slaveServer != null ) {
xml.append( slaveServer.getXML() );
}
XMLHandler.addTagValue( "joining", joining );
XMLHandler.addTagValue( "max_log_lines", maxLogLines );
XMLHandler.addTagValue( "max_log_timeout_minutes", maxLogTimeoutMinutes );
XMLHandler.addTagValue( "object_timeout_minutes", objectTimeoutMinutes );
xml.append( XMLHandler.openTag( XML_TAG_SEQUENCES ) );
for ( SlaveSequence slaveSequence : slaveSequences ) {
xml.append( XMLHandler.openTag( SlaveSequence.XML_TAG ) );
xml.append( slaveSequence.getXML() );
xml.append( XMLHandler.closeTag( SlaveSequence.XML_TAG ) );
}
xml.append( XMLHandler.closeTag( XML_TAG_SEQUENCES ) );
if ( autoSequence != null ) {
xml.append( XMLHandler.openTag( XML_TAG_AUTOSEQUENCE ) );
xml.append( autoSequence.getXML() );
xml.append( XMLHandler.addTagValue( XML_TAG_AUTO_CREATE, automaticCreationAllowed ) );
xml.append( XMLHandler.closeTag( XML_TAG_AUTOSEQUENCE ) );
}
if ( repositoryMeta != null ) {
xml.append( XMLHandler.openTag( XML_TAG_REPOSITORY ) );
xml.append( " " ).append( XMLHandler.addTagValue( "id", repositoryMeta.getId() ) );
xml.append( " " ).append( XMLHandler.addTagValue( "username", repositoryUsername ) );
xml.append( " " ).append(
XMLHandler.addTagValue( "password", Encr.encryptPasswordIfNotUsingVariables( repositoryPassword ) ) );
xml.append( XMLHandler.closeTag( XML_TAG_REPOSITORY ) );
}
xml.append( XMLHandler.closeTag( XML_TAG ) );
return xml.toString();
}
public SlaveServerConfig( LogChannelInterface log, Node node ) throws KettleXMLException {
this();
Node slaveNode = XMLHandler.getSubNode( node, SlaveServer.XML_TAG );
if ( slaveNode != null ) {
slaveServer = new SlaveServer( slaveNode );
checkNetworkInterfaceSetting( log, slaveNode, slaveServer );
}
Node mastersNode = XMLHandler.getSubNode( node, XML_TAG_MASTERS );
int nrMasters = XMLHandler.countNodes( mastersNode, SlaveServer.XML_TAG );
for ( int i = 0; i < nrMasters; i++ ) {
Node masterSlaveNode = XMLHandler.getSubNodeByNr( mastersNode, SlaveServer.XML_TAG, i );
SlaveServer masterSlaveServer = new SlaveServer( masterSlaveNode );
checkNetworkInterfaceSetting( log, masterSlaveNode, masterSlaveServer );
masterSlaveServer.setSslMode( slaveServer.isSslMode() );
masters.add( masterSlaveServer );
}
reportingToMasters = "Y".equalsIgnoreCase( XMLHandler.getTagValue( node, "report_to_masters" ) );
joining = "Y".equalsIgnoreCase( XMLHandler.getTagValue( node, "joining" ) );
maxLogLines = Const.toInt( XMLHandler.getTagValue( node, "max_log_lines" ), 0 );
maxLogTimeoutMinutes = Const.toInt( XMLHandler.getTagValue( node, "max_log_timeout_minutes" ), 0 );
objectTimeoutMinutes = Const.toInt( XMLHandler.getTagValue( node, "object_timeout_minutes" ), 0 );
// Read sequence information
//
List dbNodes = XMLHandler.getNodes( node, DatabaseMeta.XML_TAG );
for ( Node dbNode : dbNodes ) {
databases.add( new DatabaseMeta( dbNode ) );
}
Node sequencesNode = XMLHandler.getSubNode( node, "sequences" );
List seqNodes = XMLHandler.getNodes( sequencesNode, SlaveSequence.XML_TAG );
for ( Node seqNode : seqNodes ) {
slaveSequences.add( new SlaveSequence( seqNode, databases ) );
}
Node autoSequenceNode = XMLHandler.getSubNode( node, XML_TAG_AUTOSEQUENCE );
if ( autoSequenceNode != null ) {
autoSequence = new SlaveSequence( autoSequenceNode, databases );
automaticCreationAllowed =
"Y".equalsIgnoreCase( XMLHandler.getTagValue( autoSequenceNode, XML_TAG_AUTO_CREATE ) );
}
// Set Jetty Options
setUpJettyOptions( node );
Node repositoryNode = XMLHandler.getSubNode( node, XML_TAG_REPOSITORY );
repositoryId = XMLHandler.getTagValue( repositoryNode, "name" );
repositoryUsername = XMLHandler.getTagValue( repositoryNode, "username" );
repositoryPassword = XMLHandler.getTagValue( repositoryNode, "password" );
}
/** Set up jetty options to the system properties
* @param node
*/
protected void setUpJettyOptions( Node node ) {
Map jettyOptions = parseJettyOptions( node );
if ( jettyOptions != null && jettyOptions.size() > 0 ) {
for ( Entry jettyOption : jettyOptions.entrySet() ) {
System.setProperty( jettyOption.getKey(), jettyOption.getValue() );
}
}
}
/**
* Read and parse jetty options
*
* @param node
* that contains jetty options nodes
* @return map of not empty jetty options
*/
protected Map parseJettyOptions( Node node ) {
Map jettyOptions = null;
Node jettyOptionsNode = XMLHandler.getSubNode( node, XML_TAG_JETTY_OPTIONS );
if ( jettyOptionsNode != null ) {
jettyOptions = new HashMap();
if ( XMLHandler.getTagValue( jettyOptionsNode, XML_TAG_ACCEPTORS ) != null ) {
jettyOptions.put( Const.KETTLE_CARTE_JETTY_ACCEPTORS, XMLHandler.getTagValue( jettyOptionsNode, XML_TAG_ACCEPTORS ) );
}
if ( XMLHandler.getTagValue( jettyOptionsNode, XML_TAG_ACCEPT_QUEUE_SIZE ) != null ) {
jettyOptions.put( Const.KETTLE_CARTE_JETTY_ACCEPT_QUEUE_SIZE, XMLHandler.getTagValue( jettyOptionsNode,
XML_TAG_ACCEPT_QUEUE_SIZE ) );
}
if ( XMLHandler.getTagValue( jettyOptionsNode, XML_TAG_LOW_RES_MAX_IDLE_TIME ) != null ) {
jettyOptions.put( Const.KETTLE_CARTE_JETTY_RES_MAX_IDLE_TIME, XMLHandler.getTagValue( jettyOptionsNode,
XML_TAG_LOW_RES_MAX_IDLE_TIME ) );
}
}
return jettyOptions;
}
private void openRepository( String repositoryId ) throws KettleException {
try {
RepositoriesMeta repositoriesMeta = new RepositoriesMeta();
repositoriesMeta.readData();
repositoryMeta = repositoriesMeta.findRepository( repositoryId );
if ( repositoryMeta == null ) {
throw new KettleException( "Unable to find repository: " + repositoryId );
}
PluginRegistry registry = PluginRegistry.getInstance();
repository = registry.loadClass( RepositoryPluginType.class, repositoryMeta, Repository.class );
repository.init( repositoryMeta );
repository.connect( repositoryUsername, repositoryPassword );
// Add the repository MetaStore to the delegation as well.
// Set this one as active with the highest priority
//
if ( repository.getMetaStore() != null ) {
metaStore.addMetaStore( 0, repository.getMetaStore() );
metaStore.setActiveMetaStoreName( repository.getMetaStore().getName() );
}
LogChannel.GENERAL.logBasic( "Connected to repository '" + repository.getName() + "'" );
} catch ( Exception e ) {
throw new KettleException( "Unable to open repository connection", e );
}
}
public void readAutoSequences() throws KettleException {
if ( autoSequence == null ) {
return;
}
Database database = null;
try {
DatabaseMeta databaseMeta = autoSequence.getDatabaseMeta();
LoggingObjectInterface loggingInterface =
new SimpleLoggingObject( "auto-sequence", LoggingObjectType.GENERAL, null );
database = new Database( loggingInterface, databaseMeta );
database.connect();
String schemaTable =
databaseMeta.getQuotedSchemaTableCombination( autoSequence.getSchemaName(), autoSequence.getTableName() );
String seqField = databaseMeta.quoteField( autoSequence.getSequenceNameField() );
String valueField = databaseMeta.quoteField( autoSequence.getValueField() );
String sql = "SELECT " + seqField + ", " + valueField + " FROM " + schemaTable;
List rows = database.getRows( sql, 0 );
RowMetaInterface rowMeta = database.getReturnRowMeta();
for ( Object[] row : rows ) {
// Automatically create a new sequence for each sequence found...
//
String sequenceName = rowMeta.getString( row, seqField, null );
if ( !Utils.isEmpty( sequenceName ) ) {
Long value = rowMeta.getInteger( row, valueField, null );
if ( value != null ) {
SlaveSequence slaveSequence =
new SlaveSequence( sequenceName, value, databaseMeta, autoSequence.getSchemaName(), autoSequence
.getTableName(), autoSequence.getSequenceNameField(), autoSequence.getValueField() );
slaveSequences.add( slaveSequence );
LogChannel.GENERAL.logBasic( "Automatically created slave sequence '"
+ slaveSequence.getName() + "' with start value " + slaveSequence.getStartValue() );
}
}
}
} catch ( Exception e ) {
throw new KettleException( "Unable to automatically configure slave sequences", e );
} finally {
if ( database != null ) {
database.disconnect();
}
}
}
private void checkNetworkInterfaceSetting( LogChannelInterface log, Node slaveNode, SlaveServer slaveServer ) {
// See if we need to grab the network interface to use and then override the host name
//
String networkInterfaceName = XMLHandler.getTagValue( slaveNode, "network_interface" );
if ( !Utils.isEmpty( networkInterfaceName ) ) {
// OK, so let's try to get the IP address for this network interface...
//
try {
String newHostname = Const.getIPAddress( networkInterfaceName );
if ( newHostname != null ) {
slaveServer.setHostname( newHostname );
// Also change the name of the slave...
//
slaveServer.setName( slaveServer.getName() + "-" + newHostname );
log.logBasic( "Hostname for slave server ["
+ slaveServer.getName() + "] is set to [" + newHostname + "], information derived from network "
+ networkInterfaceName );
}
} catch ( SocketException e ) {
log.logError( "Unable to get the IP address for network interface "
+ networkInterfaceName + " for slave server [" + slaveServer.getName() + "]", e );
}
}
}
public SlaveServerConfig( String hostname, int port, boolean joining ) {
this();
this.joining = joining;
this.slaveServer = new SlaveServer( hostname + ":" + port, hostname, "" + port, null, null );
}
/**
* @return the list of masters to report back to if the report to masters flag is enabled.
*/
public List getMasters() {
return masters;
}
/**
* @param masters
* the list of masters to set. It is the list of masters to report back to if the report to masters flag is
* enabled.
*/
public void setMasters( List masters ) {
this.masters = masters;
}
/**
* @return the slave server.
* The user name and password defined in here are used to contact this slave by the masters.
*/
public SlaveServer getSlaveServer() {
return slaveServer;
}
/**
* @param slaveServer
* the slave server details to set.
* The user name and password defined in here are used to contact this slave by the masters.
*/
public void setSlaveServer( SlaveServer slaveServer ) {
this.slaveServer = slaveServer;
}
/**
* @return true if this slave reports to the masters
*/
public boolean isReportingToMasters() {
return reportingToMasters;
}
/**
* @param reportingToMaster
* set to true if this slave should report to the masters
*/
public void setReportingToMasters( boolean reportingToMaster ) {
this.reportingToMasters = reportingToMaster;
}
/**
* @return true if the webserver needs to join with the webserver threads (wait/block until finished)
*/
public boolean isJoining() {
return joining;
}
/**
* @param joining
* Set to true if the webserver needs to join with the webserver threads (wait/block until finished)
*/
public void setJoining( boolean joining ) {
this.joining = joining;
}
/**
* @return the maxLogLines
*/
public int getMaxLogLines() {
return maxLogLines;
}
/**
* @param maxLogLines
* the maxLogLines to set
*/
public void setMaxLogLines( int maxLogLines ) {
this.maxLogLines = maxLogLines;
}
/**
* @return the maxLogTimeoutMinutes
*/
public int getMaxLogTimeoutMinutes() {
return maxLogTimeoutMinutes;
}
/**
* @param maxLogTimeoutMinutes
* the maxLogTimeoutMinutes to set
*/
public void setMaxLogTimeoutMinutes( int maxLogTimeoutMinutes ) {
this.maxLogTimeoutMinutes = maxLogTimeoutMinutes;
}
/**
* @return the objectTimeoutMinutes
*/
public int getObjectTimeoutMinutes() {
return objectTimeoutMinutes;
}
/**
* @param objectTimeoutMinutes
* the objectTimeoutMinutes to set
*/
public void setObjectTimeoutMinutes( int objectTimeoutMinutes ) {
this.objectTimeoutMinutes = objectTimeoutMinutes;
}
/**
* @return the filename
*/
public String getFilename() {
return filename;
}
/**
* @param filename
* the filename to set
*/
public void setFilename( String filename ) {
this.filename = filename;
}
/**
* @return the databases
*/
public List getDatabases() {
return databases;
}
/**
* @param databases
* the databases to set
*/
public void setDatabases( List databases ) {
this.databases = databases;
}
/**
* @return the slaveSequences
*/
public List getSlaveSequences() {
return slaveSequences;
}
/**
* @param slaveSequences
* the slaveSequences to set
*/
public void setSlaveSequences( List slaveSequences ) {
this.slaveSequences = slaveSequences;
}
/**
* @return the autoSequence
*/
public SlaveSequence getAutoSequence() {
return autoSequence;
}
/**
* @param autoSequence
* the autoSequence to set
*/
public void setAutoSequence( SlaveSequence autoSequence ) {
this.autoSequence = autoSequence;
}
/**
* @return the automaticCreationAllowed
*/
public boolean isAutomaticCreationAllowed() {
return automaticCreationAllowed;
}
/**
* @param automaticCreationAllowed
* the automaticCreationAllowed to set
*/
public void setAutomaticCreationAllowed( boolean automaticCreationAllowed ) {
this.automaticCreationAllowed = automaticCreationAllowed;
}
/**
* @return the repository, loaded lazily
*/
public Repository getRepository() throws KettleException {
if ( !Utils.isEmpty( repositoryId ) && repository == null ) {
openRepository( repositoryId );
}
return repository;
}
/**
* @param repository
* the repository to set
*/
public void setRepository( Repository repository ) {
this.repository = repository;
}
/**
* @return the repositoryUsername
*/
public String getRepositoryUsername() {
return repositoryUsername;
}
/**
* @param repositoryUsername
* the repositoryUsername to set
*/
public void setRepositoryUsername( String repositoryUsername ) {
this.repositoryUsername = repositoryUsername;
}
/**
* @return the repositoryPassword
*/
public String getRepositoryPassword() {
return repositoryPassword;
}
/**
* @param repositoryPassword
* the repositoryPassword to set
*/
public void setRepositoryPassword( String repositoryPassword ) {
this.repositoryPassword = repositoryPassword;
}
public DelegatingMetaStore getMetaStore() {
return metaStore;
}
public void setMetaStore( DelegatingMetaStore metaStore ) {
this.metaStore = metaStore;
}
public String getPasswordFile() {
return passwordFile;
}
public void setPasswordFile( String passwordFile ) {
this.passwordFile = passwordFile;
}
public String getRepositoryId() {
return repositoryId;
}
public void setRepositoryId( String repositoryId ) {
this.repositoryId = repositoryId;
}
}