au.net.causal.maven.plugins.boxdb.db.OracleDatabase Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of boxdb-maven-plugin Show documentation
Show all versions of boxdb-maven-plugin Show documentation
Maven plugin to start databases using Docker and VMs
package au.net.causal.maven.plugins.boxdb.db;
import au.net.causal.maven.plugins.boxdb.ImageCheckerUtils;
import au.net.causal.maven.plugins.boxdb.JdbcSqlRunner;
import au.net.causal.maven.plugins.boxdb.ScriptReaderRunner;
import com.google.common.collect.ImmutableList;
import io.fabric8.maven.docker.access.DockerAccess;
import io.fabric8.maven.docker.config.ImageConfiguration;
import io.fabric8.maven.docker.config.LogConfiguration;
import io.fabric8.maven.docker.config.RunImageConfiguration;
import io.fabric8.maven.docker.config.RunVolumeConfiguration;
import io.fabric8.maven.docker.log.LogOutputSpec;
import org.apache.maven.plugin.MojoExecutionException;
import org.codehaus.plexus.util.IOUtil;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
public class OracleDatabase extends DockerDatabase
{
private final Path hostScriptDirectory;
private final String dockerImageName;
public OracleDatabase(BoxConfiguration boxConfiguration, ProjectConfiguration projectConfiguration, BoxContext context,
DockerRegistry dockerRegistry, String dockerImageName)
throws IOException
{
super(boxConfiguration, projectConfiguration, context, dockerRegistry);
Objects.requireNonNull(dockerImageName, "dockerImageName == null");
this.dockerImageName = dockerImageName;
//Generate host script directory that is used to give scripts to Oracle
hostScriptDirectory = context.getTempDirectory().resolve(containerName() + "-scripts");
if (Files.notExists(hostScriptDirectory))
Files.createDirectories(hostScriptDirectory);
}
@Override
protected void configureRunImage(RunImageConfiguration.Builder builder)
{
super.configureRunImage(builder);
getContext().getLog().debug("Oracle script directory: " + hostScriptDirectory);
RunVolumeConfiguration scriptVolume = new RunVolumeConfiguration.Builder()
.bind(Arrays.asList(hostScriptDirectory.toAbsolutePath().toString() + ":/data/scripts"))
.build();
builder.env(Collections.singletonMap("ORACLE_ALLOW_REMOTE", "true"));
builder.volumes(scriptVolume);
}
@Override
protected String dockerImageName()
{
return dockerImageName;
}
@Override
protected int containerDatabasePort()
{
return 1521;
}
private void executeSqlPlus(String args, DatabaseTarget targetDatabase, Duration timeout)
throws IOException, SqlPlusException, BoxDatabaseException
{
//Docker plugin provides no easy way to get exit code from exec call so we need to script it and
//return its value via the file system and a temp file
Path returnCodeFile = Files.createTempFile(hostScriptDirectory, "return", ".txt");
executeSqlPlus(args, targetDatabase, timeout, returnCodeFile, false);
}
protected String oracleHomePath()
{
return "";
}
private void executeSqlPlus(String args, DatabaseTarget targetDatabase, Duration timeout, Path returnCodeFile, boolean silent)
throws IOException, SqlPlusException, BoxDatabaseException
{
if (targetDatabase != null)
args = targetDatabase.user(getBoxConfiguration()) + "/" + targetDatabase.password(getBoxConfiguration()) + " " + args;
else
args = "/nolog " + args;
if (silent)
args = "-S " + args;
DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();
LogConfiguration logConfig = new LogConfiguration.Builder()
.enabled(!silent)
.prefix("sqlplus")
.build();
RunImageConfiguration sqlPlusRunConfig = new RunImageConfiguration.Builder()
.links(Collections.singletonList(containerName() + ":oracle"))
.containerNamePattern("%a")
.cmd("unused") //not used but must be non-null - is replaced later
.log(logConfig)
.build();
String sqlPlusExecutablePath = oracleHomePath() + "sqlplus";
sqlPlusRunConfig.getCmd().setExec(Arrays.asList("bash", "-i", "-c", sqlPlusExecutablePath + " -L " + args + " ; echo $? > " + "/data/scripts/" + returnCodeFile.getFileName().toString()));
getContext().getLog().debug("Executing command: " + sqlPlusRunConfig.getCmd().getExec());
sqlPlusRunConfig.getCmd().setShell(null);
ImageConfiguration sqlPlusConfig = new ImageConfiguration.Builder()
.runConfig(sqlPlusRunConfig)
.name(dockerImageName())
.alias(containerName() + "-sqlplus")
.build();
String containerId = findDockerContainer().getId();
String execContainerId = docker.createExecContainer(containerId, sqlPlusRunConfig.getCmd());
LogOutputSpec execLog;
Path logFile = null;
if (silent)
{
logFile = Files.createTempFile(returnCodeFile.getParent(), "wait", ".log");
execLog = new LogOutputSpec.Builder().logStdout(false).file(logFile.toAbsolutePath().toString()).build();
}
else
execLog = getContext().getLogSpecFactory().createSpec(execContainerId, sqlPlusConfig);
docker.startExecContainer(execContainerId, execLog);
getContext().getLog().debug("Exec container done " + execContainerId);
String exitCode = new String(Files.readAllBytes(returnCodeFile), StandardCharsets.UTF_8).trim();
getContext().getLog().debug("All done: " + exitCode);
//If we wanted silent run we won't want to keep the output so just delete these files
//otherwise we'll have a huge amount of them, especially for the waiter script
if (logFile != null)
{
if (getContext().getLog().isDebugEnabled())
getContext().getLog().debug(Files.lines(logFile).collect(Collectors.joining(System.lineSeparator())));
try
{
Files.deleteIfExists(logFile);
}
catch (IOException e)
{
//Not a huge deal if we can't delete
getContext().getLog().debug("Failed to delete temporary log file " + logFile + ": " + e, e);
}
}
if (!"0".equals(exitCode))
throw new SqlPlusException("SQLPlus exit code: " + exitCode, exitCode);
}
@Override
protected void executeScriptFile(Path scriptFile, DatabaseTarget targetDatabase, Duration timeout)
throws IOException, SQLException, BoxDatabaseException
{
if (!Files.exists(scriptFile))
throw new NoSuchFileException(scriptFile.toString());
//Copy the script file to inside our script dir mounted to the container
Path mountedScriptFile = Files.createTempFile(hostScriptDirectory, "script", ".sql");
Files.copy(scriptFile, mountedScriptFile, StandardCopyOption.REPLACE_EXISTING);
executeSqlPlus("@/data/scripts/" + mountedScriptFile.getFileName().toString(), targetDatabase, timeout);
}
@Override
public void executeScript(URL script, DatabaseTarget targetDatabase, Duration timeout)
throws IOException, SQLException, BoxDatabaseException
{
//Copy the script file to inside our script dir mounted to the container
Path mountedScriptFile = Files.createTempFile(hostScriptDirectory, "script", ".sql");
try (OutputStream os = Files.newOutputStream(mountedScriptFile);
InputStream is = script.openStream())
{
IOUtil.copy(is, os);
}
executeSqlPlus("@/data/scripts/" + mountedScriptFile.getFileName().toString(), targetDatabase, timeout);
}
@Override
public void executeSql(String sql, DatabaseTarget targetDatabase, Duration timeout)
throws IOException, SQLException, BoxDatabaseException
{
executeSqlPlus("<<< \"" + ScriptUtils.shellEscape(sql) + "\"", targetDatabase, timeout);
}
@Override
public JdbcConnectionInfo jdbcConnectionInfo(DatabaseTarget target) throws BoxDatabaseException
{
String uri = "jdbc:oracle:thin:@" +
getContext().getDockerHostAddress() +
":" + getBoxConfiguration().getDatabasePort() +
":xe";
return new JdbcConnectionInfo(uri,
target.user(getBoxConfiguration()),
target.password(getBoxConfiguration()),
getContext().getDockerHostAddress(),
getBoxConfiguration().getDatabasePort());
}
@Override
public JdbcDriverInfo jdbcDriverInfo()
throws BoxDatabaseException
{
//Drivers are backward/forward compatible so just use latest
return new JdbcDriverInfo(new RunnerDependency("com.oracle", "ojdbc7", "12.1.0.2"), "oracle.jdbc.driver.OracleDriver",
"http://www.oracle.com/technetwork/database/features/jdbc/index-091264.html");
}
@Override
public void configureNewDatabase()
throws IOException, SQLException, BoxDatabaseException
{
runFilteredScript("oracle-create-database.sql");
}
@Override
public void waitUntilStarted(Duration maxTimeToWait) throws TimeoutException, BoxDatabaseException
{
long startTime = System.currentTimeMillis();
long maxSystemTime = startTime + maxTimeToWait.toMillis();
super.waitUntilStarted(maxTimeToWait);
//Extract once
URL waitScript = OracleDatabase.class.getResource("oracle-wait.sql");
try
{
Path mountedScriptFile = Files.createTempFile(hostScriptDirectory, "wait", ".sql");
try (OutputStream os = Files.newOutputStream(mountedScriptFile);
InputStream is = waitScript.openStream())
{
IOUtil.copy(is, os);
}
waitWithScriptFile(mountedScriptFile, maxSystemTime, maxTimeToWait);
}
catch (IOException | SQLException e)
{
throw new BoxDatabaseException("Error while waiting for database to come up: " + e, e);
}
}
private void waitWithScriptFile(Path mountedScriptFile, long maxSystemTime, Duration waitTimeout)
throws BoxDatabaseException, IOException, SQLException, TimeoutException
{
//Use one return code file instead of one for each execution
Path returnCodeFile = Files.createTempFile(hostScriptDirectory, "return", ".txt");
try
{
boolean oracleReady = false;
do
{
try
{
String arg = "@/data/scripts/" + mountedScriptFile.getFileName().toString();
boolean debugOutput = getContext().getLog().isDebugEnabled();
executeSqlPlus(arg, null, waitTimeout, returnCodeFile, !debugOutput);
oracleReady = true;
}
catch (SqlPlusException e)
{
getContext().getLog().debug("Waiting for database to come up: " + e);
//This exit code is fine - it means Oracle is not yet up
if ("22".equals(e.getExitCode()))
Thread.sleep(getProjectConfiguration().getPollTime().toMillis());
else
throw e;
}
}
while (!oracleReady && System.currentTimeMillis() <= maxSystemTime); //120 seconds should be enough time for it to come up)
if (!oracleReady)
throw new TimeoutException("Timed out waiting for Oracle to start");
}
catch (InterruptedException e)
{
throw new BoxDatabaseException("Interrupted waiting", e);
}
}
protected void runFilteredScript(String scriptResourceName)
throws IOException, BoxDatabaseException, SQLException
{
URL scriptResource = OracleDatabase.class.getResource(scriptResourceName);
if (scriptResource == null)
throw new FileNotFoundException("Missing script resource: " + scriptResourceName);
ScriptReaderRunner scriptRunner = getContext().createScriptReaderRunner(this, getBoxConfiguration(), getProjectConfiguration());
ScriptReaderExecution execution = new ScriptReaderExecution();
execution.setFiltering(true);
execution.setScripts(Arrays.asList(scriptResource));
try
{
scriptRunner.execute(execution, DatabaseTarget.ADMIN, getProjectConfiguration().getScriptTimeout());
}
catch (MojoExecutionException e)
{
throw new BoxDatabaseException(e);
}
}
protected void runShellScript(URL scriptResource)
throws IOException, BoxDatabaseException
{
//Copy the script file to inside our script dir mounted to the container
Path mountedScriptFile = Files.createTempFile(hostScriptDirectory, "script", ".sh");
try (OutputStream os = Files.newOutputStream(mountedScriptFile);
InputStream is = scriptResource.openStream())
{
IOUtil.copy(is, os);
}
DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();
LogConfiguration logConfig = new LogConfiguration.Builder()
.enabled(true)
.prefix("bash")
.build();
RunImageConfiguration bashRunConfig = new RunImageConfiguration.Builder()
.links(Collections.singletonList(containerName() + ":oracle"))
.containerNamePattern("%a")
.cmd("unused") //not used but must be non-null - is replaced later
.log(logConfig)
.build();
bashRunConfig.getCmd().setExec(Arrays.asList("bash", "/data/scripts/" + mountedScriptFile.getFileName().toString()));
getContext().getLog().debug("Executing command: " + bashRunConfig.getCmd().getExec());
bashRunConfig.getCmd().setShell(null);
ImageConfiguration sqlPlusConfig = new ImageConfiguration.Builder()
.runConfig(bashRunConfig)
.name(dockerImageName())
.alias(containerName() + "-bash")
.build();
String containerId = findDockerContainer().getId();
String execContainerId = docker.createExecContainer(containerId, bashRunConfig.getCmd());
LogOutputSpec execLog = getContext().getLogSpecFactory().createSpec(execContainerId, sqlPlusConfig);
docker.startExecContainer(execContainerId, execLog);
getContext().getLog().debug("Exec container done " + execContainerId);
}
protected void executeExpDp(String args, DatabaseTarget target, Duration timeout)
throws IOException, SQLException, BoxDatabaseException
{
//Docker plugin provides no easy way to get exit code from exec call so we need to script it and
//return its value via the file system and a temp file
Path returnCodeFile = Files.createTempFile(hostScriptDirectory, "return", ".txt");
executeExpDp(args, target, timeout, returnCodeFile, false);
}
protected void executeExpDp(String args, DatabaseTarget target, Duration timeout, Path returnCodeFile, boolean silent)
throws IOException, SQLException, BoxDatabaseException
{
if (target != null)
args = target.user(getBoxConfiguration()) + "/" + target.password(getBoxConfiguration()) + " " + args;
DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();
LogConfiguration logConfig = new LogConfiguration.Builder()
.enabled(!silent)
.prefix("expdp")
.build();
RunImageConfiguration expDpRunConfig = new RunImageConfiguration.Builder()
.links(Collections.singletonList(containerName() + ":oracle"))
.containerNamePattern("%a")
.cmd("unused") //not used but must be non-null - is replaced later
.log(logConfig)
.build();
expDpRunConfig.getCmd().setExec(Arrays.asList("bash", "-i", "-c", oracleHomePath() + "expdp " + args + " ; echo $? > " + "/data/scripts/" + returnCodeFile.getFileName().toString()));
expDpRunConfig.getCmd().setShell(null);
ImageConfiguration expDpConfig = new ImageConfiguration.Builder()
.runConfig(expDpRunConfig)
.name(dockerImageName())
.alias(containerName() + "-expdp")
.build();
String containerId = findDockerContainer().getId();
String execContainerId = docker.createExecContainer(containerId, expDpRunConfig.getCmd());
LogOutputSpec execLog;
Path logFile = null;
if (silent)
{
logFile = Files.createTempFile(returnCodeFile.getParent(), "wait", ".log");
execLog = new LogOutputSpec.Builder().logStdout(false).file(logFile.toAbsolutePath().toString()).build();
}
else
execLog = getContext().getLogSpecFactory().createSpec(execContainerId, expDpConfig);
docker.startExecContainer(execContainerId, execLog);
getContext().getLog().debug("Exec container done " + execContainerId);
String exitCode = new String(Files.readAllBytes(returnCodeFile), StandardCharsets.UTF_8).trim();
getContext().getLog().debug("All done: " + exitCode);
//If we wanted silent run we won't want to keep the output so just delete these files
//otherwise we'll have a huge amount of them, especially for the waiter script
if (logFile != null)
Files.deleteIfExists(logFile);
if (!"0".equals(exitCode))
throw new SQLException("expdp exit code: " + exitCode);
}
protected void executeImpDp(String args, DatabaseTarget target, Duration timeout)
throws IOException, SQLException, BoxDatabaseException
{
//Docker plugin provides no easy way to get exit code from exec call so we need to script it and
//return its value via the file system and a temp file
Path returnCodeFile = Files.createTempFile(hostScriptDirectory, "return", ".txt");
executeImpDp(args, target, timeout, returnCodeFile, false);
}
protected void executeImpDp(String args, DatabaseTarget target, Duration timeout, Path returnCodeFile, boolean silent)
throws IOException, SQLException, BoxDatabaseException
{
if (target != null)
args = target.user(getBoxConfiguration()) + "/" + target.password(getBoxConfiguration()) + " " + args;
DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();
LogConfiguration logConfig = new LogConfiguration.Builder()
.enabled(!silent)
.prefix("impdp")
.build();
RunImageConfiguration impDpRunConfig = new RunImageConfiguration.Builder()
.links(Collections.singletonList(containerName() + ":oracle"))
.containerNamePattern("%a")
.cmd("unused") //not used but must be non-null - is replaced later
.log(logConfig)
.build();
impDpRunConfig.getCmd().setExec(Arrays.asList("bash", "-i", "-c", oracleHomePath() + "impdp " + args + " ; echo $? > " + "/data/scripts/" + returnCodeFile.getFileName().toString()));
impDpRunConfig.getCmd().setShell(null);
ImageConfiguration impDpConfig = new ImageConfiguration.Builder()
.runConfig(impDpRunConfig)
.name(dockerImageName())
.alias(containerName() + "-impdp")
.build();
String containerId = findDockerContainer().getId();
String execContainerId = docker.createExecContainer(containerId, impDpRunConfig.getCmd());
LogOutputSpec execLog;
Path logFile = null;
if (silent)
{
logFile = Files.createTempFile(returnCodeFile.getParent(), "wait", ".log");
execLog = new LogOutputSpec.Builder().logStdout(false).file(logFile.toAbsolutePath().toString()).build();
}
else
execLog = getContext().getLogSpecFactory().createSpec(execContainerId, impDpConfig);
docker.startExecContainer(execContainerId, execLog);
getContext().getLog().debug("Exec container done " + execContainerId);
String exitCode = new String(Files.readAllBytes(returnCodeFile), StandardCharsets.UTF_8).trim();
getContext().getLog().debug("All done: " + exitCode);
//If we wanted silent run we won't want to keep the output so just delete these files
//otherwise we'll have a huge amount of them, especially for the waiter script
if (logFile != null)
Files.deleteIfExists(logFile);
if (!"0".equals(exitCode))
throw new SQLException("impdp exit code: " + exitCode);
}
@Override
public void backup(Path backupFile, BackupFileTypeHint backupFileTypeHint)
throws BoxDatabaseException, IOException, SQLException
{
//This is the backup file created by Oracle, move it at the end
//If the file name specified has no extension then it gets ".dmp" by Oracle tool
//Thanks Oracle
String oracleFileName = backupFile.getFileName().toString();
if (!oracleFileName.contains("."))
oracleFileName = oracleFileName + ".dmp";
Path targetFile = hostScriptDirectory.resolve(oracleFileName);
//Set up directory and give database user permission to use it
runFilteredScript("oracle-prepare-backup.sql");
//Run the dump tool
String args = "schemas=" + getBoxConfiguration().getDatabaseUser() +
" directory=backup_dir dumpfile=" + backupFile.getFileName().toString() +
" logfile=" + backupFile.getFileName().toString() + ".log";
executeExpDp(args, DatabaseTarget.USER, getProjectConfiguration().getBackupTimeout());
//Move backup file to target
Files.move(targetFile, backupFile, StandardCopyOption.REPLACE_EXISTING);
}
@Override
public void restore(Path backupFile)
throws BoxDatabaseException, IOException, SQLException
{
//Copy backup file to host script directory so Oracle can access it
Path mountedBackupFile = Files.createTempFile(hostScriptDirectory, "backup", ".dmp");
Files.copy(backupFile, mountedBackupFile, StandardCopyOption.REPLACE_EXISTING);
//Set up directory and give database user permission to use it
runFilteredScript("oracle-prepare-backup.sql");
//Run tool
String args = "schemas=" + getBoxConfiguration().getDatabaseUser() +
" directory=backup_dir dumpfile=" + mountedBackupFile.getFileName().toString() +
" logfile=" + mountedBackupFile.getFileName().toString() + ".log";
executeImpDp(args, DatabaseTarget.USER, getProjectConfiguration().getBackupTimeout());
}
protected DataSourceBuilder dataSourceBuilder(DatabaseTarget target)
throws BoxDatabaseException
{
JdbcConnectionInfo jdbcInfo = jdbcConnectionInfo(target);
return new DataSourceBuilder(getContext())
.dataSourceClassName("oracle.jdbc.pool.OracleDataSource")
.dependencies(jdbcDriverInfo().getDependencies())
.configureDataSource("setURL", String.class, jdbcInfo.getUri())
.configureDataSource("setUser", String.class, jdbcInfo.getUser())
.configureDataSource("setPassword", String.class, jdbcInfo.getPassword());
}
@Override
public Connection createJdbcConnection(DatabaseTarget targetDatabase)
throws SQLException, BoxDatabaseException, IOException
{
return dataSourceBuilder(targetDatabase).create().getConnection();
}
@Override
public void executeJdbcScript(Reader scriptReader, DatabaseTarget targetDatabase)
throws IOException, SQLException, BoxDatabaseException
{
try (Connection con = createJdbcConnection(targetDatabase);
JdbcSqlRunner sqlRunner = new JdbcSqlRunner(con, getContext().getLog()))
{
sqlRunner.executeSql(new BufferedReader(scriptReader));
}
}
protected Path getHostScriptDirectory()
{
return hostScriptDirectory;
}
@Override
public Collection checkImage()
throws BoxDatabaseException
{
ImageComponent jdbcDriverComponent = ImageCheckerUtils.checkImageUsingMavenDependencies("JDBC driver",
getContext(),
jdbcDriverInfo().getDependencies());
ImageComponent dockerDatabaseComponent = checkDockerDatabaseImage(OracleFactory.ORACLE_11_DOCKER_REPOSITORY,
OracleFactory.ORACLE_11_DOCKER_TAG);
return ImmutableList.of(jdbcDriverComponent, dockerDatabaseComponent);
}
public static class SqlPlusException extends SQLException
{
private final String exitCode;
public SqlPlusException(String reason, String exitCode)
{
super(reason);
this.exitCode = exitCode;
}
public String getExitCode()
{
return exitCode;
}
}
}