• Home
• au.net.causal.maven.plugins
• boxdb-maven-plugin
• 1.3
• source code
• PostgresDatabase.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

au.net.causal.maven.plugins.boxdb.db.PostgresDatabase Maven / Gradle / Ivy

package au.net.causal.maven.plugins.boxdb.db;

import au.net.causal.maven.plugins.boxdb.JavaRunner;
import au.net.causal.maven.plugins.boxdb.JdbcSqlRunner;
import au.net.causal.maven.plugins.boxdb.ScriptReaderRunner;
import com.google.common.io.ByteStreams;
import io.fabric8.maven.docker.access.DockerAccess;
import io.fabric8.maven.docker.access.DockerAccessException;
import io.fabric8.maven.docker.access.PortMapping;
import io.fabric8.maven.docker.config.ImageConfiguration;
import io.fabric8.maven.docker.config.LogConfiguration;
import io.fabric8.maven.docker.config.RunImageConfiguration;
import io.fabric8.maven.docker.config.VolumeConfiguration;
import io.fabric8.maven.docker.log.LogDispatcher;
import io.fabric8.maven.docker.log.LogOutputSpec;
import io.fabric8.maven.docker.model.Container;
import io.fabric8.maven.docker.service.RunService;
import io.fabric8.maven.docker.util.PomLabel;
import org.apache.maven.plugin.MojoExecutionException;
import org.eclipse.aether.resolution.DependencyResolutionException;

import javax.sql.DataSource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.Properties;

public class PostgresDatabase extends DockerDatabase
{
    /**
     * First bytes of a Postgres binary dump file, used to detect different between text dump and binary dump.
     */
    private static final byte[] BINARY_DUMP_HEADER = "PGDMP".getBytes(StandardCharsets.US_ASCII);

    public PostgresDatabase(BoxConfiguration boxConfiguration, ProjectConfiguration projectConfiguration, BoxContext context)
    {
        super(boxConfiguration, projectConfiguration, context);
    }

    @Override
    protected int containerDatabasePort()
    {
        return 5432;
    }

    private void executePsql(VolumeConfiguration volumes, String psqlArgs, DatabaseTarget targetDatabase, Duration timeout)
    throws IOException, SQLException, BoxDatabaseException
    {
        if (targetDatabase == DatabaseTarget.USER)
            psqlArgs = psqlArgs + " " + getBoxConfiguration().getDatabaseName();

        RunService runService = getContext().getDockerServiceHub().getRunService();
        DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();

        Properties projectProperties = getProjectConfiguration().getProjectProperties();
        PomLabel pomLabel = getProjectConfiguration().getPomLabel();
        PortMapping mappedPorts = new PortMapping(Collections.emptyList(), projectProperties);

        int psqlMaxExecutionTimeSeconds = Math.toIntExact(timeout.getSeconds());
        LogConfiguration logConfig = new LogConfiguration.Builder()
                .enabled(true)
                .prefix("psql")
                .build();
        RunImageConfiguration psqlRunConfig = new RunImageConfiguration.Builder()
                .links(Collections.singletonList(containerName() + ":postgres"))
                .namingStrategy("alias")
                .cmd("unused") //not used but must be non-null - is replaced later
                .env(Collections.singletonMap("PGPASSWORD", targetDatabase.password(getBoxConfiguration())))
                .volumes(volumes)
                .log(logConfig)
                .build();
        psqlRunConfig.getCmd().setExec(Arrays.asList("sh", "-c", "psql -h postgres -U " + targetDatabase.user(getBoxConfiguration()) + " -a " + psqlArgs));
        psqlRunConfig.getCmd().setShell(null);
        ImageConfiguration psqlConfig = new ImageConfiguration.Builder()
                .runConfig(psqlRunConfig)
                .name(dockerImageName())
                .alias(containerName() + "-psql")
                .build();

        LogDispatcher dispatcher = new LogDispatcher(getContext().getDockerServiceHub().getDockerAccess());

        String containerId = runService.createAndStartContainer(psqlConfig, mappedPorts, pomLabel, projectProperties);
        Container container = null;
        try
        {
            getContext().getLog().debug("PSQL running in container " + containerId);

            dispatcher.trackContainerLog(containerId, new LogOutputSpec.Builder()
                    .logStdout(true)
                    .prefix("psql")
                    .containerId(containerId)
                    .build());
        }
        finally
        {
            try
            {
                container = waitForContainerToFinish(containerId, psqlMaxExecutionTimeSeconds);
            }
            catch (DockerAccessException e)
            {
                getContext().getLog().warn("Error shutting down PSQL container", e);
            }

            try
            {
                boolean removeVolumes = true;
                docker.removeContainer(containerId, removeVolumes);
            }
            catch (DockerAccessException e)
            {
                getContext().getLog().warn("Error removing PSQL container", e);
            }
        }

        int exitCode = readExitCodeFromContainer(container);
        if (exitCode != 0)
            throw new SQLException("PSQL exit code: " + exitCode);

        getContext().getLog().debug("All done");
    }

    @Override
    protected void executeScriptFile(Path scriptFile, DatabaseTarget targetDatabase, Duration timeout)
    throws IOException, SQLException, BoxDatabaseException
    {
        if (!Files.exists(scriptFile))
            throw new NoSuchFileException(scriptFile.toString());

        VolumeConfiguration volumes = new VolumeConfiguration.Builder()
                .bind(Arrays.asList(scriptFile.getParent().toAbsolutePath().toString() + ":/data/scripts:ro"))
                .build();

        executePsql(volumes, "-f /data/scripts/" + scriptFile.getFileName().toString(), targetDatabase, timeout);
    }

    @Override
    public void executeSql(String sql, DatabaseTarget targetDatabase, Duration timeout)
    throws IOException, SQLException, BoxDatabaseException
    {
        executePsql(new VolumeConfiguration.Builder().build(), "-c " + ScriptUtils.shellEscape(sql), targetDatabase, timeout);
    }

    private DataSource createPostgresDataSource(DatabaseTarget target)
    throws BoxDatabaseException, IOException
    {
        try
        {
            JavaRunner javaRunner = getContext().createJavaRunner("org.postgresql.ds.PGSimpleDataSource",
                    jdbcDriverInfo().getDependencies());
            Class clientDataSourceClass = javaRunner.makeClass();
            DataSource dataSource = (DataSource) clientDataSourceClass.getConstructor().newInstance();

            //Need to set some properties on the data source
            JdbcConnectionInfo jdbcInfo = jdbcConnectionInfo(target);
            configureDataSource(dataSource, "setUrl", String.class, jdbcInfo.getUri());
            configureDataSource(dataSource, "setUser", String.class, jdbcInfo.getUser());
            configureDataSource(dataSource, "setPassword", String.class, jdbcInfo.getPassword());

            return dataSource;
        }
        catch (DependencyResolutionException e)
        {
            throw new BoxDatabaseException("Failed to resolve Postgres dependency: " + e, e);
        }
        catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e)
        {
            throw new BoxDatabaseException("Failed to create Postgres data source: " + e, e);
        }
    }

    private void configureDataSource(DataSource dataSource, String methodName, Class valueType, Object value)
    throws NoSuchMethodException, InvocationTargetException, IllegalAccessException
    {
        if (value != null)
            dataSource.getClass().getMethod(methodName, valueType).invoke(dataSource, value);
    }

    @Override
    public Connection createJdbcConnection(DatabaseTarget targetDatabase)
    throws SQLException, BoxDatabaseException, IOException
    {
        return createPostgresDataSource(targetDatabase).getConnection();
    }

    @Override
    public void executeJdbcScript(Reader scriptReader, DatabaseTarget targetDatabase)
    throws IOException, SQLException, BoxDatabaseException
    {
        try (Connection con = createJdbcConnection(targetDatabase);
             JdbcSqlRunner sqlRunner = new JdbcSqlRunner(con, getContext().getLog()))
        {
            sqlRunner.executeSql(new BufferedReader(scriptReader));
        }
    }

    @Override
    public JdbcConnectionInfo jdbcConnectionInfo(DatabaseTarget target)
    throws BoxDatabaseException
    {
        String databaseName;
        if (target == DatabaseTarget.ADMIN)
            databaseName = "postgres";
        else
            databaseName = getBoxConfiguration().getDatabaseName();

        String uri =  "jdbc:postgresql://" +
                        getContext().getDockerHostAddress() +
                        ":" + getBoxConfiguration().getDatabasePort() +
                        "/" + databaseName;

        return new JdbcConnectionInfo(uri,
                        target.user(getBoxConfiguration()),
                        target.password(getBoxConfiguration()),
                        getContext().getDockerHostAddress(),
                        getBoxConfiguration().getDatabasePort());
    }

	@Override
	public JdbcDriverInfo jdbcDriverInfo()
	throws BoxDatabaseException
	{
		//Drivers are backward/forward compatible so just use latest
		return new JdbcDriverInfo(new RunnerDependency("org.postgresql", "postgresql", "9.4.1209"), "org.postgresql.Driver");
	}

	@Override
    public void configureNewDatabase()
    throws IOException, SQLException, BoxDatabaseException
    {
        URL initScript = PostgresDatabase.class.getResource("postgres-create-user.sql");
        URL initScript2 = PostgresDatabase.class.getResource("postgres-create-database.sql");

        ScriptReaderRunner scriptRunner = getContext().createScriptReaderRunner(this, getBoxConfiguration(), getProjectConfiguration());
        ScriptReaderExecution execution = new ScriptReaderExecution();
        execution.setFiltering(true);
        execution.setScripts(Arrays.asList(initScript, initScript2));

        try
        {
            scriptRunner.execute(execution, DatabaseTarget.ADMIN, getProjectConfiguration().getScriptTimeout());
        }
        catch (MojoExecutionException e)
        {
            throw new BoxDatabaseException(e);
        }
    }

    @Override
    public void backup(Path backupFile, BackupFileTypeHint backupFileTypeHint)
    throws BoxDatabaseException, IOException, SQLException
    {
        Path backupDirectory = backupFile.getParent();

        if (!Files.exists(backupDirectory))
            Files.createDirectories(backupDirectory);

        //Mount backup file as volume to /data/backup/dump
        VolumeConfiguration volumes = new VolumeConfiguration.Builder()
                                            .bind(Arrays.asList(backupDirectory.toAbsolutePath().toString() + ":/data/backup"))
                                            .build();

        //Do a pg_dump to a mounted directory
        RunService runService = getContext().getDockerServiceHub().getRunService();
        DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();

        Properties projectProperties = getProjectConfiguration().getProjectProperties();
        PomLabel pomLabel = getProjectConfiguration().getPomLabel();
        PortMapping mappedPorts = new PortMapping(Collections.emptyList(), projectProperties);

        int backupMaxExecutionTimeSeconds = Math.toIntExact(getProjectConfiguration().getBackupTimeout().getSeconds());
        LogConfiguration logConfig = new LogConfiguration.Builder()
                .enabled(true)
                .prefix("pgdump")
                .build();
        RunImageConfiguration pgDumpRunConfig = new RunImageConfiguration.Builder()
                .links(Collections.singletonList(containerName() + ":postgres"))
                .namingStrategy("alias")
                .cmd("unused") //not used but must be non-null - is replaced later
                .env(Collections.singletonMap("PGPASSWORD", getBoxConfiguration().getDatabasePassword()))
                .volumes(volumes)
                .log(logConfig)
                .build();
        getContext().getLog().debug("Backup type: " + backupFileTypeHint);
        pgDumpRunConfig.getCmd().setExec(Arrays.asList("sh", "-c", "pg_dump -h postgres -U " + getBoxConfiguration().getDatabaseUser() +
                                                    " -d " + getBoxConfiguration().getDatabaseName() +
                                                    (backupFileTypeHint == BackupFileTypeHint.COMPACT ? " -F custom" : "") +
                                                    " -f /data/backup/" + backupFile.getFileName().toString()));
        pgDumpRunConfig.getCmd().setShell(null);
        ImageConfiguration pgDumpConfig = new ImageConfiguration.Builder()
                .runConfig(pgDumpRunConfig)
                .name(dockerImageName())
                .alias(containerName() + "-pgdump")
                .build();

        LogDispatcher dispatcher = new LogDispatcher(getContext().getDockerServiceHub().getDockerAccess());

        String containerId = runService.createAndStartContainer(pgDumpConfig, mappedPorts, pomLabel, projectProperties);
        Container container = null;
        try
        {
            getContext().getLog().debug("pgdump running in container " + containerId);

            dispatcher.trackContainerLog(containerId, new LogOutputSpec.Builder()
                    .logStdout(true)
                    .prefix("pgdump")
                    .containerId(containerId)
                    .build());
        }
        finally
        {
            try
            {
                container = waitForContainerToFinish(containerId, backupMaxExecutionTimeSeconds);
            }
            catch (DockerAccessException e)
            {
                getContext().getLog().warn("Error shutting down pgdump container", e);
            }

            try
            {
                boolean removeVolumes = true;
                docker.removeContainer(containerId, removeVolumes);
            }
            catch (DockerAccessException e)
            {
                getContext().getLog().warn("Error removing pgdump container", e);
            }
        }

        int exitCode = readExitCodeFromContainer(container);
        if (exitCode != 0)
            throw new SQLException("pgdump exit code: " + exitCode);

        getContext().getLog().debug("All done");
    }

    @Override
    public void restore(Path backupFile) throws BoxDatabaseException, IOException, SQLException
    {
        if (isBinaryDump(backupFile))
            restoreBinaryDump(backupFile);
        else
            executeScriptFile(backupFile, DatabaseTarget.USER, getProjectConfiguration().getBackupTimeout());
    }

    private void restoreBinaryDump(Path backupFile)
    throws BoxDatabaseException, IOException, SQLException
    {
        Path backupDirectory = backupFile.getParent();

        //Mount backup file as volume to /data/backup/dump
        VolumeConfiguration volumes = new VolumeConfiguration.Builder()
                .bind(Arrays.asList(backupDirectory.toAbsolutePath().toString() + ":/data/backup:ro"))
                .build();

        //Do a pg_dump to a mounted directory
        RunService runService = getContext().getDockerServiceHub().getRunService();
        DockerAccess docker = getContext().getDockerServiceHub().getDockerAccess();

        Properties projectProperties = getProjectConfiguration().getProjectProperties();
        PomLabel pomLabel = getProjectConfiguration().getPomLabel();
        PortMapping mappedPorts = new PortMapping(Collections.emptyList(), projectProperties);

        int backupMaxExecutionTimeSeconds = Math.toIntExact(getProjectConfiguration().getBackupTimeout().getSeconds());
        LogConfiguration logConfig = new LogConfiguration.Builder()
                .enabled(true)
                .prefix("pgrestore")
                .build();
        RunImageConfiguration pgRestore = new RunImageConfiguration.Builder()
                .links(Collections.singletonList(containerName() + ":postgres"))
                .namingStrategy("alias")
                .cmd("unused") //not used but must be non-null - is replaced later
                .env(Collections.singletonMap("PGPASSWORD", getBoxConfiguration().getAdminPassword()))
                .volumes(volumes)
                .log(logConfig)
                .build();
        //Use admin user instead of DB user because binary dumps might have stuff like extensions that we need superuser to restore
        pgRestore.getCmd().setExec(Arrays.asList("sh", "-c", "pg_restore -e -h postgres -U " + getBoxConfiguration().getAdminUser() +
                " -d " + getBoxConfiguration().getDatabaseName() +
                " /data/backup/" + backupFile.getFileName().toString()));
        pgRestore.getCmd().setShell(null);
        ImageConfiguration pgRestoreConfig = new ImageConfiguration.Builder()
                .runConfig(pgRestore)
                .name(dockerImageName())
                .alias(containerName() + "-pgrestore")
                .build();

        LogDispatcher dispatcher = new LogDispatcher(getContext().getDockerServiceHub().getDockerAccess());

        String containerId = runService.createAndStartContainer(pgRestoreConfig, mappedPorts, pomLabel, projectProperties);
        Container container = null;
        try
        {
            getContext().getLog().debug("pgrestore running in container " + containerId);

            dispatcher.trackContainerLog(containerId, new LogOutputSpec.Builder()
                    .logStdout(true)
                    .prefix("pgrestore")
                    .containerId(containerId)
                    .build());
        }
        finally
        {
            try
            {
                container = waitForContainerToFinish(containerId, backupMaxExecutionTimeSeconds);
            }
            catch (DockerAccessException e)
            {
                getContext().getLog().warn("Error shutting down pgrestore container", e);
            }

            try
            {
                boolean removeVolumes = true;
                docker.removeContainer(containerId, removeVolumes);
            }
            catch (DockerAccessException e)
            {
                getContext().getLog().warn("Error removing pgrestore container", e);
            }
        }

        int exitCode = readExitCodeFromContainer(container);
        if (exitCode != 0)
            throw new SQLException("pgrestore exit code: " + exitCode);

        getContext().getLog().debug("All done");
    }

    private boolean isBinaryDump(Path dumpFile)
    throws IOException
    {
        //Check header of file to see if binary dump
        byte[] header = readFirstBytesFromFile(dumpFile, BINARY_DUMP_HEADER.length);
        return Arrays.equals(BINARY_DUMP_HEADER, header);
    }

    private byte[] readFirstBytesFromFile(Path file, int numBytesToRead)
    throws IOException
    {
        byte[] buf = new byte[numBytesToRead];
        try (InputStream is = Files.newInputStream(file))
        {
            int n = ByteStreams.read(is, buf, 0, numBytesToRead);
            if (n != numBytesToRead)
                buf = Arrays.copyOf(buf, n);
        }

        return buf;
    }
}