All Downloads are FREE. Search and download functionalities are using the official Maven repository.

pro.foundev.cassandra.commons.migrations.MigrationRunnerImpl Maven / Gradle / Ivy

The newest version!
/*
 * Copyright 2015 Ryan Svihla
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package pro.foundev.cassandra.commons.migrations;

import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.datastax.driver.mapping.Mapper;
import com.datastax.driver.mapping.MappingManager;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pro.foundev.cassandra.commons.core.CassandraConfiguration;
import pro.foundev.cassandra.commons.core.CassandraSessionFactory;
import pro.foundev.cassandra.commons.migrations.parsing.MigrationFileName;
import pro.foundev.cassandra.commons.migrations.parsing.MigrationFileNameParser;
import pro.foundev.cassandra.commons.migrations.script_parser.ScriptParser;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;

/**
 * Responsible for reading a given script file and apply migrations
 * Script file has implied version number
 */
public class MigrationRunnerImpl implements MigrationRunner {

    private final CassandraConfiguration configuration;
    private Logger log = LoggerFactory.getLogger("MigrationRunnerLog");
    private final MigrationFileNameParser fileParser = new MigrationFileNameParser();
    private ScriptParser scriptParser = new ScriptParser();

    public MigrationRunnerImpl(String yamlConfig) throws IOException {
        this(CassandraConfiguration.parse(yamlConfig));
    }
    public MigrationRunnerImpl(CassandraConfiguration configuration){
        this.configuration = configuration;
    }

    /**
     */
    /**
     * Executes all CQL scripts found inside of the specified path. TODO: document file finding rules
     *
     * @param scriptDirectory Contains cql scripts with the following naming convention:
     *                        [version]_[version].cql version is typically a timestamp and will be applied in
     *                        ascending order.
     * @throws FileNotFoundException occurs when unable to find the scriptDirectory.
     */
    @Override
    public void runScriptDirectory(Path scriptDirectory) throws FileNotFoundException {
        if(scriptDirectory==null || !Files.exists(scriptDirectory)){
            throw new FileNotFoundException("script directory is not present at :" + scriptDirectory);
        }
        String keyspace = configuration.getKeyspace();
        withSession(session -> {
            logAction("migrations starting");
            //slightly silly dumping this in one partition, may reconsider later and do client side sorting
            session.execute("CREATE TABLE IF NOT EXISTS " + keyspace + ".migrations (migration_set int, version bigint, name text, cql text" +
                    ", PRIMARY KEY(migration_set, version))");
            MappingManager manager = new MappingManager(session);
            final Mapper mapper = manager.mapper(Migration.class);
            ResultSet rows = session.execute("SELECT * FROM migrations where migration_set=0 ORDER BY version DESC");
            Row latestMigrationFromDb = rows.one();
            Long latestVersionFromDB = 0L;
            if (latestMigrationFromDb != null) {
                latestVersionFromDB = latestMigrationFromDb.getLong("version");
            }
            final List migrations = new ArrayList<>();
            try {
                final Long finalLatestVersionFromDB = latestVersionFromDB;
                Files.walkFileTree(scriptDirectory, new SimpleFileVisitor() {
                    @Override
                    public FileVisitResult visitFile(Path file,
                                                     BasicFileAttributes attrs) throws IOException {
                        String fileName = file.getFileName().toString();
                        if(!fileName.endsWith(".cql")){
                            return FileVisitResult.CONTINUE;
                        }
                        MigrationFileName migrationFileName = fileParser.parseFileName(fileName);
                        String name = migrationFileName.getName();
                        Long version  = migrationFileName.getVersion();
                        if (version > finalLatestVersionFromDB) {

                            List statements = scriptParser.parse(file);
                            Migration migration = new Migration();
                            migration.setMigrationSet(0);//FIXME hack for now till I figure out what this is
                            migration.setCqlToRun(statements);
                            migration.setVersion(version);
                            migration.setName(name);
                            migrations.add(migration);
                        }
                        return FileVisitResult.CONTINUE;
                    }
                });
                //not assuming filesystem returns correct order.
                migrations.sort((o1, o2) -> {
                    if(o1.getVersion()>o2.getVersion()){
                        return 1;
                    }else if(o1.getVersion() {
                    try {
                        String migrationRun = String.format("running migration: %s with version: %d",
                                m.getName(), m.getVersion());
                        logAction(migrationRun);
                        List executed = Lists.newArrayList();
                        m.getCqlToRun().forEach(s -> {
                            String trimmedStatement = null;
                            try {
                                trimmedStatement = s.toString().trim();
                                logAction(trimmedStatement);
                                session.execute(s);
                                executed.add(trimmedStatement);
                            } catch (Exception ex) {
                                if (executed.size() > 0) {
                                    String executedCQL = String.join(";",executed);
                                    int step = executed.size()+1;
                                    String.format("-- critical error: migration step #%d failed miserably. " +
                                            "We do not know how to recover. Please review the exception and figure out the error in your migration " +
                                            "script, then do the following:\n" +
                                            "1. Make a new script and a new migration\n" +
                                            "2. Fix your script and then add it to the new migration\n" +
                                            "3. Save the migration that was successful with the following " +
                                                    "CQL: INSERT INTO migrations (migrations_set, version, name, cql) values\" +\n" +
                                            "                            \" (%d, %d, '%s', '%s');" +
                                            "Keep an eye out for https://github.com/rssvihla/cassandra-commons/issues/45 which will bring resumable migrations" +
                                            " The CQL that failed was '%s", step,
                                            m.getMigrationSet(),
                                            m.getVersion(),
                                            m.getName(),
                                            executedCQL,
                                            trimmedStatement
                                            );
                                }else{
                                    throw ex;
                                }

                            }
                        });
                    } catch (Exception ex) {
                        String migrationError = String.format("failed migration: %s with version: %d",
                                m.getName(),
                                m.getVersion());
                        logError(migrationError, ex);
                        throw ex;
                    }
                    try {
                        //FIXME: need to have specific handling for the case when a save fails
                        mapper.save(m);
                    } catch (Exception ex) {
                        logError(
                                "applied migration: create_table with version: 201510220938 but it failed " +
                                        "writing the migration record to the migration table. Please review the " +
                                        "error fix and manually insert the following CQL in the keyspace you are " +
                                        "running migrations against to resolve the problem:\n" +
                                        "INSERT INTO migrations (migrations_set, version, name, cql) values " +
                                        "(%d, %d, '%s', '%s');", ex);
                    }
                    String completedRun = String.format("migration complete: %s with version: %d",
                            m.getName(), m.getVersion());
                    logAction(completedRun);
                });

            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            logAction("migrations successfully completed");
        });
    }
    private void logError(String errorDescription, Throwable ex){
        String paddedError = "-- " + errorDescription + " --";
        String errorPadding = padString(paddedError.length());
        log.error(errorPadding);
        log.error(paddedError, ex);
        log.error(errorPadding);
    }

    private void logAction(String actionDescription) {
        String paddedDescription = "-- " + actionDescription + " --";
        String completedPaddingBuffer = padString(paddedDescription.length());
        log.info(completedPaddingBuffer);
        log.info(paddedDescription);
        log.info(completedPaddingBuffer);
    }


    private String padString(int length) {
        StringBuffer runPaddingBuffer = new StringBuffer();
        for (int i = 0; i < length; i++) {
            runPaddingBuffer.append("-");
        }
        return runPaddingBuffer.toString();
    }

    private void withSession(Consumer sessionConsumer){

        CassandraSessionFactory cassandraSessionFactory = null;
        try {
            cassandraSessionFactory = new CassandraSessionFactory(configuration);
            sessionConsumer.accept(cassandraSessionFactory.getSession());
        }finally{
            if (cassandraSessionFactory != null) {
                try {
                    cassandraSessionFactory.close();
                } catch (IOException e) {
                    throw new RuntimeException("was able to close cassandra session factory");
                }
            }
        }
    }

    /**
     * drops and recreates keyspace. Today it does a simplestrategy of RF 1 on recreate. Not useful for production
     */
    @Override
    public void resetKeyspace() {
        String keyspaceName = configuration.getKeyspace();
        withSession(session->{
            Row row = session.execute("SELECT * FROM system.schema_keyspaces WHERE keyspace_name='"
                    + keyspaceName + "'").one();
            if(row!=null) {
                String strategyClass = "SimpleStrategy";
                String strategyOptions = "'replication_factor':1";
                Boolean durableWrites = row.getBool("durable_writes");
                session.execute("DROP KEYSPACE " + keyspaceName);
                String createCql = "CREATE KEYSPACE " + keyspaceName + " WITH REPLICATION = {'class':'"
                        + strategyClass + "', "+ strategyOptions + "} AND DURABLE_WRITES=" + durableWrites;
                session.execute(createCql);
            }else{
                //FIXME place holder I want a proper output message, not an exception
                throw new RuntimeException("there is no keyspace by the name of " + keyspaceName);
            }
        });
    }

    public void setLog(Logger log) {
        this.log = log;
    }


}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy