• Home
• org.helenus
• helenus-junit
• 1.0.15
• source code
• HelenusJUnit.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.helenus.driver.junit.HelenusJUnit Maven / Gradle / Ivy

/*
 * Copyright (C) 2015-2016 The Helenus Driver Project Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.helenus.driver.junit;

import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;

import java.net.ServerSocket;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.commons.lang3.tuple.Pair;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.KeyspaceMetadata;
import com.datastax.driver.core.QueryOptions;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.WriteType;
import com.datastax.driver.core.exceptions.NoHostAvailableException;
import com.datastax.driver.core.policies.LoggingRetryPolicy;
import com.datastax.driver.core.policies.RetryPolicy;

import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.StorageService;
import org.helenus.commons.collections.iterators.CombinationIterator;
import org.helenus.commons.lang3.reflect.ReflectionUtils;
import org.helenus.driver.AlterSchema;
import org.helenus.driver.AlterSchemas;
import org.helenus.driver.CreateIndex;
import org.helenus.driver.CreateKeyspace;
import org.helenus.driver.CreateSchema;
import org.helenus.driver.CreateSchemas;
import org.helenus.driver.CreateTable;
import org.helenus.driver.CreateType;
import org.helenus.driver.Delete;
import org.helenus.driver.ExcludedKeyspaceKeyException;
import org.helenus.driver.GenericStatement;
import org.helenus.driver.Group;
import org.helenus.driver.Insert;
import org.helenus.driver.Select;
import org.helenus.driver.Sequence;
import org.helenus.driver.StatementBuilder;
import org.helenus.driver.StatementManager;
import org.helenus.driver.Truncate;
import org.helenus.driver.Update;
import org.helenus.driver.impl.ClassInfoImpl;
import org.helenus.driver.impl.CreateSchemaImpl;
import org.helenus.driver.impl.GroupImpl;
import org.helenus.driver.impl.RootClassInfoImpl;
import org.helenus.driver.impl.StatementImpl;
import org.helenus.driver.impl.StatementManagerImpl;
import org.helenus.driver.impl.TypeClassInfoImpl;
import org.helenus.driver.impl.UDTClassInfoImpl;
import org.helenus.driver.impl.UDTRootClassInfoImpl;
import org.helenus.driver.impl.UDTTypeClassInfoImpl;
import org.helenus.driver.info.ClassInfo;
import org.helenus.driver.info.FieldInfo;
import org.helenus.driver.info.TableInfo;
import org.helenus.driver.info.TypeClassInfo;
import org.helenus.driver.junit.util.ReflectionJUnitUtils;
import org.helenus.driver.junit.util.Strings;
import org.helenus.driver.persistence.Keyspace;
import org.helenus.driver.persistence.Table;
import org.helenus.util.function.ERunnable;
import org.junit.rules.MethodRule;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.Statement;
import org.yaml.snakeyaml.reader.UnicodeReader;

/**
 * The HelenusJUnit class provides the JUnit 4 definition for a
 * @Rule service that will take care of initializing an embedded
 * Cassandra server and the Helenus driver.
 * 

 * Note: This file is largely based on cassandra-unit.
 *
 * @copyright 2015-2016 The Helenus Driver Project Authors
 *
 * @author  The Helenus Driver Project Authors
 * @version 1 - Jun 27, 2015 - paouelle - Creation
 *
 * @since 1.0
 */
public class HelenusJUnit implements MethodRule {
  /**
   * Holds the logger.
   *
   * @author paouelle
   */
  final static Logger logger = LogManager.getFormatterLogger(HelenusJUnit.class);

  /**
   * Holds a trace prefix for all CQL statements traced.
   *
   * @author paouelle
   */
  final static String TRACE_PREFIX = "HELENUS-JUNIT ";

  /**
   * Constant for the default timeout to wait for the Cassandra daemon to start
   * before failing.
   *
   * @author paouelle
   */
  public final static long DEFAULT_STARTUP_TIMEOUT = 60000L;

  /**
   * Constant for the default Cassandra configuration file which starts the
   * Cassandra daemon on a random free port.
   *
   * @author paouelle
   */
  public final static String DEFAULT_CFG_FILE = "cassandra.yaml";

  /**
   * Constant for the default runtime directory used by the embedded Cassandra
   * daemon.
   *
   * @author paouelle
   */
  private final static String RUNTIME_DIR = "target/helenus-junit";

  /**
   * Holds the regex pattern used to parse the Cassandra yaml configuration
   * file in search of ports.
   *
   * @author paouelle
   */
  private static Pattern portPattern = Pattern.compile(
    "^([a-z_]+_port:\\s*)([0-9]+)\\s*$", Pattern.MULTILINE
  );

  /**
   * Holds the regex pattern used to parse the Cassandra yaml configuration
   * file in search of target directories.
   *
   * @author paouelle
   */
  private static Pattern targetPattern = Pattern.compile(
    "^([a-z_]+:[\\s-]*)(target/helenus-junit)(.*)$", Pattern.MULTILINE
  );

  /**
   * Holds the thread group used to combine all Cassandra daemon threads
   * together.
   *
   * @author paouelle
   */
  private static volatile ThreadGroup group = null;

  /**
   * Holds the Cassandra config file name used to start the Cassandra daemon.
   *
   * @author paouelle
   */
  private static String config = null;

  /**
   * Holds the optional fork number which can be used to run multiple JVMs in
   * parallel to test. When specified, the directory used to store Cassandra's
   * files will have the forkNumber appended to it.
   * 

   * This can be used with the maven-surefire-plugin when using a fork count
   * greater than 1 by setting the system property "fork" with
   * "${surefire.forkNumber}".
   * 

   * Maven Example:
   *   <argLine>-Dfork=${surefire.forkNumber}</argLine>
   *
   * @author paouelle
   */
  private static String fork = System.getProperty("fork", "");

  /**
   * Holds the Cassandra daemon when started.
   *
   * @author paouelle
   */
  private static volatile CassandraDaemon daemon = null;

  /**
   * Holds the Helenus statement manager when started.
   *
   * @author paouelle
   */
  private static StatementManagerUnitImpl manager = null;

  /**
   * Holds the number of times to retry a read operation that timed out.
   * 

   * Note: Defaults to none.
   *
   * @author paouelle
   */
  static volatile int numReadRetries = 0;

  /**
   * Holds the number of times to retry a write operation that timed out.
   * 

   * Note: Defaults to none.
   *
   * @author paouelle
   */
  static volatile int numWriteRetries = 0;

  /**
   * Holds a flag to control whether to trace the full statement or part of it
   * when it exceeds 2K in size.
   *
   * @author paouelle
   */
  private static volatile boolean fullTraces = false;

  /**
   * Holds a flag indicating if all statements should be traced regardless of
   * the statement tracing setting
   * (see {@link org.helenus.driver.Statement#isTracing}).
   *
   * @author paouelle
   */
  private static volatile boolean allStatementTraces = false;

  /**
   * Holds the keyspaces created so far via create schema statements.
   *
   * @author paouelle
   */
  private static final Set> keyspaces
    = ConcurrentHashMap.newKeySet();

  /**
   * Holds the tables created so far via create schema statements.
   *
   * @author paouelle
   */
  private static final Map, Set
> tables
    = new ConcurrentHashMap<>();

  /**
   * Hold a set of pojo class for which we have loaded the schema definition
   * onto the embedded cassandra database along with the complete set of initial
   * objects to re-insert when resetting the schema. The value corresponds to a
   * future's representation of the schema creation for the corresponding class.
   *
   * @author paouelle
   */
  static final Map, SchemaFuture> schemas = new ConcurrentHashMap<>();

  /**
   * Hold the complete set of initial objects to re-insert when resetting the
   * schema along with the groups to do so.
   *
   * @author paouelle
   */
  static final Map, MutablePair, Group>> initials
    = new ConcurrentHashMap<>();

  /**
   * Holds a cache of class info structures created by previous test cases.
   *
   * @author paouelle
   */
  static final Map, ClassInfoImpl> fromPreviousTestsCacheInfoCache
    = new ConcurrentHashMap<>(64);

  /**
   * Holds the test method currently running.
   *
   * @author paouelle
   */
  private static volatile FrameworkMethod method = null;

  /**
   * Holds the test object on which the test method is currently running
   *
   * @author paouelle
   */
  private static volatile Object target = null;

  /**
   * Holds the keyspace key values for the current test method keyed by keyspace
   * key types.
   *
   * @author paouelle
   */
  private static volatile Map> keyspaceKeyValues = null;

  /**
   * Holds the capture lists. Use a concurrent list.
   *
   * @author paouelle
   */
  @SuppressWarnings("rawtypes")
  static final List> captures
    = new CopyOnWriteArrayList<>();

  /**
   * Holds the sent lists. Use a concurrent list.
   *
   * @author paouelle
   */
  static final List>> sent
    = new CopyOnWriteArrayList<>();

  /**
   * Holds a depth counter indicating if capturing is enabled or not when greater
   * than 0.
   *
   * @author paouelle
   */
  static final AtomicInteger capturing = new AtomicInteger(0);

  /**
   * Holds a depth counter indicating if we are recursing within helenus-junit
   * or not when greater than 0.
   *
   * @author paouelle
   */
  static final AtomicInteger recursing = new AtomicInteger(0);

  /**
   * Holds a flag indicating if internal CQL statements should be traced or not.
   *
   * @author paouelle
   */
  static volatile boolean traceInternalCQL = false;

  /**
   * Initializes tracing for the specified statement.
   *
   * @author paouelle
   *
   * @param  s the statement to initializes traces for
   * @return the same statement
   */
  private static > S initTrace(S s) {
    if (HelenusJUnit.traceInternalCQL) {
      s.enableTracing(HelenusJUnit.TRACE_PREFIX);
    }
    s.enableErrorTracing(HelenusJUnit.TRACE_PREFIX);
    return s;
  }

  /**
   * Gets the caller's information. This would be the information about the
   * method that called this class first.
   *
   * @author paouelle
   *
   * @return the corresponding non-null stack trace element
   * @throws IllegalStateException if no outside method called this class
   */
  private static StackTraceElement getCallerInfo() {
    final Exception e = new Exception();

    for (final StackTraceElement ste: e.getStackTrace()) {
      if (!HelenusJUnit.class.getName().equals(ste.getClassName())) {
        return ste;
      }
    }
    throw new IllegalStateException("missing caller's information");
  }

  /**
   * Reads the specified file fully based on the appropriate encoding.
   *
   * @author paouelle
   *
   * @param  file the non-null file to read
   * @return a non-null string of the complete content of the
   *         specified file
   * @throws IOException if an I/O error occurs while reading the file
   */
  private static String readFully(File file) throws IOException {
    try (
      final FileInputStream fis = new FileInputStream(file);
      final UnicodeReader ur = new UnicodeReader(fis);
    ) {
      return IOUtils.toString(ur);
    }
  }

  /**
   * Updates the specified yaml config file by searching for ports configured
   * with the value 0 and finding a free port to use instead.
   *
   * @author paouelle
   *
   * @param  cfgfile the non-null config file to update
   * @throws IOException if an I/O error occurs while updating the config file
   */
  private static void update(File cfgfile) throws IOException {
    final String yaml = HelenusJUnit.readFully(cfgfile);
    final StringBuffer sb = new StringBuffer(yaml.length() + 40);
    final Matcher pm = HelenusJUnit.portPattern.matcher(yaml);
    boolean updated = false; // until proven otherwise

    while (pm.find()) {
      final String pname = pm.group(1);
      final int port = Integer.parseInt(pm.group(2));

      if (port == 0) {
        try (
          final ServerSocket socket = new ServerSocket(0);
        ) {
          pm.appendReplacement(sb, pname + socket.getLocalPort());
          updated = true;
          logger.info("Allocated free %s%d", pname, socket.getLocalPort());
        }
      } else {
        pm.appendReplacement(sb, pm.group());
      }
    }
    pm.appendTail(sb);
    if (!StringUtils.isEmpty(HelenusJUnit.fork)) {
      final Matcher tm = HelenusJUnit.targetPattern.matcher(sb.toString());

      sb.setLength(0);
      while (tm.find()) {
        final String pname = tm.group(1);
        final String tname = tm.group(2);
        final String rest = tm.group(3);

        tm.appendReplacement(sb, pname + tname + '/' + HelenusJUnit.fork + rest);
        updated = true;
      }
      tm.appendTail(sb);
    }
    if (updated) { // update the config file with the updated ports and/or target dirs
      FileUtils.writeStringToFile(cfgfile, sb.toString(), "utf-8");
    }
  }

  /**
   * Cleans up the Cassandra runtime directories if they exist and creates
   * them if they do not exists.
   *
   * @author paouelle
   */
  private static void cleanupAndCreateCassandraDirectories() {
    try {
      DatabaseDescriptor.createAllDirectories(); // make sure they exist before cleanup up to avoid failures
      final File dir = new File(DatabaseDescriptor.getCommitLogLocation());

      if (!dir.exists()) {
        throw new FileNotFoundException(
          "missing cassandra commit directory: " + dir.getAbsolutePath()
        );
      }
      FileUtils.deleteDirectory(dir);
      for (final String dname: DatabaseDescriptor.getAllDataFileLocations()) {
        final File d = new File(dname);

        if (!d.exists()) {
          throw new FileNotFoundException(
            "missing cassandra data directory: " + d.getAbsolutePath()
          );
        }
        FileUtils.deleteDirectory(dir);
      }
      DatabaseDescriptor.createAllDirectories();
      CommitLog.instance.getContext(); // wait for commit log allocator instantiation to avoid hanging on a race condition
      CommitLog.instance.resetUnsafe(true); // cleanup screws with CommitLog, this brings it back to safe state
    } catch (IOException e) {
      throw new AssertionError(e);
    }
  }

  /**
   * Find the keyspace key values from the specified test definition that matches
   * the specified pojo class.
   *
   * @author paouelle
   *
   * @param  cinfo the non-null pojo class information
   * @return a non-null collection of all keyspace key values keyed by
   *         their names based on the specified class info (each entry in the main
   *         collection provides another collection of all possible values for a
   *         given keyspace key)
   */
  private static Collection> getKeyspaceKeyValues(
    ClassInfo cinfo
  ) {
    final Map> skvss = HelenusJUnit.keyspaceKeyValues;

    if (skvss == null) {
      return Collections.emptyList();
    }
    return skvss.entrySet().stream()
      .map(e -> {
        final FieldInfo finfo = cinfo.getKeyspaceKeyByType(e.getKey());

        if (finfo != null) { // pojo defines this keyspace key type
          return e.getValue().stream()
            .map(v -> new Strings(finfo.getKeyspaceKeyName(), v))
            .collect(Collectors.toList());
        }
        return null;
      })
      .filter(p -> p != null)
      .collect(Collectors.toList());
  }

  /**
   * Processes the {@link BeforeObjects} annotated methods by calling it and
   * inserting all returned pojo objects into the database.
   *
   * @author paouelle
   *
   * @param group the non-null group to insert the objects to
   *        create in
   * @param m the non-null method to invoke to get the initial objects
   * @param bo the non-null annotation for the method
   * @param target the test object for which we are calling the method
   * @param method the test method
   * @param kkeys the map of keyspace keys to pass to the method
   * @param onlyIfRequiresKeyspaceKeys true if the method should
   *        not be called if it doesn't require keyspace keys
   */
  private static void processBeforeObjects(
    Group group,
    Method m,
    BeforeObjects bo,
    Object target,
    FrameworkMethod method,
    Map kkeys,
    boolean onlyIfRequiresKeyspaceKeys
  ) {
    if (!ArrayUtils.isEmpty(bo.value())
        && !ArrayUtils.contains(bo.value(), method.getName())) {
      return;
    }
    try {
      final Class[] cparms = m.getParameterTypes();
      final Object ret;

      // check if the method expects a map of keyspace keys
      if (cparms.length == 0) { // doesn't care about keyspace keys
        if (!onlyIfRequiresKeyspaceKeys) {
          ret = m.invoke(target);
        } else {
          ret = null;
        }
      } else {
        final Type[] tparms = m.getGenericParameterTypes();

        if ((cparms.length != 1)
            || !Map.class.isAssignableFrom(cparms[0])
            || (tparms.length != 1)
            || !(tparms[0] instanceof ParameterizedType)) {
          throw new AssertionError(
            "expecting one Map parameter for @BeforeObjects method "
            + m.getName()
            + "("
            + m.getDeclaringClass().getName()
            + ")"
          );
        }
        final ParameterizedType ptype = (ParameterizedType)tparms[0];

        // maps will always have 2 arguments
        for (final Type atype: ptype.getActualTypeArguments()) {
          final Class aclazz = ReflectionUtils.getRawClass(atype);

          if (String.class != aclazz) {
            throw new AssertionError(
              "expecting one Map parameter for @BeforeObjects method "
              + m.getName()
              + "("
              + m.getDeclaringClass().getName()
              + ")"
            );
          }
        }
        ret = m.invoke(target, kkeys);
      }
      HelenusJUnit.processObjects(group, ret);
    } catch (IllegalAccessException e) { // should not happen
      throw new IllegalStateException(e);
    } catch (InvocationTargetException e) {
      final Throwable t = e.getTargetException();

      if (t instanceof Error) {
        throw (Error)t;
      } else if (t instanceof RuntimeException) {
        throw (RuntimeException)t;
      } else { // we don't expect any of those
        throw new IllegalStateException(t);
      }
    }
  }

  /**
   * Processes the returned object as a {@link Collection}, {@link Stream},
   * {@link Iterator}, {@link Enumeration}, {@link Iterable}, or a single object
   * and insert the object(s) in the specified batch.
   *
   * @author paouelle
   *
   * @param group the non-null group to insert the objects to
   *        create in
   * @param ret the return object to process
   */
  @SuppressWarnings("unchecked")
  private static void processObjects(Group group, Object ret) {
    if (ret == null) { // nothing to do
      return;
    }
    // validate the return type is either an array, a collection, or a stream
    final Class type = ret.getClass();

    if (type.isArray()) {
      final int l = Array.getLength(ret);

      for (int i = 0; i < l; i++) {
        group.add(HelenusJUnit.initTrace(
          StatementBuilder.insert(Array.get(ret, i)).intoAll()
        ));
      }
    } else if (ret instanceof Collection) {
      ((Collection