org.opencypher.morpheus.testing.TestSparkSession.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of morpheus-testing Show documentation
Cypher for Apache Spark - Test Utilities
The newest version!
/*
 * Copyright (c) 2016-2019 "Neo4j Sweden, AB" [https://neo4j.com]
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Attribution Notice under the terms of the Apache License 2.0
 *
 * This work was created by the collective efforts of the openCypher community.
 * Without limiting the terms of Section 6, any Derivative Work that is not
 * approved by the public consensus process of the openCypher Implementers Group
 * should not be described as “Cypher” (and Cypher® is a registered trademark of
 * Neo4j Inc.) or as "openCypher". Extensions by implementers or prototypes or
 * proposals for change that have been documented or implemented should only be
 * described as "implementation extensions to Cypher" or as "proposed changes to
 * Cypher that are not yet approved by the openCypher community".
 */
/**
  * Copyright (c) 2016-2017 "Neo4j, Inc." [https://neo4j.com]
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
package org.opencypher.morpheus.testing

import java.io.File
import java.util.UUID

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

object TestSparkSession {

  lazy val instance: SparkSession = {
    val conf = new SparkConf(true)

    conf.set("spark.sql.codegen.wholeStage", "true")
    conf.set("spark.sql.shuffle.partitions", "5")
    // Required for left outer join without join expressions in OPTIONAL MATCH (leads to cartesian product)
    conf.set("spark.sql.crossJoin.enabled", "true")

    //    setting for debug
    //    conf.set("spark.sql.shuffle.partitions", "1")
    //    conf.set("spark.default.parallelism", "1")
    //    performance
    //    conf.set("spark.sql.inMemoryColumnarStorage.compressed", "false")
    //    conf.set("spark.submit.deployMode", "client")

    // Writes spark-warehouse folder to local temp folder
    conf.set("spark.sql.warehouse.dir", s"${System.getProperty("java.io.tmpdir")}${File.separator}spark-warehouse-${System.nanoTime()}")
    // Store Hive MetaStore (derby) in memory only
    conf.set("javax.jdo.option.ConnectionURL", s"jdbc:derby:memory:;databaseName=metastore_db;create=true")
    conf.set("javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver")
    conf.set("hive.metastore.warehouse.dir", s"${System.getProperty("java.io.tmpdir")}${File.separator}hive-warehouse-${System.nanoTime()}")
    //
    // If this is slow, you might be hitting: http://bugs.java.com/view_bug.do?bug_id=8077102
    //
    val session = SparkSession
      .builder()
      .config(conf)
      .master("local[*]")
      .appName(s"morpheus-tests-${UUID.randomUUID()}")
      .enableHiveSupport()
      .getOrCreate()

    session.sparkContext.setLogLevel("WARN")
    session
  }
}