org.apache.spark.sql.RuntimeConfig.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of spark-sql_2.12 Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql

import org.apache.spark.SPARK_DOC_ROOT
import org.apache.spark.annotation.Stable
import org.apache.spark.internal.config.{ConfigEntry, OptionalConfigEntry}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.SQLConf

/**
 * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`.
 *
 * Options set here are automatically propagated to the Hadoop configuration during I/O.
 *
 * @since 2.0.0
 */
@Stable
class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {

  /**
   * Sets the given Spark runtime configuration property.
   *
   * @since 2.0.0
   */
  def set(key: String, value: String): Unit = {
    requireNonStaticConf(key)
    sqlConf.setConfString(key, value)
  }

  /**
   * Sets the given Spark runtime configuration property.
   *
   * @since 2.0.0
   */
  def set(key: String, value: Boolean): Unit = {
    set(key, value.toString)
  }

  /**
   * Sets the given Spark runtime configuration property.
   *
   * @since 2.0.0
   */
  def set(key: String, value: Long): Unit = {
    set(key, value.toString)
  }

  /**
   * Sets the given Spark runtime configuration property.
   */
  private[sql] def set[T](entry: ConfigEntry[T], value: T): Unit = {
    requireNonStaticConf(entry.key)
    sqlConf.setConf(entry, value)
  }

  /**
   * Returns the value of Spark runtime configuration property for the given key.
   *
   * @throws java.util.NoSuchElementException if the key is not set and does not have a default
   *                                          value
   * @since 2.0.0
   */
  @throws[NoSuchElementException]("if the key is not set")
  def get(key: String): String = {
    sqlConf.getConfString(key)
  }

  /**
   * Returns the value of Spark runtime configuration property for the given key.
   *
   * @since 2.0.0
   */
  def get(key: String, default: String): String = {
    sqlConf.getConfString(key, default)
  }

  /**
   * Returns the value of Spark runtime configuration property for the given key.
   */
  @throws[NoSuchElementException]("if the key is not set")
  private[sql] def get[T](entry: ConfigEntry[T]): T = {
    sqlConf.getConf(entry)
  }

  private[sql] def get[T](entry: OptionalConfigEntry[T]): Option[T] = {
    sqlConf.getConf(entry)
  }

  /**
   * Returns the value of Spark runtime configuration property for the given key.
   */
  private[sql] def get[T](entry: ConfigEntry[T], default: T): T = {
    sqlConf.getConf(entry, default)
  }

  /**
   * Returns all properties set in this conf.
   *
   * @since 2.0.0
   */
  def getAll: Map[String, String] = {
    sqlConf.getAllConfs
  }

  /**
   * Returns the value of Spark runtime configuration property for the given key.
   *
   * @since 2.0.0
   */
  def getOption(key: String): Option[String] = {
    try Option(get(key)) catch {
      case _: NoSuchElementException => None
    }
  }

  /**
   * Resets the configuration property for the given key.
   *
   * @since 2.0.0
   */
  def unset(key: String): Unit = {
    requireNonStaticConf(key)
    sqlConf.unsetConf(key)
  }

  /**
   * Indicates whether the configuration property with the given key
   * is modifiable in the current session.
   *
   * @return `true` if the configuration property is modifiable. For static SQL, Spark Core,
   *         invalid (not existing) and other non-modifiable configuration properties,
   *         the returned value is `false`.
   * @since 2.4.0
   */
  def isModifiable(key: String): Boolean = sqlConf.isModifiable(key)

  /**
   * Returns whether a particular key is set.
   */
  private[sql] def contains(key: String): Boolean = {
    sqlConf.contains(key)
  }

  private def requireNonStaticConf(key: String): Unit = {
    if (SQLConf.isStaticConfigKey(key)) {
      throw QueryCompilationErrors.cannotModifyValueOfStaticConfigError(key)
    }
    if (sqlConf.setCommandRejectsSparkCoreConfs &&
        ConfigEntry.findEntry(key) != null && !SQLConf.containsConfigKey(key)) {
      throw QueryCompilationErrors.cannotModifyValueOfSparkConfigError(key, SPARK_DOC_ROOT)
    }
  }
}