org.elasticsearch.spark.sql.streaming.SparkSqlStreamingConfigs.scala Maven / Gradle / Ivy
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.spark.sql.streaming
import java.util.UUID
import java.util.concurrent.TimeUnit
import org.apache.spark.sql.internal.SQLConf
import org.elasticsearch.hadoop.EsHadoopIllegalArgumentException
import org.elasticsearch.hadoop.cfg.Settings
import org.elasticsearch.hadoop.util.unit.TimeValue
/**
* Configurations specifically used for Spark Structured Streaming
*/
object SparkSqlStreamingConfigs {
val ES_SINK_LOG_ENABLE: String = "es.spark.sql.streaming.sink.log.enabled"
val ES_SINK_LOG_ENABLE_DEFAULT: Boolean = true
val ES_SINK_LOG_PATH: String = "es.spark.sql.streaming.sink.log.path"
val ES_INTERNAL_APP_NAME: String = "es.internal.spark.sql.streaming.appName"
val ES_INTERNAL_APP_ID: String = "es.internal.spark.sql.streaming.appID"
val ES_INTERNAL_QUERY_NAME: String = "es.internal.spark.sql.streaming.queryName"
val ES_INTERNAL_USER_CHECKPOINT_LOCATION: String = "es.internal.spark.sql.streaming.userCheckpointLocation"
val ES_INTERNAL_SESSION_CHECKPOINT_LOCATION: String = "es.internal.spark.sql.streaming.sessionCheckpointLocation"
val ES_SINK_LOG_CLEANUP_DELAY: String = "es.spark.sql.streaming.sink.log.cleanupDelay"
val ES_SINK_LOG_CLEANUP_DELAY_DEFAULT: Long = TimeUnit.MINUTES.toMillis(10)
val ES_SINK_LOG_DELETION: String = "es.spark.sql.streaming.sink.log.deletion"
val ES_SINK_LOG_DELETION_DEFAULT: Boolean = true
val ES_SINK_LOG_COMPACT_INTERVAL: String = "es.spark.sql.streaming.sink.log.compactInterval"
val ES_SINK_LOG_COMPACT_INTERVAL_DEFAULT: Int = 10
/**
* Determines if we should use the commit log for writes, or if we should go without one.
* @param settings connector settings
* @return true if we should use the commit log, false if we should not
*/
def getSinkLogEnabled(settings: Settings): Boolean = {
Option(settings.getProperty(ES_SINK_LOG_ENABLE)).map(_.toBoolean)
.getOrElse(ES_SINK_LOG_ENABLE_DEFAULT)
}
/**
* Determines the location of the streaming commit log.
* @param settings connector settings
* @return the location to use as the commit log
*/
def constructCommitLogPath(settings: Settings): String = {
val logPath = getLogPath(settings)
val userCheckpointLocation = getUserSpecifiedCheckpointLocation(settings)
val sessionCheckpointLocation = getSessionCheckpointLocation(settings)
val queryName = getQueryName(settings)
(logPath, userCheckpointLocation, sessionCheckpointLocation, queryName) match {
// Case 1) /{log.path}/{batchFiles}
// If a user gives an explicit log location with ES_SINK_LOG_PATH, then it's fine to store the commit
// files in the root of that. That directory should be just for us.
case (Some(explicitPath), _, _, _) => explicitPath
// Case 2) /{checkpointLocation}/sinks/es
// Don't store commit files for the log in the root of the checkpoint location; There are other directories
// inside that root like '/sources', '/metadata', '/state', '/offsets', etc.
// Instead, nest it under '/sinks/elasticsearch'.
case (None, Some(userCheckpoint), _, _) => s"$userCheckpoint/sinks/elasticsearch"
// Case 3) /{sessionCheckpointLocation}/{UUID}
// Spark lets you define a common location to store checkpoint locations for an entire session. Checkpoint
// locations are then keyed by their query names. In the event of no query name being present, the query manager
// creates a random UUID to store the checkpoint in. Not great, since you can't recover easily, but we'll follow
// suit anyway.
case (None, None, Some(sessionCheckpoint), None) => s"$sessionCheckpoint/${UUID.randomUUID().toString}/sinks/elasticsearch"
// Case 4) /{sessionCheckpointLocation}/{queryName}
// Same as above, but the query name is specified.
case (None, None, Some(sessionCheckpoint), Some(query)) => s"$sessionCheckpoint/$query/sinks/elasticsearch"
// Case 5) throw because we don't know where to store things...
case (None, None, None, _) => throw new EsHadoopIllegalArgumentException(
"Could not determine path for the Elasticsearch commit log. Specify the commit log location by setting the " +
"[checkpointLocation] option on your DataStreamWriter. If you do not want to persist the Elasticsearch " +
"commit log in the regular checkpoint location for your streaming query then you can specify a location to " +
s"store the log with [$ES_SINK_LOG_PATH], or disable the commit log by setting [$ES_SINK_LOG_ENABLE] to false.")
}
}
/**
* The log path, if set, is the complete log path to be used for the commit log.
* @param settings connector settings
* @return either Some log path or None
*/
def getLogPath(settings: Settings): Option[String] = {
Option(settings.getProperty(ES_SINK_LOG_PATH))
}
/**
* The name of the current Spark application, if set
* @param settings connector settings
* @return Some name or None
*/
def getAppName(settings: Settings): Option[String] = {
Option(settings.getProperty(ES_INTERNAL_APP_NAME))
}
/**
* The ID of the current Spark application, if set
* @param settings connector settings
* @return Some id or None
*/
def getAppId(settings: Settings): Option[String] = {
Option(settings.getProperty(ES_INTERNAL_APP_ID))
}
/**
* The name of the current Spark Streaming Query, if set
* @param settings connector settings
* @return Some query name or None
*/
def getQueryName(settings: Settings): Option[String] = {
Option(settings.getProperty(ES_INTERNAL_QUERY_NAME))
}
/**
* The name of the user specified checkpoint location for the current Spark Streaming Query, if set
* @param settings connector settings
* @return Some checkpoint location or None
*/
def getUserSpecifiedCheckpointLocation(settings: Settings): Option[String] = {
Option(settings.getProperty(ES_INTERNAL_USER_CHECKPOINT_LOCATION))
}
/**
* The name of the default session checkpoint location, if set
* @param settings connector settings
* @return Some checkpoint location or None
*/
def getSessionCheckpointLocation(settings: Settings): Option[String] = {
Option(settings.getProperty(ES_INTERNAL_SESSION_CHECKPOINT_LOCATION))
}
/**
* The number of milliseconds to wait before cleaning up compacted log files
* @param settings connector settings
* @return time in millis if set, or the default delay
*/
def getFileCleanupDelayMs(settings: Settings): Long =
Option(settings.getProperty(ES_SINK_LOG_CLEANUP_DELAY)).map(TimeValue.parseTimeValue(_).getMillis)
.orElse(SQLConf.FILE_SINK_LOG_CLEANUP_DELAY.defaultValue)
.getOrElse(ES_SINK_LOG_CLEANUP_DELAY_DEFAULT)
/**
*
* @param settings connector settings
* @return
*/
def getIsDeletingExpiredLog(settings: Settings): Boolean =
Option(settings.getProperty(ES_SINK_LOG_DELETION)).map(_.toBoolean)
.orElse(SQLConf.FILE_SINK_LOG_DELETION.defaultValue)
.getOrElse(ES_SINK_LOG_DELETION_DEFAULT)
/**
*
* @param settings connector settings
* @return
*/
def getDefaultCompactInterval(settings: Settings): Int =
Option(settings.getProperty(ES_SINK_LOG_COMPACT_INTERVAL)).map(_.toInt)
.orElse(SQLConf.FILE_SINK_LOG_COMPACT_INTERVAL.defaultValue)
.getOrElse(ES_SINK_LOG_COMPACT_INTERVAL_DEFAULT)
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy