org.elasticsearch.spark.sql.streaming.StructuredStreamingVersionLock.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of elasticsearch-hadoop Show documentation

Elasticsearch Hadoop

There is a newer version: 8.16.0

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.spark.sql.streaming

import org.apache.commons.logging.LogFactory
import org.apache.spark.sql.SparkSession
import org.elasticsearch.hadoop.EsHadoopException
import org.elasticsearch.hadoop.EsHadoopIllegalArgumentException

/**
 * Spark Structured Streaming just recently left "Experimental" mode in Spark 2.2.0. Its underlying interfaces have not
 * been subject to the same stability rules as other parts of the project since its debut in 2.0.0. Each point release
 * between 2.0 and 2.2 has introduced breaking changes to the api. This means that even though our 2.x artifact supports
 * all 2.0 and up versions of Spark, we need to ensure that users of Spark Structured Streaming are running on
 * Spark 2.2.0 and above and are not confused about why their streaming jobs are failing if they use versions of Spark
 * that do not match what we can reasonably support.
 */
object StructuredStreamingVersionLock {

  private [this] val LOG = LogFactory.getLog(getClass)

  private [this] val supported = """(2.[2-9].[0-9]|[3-9].[0-9].[0-9]).*""".r

  /**
   * Checks the spark session to make sure that it contains a compatible
   * version.
   */
  def checkCompatibility(session: SparkSession): Unit = {
    try {
      session.version match {
        case supported(version) => if (LOG.isDebugEnabled) LOG.debug(s"Running against supported version of Spark [$version]")
        case _ =>
          throw new EsHadoopIllegalArgumentException(s"Spark version mismatch. Expected at least Spark version [2.2.0] " +
            s"but found [${session.version}]. Spark Structured Streaming is a feature that is only supported on Spark " +
            s"[2.2.0] and up for this version of ES-Hadoop/Spark.")
      }
    } catch {
      case e: EsHadoopException =>
        throw e
      case t: Throwable =>
        throw new EsHadoopIllegalArgumentException("Could not determine the version of Spark for compatibility", t)
    }
  }
}