park-connector_2.13.1.3.3.source-code.WeaviateCluster.scala Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of spark-connector_2.13 Show documentation

Weaviate Spark Connector to use in Spark ETLs to populate a Weaviate vector database.

The newest version!

package io.weaviate.spark

import org.apache.spark.sql.connector.catalog.{SupportsWrite, TableCapability}
import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
import org.apache.spark.sql.types.StructType

import java.util
import scala.jdk.CollectionConverters._

case class WeaviateCluster(weaviateOptions: WeaviateOptions, schema: StructType) extends SupportsWrite {
  override def name(): String = weaviateOptions.className

  override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
    WeaviateWriteBuilder(weaviateOptions, info.schema())
  }
  override def capabilities(): util.Set[TableCapability] = Set(
    TableCapability.BATCH_WRITE,
    TableCapability.STREAMING_WRITE
  ).asJava
}