templates.deployment.spark-operator.spark-application-dev-values.yaml.vm Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of foundation-mda Show documentation

Model driven architecture artifacts for aiSSEMBLE

The newest version!

sparkApp:
    spec:
      image: "${projectName}-spark-worker-docker:latest"
      sparkConf:
        spark.eventLog.enabled: "true"
        #if ("${sparkExtensions}" != "")
        spark.sql.extensions: "${sparkExtensions}"
        #end
        #if (${enableHiveSupport})
        spark.sql.catalogImplementation: "hive"
        #end
        #if (${enableDeltaSupport})
        spark.sql.catalog.spark_catalog: "org.apache.spark.sql.delta.catalog.DeltaCatalog"
        #end
        #if (${useS3Local})
        spark.hadoop.fs.s3a.endpoint: "http://s3-local:4566"
        spark.eventLog.dir: "/opt/spark/spark-events"
        spark.hive.metastore.warehouse.dir: "s3a://spark-infrastructure/warehouse"
        #end
        #if (${enableElasticsearchSupport})
        spark.es.nodes: "elasticsearch-es-http"
        spark.es.port: "9200"

        # these values can be overridden in [YOUR-PROJECT]-deploy/src/main/resources/apps/elasticsearch/*.yaml
        spark.es.net.http.auth.user: "elastic"
        spark.es.net.http.auth.pass: "elastic"
        #end
        #if (${enableSedonaSupport})
        spark.serializer: "org.apache.spark.serializer.KryoSerializer"
        #if (${enablePySparkSupport})
        spark.kry.registrator: "org.apache.sedona.core.serde.SedonaKryoRegistrator"
        #else
        spark.kryo.registrator: "org.apache.sedona.viz.core.Serde.SedonaVizKryoRegistrator"
        #end
      #end
      driver:
        cores: 1
        memory: "2048m"
      executor:
        cores: 1
        memory: "2048m"