templates.deployment.spark-infrastructure.v2.spark.infrastructure.values.yaml.vm Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of foundation-mda Show documentation
Model driven architecture artifacts for aiSSEMBLE
The newest version!
# This file contains the default values for the aiSSEMBLE Spark Infrastructure Helm chart.

aissemble-spark-history-chart:
  app:
    name: "spark-history"
  eventVolume:
    enabled: true

aissemble-thrift-server-chart:
  app:
    name: "thrift-server"
  dependencies:
    packages:
      - org.apache.hadoop:hadoop-aws:3.3.4

  deployment:
    envFromSecret:
      AWS_ACCESS_KEY_ID:
        secretName: remote-auth-config
        key: AWS_ACCESS_KEY_ID
      AWS_SECRET_ACCESS_KEY:
        secretName: remote-auth-config
        key: AWS_SECRET_ACCESS_KEY

  sparkConf: |
    spark.hadoop.fs.s3a.endpoint=http://s3-local:4566
    spark.hadoop.fs.s3a.access.key=#[[${env:AWS_ACCESS_KEY_ID}]]#
    spark.hadoop.fs.s3a.secret.key=#[[${env:AWS_SECRET_ACCESS_KEY}]]#
    spark.hadoop.fs.s3.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
    spark.hive.server2.thrift.port=10000
    spark.hive.server2.thrift.http.port=10001
    spark.hive.server2.transport.mode=http
    spark.hive.metastore.warehouse.dir=s3a://spark-infrastructure/warehouse
    spark.hadoop.fs.s3a.path.style.access=true
    spark.hive.server2.thrift.http.path=cliservice
    spark.hive.metastore.schema.verification=false
    spark.hive.metastore.uris=thrift://hive-metastore-service:9083/default

  hiveSite: |
    
         
             datanucleus.schema.autoCreateAll
             true
             Creates necessary schema on a startup if one does not exist
         
     

aissemble-hive-metastore-service-chart:
  mysql:
    auth:
      # Note: Changing these values requires removal of the `hive-metastore-db-0` PVC, or manual modification of the
      # persisted database.
      username: hive
      # NB: UCS
      rootPassword: hive
      replicationPassword: hive
      password: hive

  deployment:
    env:
      - name: AWS_SECRET_ACCESS_KEY
        valueFrom:
          secretKeyRef:
            name: remote-auth-config
            key: AWS_SECRET_ACCESS_KEY
      - name: AWS_ACCESS_KEY_ID
        valueFrom:
          secretKeyRef:
            name: remote-auth-config
            key: AWS_ACCESS_KEY_ID

  configMap:
    metastoreServiceConfig:
      properties:
        - name: javax.jdo.option.ConnectionUserName
          value: hive
          description: Username to use against metastore database
          # NB: UCS
        - name: javax.jdo.option.ConnectionPassword
          value: hive
          description: Password to use against metastore database
        - name: metastore.warehouse.dir
          value: s3a://spark-infrastructure/warehouse
        - name: fs.s3a.endpoint
          value: http://s3-local:4566
        - name: fs.s3a.path.style.access
          value: true