schema.sparkCluster.json Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of spark-operator Show documentation
There is a newer version: 1.0.4
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "description": "A Spark cluster configuration",
  "dependencies": null,
  "type": "object",
  "extends": {
    "type": "object",
    "existingJavaType": "io.radanalytics.operator.common.EntityInfo"
  },
  "properties": {
    "master": {
      "type": "object",
      "properties": {
        "instances": {
          "type": "integer",
          "default": "1",
          "minimum": "1"
        },
        "memory": {
          "type": "string"
        },
        "cpu": {
          "type": "string"
        },
        "labels": {
          "existingJavaType": "java.util.Map",
          "type": "string",
          "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]"
        },
        "command": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "commandArgs": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "worker": {
      "type": "object",
      "properties": {
        "instances": {
          "type": "integer",
          "default": "1",
          "minimum": "0"
        },
        "memory": {
          "type": "string"
        },
        "cpu": {
          "type": "string"
        },
        "labels": {
          "existingJavaType": "java.util.Map",
          "type": "string",
          "pattern": "([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]"
        },
        "command": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "commandArgs": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "customImage": {
      "type": "string"
    },
    "metrics": {
      "type": "boolean",
      "default": "false"
    },
    "sparkWebUI": {
      "type": "boolean",
      "default": "true"
    },
    "sparkConfigurationMap": {
      "type": "string"
    },
    "env": {
      "type": "array",
      "items": {
        "type": "object",
        "javaType": "io.radanalytics.types.Env",
        "properties": {
          "name": { "type": "string" },
          "value": { "type": "string" }
        },
        "required": ["name", "value"]
      }
    },
    "sparkConfiguration": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": { "type": "string" },
          "value": { "type": "string" }
        },
        "required": ["name", "value"]
      }
    },
    "labels": {
      "type": "object",
      "existingJavaType": "java.util.Map"
    },
    "historyServer": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string"
        },
        "type": {
          "type": "string",
          "default": "sharedVolume",
          "enum": [
            "sharedVolume",
            "remoteStorage"
          ],
          "javaEnumNames": [
            "sharedVolume",
            "remoteStorage"
          ]
        },

        "sharedVolume": {
          "type": "object",
          "properties": {
            "size": {
              "type": "string",
              "default": "0.3Gi"
            },
            "mountPath": {
              "type": "string",
              "default": "/history/spark-events"
            },
            "matchLabels": {
              "type": "object",
              "existingJavaType": "java.util.Map"
            }
          }
        },
        "remoteURI": {
          "type": "string",
          "description": "s3 bucket or hdfs path"
        }
      }
    },
    "downloadData": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "url": {
            "type": "string"
          },
          "to": {
            "type": "string"
          }
        },
        "required": [
          "url",
          "to"
        ]
      }
    }
  },
  "required": []
}