com.pulumi.aws.glue.Job Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of aws Show documentation
Show all versions of aws Show documentation
A Pulumi package for creating and managing Amazon Web Services (AWS) cloud resources.
// *** WARNING: this file was generated by pulumi-java-gen. ***
// *** Do not edit by hand unless you're certain you know what you are doing! ***
package com.pulumi.aws.glue;
import com.pulumi.aws.Utilities;
import com.pulumi.aws.glue.JobArgs;
import com.pulumi.aws.glue.inputs.JobState;
import com.pulumi.aws.glue.outputs.JobCommand;
import com.pulumi.aws.glue.outputs.JobExecutionProperty;
import com.pulumi.aws.glue.outputs.JobNotificationProperty;
import com.pulumi.core.Output;
import com.pulumi.core.annotations.Export;
import com.pulumi.core.annotations.ResourceType;
import com.pulumi.core.internal.Codegen;
import java.lang.Boolean;
import java.lang.Double;
import java.lang.Integer;
import java.lang.String;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;
/**
* Provides a Glue Job resource.
*
* > Glue functionality, such as monitoring and logging of jobs, is typically managed with the `default_arguments` argument. See the [Special Parameters Used by AWS Glue](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html) topic in the Glue developer guide for additional information.
*
* ## Example Usage
*
* ### Python Job
*
* <!--Start PulumiCodeChooser -->
*
* {@code
* package generated_program;
*
* import com.pulumi.Context;
* import com.pulumi.Pulumi;
* import com.pulumi.core.Output;
* import com.pulumi.aws.glue.Job;
* import com.pulumi.aws.glue.JobArgs;
* import com.pulumi.aws.glue.inputs.JobCommandArgs;
* import java.util.List;
* import java.util.ArrayList;
* import java.util.Map;
* import java.io.File;
* import java.nio.file.Files;
* import java.nio.file.Paths;
*
* public class App {
* public static void main(String[] args) {
* Pulumi.run(App::stack);
* }
*
* public static void stack(Context ctx) {
* var example = new Job("example", JobArgs.builder()
* .name("example")
* .roleArn(exampleAwsIamRole.arn())
* .command(JobCommandArgs.builder()
* .scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
* .build())
* .build());
*
* }
* }
* }
*
* <!--End PulumiCodeChooser -->
*
* ### Ray Job
*
* <!--Start PulumiCodeChooser -->
*
* {@code
* package generated_program;
*
* import com.pulumi.Context;
* import com.pulumi.Pulumi;
* import com.pulumi.core.Output;
* import com.pulumi.aws.glue.Job;
* import com.pulumi.aws.glue.JobArgs;
* import com.pulumi.aws.glue.inputs.JobCommandArgs;
* import java.util.List;
* import java.util.ArrayList;
* import java.util.Map;
* import java.io.File;
* import java.nio.file.Files;
* import java.nio.file.Paths;
*
* public class App {
* public static void main(String[] args) {
* Pulumi.run(App::stack);
* }
*
* public static void stack(Context ctx) {
* var example = new Job("example", JobArgs.builder()
* .name("example")
* .roleArn(exampleAwsIamRole.arn())
* .glueVersion("4.0")
* .workerType("Z.2X")
* .command(JobCommandArgs.builder()
* .name("glueray")
* .pythonVersion("3.9")
* .runtime("Ray2.4")
* .scriptLocation(String.format("s3://%s/example.py", exampleAwsS3Bucket.bucket()))
* .build())
* .build());
*
* }
* }
* }
*
* <!--End PulumiCodeChooser -->
*
* ### Scala Job
*
* <!--Start PulumiCodeChooser -->
*
* {@code
* package generated_program;
*
* import com.pulumi.Context;
* import com.pulumi.Pulumi;
* import com.pulumi.core.Output;
* import com.pulumi.aws.glue.Job;
* import com.pulumi.aws.glue.JobArgs;
* import com.pulumi.aws.glue.inputs.JobCommandArgs;
* import java.util.List;
* import java.util.ArrayList;
* import java.util.Map;
* import java.io.File;
* import java.nio.file.Files;
* import java.nio.file.Paths;
*
* public class App {
* public static void main(String[] args) {
* Pulumi.run(App::stack);
* }
*
* public static void stack(Context ctx) {
* var example = new Job("example", JobArgs.builder()
* .name("example")
* .roleArn(exampleAwsIamRole.arn())
* .command(JobCommandArgs.builder()
* .scriptLocation(String.format("s3://%s/example.scala", exampleAwsS3Bucket.bucket()))
* .build())
* .defaultArguments(Map.of("--job-language", "scala"))
* .build());
*
* }
* }
* }
*
* <!--End PulumiCodeChooser -->
*
* ### Streaming Job
*
* <!--Start PulumiCodeChooser -->
*
* {@code
* package generated_program;
*
* import com.pulumi.Context;
* import com.pulumi.Pulumi;
* import com.pulumi.core.Output;
* import com.pulumi.aws.glue.Job;
* import com.pulumi.aws.glue.JobArgs;
* import com.pulumi.aws.glue.inputs.JobCommandArgs;
* import java.util.List;
* import java.util.ArrayList;
* import java.util.Map;
* import java.io.File;
* import java.nio.file.Files;
* import java.nio.file.Paths;
*
* public class App {
* public static void main(String[] args) {
* Pulumi.run(App::stack);
* }
*
* public static void stack(Context ctx) {
* var example = new Job("example", JobArgs.builder()
* .name("example streaming job")
* .roleArn(exampleAwsIamRole.arn())
* .command(JobCommandArgs.builder()
* .name("gluestreaming")
* .scriptLocation(String.format("s3://%s/example.script", exampleAwsS3Bucket.bucket()))
* .build())
* .build());
*
* }
* }
* }
*
* <!--End PulumiCodeChooser -->
*
* ### Enabling CloudWatch Logs and Metrics
*
* <!--Start PulumiCodeChooser -->
*
* {@code
* package generated_program;
*
* import com.pulumi.Context;
* import com.pulumi.Pulumi;
* import com.pulumi.core.Output;
* import com.pulumi.aws.cloudwatch.LogGroup;
* import com.pulumi.aws.cloudwatch.LogGroupArgs;
* import com.pulumi.aws.glue.Job;
* import com.pulumi.aws.glue.JobArgs;
* import java.util.List;
* import java.util.ArrayList;
* import java.util.Map;
* import java.io.File;
* import java.nio.file.Files;
* import java.nio.file.Paths;
*
* public class App {
* public static void main(String[] args) {
* Pulumi.run(App::stack);
* }
*
* public static void stack(Context ctx) {
* var example = new LogGroup("example", LogGroupArgs.builder()
* .name("example")
* .retentionInDays(14)
* .build());
*
* var exampleJob = new Job("exampleJob", JobArgs.builder()
* .defaultArguments(Map.ofEntries(
* Map.entry("--continuous-log-logGroup", example.name()),
* Map.entry("--enable-continuous-cloudwatch-log", "true"),
* Map.entry("--enable-continuous-log-filter", "true"),
* Map.entry("--enable-metrics", "")
* ))
* .build());
*
* }
* }
* }
*
* <!--End PulumiCodeChooser -->
*
* ## Import
*
* Using `pulumi import`, import Glue Jobs using `name`. For example:
*
* ```sh
* $ pulumi import aws:glue/job:Job MyJob MyJob
* ```
*
*/
@ResourceType(type="aws:glue/job:Job")
public class Job extends com.pulumi.resources.CustomResource {
/**
* Amazon Resource Name (ARN) of Glue Job
*
*/
@Export(name="arn", refs={String.class}, tree="[0]")
private Output arn;
/**
* @return Amazon Resource Name (ARN) of Glue Job
*
*/
public Output arn() {
return this.arn;
}
/**
* The command of the job. Defined below.
*
*/
@Export(name="command", refs={JobCommand.class}, tree="[0]")
private Output command;
/**
* @return The command of the job. Defined below.
*
*/
public Output command() {
return this.command;
}
/**
* The list of connections used for this job.
*
*/
@Export(name="connections", refs={List.class,String.class}, tree="[0,1]")
private Output> connections;
/**
* @return The list of connections used for this job.
*
*/
public Output>> connections() {
return Codegen.optional(this.connections);
}
/**
* The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide.
*
*/
@Export(name="defaultArguments", refs={Map.class,String.class}, tree="[0,1,1]")
private Output> defaultArguments;
/**
* @return The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide.
*
*/
public Output>> defaultArguments() {
return Codegen.optional(this.defaultArguments);
}
/**
* Description of the job.
*
*/
@Export(name="description", refs={String.class}, tree="[0]")
private Output description;
/**
* @return Description of the job.
*
*/
public Output> description() {
return Codegen.optional(this.description);
}
/**
* Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`.
*
*/
@Export(name="executionClass", refs={String.class}, tree="[0]")
private Output executionClass;
/**
* @return Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources. Valid value: `FLEX`, `STANDARD`.
*
*/
public Output> executionClass() {
return Codegen.optional(this.executionClass);
}
/**
* Execution property of the job. Defined below.
*
*/
@Export(name="executionProperty", refs={JobExecutionProperty.class}, tree="[0]")
private Output executionProperty;
/**
* @return Execution property of the job. Defined below.
*
*/
public Output executionProperty() {
return this.executionProperty;
}
/**
* The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html).
*
*/
@Export(name="glueVersion", refs={String.class}, tree="[0]")
private Output glueVersion;
/**
* @return The version of glue to use, for example "1.0". Ray jobs should set this to 4.0 or greater. For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html).
*
*/
public Output glueVersion() {
return this.glueVersion;
}
/**
* Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
*
*/
@Export(name="jobRunQueuingEnabled", refs={Boolean.class}, tree="[0]")
private Output jobRunQueuingEnabled;
/**
* @return Specifies whether job run queuing is enabled for the job runs for this job. A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
*
*/
public Output> jobRunQueuingEnabled() {
return Codegen.optional(this.jobRunQueuingEnabled);
}
/**
* Specifies the day of the week and hour for the maintenance window for streaming jobs.
*
*/
@Export(name="maintenanceWindow", refs={String.class}, tree="[0]")
private Output maintenanceWindow;
/**
* @return Specifies the day of the week and hour for the maintenance window for streaming jobs.
*
*/
public Output> maintenanceWindow() {
return Codegen.optional(this.maintenanceWindow);
}
/**
* The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `number_of_workers` and `worker_type` arguments instead with `glue_version` `2.0` and above.
*
*/
@Export(name="maxCapacity", refs={Double.class}, tree="[0]")
private Output maxCapacity;
/**
* @return The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. `Required` when `pythonshell` is set, accept either `0.0625` or `1.0`. Use `number_of_workers` and `worker_type` arguments instead with `glue_version` `2.0` and above.
*
*/
public Output maxCapacity() {
return this.maxCapacity;
}
/**
* The maximum number of times to retry this job if it fails.
*
*/
@Export(name="maxRetries", refs={Integer.class}, tree="[0]")
private Output maxRetries;
/**
* @return The maximum number of times to retry this job if it fails.
*
*/
public Output> maxRetries() {
return Codegen.optional(this.maxRetries);
}
/**
* The name you assign to this job. It must be unique in your account.
*
*/
@Export(name="name", refs={String.class}, tree="[0]")
private Output name;
/**
* @return The name you assign to this job. It must be unique in your account.
*
*/
public Output name() {
return this.name;
}
/**
* Non-overridable arguments for this job, specified as name-value pairs.
*
*/
@Export(name="nonOverridableArguments", refs={Map.class,String.class}, tree="[0,1,1]")
private Output> nonOverridableArguments;
/**
* @return Non-overridable arguments for this job, specified as name-value pairs.
*
*/
public Output>> nonOverridableArguments() {
return Codegen.optional(this.nonOverridableArguments);
}
/**
* Notification property of the job. Defined below.
*
*/
@Export(name="notificationProperty", refs={JobNotificationProperty.class}, tree="[0]")
private Output notificationProperty;
/**
* @return Notification property of the job. Defined below.
*
*/
public Output notificationProperty() {
return this.notificationProperty;
}
/**
* The number of workers of a defined workerType that are allocated when a job runs.
*
*/
@Export(name="numberOfWorkers", refs={Integer.class}, tree="[0]")
private Output numberOfWorkers;
/**
* @return The number of workers of a defined workerType that are allocated when a job runs.
*
*/
public Output numberOfWorkers() {
return this.numberOfWorkers;
}
/**
* The ARN of the IAM role associated with this job.
*
*/
@Export(name="roleArn", refs={String.class}, tree="[0]")
private Output roleArn;
/**
* @return The ARN of the IAM role associated with this job.
*
*/
public Output roleArn() {
return this.roleArn;
}
/**
* The name of the Security Configuration to be associated with the job.
*
*/
@Export(name="securityConfiguration", refs={String.class}, tree="[0]")
private Output securityConfiguration;
/**
* @return The name of the Security Configuration to be associated with the job.
*
*/
public Output> securityConfiguration() {
return Codegen.optional(this.securityConfiguration);
}
/**
* Key-value map of resource tags. If configured with a provider `default_tags` configuration block present, tags with matching keys will overwrite those defined at the provider-level.
*
*/
@Export(name="tags", refs={Map.class,String.class}, tree="[0,1,1]")
private Output> tags;
/**
* @return Key-value map of resource tags. If configured with a provider `default_tags` configuration block present, tags with matching keys will overwrite those defined at the provider-level.
*
*/
public Output>> tags() {
return Codegen.optional(this.tags);
}
/**
* A map of tags assigned to the resource, including those inherited from the provider `default_tags` configuration block.
*
* @deprecated
* Please use `tags` instead.
*
*/
@Deprecated /* Please use `tags` instead. */
@Export(name="tagsAll", refs={Map.class,String.class}, tree="[0,1,1]")
private Output> tagsAll;
/**
* @return A map of tags assigned to the resource, including those inherited from the provider `default_tags` configuration block.
*
*/
public Output> tagsAll() {
return this.tagsAll;
}
/**
* The job timeout in minutes. The default is 2880 minutes (48 hours) for `glueetl` and `pythonshell` jobs, and null (unlimited) for `gluestreaming` jobs.
*
*/
@Export(name="timeout", refs={Integer.class}, tree="[0]")
private Output timeout;
/**
* @return The job timeout in minutes. The default is 2880 minutes (48 hours) for `glueetl` and `pythonshell` jobs, and null (unlimited) for `gluestreaming` jobs.
*
*/
public Output timeout() {
return this.timeout;
}
/**
* The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
* * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
* * For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
* * For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
* * For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
* * For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
* * For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
* * For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
*
*/
@Export(name="workerType", refs={String.class}, tree="[0]")
private Output workerType;
/**
* @return The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
* * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
* * For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
* * For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs.
* * For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
* * For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).
* * For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0.
* * For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler.
*
*/
public Output workerType() {
return this.workerType;
}
/**
*
* @param name The _unique_ name of the resulting resource.
*/
public Job(java.lang.String name) {
this(name, JobArgs.Empty);
}
/**
*
* @param name The _unique_ name of the resulting resource.
* @param args The arguments to use to populate this resource's properties.
*/
public Job(java.lang.String name, JobArgs args) {
this(name, args, null);
}
/**
*
* @param name The _unique_ name of the resulting resource.
* @param args The arguments to use to populate this resource's properties.
* @param options A bag of options that control this resource's behavior.
*/
public Job(java.lang.String name, JobArgs args, @Nullable com.pulumi.resources.CustomResourceOptions options) {
super("aws:glue/job:Job", name, makeArgs(args, options), makeResourceOptions(options, Codegen.empty()), false);
}
private Job(java.lang.String name, Output id, @Nullable JobState state, @Nullable com.pulumi.resources.CustomResourceOptions options) {
super("aws:glue/job:Job", name, state, makeResourceOptions(options, id), false);
}
private static JobArgs makeArgs(JobArgs args, @Nullable com.pulumi.resources.CustomResourceOptions options) {
if (options != null && options.getUrn().isPresent()) {
return null;
}
return args == null ? JobArgs.Empty : args;
}
private static com.pulumi.resources.CustomResourceOptions makeResourceOptions(@Nullable com.pulumi.resources.CustomResourceOptions options, @Nullable Output id) {
var defaultOptions = com.pulumi.resources.CustomResourceOptions.builder()
.version(Utilities.getVersion())
.build();
return com.pulumi.resources.CustomResourceOptions.merge(defaultOptions, options, id);
}
/**
* Get an existing Host resource's state with the given name, ID, and optional extra
* properties used to qualify the lookup.
*
* @param name The _unique_ name of the resulting resource.
* @param id The _unique_ provider ID of the resource to lookup.
* @param state
* @param options Optional settings to control the behavior of the CustomResource.
*/
public static Job get(java.lang.String name, Output id, @Nullable JobState state, @Nullable com.pulumi.resources.CustomResourceOptions options) {
return new Job(name, id, state, options);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy