gobblin.gobblin-example.0.7.0.source-code.simplejson.pull Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of gobblin-example Show documentation

Gobblin Ingestion Framework

There is a newer version: 0.11.0

#########################################################################
###################### Demo job configuration file ######################
#########################################################################

job.name=GobblinDemo
job.group=demo
job.description=A Gobblin job for demo purpose

source.class=gobblin.example.simplejson.SimpleJsonSource
converter.classes=gobblin.example.simplejson.SimpleJsonConverter
extract.namespace=gobblin.example.simplejson

# source configuration properties
# comma-separated list of file URIs (supporting different schemes, e.g., file://, ftp://, sftp://, http://, etc)
source.filebased.files.to.pull=
# whether to use authentication or not (default is false)
source.conn.use.authentication=
# credential for authentication purpose (optional)
source.conn.domain=
source.conn.username=
source.conn.password=
# source data schema
source.schema={"namespace":"example.avro", "type":"record", "name":"User", "fields":[{"name":"name", "type":"string"}, {"name":"favorite_number",  "type":"int"}, {"name":"favorite_color", "type":"string"}]}

# quality checker configuration properties
qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy
qualitychecker.task.policy.types=OPTIONAL,OPTIONAL
qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy
qualitychecker.row.policy.types=OPTIONAL
qualitychecker.row.err.file=test/jobOutput

# data publisher class to be used
data.publisher.type=gobblin.publisher.BaseDataPublisher

# writer configuration properties
writer.destination.type=HDFS
writer.output.format=AVRO