scaffolding.defaultFlow.flow.json Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of marklogic-data-hub Show documentation
Library for Creating an Operational Data Hub on MarkLogic
There is a newer version: 6.1.1
{
  "name": "%%mlFlowName%%",
  "description": "This is the default flow containing all of the default steps",
  "batchSize": 100,
  "threadCount": 4,
  "steps": {
    "1": {
      "name": "ingestion-step",
      "description": "This is the default ingestion step",
      "stepDefinitionName": "default-ingestion",
      "stepDefinitionType": "ingestion",
      "batchSize" : 100,
      "threadCount" : 4,
      "fileLocations": {
        "inputFilePath": "path/to/folder",
        "outputURIReplacement": "output/URI,'substitute/URI'",
        "inputFileType": "json"
      },
      "options": {
        "targetDatabase": "%%mlStagingDbName%%",
        "permissions": "data-hub-common,read,data-hub-common,update",
        "outputFormat": "json",
        "collections": [
          "default-ingestion"
        ],
        "headers": {
          "sources": [{"name":  "%%mlFlowName%%"}],
          "createdOn" : "currentDateTime",
          "createdBy" : "currentUser"
        }
      }
    },
    "2": {
      "name": "mapping-step",
      "description": "This is the default mapping step",
      "stepDefinitionName": "entity-services-mapping",
      "stepDefinitionType": "mapping",
      "batchSize" : 100,
      "threadCount" : 4,
      "options": {
        "sourceDatabase": "%%mlStagingDbName%%",
        "targetDatabase": "%%mlFinalDbName%%",
        "sourceQuery": "cts.collectionQuery('default-ingestion')",
        "permissions": "data-hub-common,read,data-hub-common,update",
        "outputFormat": "json",
        "collections": [
          "default-mapping",
          "mdm-content"
        ],
        "targetEntity": "Change this to a valid entity type name; e.g. Customer",
        "mapping": {
          "name": "mapping-name",
          "version": 1
        },
        "validateEntity": false
      }
    },
    "3": {
      "name": "matching-step",
      "description": "This is the default matching step",
      "stepDefinitionName": "default-matching",
      "stepDefinitionType": "matching",
      "batchSize" : 100,
      "threadCount" : 1,
      "options": {
        "sourceDatabase": "%%mlFinalDbName%%",
        "targetDatabase": "%%mlFinalDbName%%",
        "targetEntity": "Change this to a valid entity type name; e.g. Customer",
        "sourceQuery": "cts.collectionQuery('default-mapping')",
        "collections": [
          "mastering-summary"
        ],
        "permissions": "data-hub-common,read,data-hub-common,update",
        "outputFormat": "json",
        "matchOptions" : {
          "dataFormat": "json",
          "propertyDefs": {
            "property": []
          },
          "algorithms": {
            "algorithm": []
          },
          "collections": {
            "content": []
          },
          "matchRulesets":  [],
          "thresholds": [],
          "tuning": {
            "maxScan": 200
          }
        }
      }
    },
    "4": {
      "name": "merging-step",
      "description": "This is the default merging step",
      "stepDefinitionName": "default-merging",
      "stepDefinitionType": "merging",
      "batchSize" : 100,
      "threadCount" : 1,
      "options": {
        "sourceDatabase": "%%mlFinalDbName%%",
        "targetDatabase": "%%mlFinalDbName%%",
        "targetEntity": "Change this to a valid entity type name; e.g. Customer",
        "sourceQuery": "cts.collectionQuery('mastering-summary')",
        "collections": [
          "default-merging","mastered"
        ],
        "permissions": "data-hub-common,read,data-hub-common,update",
        "outputFormat": "json",
        "mergeOptions" : {
          "propertyDefs" : {
            "properties" : [ ],
            "namespaces" : { }
          },
          "algorithms" : {
            "stdAlgorithm" : {
              "timestamp" : { }
            },
            "custom" : [ ],
            "collections" : { }
          },
          "mergeStrategies" : [ ],
          "merging" : [ ]
        }
      }
    }
  }
}