schema.deltafi.graphql Maven / Gradle / Ivy

Go to download
#
#    DeltaFi - Data transformation and enrichment platform
#
#    Copyright 2022 DeltaFi Contributors 
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#

scalar DateTime
scalar JSON
scalar Long

type KeyValue {
  key: String!
  value: String
}

input KeyValueInput {
  key: String!
  value: String
}

type UniqueKeyValues {
  key: String!
  values: [String]!
}

type Domain {
  name: String!
  value: String
  mediaType: String!
}

input DomainInput {
  name: String!
  value: String
  mediaType: String!
}

type Enrichment {
  name: String!
  value: String
  mediaType: String!
}

input EnrichmentInput {
  name: String!
  value: String
  mediaType: String!
}

type SourceInfo {
  filename: String!
  flow: String!
  metadata: [KeyValue!]
}

input SourceInfoInput {
  filename: String!
  flow: String!
  metadata: [KeyValueInput!]
}

input SourceInfoFilter {
  filename: String
  flow: String
  metadata: [KeyValueInput!]
}

type FlowAssignmentRule {
  name: String!
  flow: String!
  priority: Int!
  filenameRegex: String
  requiredMetadata: [KeyValue!]
}

input FlowAssignmentRuleInput {
  name: String!
  flow: String!
  priority: Int
  filenameRegex: String
  requiredMetadata: [KeyValueInput!]
}

type Content {
  name: String
  metadata: [KeyValue!]
  contentReference: ContentReference!
}

input ContentInput {
  name: String
  metadata: [KeyValueInput!]
  contentReference: ContentReferenceInput!
}

"""
A ContentReference is a reference to stored content for a delta file.
"""
type ContentReference {
  uuid: String!
  offset: Long!
  size: Long!
  did: String!
  mediaType: String!
}

input ContentReferenceInput {
  uuid: String!
  offset: Long!
  size: Long!
  did: String!
  mediaType: String!
}

"""
Ingress and transform actions create contentReferences and express additional metadata through ProtocolLayers
Followon actions get the latest "state" of the transformed object from the top of the ProtocolLayer stack
"""
type ProtocolLayer {
  "The type of ProtocolLayer"
  type: String!
  "The action that produced this ProtocolLayer"
  action: String!
  "Content produced by the action"
  content: [Content!]!
  "Additional metadata"
  metadata: [KeyValue!]
}

input ProtocolLayerInput {
  type: String!
  action: String!
  content: [ContentInput!]!
  metadata: [KeyValueInput!]
}

type FormattedData {
  filename: String!
  formatAction: String!
  contentReference: ContentReference!
  metadata: [KeyValue!]
  egressActions: [String!]!
  validateActions: [String!]!
}

input FormattedDataFilter {
  filename: String
  formatAction: String
  metadata: [KeyValueInput!]
  egressActions: [String!]
}

enum DeltaFileStage {
  INGRESS
  ENRICH
  EGRESS
  COMPLETE
  ERROR
}

enum ActionState {
  QUEUED
  COMPLETE
  ERROR
  RETRIED
  FILTERED
  SPLIT
}

type Action {
  name: String!
  state: ActionState!
  created: DateTime!
  queued: DateTime
  start: DateTime
  stop: DateTime
  modified: DateTime!
  errorCause: String
  errorContext: String
}

enum ActionEventType {
  TRANSFORM
  LOAD
  ENRICH
  FORMAT
  FORMAT_MANY
  VALIDATE
  EGRESS
  ERROR
  FILTER
  SPLIT
}

input ActionEventInput {
  did: String!
  action: String!
  start: DateTime!
  stop: DateTime!
  # deprecated -- remove after a few releases have bumped through (5/3/22)
  time: DateTime
  type: ActionEventType!
  transform: TransformInput
  load: LoadInput
  enrich: EnrichInput
  format: FormatInput
  formatMany: [FormatInput]
  error: ErrorInput
  filter: FilterInput
  split: [SplitInput]
}

type DeltaFile {
  did: String!
  parentDids: [String!]!
  childDids: [String!]!
  # TODO: Make this a required field in a future release.  Optional for now for backward compatibility.
  totalBytes: Long
  stage: DeltaFileStage!
  actions: [Action!]!
  sourceInfo: SourceInfo!
  protocolStack: [ProtocolLayer!]!
  domains: [Domain!]!
  enrichment: [Enrichment!]!
  formattedData: [FormattedData!]!
  created: DateTime!
  modified: DateTime!
  contentDeleted: DateTime
  contentDeletedReason: String
  errorAcknowledged: DateTime
  errorAcknowledgedReason: String
  egressed: Boolean
  filtered: Boolean
  replayed: DateTime
  replayDid: String
}

input IngressInput {
  did: String!
  sourceInfo: SourceInfoInput!
  content: [ContentInput!]!
  created: DateTime!
}

input TransformInput {
  protocolLayer: ProtocolLayerInput!
}

input LoadInput {
  domains: [DomainInput!]!
  protocolLayer: ProtocolLayerInput!
}

input EnrichInput {
  enrichments: [EnrichmentInput!]!
}

input FormatInput {
  filename: String!
  contentReference: ContentReferenceInput!
  metadata: [KeyValueInput!]
}

input FilterInput {
  message: String!
}

input SplitInput {
  sourceInfo: SourceInfoInput!
  content: [ContentInput!]!
}

type DeltaFiles {
  offset: Int
  count: Int
  totalCount: Int
  deltaFiles: [DeltaFile!]!
}

input DeltaFileOrder {
  direction: DeltaFileDirection!
  # The field name is passed directly to the DB for sort
  # this is not ideal, but better than requiring hard-coding every possibility
  field: String!
}

enum DeltaFileDirection {
  ASC
  DESC
}

input DeltaFilesFilter {
  dids: [String!]
  parentDid: String
  createdAfter: DateTime
  createdBefore: DateTime
  domains: [String!]
  enrichment: [String!]
  contentDeleted: Boolean
  modifiedAfter: DateTime
  modifiedBefore: DateTime
  sourceInfo: SourceInfoFilter
  stage: DeltaFileStage
  actions: [String!]
  formattedData: FormattedDataFilter
  errorAcknowledged: Boolean
  egressed: Boolean
  filtered: Boolean
}

type RetryResult {
  did: String!
  success: Boolean!
  error: String
}

type AcknowledgeResult {
  did: String!
  success: Boolean!
  error: String
}

type Result {
  success: Boolean!
  errors: [String]
}

type Query {
  getIngressFlowPlan(planName: String!): IngressFlowPlan!
  getEgressFlowPlan(planName: String!): EgressFlowPlan!
  getEnrichFlowPlan(planName: String!): EnrichFlowPlan!
  getIngressFlow(flowName: String!): IngressFlow!
  getEgressFlow(flowName: String!): EgressFlow!
  getEnrichFlow(flowName: String!): EnrichFlow!
  validateIngressFlow(flowName: String!): IngressFlow!
  validateEgressFlow(flowName: String!): EgressFlow!
  validateEnrichFlow(flowName: String!): EnrichFlow!
  getFlows: [Flows]!
  getAllFlows: SystemFlows!
  getAllFlowPlans: SystemFlowPlans!
  getActionNamesByFamily: [ActionFamily]!

  getPropertySets: [PropertySet]!

  resolveFlowFromFlowAssignmentRules(sourceInfo: SourceInfoInput!): String
  getAllFlowAssignmentRules: [FlowAssignmentRule]!
  getFlowAssignmentRule(name: String!): FlowAssignmentRule

  deltaFiles(offset: Int, limit: Int, filter: DeltaFilesFilter, orderBy: DeltaFileOrder): DeltaFiles!
  deltaFile(did: String!): DeltaFile
  lastCreated(last: Int) : [DeltaFile]!
  lastModified(last: Int) : [DeltaFile]!
  lastErrored(last: Int) : [DeltaFile]!
  lastWithFilename(filename: String!) : DeltaFile

  sourceMetadataUnion(dids: [String!]!) : [UniqueKeyValues!]!

  deltaFiConfigs(configQuery: ConfigQueryInput) : [DeltaFiConfiguration]!
  actionSchemas: [ActionSchema!]!
  exportConfigAsYaml: String!

  plugins:  [Plugin!]!
}

type Mutation {
  loadFlowAssignmentRules(replaceAll: Boolean!, rules: [FlowAssignmentRuleInput!]!): [Result!]!
  removeFlowAssignmentRule(name: String!): Boolean!

  saveIngressFlowPlan(ingressFlowPlan: IngressFlowPlanInput!) : IngressFlow!
  saveEnrichFlowPlan(enrichFlowPlan: EnrichFlowPlanInput!) : EnrichFlow!
  saveEgressFlowPlan(egressFlowPlan: EgressFlowPlanInput!) : EgressFlow!
  removeIngressFlowPlan(name: String!) : Boolean!
  removeEnrichFlowPlan(name: String!) : Boolean!
  removeEgressFlowPlan(name: String!) : Boolean!

  savePluginVariables(pluginVariablesInput: PluginVariablesInput!) : Boolean!
  setPluginVariableValues(pluginCoordinates: PluginCoordinatesInput!, variables: [KeyValueInput]!) : Boolean!

  startIngressFlow(flowName: String!) : Boolean!
  stopIngressFlow(flowName: String!) : Boolean!
  startEgressFlow(flowName: String!) : Boolean!
  stopEgressFlow(flowName: String!) : Boolean!
  startEnrichFlow(flowName: String!) : Boolean!
  stopEnrichFlow(flowName: String!) : Boolean!

  ingress(input: IngressInput!): DeltaFile!
  actionEvent(event: ActionEventInput!): DeltaFile!

  resume(dids: [String!]!, replaceFilename: String, replaceFlow: String, removeSourceMetadata: [String!], replaceSourceMetadata: [KeyValueInput!]) : [RetryResult!]!
  # deprecated, use resume instead
  retry(dids: [String!]!, replaceFilename: String, replaceFlow: String, removeSourceMetadata: [String!], replaceSourceMetadata: [KeyValueInput!]) : [RetryResult!]!
  replay(dids: [String!]!, replaceFilename: String, replaceFlow: String, removeSourceMetadata: [String!], replaceSourceMetadata: [KeyValueInput!]) : [RetryResult!]!
  acknowledge(dids: [String!]!, reason: String) : [AcknowledgeResult!]!

  registerActions(actionRegistration: ActionRegistrationInput!): Int!

  registerPlugin(pluginInput: PluginInput!): Result!
  uninstallPlugin(dryRun: Boolean!, pluginCoordinatesInput: PluginCoordinatesInput!): Result!


  updateProperties(updates: [PropertyUpdate]!): Int!
  removePropertyOverrides(propertyIds: [PropertyId]!): Int!

  # use when adding or removing a plugin
  addPluginPropertySet(propertySet: PropertySetInput!) : Boolean!
  removePluginPropertySet(propertySetId: String!): Boolean!
}