core_ingest.barrier.pure Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of legend-engine-xt-ingest-pure Show documentation

There is a newer version: 4.67.9

Show newest version

import meta::external::ingest::barrier::*;
import meta::external::ingest::metamodel::*;

function meta::external::ingest::barrier::isDatasetAvailable(ingestConfig: IngestConfig[1], prevPublishedWatermark: Watermark[1], currentStateWatermark: Watermark[1]):Boolean[1]
{
  let prevHighBatchId = getBatchId($ingestConfig.datasetName, $prevPublishedWatermark);
  let currentBatchId = getBatchId($ingestConfig.datasetName, $currentStateWatermark);   
  ($currentBatchId > $prevHighBatchId) ;                
}

function meta::external::ingest::barrier::isDatasetAvailable(ingestConfig: IngestConfig[1],filterFunc:Function<{T[1]->Boolean[1]}>[1], prevPublishedWatermark: Watermark[1], currentStateWatermark: Watermark[1]):Boolean[1]
{
  let prevHighBatchId = getBatchId($ingestConfig.datasetName, $prevPublishedWatermark);
  let currentBatchId = getBatchId($ingestConfig.datasetName, $currentStateWatermark); 

//TODO : Change to T , likely need to uplift getAll to handle T properly
  meta::pure::functions::collection::getAll(BatchMetadata)
                  ->filter(x | $x.datasetName == $ingestConfig.datasetName)
                  ->filter(y | ($y.batchId > $prevHighBatchId) && ($y.batchId <= $currentBatchId))
                  ->cast(@T)
                  ->filter($filterFunc)
                  ->isNotEmpty();
}


function <> meta::external::ingest::barrier::getBatchId(datasetName: String[1], watermark: Watermark[1]):Integer[1]
{
  $watermark.datasetMarks->filter(w | $w.datasetName == $datasetName)->toOne().batchId;
}