All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.finos.legend.engine.generation.dataquality.DataQualityValidationArtifactGenerationExtension Maven / Gradle / Ivy

There is a newer version: 4.67.8
Show newest version
// Copyright 2021 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.engine.generation.dataquality;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.api.factory.Lists;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact;
import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension;
import org.finos.legend.engine.plan.generation.PlanGenerator;
import org.finos.legend.engine.plan.generation.transformers.LegendPlanTransformers;
import org.finos.legend.engine.plan.platform.PlanPlatform;
import org.finos.legend.engine.protocol.dataquality.model.DataQualityMetadata;
import org.finos.legend.engine.protocol.dataquality.model.DataQualityRule;
import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan;
import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
import org.finos.legend.engine.shared.core.ObjectMapperFactory;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality;
import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRule;
import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
import org.finos.legend.pure.generated.core_dataquality_generation_dataquality;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction;
import org.slf4j.Logger;

import java.util.List;

public class DataQualityValidationArtifactGenerationExtension implements ArtifactGenerationExtension
{
    private final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(DataQualityValidationArtifactGenerationExtension.class);
    private final ObjectMapper mapper = ObjectMapperFactory.withStandardConfigurations(PureProtocolObjectMapperFactory.withPureProtocolExtensions(new ObjectMapper()));

    public static final String ROOT_PATH = "dataQualityValidation";
    public static final String META_DATA_FILE_NAME = "dataQualityRulesMetadata.json";
    public static final String EXECUTION_PLAN_FILE_NAME = "dataQualityValidationExecutionPlan.json";


    @Override
    public String getKey()
    {
        return ROOT_PATH;
    }

    @Override
    public boolean canGenerate(PackageableElement element)
    {
       return element instanceof Root_meta_external_dataquality_DataQuality
                && ((Root_meta_external_dataquality_DataQuality) element)._validationTree() != null;
    }

    @Override
    public List generate(PackageableElement packageableElement, PureModel pureModel, PureModelContextData pureModelContextData, String clientVersion)
    {
        List artifacts = Lists.mutable.empty();

        try
        {
            RichIterable dataQualityRules =
                    core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDQMetaDataForDQValidation_DataQuality_1__DataQualityRule_MANY_((Root_meta_external_dataquality_DataQuality)packageableElement, pureModel.getExecutionSupport());
            List dataQualityRuleList = Lists.mutable.withAll(dataQualityRules).collect(d -> new DataQualityRule(d._constraintName(), d._constraintType(), d._constraintGrammar(), d._propertyPath()));
            DataQualityMetadata dataQualityMetadata = new DataQualityMetadata();
            dataQualityMetadata.dqRules = dataQualityRuleList;
            artifacts.add(new Artifact(mapper.writeValueAsString(dataQualityMetadata), META_DATA_FILE_NAME, "json"));

            Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport()));
            // 1. call DQ PURE func to generate lambda
            LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, routerExtensions);
            // 2. Generate Plan from the lambda generated in the previous step
            SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null,
                    routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context
            artifacts.add(new Artifact(mapper.writeValueAsString(singleExecutionPlan), EXECUTION_PLAN_FILE_NAME, "json"));
        }
        catch (Exception e)
        {
            LOGGER.error("Unable to compute dataQuality validation artifact for dqValidation: " + packageableElement.getName() + ". Exception:" + e.getMessage());
        }
        return artifacts;
    }

}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy