All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.boozallen.aiops.mda.ManualActionNotificationService Maven / Gradle / Ivy

The newest version!
package com.boozallen.aiops.mda;

/*-
 * #%L
 * aiSSEMBLE::Foundation::MDA
 * %%
 * Copyright (C) 2021 Booz Allen
 * %%
 * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
 * #L%
 */

import com.boozallen.aiops.mda.generator.util.MavenUtil;
import com.boozallen.aiops.mda.generator.util.PipelineUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.codehaus.plexus.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.technologybrewery.fermenter.mda.generator.GenerationContext;
import org.technologybrewery.fermenter.mda.generator.GenerationException;
import org.technologybrewery.fermenter.mda.notification.Notification;
import org.technologybrewery.fermenter.mda.notification.NotificationCollector;
import org.technologybrewery.fermenter.mda.notification.VelocityNotification;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import javax.inject.Named;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
 * Class to aid in notifying users of required actions.
 */
@Named
public class ManualActionNotificationService {
    private static final Logger logger = LoggerFactory.getLogger(ManualActionNotificationService.class);
    private static final String EMPTY_LINE = "\n";
    private static final String SUPPRESS_WARNINGS = "maven-suppress-warnings";
    public static final String GROUP_TILT = "tilt";

    public void addSchemaElementDeprecationNotice(String illegalElement, String objectType) {
        final String SCHEMA_ELEMENT_DEPRECATION_KEY = "schema_element_deprecation";

        VelocityNotification notification = new VelocityNotification(getMessageKey(SCHEMA_ELEMENT_DEPRECATION_KEY, illegalElement), "schemaelementdeprecation", new HashSet(), "templates/notifications/notification.schema.element.deprecation.vm");
        notification.addToVelocityContext("objectType", objectType);
        notification.addToVelocityContext("illegalElement", illegalElement);
        addManualAction(SCHEMA_ELEMENT_DEPRECATION_KEY, notification);
    }

    /**
     * Adds message to build output indicating that no pipelines have been defined but are required.
     *
     * @param context contextual information about what is being generated
     */
    public void addNoticeToAddPipelines(GenerationContext context) {

        final String ADD_PIPELINES_KEY = "add_pipelines";

        final String pathToReadMe = context.getProjectDirectory().toString()
                .replace("pipelines", "pipeline-models/pipelines/README.md");

        VelocityNotification notification = new VelocityNotification(ADD_PIPELINES_KEY, new HashSet<>(), "templates/notifications/notification.pipelines.vm");
        notification.addToVelocityContext("pathToReadMe", pathToReadMe);
        addManualAction(ADD_PIPELINES_KEY, notification);
    }

    /**
     * Checks if updates to the modules for a POM file are necessary and adds a notification if so.
     * Using submoduleDepth is a placeholder solution that will be replaced with a more long-term solution (such as updating Fermenter).
     *
     * @param context    the generation context
     * @param artifactId the artifact ID
     * @param moduleType the module type
     */
    public void addNoticeToAddModuleToParentBuild(GenerationContext context, String artifactId, String moduleType) {
        final String pomFilePath = context.getProjectDirectory() + File.separator + "pom.xml";
        final String query = "" + artifactId + "";
        boolean alreadyExists = existsInFile(pomFilePath, query);

        String relativePomFilePath = getRelativePathToProjectRoot(context.getExecutionRootDirectory(), new File(pomFilePath));

        if (!alreadyExists) {
            final String key = getMessageKey(relativePomFilePath, "module");

            HashSet items = new HashSet();
            items.add("" + (artifactId) + "");

            VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.module.to.parent.vm");
            notification.addToVelocityContext("moduleType", moduleType);
            notification.addToVelocityContext("displayPomFilePath", relativePomFilePath);
            notification.addToVelocityContext("artifactId", artifactId);
            addManualAction(pomFilePath, notification);
        }

    }

    /**
     * Checks if updates are needed to dependencies in a POM file.
     *
     * @param context     the generation context
     * @param artifactId  the artifact ID
     * @param persistType the persist type
     */
    public void addNoticeToAddDependency(GenerationContext context, String artifactId, String persistType) {

        final String pomFilePath = context.getProjectDirectory() + File.separator + artifactId + File.separator + "pom.xml";
        boolean alreadyExists = existsInFile(pomFilePath, String.format("extensions-data-delivery-spark-%s", persistType));

        if (!alreadyExists) {
            String relativePomFilePath = getRelativePathToProjectRoot(context.getExecutionRootDirectory(), new File(pomFilePath));
            final String key = getMessageKey(relativePomFilePath, "extensions-data-delivery-spark", persistType);

            VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.dependency.vm");
            notification.addToVelocityContext("dependencyArtifactId", String.format("extensions-data-delivery-spark-%s", persistType));
            notification.addToVelocityContext("artifactId", artifactId);
            addManualAction(pomFilePath, notification);
        }
    }

    /**
     * Checks if there are deployment changes necessary for the live update feature in
     * Tiltfile and adds a message if so.
     *
     * @param context                     the generation context
     * @param dockerArtifactId            the docker artifact ID
     * @param dockerApplicationArtifactId the docker application artifact ID
     * @param pipelineName                the pipeline's name
     * @param stepName                    the pipeline step's name
     * @param includeHelmBuild whether to include the helm build in the message
     */
    public void addDockerBuildWithLiveUpdateTiltFileMessage(final GenerationContext context, final String dockerArtifactId, final String dockerApplicationArtifactId, 
                                                            final String pipelineName, final String stepName, final boolean includeHelmBuild) {
        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Dockerbuild updates to Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String pipelineArtifactId = dockerArtifactId.replace("-docker", "-pipelines");
            final String stepNameSnakeCase = PipelineUtils.deriveLowerSnakeCaseNameFromHyphenatedString(stepName);
            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, dockerApplicationArtifactId);
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "docker-build", dockerApplicationArtifactId);

                addLocalResourceTiltFileMessage(context, dockerArtifactId, dockerApplicationArtifactId, stepName, pipelineName + "/" + stepName, true);

                VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.docker.live.vm");
                notification.addToVelocityContext("dockerApplicationArtifactId", dockerApplicationArtifactId);
                notification.addToVelocityContext("stepName", stepName);
                notification.addToVelocityContext("pipelineArtifactId", pipelineArtifactId);
                notification.addToVelocityContext("pipelineName", pipelineName);
                notification.addToVelocityContext("dockerArtifactId", dockerArtifactId);
                notification.addToVelocityContext("referenceName", dockerApplicationArtifactId);
                notification.addToVelocityContext("stepNameSnakeCase", stepNameSnakeCase);
                notification.addToVelocityContext("includeHelmBuild", includeHelmBuild);
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    /**
     * Checks if there are deployment changes necessary in either the Maven POM file or Tiltfile and
     * adds a message if so.
     *
     * @param context                     the generation context
     * @param appName                     the application name
     * @param dockerApplicationArtifactId the docker application artifact ID
     * @param dockerArtifactId            the docker artifact ID
     * @param dockerImage                 use custom image instead of dockerApplicationArtifactId
     * @param deployedAppName             the application name after deployment
     * @param includeHelmBuild            whether to include the helm build in the message
     * @param includeLatestTag            whether to include the latest tag as a tilt extra tag
     */
    public void addDockerBuildTiltFileMessage(DockerBuildParams params) {

        final File rootDir = params.getContext().getExecutionRootDirectory();

        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Dockerbuild updates to Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String deployArtifactId = params.getDockerArtifactId().replace("-docker", "-deploy");
            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, params.getDockerApplicationArtifactId());

            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tilefile", "docker-build", params.getDockerApplicationArtifactId());
                VelocityNotification notification = new VelocityNotification(key,
                        GROUP_TILT, new HashSet<>(), "templates/notifications/notification.docker.tilt.vm");
                notification.addToVelocityContext("appNameTitle", params.getAppName());
                notification.addToVelocityContext("dockerArtifactId", params.getDockerArtifactId());
                notification.addToVelocityContext("dockerApplicationArtifactId", params.getDockerApplicationArtifactId());
                notification.addToVelocityContext("includeHelmBuild", params.isIncludeHelmBuild());
                notification.addToVelocityContext("appName", params.getDeployedAppName());
                notification.addToVelocityContext("deployArtifactId", deployArtifactId);
                notification.addToVelocityContext("includeLatestTag", params.isIncludeLatestTag());
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    /**
     * Notification to add the function which sets the habushu dist artifact version property in the root pom for
     * access across modules.
     *
     * @param context generation context
     */
    public void addHabushuRegexPluginInvocation(final GenerationContext context) {
        if(!existsInFile("pom.xml", "set-habushu-dist-artifact-version")) {
            VelocityNotification notification = new VelocityNotification(
                    getMessageKey("pom.xml", "set-habushu-dist-artifact-version"),
                    "root-plugins",
                    new HashSet<>(),
                    "templates/notifications/notification.root.habushu.regex.plugin.vm"
            );
            addManualAction(context.getRootArtifactId(), notification);
        }
    }

    /**
     * Adds the maven-clean-plugin to the deploy pom.  Needed in order to reset the content of app target directories.
     *
     * @param deployArtifactId Deploy module artifact ID
     * @param context generation context
     */
    public void addCleanPluginNotification(final String deployArtifactId, final GenerationContext context) {
        final File rootDir = context.getExecutionRootDirectory();
        final String deployPom = rootDir.getAbsolutePath() + File.separator + deployArtifactId +  File.separator + "pom.xml";

        if(deployArtifactId != null && !existsInFile(deployPom, "maven-clean-plugin")) {
            String relativePomFilePath = getRelativePathToProjectRoot(rootDir, new File(deployPom));
            VelocityNotification notification = new VelocityNotification(
                    getMessageKey(relativePomFilePath, "clean-deploy-apps-targets"),
                    "deploy-plugins",
                    new HashSet<>(),
                    "templates/notifications/notification.deploy.clean.app.target.vm"
            );
            addManualAction(deployPom, notification);
        }
    }

    /**
     * Manual action for adding the pipeline invocation service deployment execution and all associated plugin
     * executions.
     * @param context generation context
     */
    public void addPipelineInvocationServiceDeployment(final GenerationContext context) {
        final File rootDir = context.getExecutionRootDirectory();
        String deployArtifactId = MavenUtil.getDeployModuleName(rootDir);
        final String deployPom = rootDir.getAbsolutePath() + File.separator + deployArtifactId + File.separator + "pom.xml";

        if(deployArtifactId != null && !existsInFile(deployPom, "mda-maven-plugin")) {
            String relativePomFilePath = getRelativePathToProjectRoot(rootDir, new File(deployPom));
            VelocityNotification notification = new VelocityNotification(
                    getMessageKey(relativePomFilePath, "pipeline-invocation-service-spark-apps"),
                    "deploy-plugins",
                    new HashSet<>(),
                    "templates/notifications/notification.mda.maven.pipeline.invocation.execution.vm"
            );
            notification.addToExternalVelocityContextProperties("deployArtifactId", deployArtifactId);
            addManualAction(deployPom, notification);
            addDeployPomMessage(context, "pipeline-invocation-service-v2", "pipeline-invocation-service");
        }
        addCleanPluginNotification(deployArtifactId, context);
        addHabushuRegexPluginInvocation(context);
    }

    // TODO: Standardize the spark, inference, and training docker folders and dockerfile to use the same structure so
    // that live updates can be applied with the same pattern copyFullPath variable should be removed after the
    // structure for target folder is established.
    /**
     * Checks if there are deployment changes necessary for live updating a specific python step module's source code in
     * Tiltfile and adds a message if not.
     *
     * @param context                     the generation context
     * @param dockerArtifactId            the project's docker module
     * @param dockerApplicationArtifactId the docker image module that should be live-updated with changes
     * @param moduleName                  the module name to compile for live updates
     * @param pipelinesModulePath         the path to the module under the project's "pipelines" module (e.g. "ml-pipeline/training-step")
     * @param copyFullPath                this variable is temporary and should be removed (see TODO); determines whether to copy the generated output to a different folder path
     */
    public void addLocalResourceTiltFileMessage(final GenerationContext context, final String dockerArtifactId,
                                                final String dockerApplicationArtifactId, final String moduleName,
                                                final String pipelinesModulePath, final boolean copyFullPath) {
        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Dockerbuild updates to Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String pipelinesArtifactId = dockerArtifactId.replace("-docker", "-pipelines");
            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, "compile-" + moduleName);
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "local-resource", moduleName);

                VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.local.resource.tilt.vm");
                notification.addToVelocityContext("srcModuleName", moduleName);
                notification.addToVelocityContext("srcModulePath", pipelinesArtifactId + "/" + pipelinesModulePath);
                notification.addToVelocityContext("dockerModulePath", dockerArtifactId + "/" + dockerApplicationArtifactId);
                notification.addToVelocityContext("copyFullPath", copyFullPath);
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    /**
     * Checks if there are deployment changes necessary in the Tiltfile for Spark Worker and
     * adds a message if so.
     *
     * @param context                     the generation context
     * @param appName                     the application name
     * @param dockerApplicationArtifactId the docker application artifact ID
     * @param dockerArtifactId            the docker artifact ID
     * @param includeHelmBuild            whether to include the helm build in the message
     * @param includeLatestTag            whether to include the latest tag as a tilt extra tag
     */
    public void addSparkWorkerDockerBuildTiltMessage(final GenerationContext context, final String appName, final String dockerApplicationArtifactId,
                                                     final String dockerArtifactId) {

        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Spark Worker resources to Tiltfile");
        } else {
            //creates the instructions for the spark worker image
            DockerBuildParams params = new DockerBuildParams.ParamBuilder().setContext(context)
                    .setAppName(appName)
                    .setDockerApplicationArtifactId(dockerApplicationArtifactId)
                    .setDockerArtifactId(dockerArtifactId)
                    .setIncludeHelmBuild(false)
                    .setIncludeLatestTag(true)
                    .build();
            addDockerBuildTiltFileMessage(params);

            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String text = "k8s_kind('SparkApplication'";

            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text);
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "spark-worker-docker-build");

                VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet<>(),
                        "templates/notifications/notification.spark.worker.docker.build.tilt.vm");
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    /**
     * Adds a notification to update the Tiltfile.
     *
     * @param context          the generation context
     * @param appName          the application name
     * @param deployArtifactId the deploy artifact ID
     * @param isConfigStore    whether the deployment is the config store
     */
    public void addHelmTiltFileMessage(final GenerationContext context, final String appName,
                                       final String deployArtifactId) {

        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Helm updates to for Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String text = "apps/" + appName + "'";

            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text);
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "helm", appName);
                VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet(), "templates/notifications/notification.helm.tilt.vm");
                notification.addToVelocityContext("appName", appName);
                notification.addToVelocityContext("deployArtifactId", deployArtifactId);
                addManualAction(tiltFilePath, notification);
            }

        }
    }


    /**
     * Adds a notification to update the Tiltfile for resources dependent on another resource
     *
     * @param context         the generation context
     * @param appName         the application name
     * @param appDependencies application names this resource is dependent on
     */
    public void addResourceDependenciesTiltFileMessage(GenerationContext context, String appName, List appDependencies) {
        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual resource dependency updates to for Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";

            final StringBuilder item = new StringBuilder();
            final String formattedAppDependencies = appDependencies.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
            item.append(String.format("k8s_resource('%s', resource_deps=[%s])\n\n", appName, formattedAppDependencies));

            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, item.toString().trim());
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "resource-dependencies", appName);

                VelocityNotification notification = new VelocityNotification(key, new HashSet<>(), "templates/notifications/notification.resource.tilt.vm");
                notification.addToVelocityContext("appName", appName);
                notification.addToVelocityContext("formattedAppDependencies", formattedAppDependencies);
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    private void appendTiltHelmBuild(String appName, String deployArtifactId, StringBuilder builder) {
        builder.append("yaml = helm(\n");
        builder.append("   '")
                .append(deployArtifactId)
                .append("/src/main/resources/apps/")
                .append(appName)
                .append("',\n");
        builder.append("   values=['")
                .append(deployArtifactId)
                .append("/src/main/resources/apps/")
                .append(appName)
                .append("/values.yaml',\n");
        builder.append("       '")
                .append(deployArtifactId)
                .append("/src/main/resources/apps/")
                .append(appName)
                .append("/values-dev.yaml']\n");
        builder.append(")\n");
        builder.append("k8s_yaml(yaml)\n");
        builder.append(EMPTY_LINE);
    }

    /**
     * Adds a notification to update the Tiltfile.
     * NOTE: This "k8s_yaml" line is also being output by appendTiltHelmBuild() and addSparkWorkerTiltResources() so
     * we should consider refactoring to extract out common code.
     *
     * @param context          the generation context
     * @param appName          the application name
     * @param deployArtifactId the deploy artifact ID
     * @param yamlFileName     the deploy artifact ID
     */
    public void addYamlTiltFileMessage(final GenerationContext context, final String appName,
                                       final String deployArtifactId, final String yamlFileName) {

        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Helm updates to for Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String yamlFilePath = deployArtifactId + "/src/main/resources/apps/" + appName + "/" + yamlFileName;
            final String text = "apps/" + appName;

            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text);
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "yaml", appName);

                HashSet items = new HashSet();
                items.add(yamlFilePath);

                VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, items,
                        "templates/notifications/notification.yaml.tiltfile.vm");
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    /**
     * Adds a notification to update the tiltfile with necessary elasticsearch resources
     *
     * @param context    the generation context
     * @param appName    the application name
     * @param artifactId the artifact ID
     */
    public void addElasticsearchTiltResources(final GenerationContext context, final String appName, final String artifactId) {
        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Helm updates to for Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String[] elasticSearchResources = {"k8s_kind('Elasticsearch')", "k8s_resource('elasticsearch')"};

            HashSet items = new HashSet();

            //loop through each resource to see if it's in the tiltfile
            for (String resource : elasticSearchResources) {
                boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, resource);
                if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                    items.add(resource);
                }
            }

            //add a manual action for any resources that may need to be added
            if (items.size() > 0) {
                final String key = getMessageKey("Tiltfile", "elasticsearch");

                VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, items,
                        "templates/notifications/notification.elastic.search.tilt.vm");
                addManualAction(tiltFilePath, notification);
            }
        }
    }


    /**
     * Adds a notification to update the tiltfile with necessary spark worker resources
     * NOTE: Consider refactor to leverage addYamlTiltFileMessage() for "k8s_yaml" line
     *
     * @param context                the generation context
     * @param parentArtifactId       the name of the parent directory the pipelines are in
     * @param pipelineArtifactId     the artifact id of the pipeline
     * @param pipelineImplementation the implementation of the pipeline
     */
    public void addSparkWorkerTiltResources(final GenerationContext context, final String parentArtifactId,
                                            final String pipelineArtifactId, final String pipelineImplementation) {

        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !tiltFileFound(rootDir)) {
            logger.warn("Unable to find Tiltfile. Will not be able to direct manual Spark Worker resources to Tiltfile");
        } else {
            final String tiltFilePath = rootDir.getAbsolutePath() + File.separator + "Tiltfile";
            final String text = "--values " + parentArtifactId + "/" + pipelineArtifactId;

            boolean tiltFileContainsArtifact = existsInFile(tiltFilePath, text);
            if (!tiltFileContainsArtifact && showWarnings(tiltFilePath)) {
                final String key = getMessageKey("Tiltfile", "spark-application");

                VelocityNotification notification = new VelocityNotification(key, GROUP_TILT, new HashSet<>(),
                        "templates/notifications/notification.spark.worker.tilt.vm");
                notification.addToVelocityContext("parentArtifactId", parentArtifactId);
                notification.addToVelocityContext("pipelineArtifactId", pipelineArtifactId);
                notification.addToVelocityContext("pipelineImplementation", pipelineImplementation);
                notification.addToVelocityContext("pythonPipelineArtifactId", PipelineUtils.deriveLowerSnakeCaseNameFromHyphenatedString(pipelineArtifactId));
                notification.addToVelocityContext("aissembleVersion", PipelineUtils.getAiSSEMBLEVersion(context));
                addManualAction(tiltFilePath, notification);
            }
        }
    }

    /**
     * Adds a notification to update the pom.xml for the deploy module.
     *
     * @param context the generation context
     * @param profile the profile to add
     * @param appName the application name to add
     */
    public void addDeployPomMessage(final GenerationContext context, final String profile, final String appName) {
        addDeployPomMessage(context, profile, appName, null);
    }

    /**
     * Adds a notification to update the pom.xml for the docker module.
     *
     * @param context         the generation context
     * @param profile         the profile to add
     * @param appName         the application name to add
     * @param appDependencies application names this resource is dependent on
     */
    public void addDeployPomMessage(final GenerationContext context, final String profile, final String appName, List appDependencies) {
        final File rootDir = context.getExecutionRootDirectory();
        if (!rootDir.exists() || !deployModuleFoundPom(rootDir)) {
            logger.warn("Unable to find Docker module. Will not be able to direct manual updates for the deploy module's POM.xml");
        } else {
            if (StringUtils.isNotEmpty(profile)) {
                NotificationParams params = configureNotification(rootDir, profile, appName, MavenUtil::getDeployModuleName);
                boolean hasAppDependencies = appDependencies != null;

                if (!params.isExistsInFileOrNotification()) {

                    VelocityNotification notification = new VelocityNotification(params.getKey(), "deploypom", new HashSet<>(), "templates/notifications/notification.deploy.pom.vm");
                    notification.addToVelocityContext("appName", appName);
                    notification.addToVelocityContext(("profile"), profile);
                    notification.addToVelocityContext("basePackage", context.getBasePackage());
                    notification.addToVelocityContext("profileConfiguration", params.getProfileConfiguration());
                    notification.addToVelocityContext("hasAppDependencies", hasAppDependencies);
                    notification.addToVelocityContext("appDependencies", appDependencies);

                    notification.addToExternalVelocityContextProperties("deployArtifactId", params.getArtifactId());

                    addManualAction(params.getPomFilePath(), notification);
                } else if (hasAppDependencies && !propertyVariableExistsInPomFile(params.getPomFile(), appName, "appDependencies", String.join(",", appDependencies))) {

                    params.setKey(getMessageKey(params.getPomFilePath(), "execution", "appDependencies", appName));
                    VelocityNotification notification = new VelocityNotification(params.getKey(), new HashSet(), "templates/notifications/notification.deploy.pom.property.variables.vm");
                    notification.addToVelocityContext("profile", profile);
                    notification.addToVelocityContext("deployArtifactId", params.getArtifactId());
                    notification.addToVelocityContext("appDependencies", appDependencies);
                    addManualAction(params.getPomFilePath(), notification);
                }
            }
        }
    }

    /**
     * Adds a notification to update the pom.xml for the docker module.
     *
     * @param context the generation context
     * @param profile the profile to add
     */
    public void addDockerPomMessage(final GenerationContext context, final String profile, final String artifactId) {
        final File optionalRoot = context.getExecutionRootDirectory();
        if (!optionalRoot.exists() || !dockerModuleFoundPom(optionalRoot)) {
            logger.warn("Unable to find Docker module. Will not be able to direct manual updates for the deploy module's POM.xml");
        } else {
            NotificationParams params = configureNotification(optionalRoot, profile, artifactId, MavenUtil::getDockerModuleName);
            if (StringUtils.isNotEmpty(profile)) {
                if (!params.isExistsInFileOrNotification()) {
                    VelocityNotification notification = new VelocityNotification(params.getKey(), "dockerpom", new HashSet<>(), "templates/notifications/notification.docker.pom.vm");
                    notification.addToVelocityContext(("profile"), profile);
                    notification.addToVelocityContext("appName", artifactId);
                    notification.addToVelocityContext("basePackage", context.getBasePackage());
                    notification.addToVelocityContext("profileConfiguration", params.getProfileConfiguration());
                    notification.addToExternalVelocityContextProperties("dockerArtifactId", params.getArtifactId());

                    addManualAction(params.getPomFilePath(), notification);
                }
            }
        }
    }
    
    private boolean executionAppExistsInPomFile(File file, String appName) {
        return propertyVariableExistsInPomFile(file, appName, "appName", appName);
    }

    private boolean propertyVariableExistsInPomFile(File file, String appName, String propertyVariableName, String propertyVariableValue) {
        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
        DocumentBuilder db;
        boolean exists = false;
        if (file.exists() && !file.isDirectory()) {
            try {
                db = dbf.newDocumentBuilder();

                Document doc = db.parse(file);
                XPath xPath = XPathFactory.newInstance().newXPath();
                String expression = "/project/build/plugins/plugin/executions/execution";
                NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(
                        doc, XPathConstants.NODESET);
                for (int i = 0; i < nodeList.getLength(); i++) {
                    Node nNode = nodeList.item(i);
                    if (nNode.getNodeType() == Node.ELEMENT_NODE) {
                        Node cNode = getChildNodeByName(nNode, "configuration");
                        cNode = getChildNodeByName(cNode, "propertyVariables");
                        if (cNode != null) {
                            Node appNode = getChildNodeByName(cNode, "appName");
                            if (appNode != null && appName.equals(appNode.getTextContent())) {
                                Node pvNode = getChildNodeByName(cNode, propertyVariableName);
                                if (pvNode != null && propertyVariableValue.equals(pvNode.getTextContent())) {
                                    exists = true;
                                }
                            }
                        }
                    }
                }

            } catch (ParserConfigurationException | SAXException | IOException | XPathExpressionException e) {
                logger.error("Error while trying to find propertyVariable {} for appName {}", propertyVariableName, appName, e);
            }
        }
        return exists;
    }

    private Node getChildNodeByName(Node nNode, String name) {
        Element eElement = (Element) nNode;
        if (eElement != null) {
            return eElement.getElementsByTagName(name).item(0);
        }
        return null;
    }

    /**
     * Adds a notification to update the {@code pyproject.toml} configuration.
     *
     * @param context      the generation context
     * @param dependencies the dependencies to add
     * @param description  description of what the dependencies are for
     */
    public void addNoticeToAddPythonDependencies(GenerationContext context, Set dependencies, String description) {

        final String pyprojectFilePath = context.getProjectDirectory() + File.separator + "pyproject.toml";
        final Set dependenciesToAdd = new LinkedHashSet<>();
        for (String dependency : dependencies) {
            if (!existsInFile(pyprojectFilePath, dependency)) {
                dependenciesToAdd.add(dependency);
            }
        }

        if (CollectionUtils.isNotEmpty(dependenciesToAdd)) {
            String relativePyprojectFilePath = getRelativePathToProjectRoot(context.getExecutionRootDirectory(), context.getProjectDirectory()) + File.separator + "pyproject.toml";
            final String key = getMessageKey(relativePyprojectFilePath, "requirement", context.getArtifactId());

            HashSet items = new HashSet();
            for (String dependency : dependenciesToAdd) {
                items.add(dependency);
            }

            VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.python.dependencies.vm");
            notification.addToVelocityContext("description", description);
            notification.addToVelocityContext("artifactId", context.getArtifactId());
            addManualAction(pyprojectFilePath, notification);
        }

    }

    /**
     * Adds a notification to update the s3-local deploy values.yaml file with a required object
     *
     * @param context     the generation context
     * @param objectNames the name of the object to add
     */
    public void addNoticeToUpdateS3LocalConfig(final GenerationContext context, final String bucketName, final List objectNames) {
        final File rootDir = context.getExecutionRootDirectory();
        if (rootDir.exists() && deployModuleFoundPom(rootDir)) {
            Path rootPath = rootDir.toPath();
            Path s3ValuesPath = rootPath.resolve(
                    Paths.get(MavenUtil.getDeployModuleName(rootDir),
                            "src", "main", "resources",
                            "apps", "s3-local", "values.yaml")).toAbsolutePath().normalize();
            try {
                if (showWarnings(s3ValuesPath.toString())) {
                    String relativeValuesFilePath = getRelativePathToProjectRoot(rootDir, s3ValuesPath.toFile());

                    if (!existsInFile(s3ValuesPath.toString(), "- name: " + bucketName)) {

                        HashSet items = new HashSet();
                        for (String objectName : objectNames) {
                            items.add(objectName);
                        }
                        final String key = getMessageKey(relativeValuesFilePath, "buckets", bucketName);
                        VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.s3.local.buckets.vm");
                        notification.addToVelocityContext("rootPath", rootPath.relativize(s3ValuesPath).toString());
                        notification.addToVelocityContext("bucketName", bucketName);
                        addManualAction(s3ValuesPath.toString(), notification);
                    } else {
                        HashSet items = new HashSet();
                        boolean pathExistsInFile = false;
                        for (String objectName : objectNames) {
                            if (!existsInFile(s3ValuesPath.toString(), "- " + objectName)) {
                                items.add(objectName);
                                pathExistsInFile = true;
                            }
                        }

                        if (pathExistsInFile) {
                            final String key = getMessageKey(relativeValuesFilePath, "bucketobjects", bucketName);

                            VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.s3.local.bucketobjects.vm");
                            notification.addToVelocityContext("rootPath", rootPath.relativize(s3ValuesPath).toString());
                            notification.addToVelocityContext("bucketName", bucketName);
                            addManualAction(s3ValuesPath.toString(), notification);
                        }
                    }
                }
            } catch (GenerationException e) {
                logger.warn("Failed to validate s3-local buckets and objects. Check that the {} file exists", s3ValuesPath);
            }
        }
    }

    /**
     * Adds a notification to update the Kafka deploy values.yaml file with step messaging topics
     *
     * @param context   the generation context
     * @param topicName the kafka topic name to add
     */
    public void addNoticeToUpdateKafkaConfig(final GenerationContext context, final String topicName) {
        final File rootDir = context.getExecutionRootDirectory();
        if (rootDir.exists() && deployModuleFoundPom(rootDir)) {
            Path rootPath = rootDir.toPath();
            Path kafkaValuesPath = rootPath.resolve(
                    Paths.get(rootPath.getFileName() + "-deploy",
                            "src", "main", "resources",
                            "apps", "kafka-cluster", "values.yaml")).toAbsolutePath().normalize();
            try {
                if (showWarnings(kafkaValuesPath.toString())
                        && !existsInFile(kafkaValuesPath.toString(), topicName + ":")
                        && existsInFile(kafkaValuesPath.toString(), "KAFKA_CREATE_TOPICS")) {
                    
                    String relativeValuesFilePath = getRelativePathToProjectRoot(rootDir, kafkaValuesPath.toFile());
                    final String key = getMessageKey(relativeValuesFilePath, "kafka-topics");

                    HashSet items = new HashSet();
                    items.add(topicName);

                    VelocityNotification notification = new VelocityNotification(key, items, "templates/notifications/notification.kafka.config.vm");
                    notification.addToVelocityContext("rootPath", rootPath.relativize(kafkaValuesPath).toString());
                    addManualAction(kafkaValuesPath.toString(), notification);
                }
            } catch (GenerationException e) {
                logger.warn("Failed to validate Kafka topics. Check that the {} file exists", kafkaValuesPath);
            }
        }
    }

    private boolean showWarnings(String filePath) {
        return !existsInFile(filePath, SUPPRESS_WARNINGS);
    }

    private boolean existsInFile(final String filePath, final String text) {
        try (Stream stream = Files.lines(Paths.get(filePath))) {
            return stream.anyMatch(lines -> lines.contains(text));
        } catch (IOException e) {
            throw new GenerationException("Could not introspect file: " + filePath, e);
        }
    }

    /**
     * Returns the path of the current project directory relative to the project root.
     * Given:
     *   my/test/project
     *   my/test/project/submodule1/submodule2
     * Returns:
     *   submodule1/submodule2
     * 
     * @param projectRoot The root directory
     * @param currentDirectory The current directory
     * @return The relative path as a {@link String}
     */
    private String getRelativePathToProjectRoot(File projectRoot, File currentDirectory) {
        return projectRoot.toPath().relativize(currentDirectory.toPath()).toString();
    }

    private String getMessageKey(final String... keyComponents) {
        if (keyComponents.length > 0) {
            return String.join("_", keyComponents);
        }
        return "";
    }

    private boolean moduleFoundPom(final File rootProjectDirectory, String moduleName) {
        return moduleName != null && MavenUtil.fileExists(rootProjectDirectory, moduleName);
    }

    private boolean deployModuleFoundPom(final File rootProjectDirectory) {
        return moduleFoundPom(rootProjectDirectory, MavenUtil.getDeployModuleName(rootProjectDirectory));
    }

    private boolean dockerModuleFoundPom(final File rootProjectDirectory) {
        return moduleFoundPom(rootProjectDirectory, MavenUtil.getDockerModuleName(rootProjectDirectory));
    }

    private boolean tiltFileFound(final File rootProjectDirectory) {
        return MavenUtil.fileExists(rootProjectDirectory, "Tiltfile");
    }

    private void addManualAction(String file, Notification notification) {
        NotificationCollector.addNotification(file, notification);
    }

    /**
     * Check if the appName already exists in the notification items with the given file and key.
     *
     * @param file    the file the notification applies to
     * @param key     the key of the notification to be added
     * @param appName the application name to add
     */
    public boolean hasNotificationWithAppName(final String file, final String key, final String appName) {
        final Map notificationsForFile = NotificationCollector.getNotifications().
                computeIfAbsent(file, m -> new ConcurrentHashMap<>());
        boolean found = false;
        if (notificationsForFile.containsKey(key)) {
            Set items = notificationsForFile.get(key).getItems();
            for (String item : items) {
                if (item.contains("" + appName + "")) {
                    found = true;
                    break;
                }
            }
        }
        return found;
    }

    /**
     * Check if the execution item with specific appName already exists in the notification with given file and the key
     *
     * @param file    the file the notification applies to
     * @param key     the key of the notification to be added
     * @param appName the application name to add
     */
    private final boolean executionAppExistsInNotification(final String file, final String key, final String appName) {
        return hasNotificationWithAppName(file, key, appName);
    }

    /**
     * Performs some common logic to set up information for building notifications to update pom files.
     *
     * @param rootDir      The root directory of the files needed for this notification
     * @param profile      The generation profile
     * @param appName      The app name to add to the pom file
     * @param getModule    A function to find the artifact id for the notification, varies by module type
     * @return An object containing the parameters needed for the rest of the notification generation
     */
    private NotificationParams configureNotification(File rootDir, String profile, String appName, Function getModule) {
        NotificationParams params = new NotificationParams();

        params.setArtifactId(getModule.apply(rootDir));
        final String deployDir = rootDir.getAbsolutePath() + File.separator + params.getArtifactId();

        params.setPomFilePath(deployDir + File.separator + "pom.xml");
        params.setProfileConfiguration("" + profile + "");
        params.setPomFile(new File(params.getPomFilePath()));

        String relativePomFilePath = getRelativePathToProjectRoot(rootDir, params.getPomFile());
        params.setKey(getMessageKey(relativePomFilePath, "execution", appName));

        params.setExistsInFileOrNotification(executionAppExistsInPomFile(params.getPomFile(), appName) || executionAppExistsInNotification(params.getPomFilePath(), params.getKey(), appName));

        return params;
    }

    /**
     * Helper class containing the parameters for configuring notifications
     */
    private class NotificationParams {
        private String artifactId;
        private String pomFilePath;
        private String profileConfiguration;
        private File pomFile;
        private String key;
        private boolean existsInFileOrNotification;

        public String getArtifactId() {
            return artifactId;
        }

        public void setArtifactId(String artifactId) {
            this.artifactId = artifactId;
        }

        public String getPomFilePath() {
            return pomFilePath;
        }

        public void setPomFilePath(String pomFilePath) {
            this.pomFilePath = pomFilePath;
        }

        public String getProfileConfiguration() {
            return profileConfiguration;
        }

        public void setProfileConfiguration(String profileConfiguration) {
            this.profileConfiguration = profileConfiguration;
        }

        public File getPomFile() {
            return pomFile;
        }

        public void setPomFile(File pomFile) {
            this.pomFile = pomFile;
        }

        public String getKey() {
            return key;
        }

        public void setKey(String key) {
            this.key = key;
        }

        public boolean isExistsInFileOrNotification() {
            return existsInFileOrNotification;
        }

        public void setExistsInFileOrNotification(boolean existsInFileOrNotification) {
            this.existsInFileOrNotification = existsInFileOrNotification;
        }
    }

}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy