All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.marklogic.hub.deploy.commands.GenerateHubTDETemplateCommand Maven / Gradle / Ivy

There is a newer version: 6.1.1
Show newest version
/*
 * Copyright (c) 2021 MarkLogic Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.marklogic.hub.deploy.commands;

import com.marklogic.appdeployer.AppConfig;
import com.marklogic.appdeployer.command.CommandContext;
import com.marklogic.appdeployer.command.es.GenerateModelArtifactsCommand;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.eval.EvalResultIterator;
import com.marklogic.client.ext.es.CodeGenerationRequest;
import com.marklogic.client.ext.es.GeneratedCode;
import com.marklogic.hub.DatabaseKind;
import com.marklogic.hub.HubConfig;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.FileCopyUtils;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;

import static java.util.stream.Collectors.toMap;


public class GenerateHubTDETemplateCommand extends GenerateModelArtifactsCommand {
    private static final String ENTITY_FILE_EXTENSION = ".entity.json";
    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    private final HubConfig hubConfig;
    private final Path userFinalSchemasTDEs;

    private String entityNames;
    @SuppressWarnings("unchecked")
    private List entityNamesList = Collections.EMPTY_LIST;

    public GenerateHubTDETemplateCommand(HubConfig hubConfig) {
        this.hubConfig = hubConfig;
        this.userFinalSchemasTDEs = hubConfig.getHubProject().getUserDatabaseDir().resolve(hubConfig.getDbName(DatabaseKind.FINAL_SCHEMAS)).resolve("schemas").resolve("tde");
    }

    @Override
    public void execute(CommandContext context) {
        AppConfig appConfig = context.getAppConfig();
        DatabaseClient client = appConfig.newDatabaseClient();
        EntityServicesManager mgr = new EntityServicesManager(client);

        CodeGenerationRequest request = createCodeGenerationRequest();

        List entityFiles  = findEntityFiles();

        if (!entityFiles.isEmpty()) {
            //create map of entity name -> entity definition file
            Map entityNameFileMap = createEntityNameFileMap(entityFiles);

            logger.debug("Found the following entities->files: {} " + entityNameFileMap);

            //filterEntities(entityNameFileMap);

            if (!entityNameFileMap.isEmpty()) {
                logger.warn("About to generate a template for the following entities: {}",
                    this.entityNamesList.isEmpty() ? entityNameFileMap.keySet() : this.entityNamesList);
                List generatedCodes = new ArrayList<>();
                for (File f : entityNameFileMap.values()) {
                    File esModel;
                    String modelName;
                    try {
                        //Write the ES model to a temp file
                        String tempDir = System.getProperty("java.io.tmpdir");
                        String fileName = f.getName();
                        modelName = EntityServicesManager.extractEntityNameFromURI(fileName).get();
                        esModel = new File(tempDir, fileName);
                        String modelString = generateModel(f);
                        if(modelString == null) {
                            logger.warn(f.getName() + " is not deployed to the database");
                            continue;
                        }
                        FileUtils.writeStringToFile(esModel, modelString);
                    } catch (IOException e) {
                        throw new RuntimeException("Unable to generate ES model");
                    }

                    GeneratedCode code;
                    try {
                        code = loadModelDefinition(request, esModel, mgr);
                    } catch (RuntimeException e) {
                        throw new RuntimeException("Unable to read model definition from file: " + f.getAbsolutePath(), e);
                    }
                    finally {
                        FileUtils.deleteQuietly(esModel);
                    }
                    if (this.entityNamesList.isEmpty() || this.entityNamesList.contains(modelName)) {
                        generatedCodes.add(code);
                    }
                }
                for (GeneratedCode code: generatedCodes) {
                    generateExtractionTemplate(appConfig, code);
                }
            }

        } else {
            logger.info("No data hub entity files found under {} or its sub-directories.",
                hubConfig.getHubEntitiesDir());
        }

    }

    //Method to obtain es-style model
    private String generateModel(File f) {
        String xquery = "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\"\n" +
            "at \"/data-hub/5/impl/hub-entities.xqy\";\n" +
            String.format("hent:get-model(\"%s\")", extractEntityNameFromFilename(f.getName()).get());
        try (EvalResultIterator resp = hubConfig.newStagingClient().newServerEval().xquery(xquery).eval()) {
            if (resp.hasNext()) {
                return resp.next().getString();
            }
        }
        return null;
    }

    public String getEntityNames() {
        return entityNames;
    }

    public void setEntityNames(String entityNames) {
        this.entityNames = entityNames;
        if (entityNames != null) {
            this.entityNamesList = Arrays.asList(entityNames.split(","));
        }
    }

    protected void filterEntities(Map entityNameFileMap) {
        Set entityNameFileMapKeys = entityNameFileMap.keySet();

        //filter on entityNames parameter if specified
        if (entityNames!=null&&!entityNames.isEmpty()) {
            List entityNamesAsList = Arrays.asList(entityNames.split(","));
            logger.info("Entities specified for TDE Generation: {} " + entityNamesAsList);

            //this will only keep keys in the map that are also in the entityNamesAsList
            entityNameFileMapKeys.retainAll(entityNamesAsList);

            if (entityNameFileMapKeys.isEmpty()) {
                logger.warn("No entities files found under {} or its sub-directories with the entity name(s) {}", hubConfig.getHubEntitiesDir(),entityNamesAsList);
            }
        }
    }

    protected static Map createEntityNameFileMap(List entityFiles) {
        if (entityFiles==null) {
            return Collections.emptyMap();
        }
        return entityFiles.stream().collect(
            toMap(extractEntityNameFunction(),Function.identity()));
    }

    protected List findEntityFiles() {
        List entities = new ArrayList<>();
        Path entitiesPath = hubConfig.getHubEntitiesDir();
        File[] entityDefs = entitiesPath.toFile().listFiles(pathname -> pathname.toString().endsWith(ENTITY_FILE_EXTENSION) && !pathname.isHidden());
        if (entityDefs != null) {
            entities.addAll(Arrays.asList(entityDefs));
        }
        return entities;
    }

    // Overriding to insert schemas into the final DB schemas folder.
    @Override
    protected void generateExtractionTemplate(AppConfig appConfig, GeneratedCode code) {
        String template = code.getExtractionTemplate();
        if (template != null) {
            File dir = userFinalSchemasTDEs.toFile();
            if (!(dir.mkdirs() || dir.exists())) {
                logger.warn("Unable to create directory for TDE templates: " + dir.getAbsolutePath());
                return;
            }
            File out = new File(dir, code.getTitle() + "-" + code.getVersion() + ".tdex");
            String logMessage = "Wrote extraction template to: ";
            if (out.exists()) {
                if (!fileHasDifferentContent(out, template)) {
                    if (logger.isInfoEnabled()) {
                        logger.info("Extraction template matches file, so not modifying: " + out.getAbsolutePath());
                    }
                    return;
                }
                out = new File(dir, code.getTitle() + "-" + code.getVersion() + "-GENERATED.tdex");
                logMessage = "Extraction template does not match existing file, so writing to: ";
            }
            try {
                FileCopyUtils.copy(template.getBytes(StandardCharsets.UTF_8), out);
                if (logger.isInfoEnabled()) {
                    logger.info(logMessage + out.getAbsolutePath());
                }
            } catch (IOException e) {
                throw new RuntimeException("Unable to write extraction template to file: " + out.getAbsolutePath(), e);
            }
        }
    }

    protected static Optional extractEntityNameFromFilename(String filename) {
        return EntityServicesManager.extractEntityNameFromURI(filename);
    }

    private static Function extractEntityNameFunction() {
        Function fileName = File::getName;
        return fileName.andThen(name -> extractEntityNameFromFilename(name).get());
    }

    private static final CodeGenerationRequest createCodeGenerationRequest() {
        CodeGenerationRequest request = new CodeGenerationRequest();
        request.setGenerateExtractionTemplate(true);
        request.setGenerateDatabaseProperties(false);
        request.setGenerateInstanceConverter(false);
        request.setGenerateSchema(false);
        request.setGenerateSearchOptions(false);
        return request;
    }
}

/**
 * Was formerly located at com.marklogic.hub.es; moved here as this class file is the only one that depended on this
 * class. Overrides the functionality for generating a TDE so that DHF-specific code can be used.
 */
class EntityServicesManager extends com.marklogic.client.ext.es.EntityServicesManager {
    protected DatabaseClient client;
    private static final String ENTITY_FILE_EXTENSION = ".entity.json";

    public EntityServicesManager(DatabaseClient client) {
        super(client);
        this.client = client;
    }

    @Override
    protected String generateCode(String modelUri, String functionName) {
        if ("extraction-template-generate".equals(functionName)) {
            String xquery = "import module namespace es = \"http://marklogic.com/entity-services\" at \"/MarkLogic/entity-services/entity-services.xqy\"; \n" +
                "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\" at \"/data-hub/5/impl/hub-entities.xqy\";\n" +
                "declare variable $entity-title external; \n" +
                "hent:dump-tde(json:to-array(es:model-validate(hent:get-model($entity-title))))";
            try (EvalResultIterator result = client.newServerEval().xquery(xquery).addVariable("entity-title", extractEntityNameFromURI(modelUri).get()).eval()) {
                return result.next().getString();
            }
        } else {
            return super.generateCode(modelUri, functionName);
        }
    }

    public static Optional extractEntityNameFromURI(String filename) {
        if (filename==null || filename.trim().isEmpty()) {
            return Optional.of(null);
        }
        int pathIndex = filename.lastIndexOf('/');
        if (pathIndex >= 0) {
            filename = filename.substring(pathIndex + 1);
        }
        int index = filename.indexOf(ENTITY_FILE_EXTENSION);
        if (index<0) {
            //not found
            return Optional.of(null);
        }
        return Optional.of(filename.substring(0,index));
    }
}




© 2015 - 2024 Weber Informatics LLC | Privacy Policy