com.marklogic.hub.deploy.commands.GenerateHubTDETemplateCommand Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of marklogic-data-hub Show documentation
Show all versions of marklogic-data-hub Show documentation
Library for Creating an Operational Data Hub on MarkLogic
/*
* Copyright (c) 2021 MarkLogic Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.marklogic.hub.deploy.commands;
import com.marklogic.appdeployer.AppConfig;
import com.marklogic.appdeployer.command.CommandContext;
import com.marklogic.appdeployer.command.es.GenerateModelArtifactsCommand;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.eval.EvalResultIterator;
import com.marklogic.client.ext.es.CodeGenerationRequest;
import com.marklogic.client.ext.es.GeneratedCode;
import com.marklogic.hub.DatabaseKind;
import com.marklogic.hub.HubConfig;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.FileCopyUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import static java.util.stream.Collectors.toMap;
public class GenerateHubTDETemplateCommand extends GenerateModelArtifactsCommand {
private static final String ENTITY_FILE_EXTENSION = ".entity.json";
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
private final HubConfig hubConfig;
private final Path userFinalSchemasTDEs;
private String entityNames;
@SuppressWarnings("unchecked")
private List entityNamesList = Collections.EMPTY_LIST;
public GenerateHubTDETemplateCommand(HubConfig hubConfig) {
this.hubConfig = hubConfig;
this.userFinalSchemasTDEs = hubConfig.getHubProject().getUserDatabaseDir().resolve(hubConfig.getDbName(DatabaseKind.FINAL_SCHEMAS)).resolve("schemas").resolve("tde");
}
@Override
public void execute(CommandContext context) {
AppConfig appConfig = context.getAppConfig();
DatabaseClient client = appConfig.newDatabaseClient();
EntityServicesManager mgr = new EntityServicesManager(client);
CodeGenerationRequest request = createCodeGenerationRequest();
List entityFiles = findEntityFiles();
if (!entityFiles.isEmpty()) {
//create map of entity name -> entity definition file
Map entityNameFileMap = createEntityNameFileMap(entityFiles);
logger.debug("Found the following entities->files: {} " + entityNameFileMap);
//filterEntities(entityNameFileMap);
if (!entityNameFileMap.isEmpty()) {
logger.warn("About to generate a template for the following entities: {}",
this.entityNamesList.isEmpty() ? entityNameFileMap.keySet() : this.entityNamesList);
List generatedCodes = new ArrayList<>();
for (File f : entityNameFileMap.values()) {
File esModel;
String modelName;
try {
//Write the ES model to a temp file
String tempDir = System.getProperty("java.io.tmpdir");
String fileName = f.getName();
modelName = EntityServicesManager.extractEntityNameFromURI(fileName).get();
esModel = new File(tempDir, fileName);
String modelString = generateModel(f);
if(modelString == null) {
logger.warn(f.getName() + " is not deployed to the database");
continue;
}
FileUtils.writeStringToFile(esModel, modelString);
} catch (IOException e) {
throw new RuntimeException("Unable to generate ES model");
}
GeneratedCode code;
try {
code = loadModelDefinition(request, esModel, mgr);
} catch (RuntimeException e) {
throw new RuntimeException("Unable to read model definition from file: " + f.getAbsolutePath(), e);
}
finally {
FileUtils.deleteQuietly(esModel);
}
if (this.entityNamesList.isEmpty() || this.entityNamesList.contains(modelName)) {
generatedCodes.add(code);
}
}
for (GeneratedCode code: generatedCodes) {
generateExtractionTemplate(appConfig, code);
}
}
} else {
logger.info("No data hub entity files found under {} or its sub-directories.",
hubConfig.getHubEntitiesDir());
}
}
//Method to obtain es-style model
private String generateModel(File f) {
String xquery = "import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\"\n" +
"at \"/data-hub/5/impl/hub-entities.xqy\";\n" +
String.format("hent:get-model(\"%s\")", extractEntityNameFromFilename(f.getName()).get());
try (EvalResultIterator resp = hubConfig.newStagingClient().newServerEval().xquery(xquery).eval()) {
if (resp.hasNext()) {
return resp.next().getString();
}
}
return null;
}
public String getEntityNames() {
return entityNames;
}
public void setEntityNames(String entityNames) {
this.entityNames = entityNames;
if (entityNames != null) {
this.entityNamesList = Arrays.asList(entityNames.split(","));
}
}
protected void filterEntities(Map entityNameFileMap) {
Set entityNameFileMapKeys = entityNameFileMap.keySet();
//filter on entityNames parameter if specified
if (entityNames!=null&&!entityNames.isEmpty()) {
List entityNamesAsList = Arrays.asList(entityNames.split(","));
logger.info("Entities specified for TDE Generation: {} " + entityNamesAsList);
//this will only keep keys in the map that are also in the entityNamesAsList
entityNameFileMapKeys.retainAll(entityNamesAsList);
if (entityNameFileMapKeys.isEmpty()) {
logger.warn("No entities files found under {} or its sub-directories with the entity name(s) {}", hubConfig.getHubEntitiesDir(),entityNamesAsList);
}
}
}
protected static Map createEntityNameFileMap(List entityFiles) {
if (entityFiles==null) {
return Collections.emptyMap();
}
return entityFiles.stream().collect(
toMap(extractEntityNameFunction(),Function.identity()));
}
protected List findEntityFiles() {
List entities = new ArrayList<>();
Path entitiesPath = hubConfig.getHubEntitiesDir();
File[] entityDefs = entitiesPath.toFile().listFiles(pathname -> pathname.toString().endsWith(ENTITY_FILE_EXTENSION) && !pathname.isHidden());
if (entityDefs != null) {
entities.addAll(Arrays.asList(entityDefs));
}
return entities;
}
// Overriding to insert schemas into the final DB schemas folder.
@Override
protected void generateExtractionTemplate(AppConfig appConfig, GeneratedCode code) {
String template = code.getExtractionTemplate();
if (template != null) {
File dir = userFinalSchemasTDEs.toFile();
if (!(dir.mkdirs() || dir.exists())) {
logger.warn("Unable to create directory for TDE templates: " + dir.getAbsolutePath());
return;
}
File out = new File(dir, code.getTitle() + "-" + code.getVersion() + ".tdex");
String logMessage = "Wrote extraction template to: ";
if (out.exists()) {
if (!fileHasDifferentContent(out, template)) {
if (logger.isInfoEnabled()) {
logger.info("Extraction template matches file, so not modifying: " + out.getAbsolutePath());
}
return;
}
out = new File(dir, code.getTitle() + "-" + code.getVersion() + "-GENERATED.tdex");
logMessage = "Extraction template does not match existing file, so writing to: ";
}
try {
FileCopyUtils.copy(template.getBytes(StandardCharsets.UTF_8), out);
if (logger.isInfoEnabled()) {
logger.info(logMessage + out.getAbsolutePath());
}
} catch (IOException e) {
throw new RuntimeException("Unable to write extraction template to file: " + out.getAbsolutePath(), e);
}
}
}
protected static Optional extractEntityNameFromFilename(String filename) {
return EntityServicesManager.extractEntityNameFromURI(filename);
}
private static Function extractEntityNameFunction() {
Function fileName = File::getName;
return fileName.andThen(name -> extractEntityNameFromFilename(name).get());
}
private static final CodeGenerationRequest createCodeGenerationRequest() {
CodeGenerationRequest request = new CodeGenerationRequest();
request.setGenerateExtractionTemplate(true);
request.setGenerateDatabaseProperties(false);
request.setGenerateInstanceConverter(false);
request.setGenerateSchema(false);
request.setGenerateSearchOptions(false);
return request;
}
}
/**
* Was formerly located at com.marklogic.hub.es; moved here as this class file is the only one that depended on this
* class. Overrides the functionality for generating a TDE so that DHF-specific code can be used.
*/
class EntityServicesManager extends com.marklogic.client.ext.es.EntityServicesManager {
protected DatabaseClient client;
private static final String ENTITY_FILE_EXTENSION = ".entity.json";
public EntityServicesManager(DatabaseClient client) {
super(client);
this.client = client;
}
@Override
protected String generateCode(String modelUri, String functionName) {
if ("extraction-template-generate".equals(functionName)) {
String xquery = "import module namespace es = \"http://marklogic.com/entity-services\" at \"/MarkLogic/entity-services/entity-services.xqy\"; \n" +
"import module namespace hent = \"http://marklogic.com/data-hub/hub-entities\" at \"/data-hub/5/impl/hub-entities.xqy\";\n" +
"declare variable $entity-title external; \n" +
"hent:dump-tde(json:to-array(es:model-validate(hent:get-model($entity-title))))";
try (EvalResultIterator result = client.newServerEval().xquery(xquery).addVariable("entity-title", extractEntityNameFromURI(modelUri).get()).eval()) {
return result.next().getString();
}
} else {
return super.generateCode(modelUri, functionName);
}
}
public static Optional extractEntityNameFromURI(String filename) {
if (filename==null || filename.trim().isEmpty()) {
return Optional.of(null);
}
int pathIndex = filename.lastIndexOf('/');
if (pathIndex >= 0) {
filename = filename.substring(pathIndex + 1);
}
int index = filename.indexOf(ENTITY_FILE_EXTENSION);
if (index<0) {
//not found
return Optional.of(null);
}
return Optional.of(filename.substring(0,index));
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy