com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of marklogic-data-hub Show documentation
Show all versions of marklogic-data-hub Show documentation
Library for Creating an Operational Data Hub on MarkLogic
/*
* Copyright (c) 2021 MarkLogic Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.marklogic.hub.deploy.commands;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.marklogic.appdeployer.AppConfig;
import com.marklogic.appdeployer.command.AbstractCommand;
import com.marklogic.appdeployer.command.CommandContext;
import com.marklogic.client.document.DocumentWriteSet;
import com.marklogic.client.document.JSONDocumentManager;
import com.marklogic.client.ext.modulesloader.Modules;
import com.marklogic.client.ext.modulesloader.ModulesFinder;
import com.marklogic.client.ext.modulesloader.impl.EntityDefModulesFinder;
import com.marklogic.client.ext.modulesloader.impl.MappingDefModulesFinder;
import com.marklogic.client.ext.tokenreplacer.TokenReplacer;
import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser;
import com.marklogic.client.ext.util.DocumentPermissionsParser;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.JacksonHandle;
import com.marklogic.hub.HubClient;
import com.marklogic.hub.HubConfig;
import com.marklogic.hub.dataservices.ArtifactService;
import com.marklogic.hub.dataservices.ConceptService;
import com.marklogic.hub.dataservices.ModelsService;
import com.marklogic.hub.dataservices.StepService;
import com.marklogic.hub.hubcentral.HubCentralManager;
import com.marklogic.mgmt.resource.hosts.HostManager;
import com.marklogic.mgmt.util.ObjectMapperFactory;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Iterator;
import java.util.regex.Pattern;
/**
* Loads user artifacts like mappings and entities. This will be deployed after triggers
*/
@Component
public class LoadUserArtifactsCommand extends AbstractCommand {
private static final Pattern stepExtension = Pattern.compile(".step.json", Pattern.LITERAL);
@Autowired
HubConfig hubConfig;
private final DocumentPermissionsParser documentPermissionsParser = new DefaultDocumentPermissionsParser();
private ObjectMapper objectMapper;
private TokenReplacer tokenReplacer;
public LoadUserArtifactsCommand() {
super();
this.objectMapper = ObjectMapperFactory.getObjectMapper();
setExecuteSortOrder(LoadHubArtifactsCommand.SORT_ORDER + 1);
}
/**
* For use outside of a Spring container.
*
* @param hubConfig
*/
public LoadUserArtifactsCommand(HubConfig hubConfig) {
this();
this.hubConfig = hubConfig;
}
static boolean isArtifactDir(Path dir, Path startPath) {
String dirStr = dir.toString();
String startPathStr = Pattern.quote(startPath.toString());
String regex = startPathStr + "[/\\\\][^/\\\\]+$";
return dirStr.matches(regex);
}
@Override
public void execute(CommandContext context) {
tokenReplacer = context.getAppConfig().buildTokenReplacer();
loadUserArtifacts();
}
/**
* The CommandContext has no bearing on how user artifacts are loaded, so this method is easier to use when this
* class is used outside a deployment context.
*/
public void loadUserArtifacts() {
HubClient hubClient = hubConfig.newHubClient();
try {
long start = System.currentTimeMillis();
loadModels(hubClient);
logger.info("Loaded models, time: " + (System.currentTimeMillis() - start) + "ms");
start = System.currentTimeMillis();
loadLegacyMappings(hubClient);
loadFlows(hubClient);
loadStepDefinitions(hubClient);
loadSteps(hubClient);
loadExclusionLists(hubClient);
loadHubCentralConfig(hubClient);
loadHubCentralConcepts(hubClient);
logger.info("Loaded flows, mappings, step definitions, steps, exclusion lists, hubcentral config, and concepts. time: " + (System.currentTimeMillis() - start) + "ms");
}
catch (IOException e) {
throw new RuntimeException("Unable to load user artifacts, cause: " + e.getMessage(), e);
}
}
/**
* Load the models into MarkLogic and assure that the expanded tree cache is cleared to ensure old schema isn't cached.
*
* @param hubClient
* @throws IOException
*/
private void loadModels(HubClient hubClient) throws IOException {
final File modelsDir = hubConfig.getHubEntitiesDir().toFile();
EntityDefModulesFinder modulesFinder = new EntityDefModulesFinder();
logger.info("Loading models from directory " + modelsDir);
ArrayNode modelsArray = objectMapper.createArrayNode();
modulesFinder.findModules(modelsDir.toString()).getAssets().forEach(r -> {
try {
logger.info("Loading model from file: " + r.getFilename());
modelsArray.add(readArtifact(r.getFile()));
}
catch (IOException e) {
throw new RuntimeException("Unable to read model file: " + r.getFilename() + "; cause: " + e.getMessage(), e);
}
});
if (!modelsArray.isEmpty()) {
ModelsService.on(hubClient.getStagingClient()).saveModels(modelsArray);
clearExpandedTreeCache(hubClient);
}
}
/**
* Attempts to clear the expanded tree cache for each node in the cluster.
*
* @param hubClient
*/
private void clearExpandedTreeCache(HubClient hubClient) {
AppConfig mlAppConfig = hubConfig.getAppConfig();
String originalHost = mlAppConfig.getHost();
try {
new HostManager(hubClient.getManageClient()).getHostNames().forEach((host) -> {
mlAppConfig.setHost(host);
logger.info("Clearing expanded tree cache on host: " + host);
mlAppConfig.newAppServicesDatabaseClient("Documents").newServerEval().xquery("xdmp:expanded-tree-cache-clear()").evalAs(String.class);
});
} catch (Exception e) {
logger.info("Failed to clear expanded tree cache: " + e.getMessage());
} finally {
mlAppConfig.setHost(originalHost);
}
}
/**
* "Legacy" = pre-5.3 mappings that are stored in documents outside of flows, but are not mapping steps.
*
* @param hubClient
* @throws IOException
*/
private void loadLegacyMappings(HubClient hubClient) throws IOException {
Path mappingsPath = hubConfig.getHubMappingsDir();
if (mappingsPath.toFile().exists()) {
JSONDocumentManager finalDocMgr = hubClient.getFinalClient().newJSONDocumentManager();
JSONDocumentManager stagingDocMgr = hubClient.getStagingClient().newJSONDocumentManager();
DocumentWriteSet stagingMappingDocumentWriteSet = stagingDocMgr.newWriteSet();
DocumentWriteSet finalMappingDocumentWriteSet = finalDocMgr.newWriteSet();
ResourceToURI mappingResourceToURI = new ResourceToURI(){
public String toURI(Resource r) throws IOException {
return "/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename();
}
};
MappingDefModulesFinder mappingDefModulesFinder = new MappingDefModulesFinder();
Files.walkFileTree(mappingsPath, new SimpleFileVisitor() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
if (isArtifactDir(dir, mappingsPath.toAbsolutePath())) {
executeWalk(
dir,
mappingDefModulesFinder,
mappingResourceToURI,
buildMetadata(hubConfig.getMappingPermissions(), "http://marklogic.com/data-hub/mappings", "hub-artifact"),
stagingMappingDocumentWriteSet,
finalMappingDocumentWriteSet
);
return FileVisitResult.CONTINUE;
}
else {
return FileVisitResult.CONTINUE;
}
}
});
if (!stagingMappingDocumentWriteSet.isEmpty()) {
stagingDocMgr.write(stagingMappingDocumentWriteSet);
finalDocMgr.write(finalMappingDocumentWriteSet);
}
}
}
void executeWalk(
Path dir,
ModulesFinder modulesFinder,
ResourceToURI resourceToURI,
DocumentMetadataHandle metadata,
DocumentWriteSet... writeSets
) throws IOException {
Modules modules = modulesFinder.findModules(dir.toString());
for (Resource r : modules.getAssets()) {
addResourceToWriteSets(
r,
resourceToURI.toURI(r),
metadata,
writeSets
);
}
}
/**
* As of 5.2.0, artifact permissions are separate from module permissions. If artifact permissions
* are not defined, then it falls back to using default permissions.
*
* @param permissions
* @param collection
* @return
*/
protected DocumentMetadataHandle buildMetadata(String permissions, String... collection) {
DocumentMetadataHandle meta = new DocumentMetadataHandle();
meta.getCollections().addAll(collection);
documentPermissionsParser.parsePermissions(permissions, meta.getPermissions());
return meta;
}
private void addResourceToWriteSets(
Resource r,
String docId,
DocumentMetadataHandle meta,
DocumentWriteSet... writeSets
) throws IOException {
JsonNode json = readArtifact(r.getFile());
if (json instanceof ObjectNode && json.has("language")) {
json = replaceLanguageWithLang((ObjectNode)json);
try {
objectMapper.writeValue(r.getFile(), json);
} catch (Exception ex) {
logger.warn("Unable to replace 'language' with 'lang' in artifact file: " + r.getFile().getAbsolutePath()
+ ". You should replace 'language' with 'lang' yourself in this file. Error cause: " + ex.getMessage(), ex);
}
}
for (DocumentWriteSet writeSet : writeSets) {
writeSet.add(docId, meta, new JacksonHandle(json));
}
}
/**
* Loads steps, where the assumption is that the name of each directory under the steps path corresponds to a step
* definition type. And thus each .step.json file in that directory should be loaded as a step.
*
* @param hubClient
* @throws IOException
*/
private void loadSteps(HubClient hubClient) {
final Path stepsPath = hubConfig.getHubProject().getStepsPath();
if (stepsPath.toFile().exists()) {
StepService stepService = StepService.on(hubClient.getStagingClient());
File[] stepTypeDirectories = stepsPath.toFile().listFiles(File::isDirectory);
if (stepTypeDirectories == null) {
return;
}
for (File stepTypeDir : stepTypeDirectories) {
final String stepType = stepTypeDir.getName();
File[] stepFiles = stepTypeDir.listFiles((File d, String name) -> name.endsWith(".step.json"));
if (stepFiles == null) {
continue;
}
for (File stepFile : stepFiles) {
JsonNode step = readArtifact(stepFile);
if (!step.has("name")) {
throw new RuntimeException("Unable to load step from file: " + stepFile + "; no 'name' property found");
}
final String stepName = step.get("name").asText();
logger.info(format("Loading step of type '%s' with name '%s'", stepType, stepName));
//We want the contents of file in the project to overwrite the step if it's already present. Hence
//steps are deployed with 'overwrite' flag set to true and 'throwErrorIfStepIsPresent' set to false.
stepService.saveStep(stepType, step, true, false);
}
}
}
}
private void loadFlows(HubClient hubClient) {
final Path flowsPath = hubConfig.getHubProject().getFlowsDir();
if (flowsPath.toFile().exists()) {
ArtifactService service = ArtifactService.on(hubClient.getStagingClient());
File[] flowFiles = flowsPath.toFile().listFiles(f -> f.isFile() && f.getName().endsWith(".flow.json"));
if (flowFiles == null) {
return;
}
for (File file : flowFiles) {
JsonNode flow = readArtifact(file);
if (!flow.has("name")) {
throw new RuntimeException("Unable to load flow from file: " + file + "; no 'name' property found");
}
final String flowName = flow.get("name").asText();
logger.info(format("Loading flow with name '%s'", flowName));
service.setArtifact("flow", flowName, flow, "");
}
}
}
private void loadExclusionLists(HubClient hubClient) {
final Path listsPath = hubConfig.getHubProject().getProjectDir().resolve("exclusionLists");
if (listsPath.toFile().exists()) {
ArtifactService service = ArtifactService.on(hubClient.getStagingClient());
for (File file : listsPath.toFile().listFiles(f -> f.isFile() && f.getName().endsWith(".exclusionList.json"))) {
JsonNode exclusionList = readArtifact(file);
if (!exclusionList.has("name")) {
throw new RuntimeException("Unable to load exclusion list from file: " + file + "; no 'name' property found");
}
final String listName = exclusionList.get("name").asText();
logger.info(format("Loading exclusion list with name '%s'", listName));
service.setArtifact("exclusionList", listName, exclusionList, "/exclusionLists/");
}
}
}
private void loadHubCentralConfig(HubClient hubClient) {
final Path configPath = hubConfig.getHubProject().getHubCentralConfigPath();
if (configPath.toFile().exists()) {
File[] configFiles = configPath.toFile().listFiles(File::isFile);
if (configFiles == null) {
return;
}
for (File file : configFiles) {
JsonNode hubCentralConfig = readArtifact(file);
String docUri = "/config/".concat(file.getName());
HubCentralManager.deployHubCentralConfig(hubClient, hubCentralConfig, docUri);
}
}
}
private void loadHubCentralConcepts(HubClient hubClient) {
final Path configPath = hubConfig.getHubProject().getHubCentralConceptsPath();
ArrayNode conceptsArray = objectMapper.createArrayNode();
if (configPath.toFile().exists()) {
File[] conceptFiles = configPath.toFile().listFiles(f -> f.isFile() && f.getName().endsWith(".concept.json"));
if (conceptFiles == null) {
return;
}
for (File file : conceptFiles) {
conceptsArray.add(readArtifact(file));
}
}
if (!conceptsArray.isEmpty()) {
ConceptService.on(hubClient.getStagingClient()).saveConceptModels(conceptsArray);
clearExpandedTreeCache(hubClient);
}
}
private void loadStepDefinitions(HubClient hubClient) {
final Path stepDefsPath = hubConfig.getHubProject().getStepDefinitionsDir();
if (stepDefsPath.toFile().exists()) {
ArtifactService service = ArtifactService.on(hubClient.getStagingClient());
File[] typeDirectories = stepDefsPath.toFile().listFiles(File::isDirectory);
if (typeDirectories == null) {
return;
}
for (File typeDir : typeDirectories) {
final String stepDefType = typeDir.getName();
File[] definitionDirectories = typeDir.listFiles(File::isDirectory);
if (definitionDirectories == null) {
continue;
}
for (File defDir : definitionDirectories) {
String[] fileNames = defDir.list();
if (fileNames == null) {
continue;
}
for (String stepDefFileName : fileNames) {
File stepDefFile = new File(defDir, stepDefFileName);
if (stepDefFile.exists()) {
JsonNode stepDef = readArtifact(stepDefFile);
if (!stepDef.has("name")) {
throw new RuntimeException("Unable to load step definition from file: " + stepDefFile +
"; no 'name' property was found");
}
final String stepDefName = stepDef.get("name").asText();
logger.info(format("Loading step definition with type '%s' and name '%s'", stepDefType, stepDefName));
service.setArtifact("stepDefinition", stepDefName, stepDef, stepExtension.matcher(stepDefFileName).replaceAll(""));
} else {
logger.warn(format("Found step definition directory '%s', but did not find expected " +
"step definition file: '%s'", defDir.getAbsolutePath(), stepDefFile.getName()));
}
}
}
}
}
}
/**
* Per DHFPROD-3193 and an update to MarkLogic 10.0-2, "lang" must now be used instead of "language". To ensure that
* a user artifact is never loaded with "language", this command handles both updating the JSON that will be loaded
* into MarkLogic and updating the artifact file.
*
* @param object
* @return
*/
protected ObjectNode replaceLanguageWithLang(ObjectNode object) {
ObjectNode newObject = objectMapper.createObjectNode();
newObject.put("lang", object.get("language").asText());
Iterator fieldNames = object.fieldNames();
while (fieldNames.hasNext()) {
String fieldName = fieldNames.next();
if (!"language".equals(fieldName)) {
newObject.set(fieldName, object.get(fieldName));
}
}
return newObject;
}
/**
* Reads the artifact file, replaces tokens and then returns the content as a JsonNode.
*
* @param file
* @return
*/
private JsonNode readArtifact(File file) {
JsonNode jsonNode;
try {
String artifact = tokenReplacer.replaceTokens(FileUtils.readFileToString(file));
jsonNode = objectMapper.readTree(artifact);
}
catch (Exception e) {
throw new RuntimeException("Unable to read file " + file.getName() + " + as JSON; cause: " + e.getMessage(), e);
}
return jsonNode;
}
public void setHubConfig(HubConfig hubConfig) {
this.hubConfig = hubConfig;
}
public void setObjectMapper(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
abstract static class ResourceToURI {
public abstract String toURI(Resource r) throws IOException;
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy