com.marklogic.hub.impl.DataHubImpl Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of marklogic-data-hub Show documentation
Show all versions of marklogic-data-hub Show documentation
Library for Creating an Operational Data Hub on MarkLogic
/*
* Copyright (c) 2021 MarkLogic Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.marklogic.hub.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.marklogic.appdeployer.AppConfig;
import com.marklogic.appdeployer.command.Command;
import com.marklogic.appdeployer.command.CommandMapBuilder;
import com.marklogic.appdeployer.command.appservers.UpdateRestApiServersCommand;
import com.marklogic.appdeployer.command.databases.DeployOtherDatabasesCommand;
import com.marklogic.appdeployer.command.forests.DeployCustomForestsCommand;
import com.marklogic.appdeployer.command.modules.DeleteTestModulesCommand;
import com.marklogic.appdeployer.command.modules.LoadModulesCommand;
import com.marklogic.appdeployer.command.security.DeployAmpsCommand;
import com.marklogic.appdeployer.command.security.DeployCertificateAuthoritiesCommand;
import com.marklogic.appdeployer.command.security.DeployCertificateTemplatesCommand;
import com.marklogic.appdeployer.command.security.DeployExternalSecurityCommand;
import com.marklogic.appdeployer.command.security.DeployPrivilegesCommand;
import com.marklogic.appdeployer.command.security.DeployProtectedCollectionsCommand;
import com.marklogic.appdeployer.command.security.DeployProtectedPathsCommand;
import com.marklogic.appdeployer.command.security.DeployQueryRolesetsCommand;
import com.marklogic.appdeployer.command.security.DeployRolesCommand;
import com.marklogic.appdeployer.command.security.DeployUsersCommand;
import com.marklogic.appdeployer.command.security.InsertCertificateHostsTemplateCommand;
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
import com.marklogic.client.DatabaseClient;
import com.marklogic.client.admin.QueryOptionsManager;
import com.marklogic.client.admin.ResourceExtensionsManager;
import com.marklogic.client.admin.ServerConfigurationManager;
import com.marklogic.client.admin.TransformExtensionsManager;
import com.marklogic.client.document.DocumentWriteOperation;
import com.marklogic.client.document.DocumentWriteSet;
import com.marklogic.client.document.JSONDocumentManager;
import com.marklogic.client.eval.EvalResultIterator;
import com.marklogic.client.eval.ServerEvaluationCall;
import com.marklogic.client.impl.DocumentWriteOperationImpl;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.JacksonHandle;
import com.marklogic.client.io.QueryOptionsListHandle;
import com.marklogic.client.query.DeleteQueryDefinition;
import com.marklogic.client.query.QueryManager;
import com.marklogic.hub.DataHub;
import com.marklogic.hub.DatabaseKind;
import com.marklogic.hub.HubClient;
import com.marklogic.hub.HubConfig;
import com.marklogic.hub.HubProject;
import com.marklogic.hub.InstallInfo;
import com.marklogic.hub.MarkLogicVersion;
import com.marklogic.hub.dataservices.ArtifactService;
import com.marklogic.hub.deploy.commands.CheckSecurityConfiguration;
import com.marklogic.hub.deploy.commands.ConfigureAppServerBasePaths;
import com.marklogic.hub.deploy.commands.CreateGranularPrivilegesCommand;
import com.marklogic.hub.deploy.commands.DeployDatabaseFieldCommand;
import com.marklogic.hub.deploy.commands.DeployHubTriggersCommand;
import com.marklogic.hub.deploy.commands.FinishHubDeploymentCommand;
import com.marklogic.hub.deploy.commands.GenerateFunctionMetadataCommand;
import com.marklogic.hub.deploy.commands.HubDeployDatabaseCommandFactory;
import com.marklogic.hub.deploy.commands.LoadHubArtifactsCommand;
import com.marklogic.hub.deploy.commands.LoadHubModulesCommand;
import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand;
import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
import com.marklogic.hub.error.DataHubConfigurationException;
import com.marklogic.hub.error.InvalidDBOperationError;
import com.marklogic.hub.error.ServerValidationException;
import com.marklogic.hub.flow.FlowRunner;
import com.marklogic.mgmt.ManageClient;
import com.marklogic.mgmt.admin.AdminManager;
import com.marklogic.mgmt.resource.appservers.ServerManager;
import com.marklogic.mgmt.resource.databases.DatabaseManager;
import com.marklogic.rest.util.Fragment;
import com.marklogic.rest.util.ResourcesFragment;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.ResourceAccessException;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Stream;
@Component
public class DataHubImpl implements DataHub, InitializingBean {
private static final Pattern xmlPattern = Pattern.compile(".xml", Pattern.LITERAL);
private static final Pattern modulePattern = Pattern.compile("\\.(sjs|mjs|xqy)$");
@Autowired
private HubConfig hubConfig;
private HubClient hubClient;
private LoadHubModulesCommand loadHubModulesCommand;
private LoadUserModulesCommand loadUserModulesCommand;
private LoadUserArtifactsCommand loadUserArtifactsCommand;
private LoadHubArtifactsCommand loadHubArtifactsCommand;
private GenerateFunctionMetadataCommand generateFunctionMetadataCommand;
private Versions versions;
@Autowired
FlowRunner flowRunner;
@Autowired
FlowManagerImpl flowManager;
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
public DataHubImpl() {
super();
}
public DataHubImpl(HubConfig hubConfig) {
this();
this.hubConfig = hubConfig;
afterPropertiesSet();
}
/**
* Only use this constructor for the clearUserData operation, which does not depend on a HubConfig.
*
* @param hubClient
*/
public DataHubImpl(HubClient hubClient) {
this();
this.hubClient = hubClient;
}
public void afterPropertiesSet() {
this.versions = new Versions(hubConfig);
this.generateFunctionMetadataCommand = new GenerateFunctionMetadataCommand(hubConfig);
this.loadHubModulesCommand = new LoadHubModulesCommand(hubConfig);
this.loadUserModulesCommand = new LoadUserModulesCommand(hubConfig);
this.loadUserArtifactsCommand = new LoadUserArtifactsCommand(hubConfig);
this.loadHubArtifactsCommand = new LoadHubArtifactsCommand(hubConfig);
}
/**
* Need to account for the group name in case the user has overridden the name of the "Default" group.
*
* @param hubConfig hubConfig object
* @return constructed ServerManager object
*/
protected static ServerManager constructServerManager(HubConfig hubConfig) {
AppConfig appConfig = hubConfig.getAppConfig();
return appConfig != null ?
new ServerManager(hubConfig.getManageClient(), appConfig.getGroupName()) :
new ServerManager(hubConfig.getManageClient());
}
private AdminManager getAdminManager() {
return hubConfig.getAdminManager();
}
private ManageClient getManageClient() {
return hubConfig.getManageClient();
}
private DatabaseManager getDatabaseManager() {
return new DatabaseManager(getManageClient());
}
@Override
public FlowRunner getFlowRunner() {
return this.flowRunner;
}
@Override
public InstallInfo isInstalled() throws ResourceAccessException {
InstallInfo installInfo = InstallInfo.create();
if (hubConfig.getIsProvisionedEnvironment()) {
return assumedProvisionedInstallInfo(installInfo);
} else {
ResourcesFragment srf = null;
try {
srf = constructServerManager(hubConfig).getAsXml();
} catch (HttpClientErrorException e) {
if (e.getStatusCode() == HttpStatus.UNAUTHORIZED) {
throw new RuntimeException("Unable to determine if Data Hub is already installed due to " +
"unauthorized user; please verify the username and password (mlUsername and mlPassword if using Gradle)");
}
}
if (srf != null) {
installInfo.setAppServerExistent(DatabaseKind.STAGING, srf.resourceExists(hubConfig.getHttpName(DatabaseKind.STAGING)));
installInfo.setAppServerExistent(DatabaseKind.FINAL, srf.resourceExists(hubConfig.getHttpName(DatabaseKind.FINAL)));
installInfo.setAppServerExistent(DatabaseKind.JOB, srf.resourceExists(hubConfig.getHttpName(DatabaseKind.JOB)));
}
ResourcesFragment drf = getDatabaseManager().getAsXml();
installInfo.setDbExistent(DatabaseKind.STAGING, drf.resourceExists(hubConfig.getDbName(DatabaseKind.STAGING)));
installInfo.setDbExistent(DatabaseKind.FINAL, drf.resourceExists(hubConfig.getDbName(DatabaseKind.FINAL)));
installInfo.setDbExistent(DatabaseKind.JOB, drf.resourceExists(hubConfig.getDbName(DatabaseKind.JOB)));
installInfo.setDbExistent(DatabaseKind.MODULES, drf.resourceExists(hubConfig.getDbName(DatabaseKind.MODULES)));
installInfo.setDbExistent(DatabaseKind.STAGING_SCHEMAS, drf.resourceExists(hubConfig.getDbName(DatabaseKind.STAGING_SCHEMAS)));
installInfo.setDbExistent(DatabaseKind.STAGING_TRIGGERS, drf.resourceExists(hubConfig.getDbName(DatabaseKind.STAGING_TRIGGERS)));
if (installInfo.isDbExistent(DatabaseKind.STAGING)) {
Fragment f = getDatabaseManager().getPropertiesAsXml(hubConfig.getDbName(DatabaseKind.STAGING));
installInfo.setTripleIndexOn(DatabaseKind.STAGING, Boolean.parseBoolean(f.getElementValue("//m:triple-index")));
installInfo.setCollectionLexiconOn(DatabaseKind.STAGING, Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon")));
installInfo.setForestsExistent(DatabaseKind.STAGING, (!f.getElements("//m:forest").isEmpty()));
}
if (installInfo.isDbExistent(DatabaseKind.FINAL)) {
Fragment f = getDatabaseManager().getPropertiesAsXml(hubConfig.getDbName(DatabaseKind.FINAL));
installInfo.setTripleIndexOn(DatabaseKind.FINAL, Boolean.parseBoolean(f.getElementValue("//m:triple-index")));
installInfo.setCollectionLexiconOn(DatabaseKind.FINAL, Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon")));
installInfo.setForestsExistent(DatabaseKind.FINAL, (!f.getElements("//m:forest").isEmpty()));
}
if (installInfo.isDbExistent(DatabaseKind.JOB)) {
Fragment f = getDatabaseManager().getPropertiesAsXml(hubConfig.getDbName(DatabaseKind.JOB));
installInfo.setForestsExistent(DatabaseKind.JOB, (!f.getElements("//m:forest").isEmpty()));
}
logger.info(installInfo.toString());
return installInfo;
}
}
// this InstallInfo is used as a dummy to return DHS provisioned information
private static InstallInfo assumedProvisionedInstallInfo(InstallInfo installInfo) {
installInfo.setAppServerExistent(DatabaseKind.STAGING, true);
installInfo.setAppServerExistent(DatabaseKind.FINAL, true);
installInfo.setAppServerExistent(DatabaseKind.JOB, true);
installInfo.setDbExistent(DatabaseKind.STAGING, true);
installInfo.setDbExistent(DatabaseKind.FINAL, true);
installInfo.setDbExistent(DatabaseKind.JOB, true);
installInfo.setDbExistent(DatabaseKind.MODULES, true);
installInfo.setDbExistent(DatabaseKind.STAGING_SCHEMAS, true);
installInfo.setDbExistent(DatabaseKind.STAGING_TRIGGERS, true);
installInfo.setTripleIndexOn(DatabaseKind.STAGING, true);
installInfo.setCollectionLexiconOn(DatabaseKind.STAGING, true);
installInfo.setForestsExistent(DatabaseKind.STAGING, true);
installInfo.setTripleIndexOn(DatabaseKind.FINAL, true);
installInfo.setCollectionLexiconOn(DatabaseKind.FINAL, true);
installInfo.setForestsExistent(DatabaseKind.FINAL, true);
installInfo.setForestsExistent(DatabaseKind.JOB, true);
return installInfo;
}
@Override
public boolean isServerVersionValid(String versionString) {
return new MarkLogicVersion(versionString).supportsDataHubFramework();
}
@Override
public void initProject() {
logger.info("Initializing the Hub Project");
hubConfig.initHubProject();
}
@Override
public void clearUserModules() {
clearUserModules(null);
}
/**
*
* @param resourceNamesToNotDelete optional list of names of resources that should not be deleted; can be null;
* introduced for the sake of DHF tests so that marklogic-unit-test will not be deleted
*/
public void clearUserModules(List resourceNamesToNotDelete) {
long start = System.currentTimeMillis();
logger.info("Clearing user modules");
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(DataHub.class.getClassLoader());
try {
HashSet dataHubOptions = new HashSet<>();
for (Resource r : resolver.getResources("classpath*:/ml-modules/options/*.xml")) {
dataHubOptions.add(xmlPattern.matcher(r.getFilename()).replaceAll(""));
}
for (Resource r : resolver.getResources("classpath*:/ml-modules-final/options/*.xml")) {
dataHubOptions.add(xmlPattern.matcher(r.getFilename()).replaceAll(""));
}
for (Resource r : resolver.getResources("classpath*:/ml-modules-traces/options/*.xml")) {
dataHubOptions.add(xmlPattern.matcher(r.getFilename()).replaceAll(""));
}
for (Resource r : resolver.getResources("classpath*:/ml-modules-jobs/options/*.xml")) {
dataHubOptions.add(xmlPattern.matcher(r.getFilename()).replaceAll(""));
}
HashSet dataHubServices = new HashSet<>();
for (Resource r : resolver.getResources("classpath*:/ml-modules/services/*")) {
dataHubServices.add(modulePattern.matcher(r.getFilename()).replaceAll(""));
}
HashSet dataHubTransforms = new HashSet<>();
for (Resource r : resolver.getResources("classpath*:/ml-modules/transforms/*")) {
dataHubTransforms.add(modulePattern.matcher(r.getFilename()).replaceAll(""));
}
ServerConfigurationManager configMgr = hubConfig.newStagingClient().newServerConfigManager();
QueryOptionsManager stagingOptionsManager = configMgr.newQueryOptionsManager();
// remove options using mgr.
QueryOptionsListHandle handle = stagingOptionsManager.optionsList(new QueryOptionsListHandle());
Map optionsMap = handle.getValuesMap();
optionsMap.keySet().forEach(
optionsName -> {
if (!dataHubOptions.contains(optionsName)) {
stagingOptionsManager.deleteOptions(optionsName);
}
}
);
ServerConfigurationManager finalConfigMgr = hubConfig.newFinalClient().newServerConfigManager();
QueryOptionsManager finalOptionsManager = finalConfigMgr.newQueryOptionsManager();
// remove options using mgr.
QueryOptionsListHandle finalHandle = finalOptionsManager.optionsList(new QueryOptionsListHandle());
Map finalOptionsMap = finalHandle.getValuesMap();
finalOptionsMap.keySet().forEach(
optionsName -> {
if (!dataHubOptions.contains(optionsName)) {
finalOptionsManager.deleteOptions(optionsName);
}
}
);
// remove transforms using amped channel
TransformExtensionsManager transformExtensionsManager = configMgr.newTransformExtensionsManager();
JsonNode transformsList = transformExtensionsManager.listTransforms(new JacksonHandle(), false).get();
transformsList.findValuesAsText("name").forEach(
x -> {
if (!(dataHubTransforms.contains(x) || x.startsWith("ml"))) {
transformExtensionsManager.deleteTransform(x);
}
}
);
// remove resource extensions
ResourceExtensionsManager resourceExtensionsManager = configMgr.newResourceExtensionsManager();
JsonNode resourceExtensions = resourceExtensionsManager.listServices(new JacksonHandle(), false).get();
if (resourceNamesToNotDelete == null) {
resourceNamesToNotDelete = new ArrayList<>(); // makes the boolean logic below simpler
}
for (String resourceName : resourceExtensions.findValuesAsText("name")) {
if (!dataHubServices.contains(resourceName) && !resourceName.startsWith("ml") && !resourceNamesToNotDelete.contains(resourceName)) {
resourceExtensionsManager.deleteServices(resourceName);
}
}
String query =
"cts:uris((),(),cts:not-query(cts:collection-query('hub-core-module')))[\n" +
" fn:not(\n" +
" fn:matches(., \"^.+options/(" + String.join("|", dataHubOptions) + ").xml$\") or\n" +
" fn:starts-with(., '/marklogic.rest.') or\n" +
// Retain compiled mappings for OOTB mapping functions
" fn:starts-with(., '/data-hub/5/mapping-functions/')\n" +
" )\n" +
"] ! xdmp:document-delete(.)\n";
runInDatabase(query, hubConfig.getDbName(DatabaseKind.MODULES));
} catch (Exception e) {
throw new RuntimeException("Failed to clear user modules, cause: " + e.getMessage(), e);
}
logger.info("Finished clearing user modules; time elapsed: " + (System.currentTimeMillis() - start));
}
public void clearUserArtifacts(){
final HubClient hubClientToUse = hubClient != null ? hubClient : hubConfig.newHubClient();
long start = System.currentTimeMillis();
logger.info("Clearing user artifacts as user: " + hubClientToUse.getUsername());
ArtifactService.on(hubClientToUse.getStagingClient()).clearUserArtifacts();
ArtifactService.on(hubClientToUse.getFinalClient()).clearUserArtifacts();
logger.info("Finished clearing user artifacts; time elapsed: " + (System.currentTimeMillis() - start));
}
public List buildListOfCommands() {
Map> commandMap = buildCommandMap();
List commands = new ArrayList<>();
for (Map.Entry> entry: commandMap.entrySet()) {
commands.addAll(entry.getValue());
}
return commands;
}
public List getSecurityCommandList() {
Map> commandMap = getSecurityCommands();
List commands = new ArrayList<>();
for (Map.Entry> entry: commandMap.entrySet()) {
commands.addAll(entry.getValue());
}
return commands;
}
@Override
public Map runPreInstallCheck() {
return runPreInstallCheck(constructServerManager(hubConfig));
}
/**
* This overloaded version was added to facilitate mock testing - i.e. to be able to mock which ports are available.
*
* @param serverManager
* @return
*/
protected Map runPreInstallCheck(ServerManager serverManager) {
Map portsInUse;
try {
portsInUse = getServerPortsInUse(serverManager);
} catch (HttpClientErrorException e) {
logger.warn("Used non-existing user to verify data hub. Usually this means a fresh system, ready to install.");
Map response = new HashMap<>();
response.put("serverVersion", serverVersion);
// no server means give it a shot.
response.put("serverVersionOk", true);
response.put("stagingPortInUse", stagingPortInUse);
response.put("stagingPortInUseBy", stagingPortInUseBy);
response.put("finalPortInUse", finalPortInUse);
response.put("finalPortInUseBy", finalPortInUseBy);
response.put("jobPortInUse", jobPortInUse);
response.put("jobPortInUseBy", jobPortInUseBy);
response.put("safeToInstall", true);
return response;
}
Set ports = portsInUse.keySet();
String serverName = portsInUse.get(hubConfig.getPort(DatabaseKind.STAGING));
stagingPortInUse = ports.contains(hubConfig.getPort(DatabaseKind.STAGING)) && serverName != null && !serverName.equals(hubConfig.getHttpName(DatabaseKind.STAGING));
if (stagingPortInUse) {
stagingPortInUseBy = serverName;
}
serverName = portsInUse.get(hubConfig.getPort(DatabaseKind.FINAL));
finalPortInUse = ports.contains(hubConfig.getPort(DatabaseKind.FINAL)) && serverName != null && !serverName.equals(hubConfig.getHttpName(DatabaseKind.FINAL));
if (finalPortInUse) {
finalPortInUseBy = serverName;
}
serverName = portsInUse.get(hubConfig.getPort(DatabaseKind.JOB));
jobPortInUse = ports.contains(hubConfig.getPort(DatabaseKind.JOB)) && serverName != null && !serverName.equalsIgnoreCase(hubConfig.getHttpName(DatabaseKind.JOB));
if (jobPortInUse) {
jobPortInUseBy = serverName;
}
serverVersion = new MarkLogicVersion(hubConfig.getManageClient()).getVersionString();
serverVersionOk = isServerVersionValid(serverVersion);
Map response = new HashMap<>();
response.put("serverVersion", serverVersion);
response.put("serverVersionOk", serverVersionOk);
response.put("stagingPortInUse", stagingPortInUse);
response.put("stagingPortInUseBy", stagingPortInUseBy);
response.put("finalPortInUse", finalPortInUse);
response.put("finalPortInUseBy", finalPortInUseBy);
response.put("jobPortInUse", jobPortInUse);
response.put("jobPortInUseBy", jobPortInUseBy);
response.put("safeToInstall", isSafeToInstall());
return response;
}
/**
* Installs the data hub configuration and server-side config files into MarkLogic
*/
@Override
public void install() {
if (!hubConfig.getHubProject().isInitialized()) {
initProject();
}
logger.warn("Installing the Data Hub into MarkLogic");
AppConfig appConfig = hubConfig.getAppConfig();
disableSomeCmaUsage(appConfig);
// When setting up the test application, there's a chance of having duplicate modules due to modules existing in
// multiple places on the classpath. For the purpose of loading DH modules, this can be avoided by setting the
// batch size for loading modules to 1.
appConfig.setModulesLoaderBatchSize(1);
// in AWS setting this fails...
// for now putting in try/catch
try {
SimpleAppDeployer roleDeployer = new SimpleAppDeployer(hubConfig.getManageClient(), hubConfig.getAdminManager());
roleDeployer.setCommands(getSecurityCommandList());
roleDeployer.deploy(appConfig);
} catch (HttpServerErrorException e) {
if (e.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
logger.warn("No manage client for security installs. Assuming DHS provisioning already there");
} else {
throw new DataHubConfigurationException(e);
}
}
SimpleAppDeployer deployer = new SimpleAppDeployer(hubConfig.getManageClient(), hubConfig.getAdminManager());
deployer.setCommands(buildListOfCommands());
deployer.deploy(appConfig);
}
/**
* Turns off CMA for some resources that have bbugs in ML 9.0-7/8.
*
* @param appConfig
*/
protected static void disableSomeCmaUsage(AppConfig appConfig) {
appConfig.getCmaConfig().setCombineRequests(false);
appConfig.getCmaConfig().setDeployDatabases(false);
appConfig.getCmaConfig().setDeployRoles(false);
appConfig.getCmaConfig().setDeployUsers(false);
}
/**
* Note that this differs from how "mlUpdateIndexes" works in ml-gradle. This is not stripping out any "non-index"
* properties from each payload - it's just updating every database.
*
* This does however disable forest creation which speeds up the process so that the only calls made are to
* update the databases.
*/
@Override
public void updateIndexes() {
// First deploy protected paths (can add more resources here in the future)
AppConfig appConfig = hubConfig.getAppConfig();
new SimpleAppDeployer(getManageClient(), getAdminManager(), new DeployProtectedPathsCommand()).deploy(appConfig);
// Then deploy databases, utilizing a pattern for filenames when in a provisioned environment
SimpleAppDeployer deployer = new SimpleAppDeployer(getManageClient(), getAdminManager());
Map> commandMap = buildCommandMap();
List indexRelatedCommands = new ArrayList<>(commandMap.get("mlDatabaseCommands"));
deployer.setCommands(indexRelatedCommands);
final boolean originalCreateForests = appConfig.isCreateForests();
final Pattern originalIncludePattern = appConfig.getResourceFilenamesIncludePattern();
try {
appConfig.setCreateForests(false);
if (hubConfig.getIsProvisionedEnvironment()) {
appConfig.setResourceFilenamesIncludePattern(buildPatternForDatabasesToUpdateIndexesFor());
}
deployer.deploy(appConfig);
} finally {
appConfig.setCreateForests(originalCreateForests);
appConfig.setResourceFilenamesIncludePattern(originalIncludePattern);
}
}
/**
* In a provisioned environment, only the databases defined by this pattern can be updated.
*
* @return database name pattern
*/
protected static Pattern buildPatternForDatabasesToUpdateIndexesFor() {
return Pattern.compile("(staging|final|job)-database.json");
}
private void runInDatabase(String query, String databaseName) {
ServerEvaluationCall eval = hubConfig.newModulesDbClient().newServerEval();
String xqy =
"xdmp:invoke-function(function() {" +
query +
"}," +
"" +
" {xdmp:database(\"" + databaseName + "\")} " +
" update-auto-commit " +
" )";
eval.xquery(xqy).eval().close();
}
/**
*
* @return a list of commands that equate to what a full deployment will run, minus any commands that can write to
* any of the 8 DHF databases (staging, final, jobs, modules, and then the 4 triggers and schemas databases)
*/
public List buildCommandsForDeployingToReplica() {
Map> commandMap = new CommandMapBuilder().buildCommandMapForReplicaCluster();
applyDataHubChangesToCommands(commandMap, true);
return commandMap.values().stream().reduce(new ArrayList<>(), (a, b) -> { a.addAll(b); return a;});
}
/**
*
* @return a map of command groups. The initial map is based on what ml-app-deployer constructs, and then modifications
* are made to many of the groups based on DHF requirements.
*/
public Map> buildCommandMap() {
Map> commandMap = new CommandMapBuilder().buildCommandMap();
applyDataHubChangesToCommands(commandMap, false);
return commandMap;
}
/**
*
* @param commandMap
* @param isDeployingToReplica if true, then certain changes won't be made because they involve commands that
* write to a database, which isn't allowed when deploying to a replica cluster
*/
private void applyDataHubChangesToCommands(Map> commandMap, boolean isDeployingToReplica) {
updateSecurityCommandList(commandMap);
updateDatabaseCommandList(commandMap);
updateServerCommandList(commandMap);
// DHF has no use case for the "deploy REST API server" commands provided by ml-gradle
commandMap.remove("mlRestApiCommands");
// DHF has a custom property named "mlCustomForestPath" that has to be set on this command.
List forestCommands = commandMap.get("mlForestCommands");
DeployCustomForestsCommand deployCustomForestsCommand = (DeployCustomForestsCommand) forestCommands.get(0);
deployCustomForestsCommand.setCustomForestsPath(hubConfig.getCustomForestPath());
List granularPrivilegeCommands = new ArrayList<>();
granularPrivilegeCommands.add(new CreateGranularPrivilegesCommand(hubConfig));
commandMap.put("hubGranularPrivilegeCommands", granularPrivilegeCommands);
if (!isDeployingToReplica) {
updateTriggersCommandList(commandMap);
updateModuleCommandList(commandMap);
List finishHubDeploymentCommands = new ArrayList<>();
finishHubDeploymentCommands.add(new FinishHubDeploymentCommand(hubConfig));
commandMap.put("finishHubDeploymentCommands", finishHubDeploymentCommands);
}
}
/**
* DeployAmpsCommand should run before we run LoadHubModulesCommand as we require the amp for getting the default
* rewriter to be present so that it can be used to generate the custom rewriter.
*
* @param commandMap
*/
private static void updateSecurityCommandList(Map> commandMap) {
for (Command c : commandMap.get("mlSecurityCommands")) {
if (c instanceof DeployAmpsCommand) {
((DeployAmpsCommand) c).setExecuteSortOrder(new LoadHubModulesCommand().getExecuteSortOrder() - 1);
}
}
}
/**
* DHF doesn't need the default commands for deploying a specific content/triggers/schemas database. It does want to
* preserve any other commands, with the one addition being that it needs to modify DeployOtherDatabaseCommand so
* that a custom DeployDatabaseCommand implementation is used.
*
* @param commandMap
*/
private void updateDatabaseCommandList(Map> commandMap) {
List dbCommands = new ArrayList<>();
for (Command c : commandMap.get("mlDatabaseCommands")) {
dbCommands.add(c);
if (c instanceof DeployOtherDatabasesCommand) {
((DeployOtherDatabasesCommand)c).setDeployDatabaseCommandFactory(new HubDeployDatabaseCommandFactory(hubConfig));
}
}
// This ensures that this command is run when mlDeployDatabases is run
dbCommands.add(new DeployDatabaseFieldCommand());
commandMap.put("mlDatabaseCommands", dbCommands);
}
private void updateServerCommandList(Map> commandMap) {
final String key = "mlServerCommands";
List newCommands = new ArrayList<>();
for (Command c : commandMap.get(key)) {
/**
* DHF doesn't need the "Update REST API" command that ml-gradle includes because DHF isn't using ml-gradle's support
* for a default REST API server.
*/
if (c instanceof UpdateRestApiServersCommand) {
continue;
}
newCommands.add(c);
}
newCommands.add(new ConfigureAppServerBasePaths(hubConfig));
commandMap.put(key, newCommands);
}
/**
* The existing "DeployTriggersCommand" is based on the ml-config path and the AppConfig object should set the default
* triggers database name to that of the final triggers database. Thus, we just need to add a hub-specific command for
* loading staging triggers into the staging triggers database.
*
*/
private void updateTriggersCommandList(Map> commandMap) {
List commands = commandMap.get("mlTriggerCommands");
commands.add(new DeployHubTriggersCommand(hubConfig, hubConfig.getStagingTriggersDbName()));
}
/**
* This affects what mlLoadModules does. We want it to load all modules, including hub modules. This supports a
* scenario where a user may clear her modules database; mlLoadModules should then load everything in.
*
* @param commandsMap
*/
private void updateModuleCommandList(Map> commandsMap) {
List commands = new ArrayList<>();
commands.add(loadHubModulesCommand);
commands.add(loadUserModulesCommand);
commands.add(loadUserArtifactsCommand);
commands.add(loadHubArtifactsCommand);
commands.add(generateFunctionMetadataCommand);
for (Command c : commandsMap.get("mlModuleCommands")) {
if (c instanceof LoadModulesCommand) {
// Don't want this, since our custom command above extends LoadModulesCommand
continue;
}
if (c instanceof DeleteTestModulesCommand) {
// Make sure this runs after our custom command for loading modules
((DeleteTestModulesCommand) c).setExecuteSortOrder(loadUserModulesCommand.getExecuteSortOrder() + 1);
}
commands.add(c);
}
commandsMap.put("mlModuleCommands", commands);
}
private static Map getServerPortsInUse(ServerManager serverManager) {
Map portsInUse = new HashMap<>();
ResourcesFragment srf = serverManager.getAsXml();
srf.getListItemNameRefs().forEach(s -> {
Fragment fragment = serverManager.getPropertiesAsXml(s);
int port = Integer.parseInt(fragment.getElementValue("//m:port"));
portsInUse.put(port, s);
});
return portsInUse;
}
private Map> getSecurityCommands() {
Map> commandMap = new HashMap<>();
List securityCommands = new ArrayList<>();
securityCommands.add(new CheckSecurityConfiguration(this.getHubConfig()));
securityCommands.add(new DeployRolesCommand());
securityCommands.add(new DeployUsersCommand());
securityCommands.add(new DeployCertificateTemplatesCommand());
securityCommands.add(new DeployCertificateAuthoritiesCommand());
securityCommands.add(new InsertCertificateHostsTemplateCommand());
securityCommands.add(new DeployExternalSecurityCommand());
securityCommands.add(new DeployPrivilegesCommand());
securityCommands.add(new DeployProtectedCollectionsCommand());
securityCommands.add(new DeployProtectedPathsCommand());
securityCommands.add(new DeployQueryRolesetsCommand());
commandMap.put("mlSecurityCommands", securityCommands);
return commandMap;
}
// Here is the former PreCheckInstall class stuff
// We should probably move this into a sub class OR its own class and interface, and create a super at the
// datahub level
private boolean stagingPortInUse;
private String stagingPortInUseBy;
private boolean finalPortInUse;
private String finalPortInUseBy;
private boolean jobPortInUse;
private String jobPortInUseBy;
private boolean serverVersionOk;
private String serverVersion;
@Override
public boolean isSafeToInstall() {
return !(isPortInUse(DatabaseKind.FINAL) ||
isPortInUse(DatabaseKind.STAGING) ||
isPortInUse(DatabaseKind.JOB)) && isServerVersionOk();
}
@Override
public boolean isPortInUse(DatabaseKind kind) {
boolean inUse;
switch (kind) {
case STAGING:
inUse = stagingPortInUse;
break;
case FINAL:
inUse = finalPortInUse;
break;
case JOB:
inUse = jobPortInUse;
break;
default:
throw new InvalidDBOperationError(kind, "check for port use");
}
return inUse;
}
@Override
public String getPortInUseBy(DatabaseKind kind) {
String inUseBy;
switch (kind) {
case STAGING:
inUseBy = stagingPortInUseBy;
break;
case FINAL:
inUseBy = finalPortInUseBy;
break;
case JOB:
inUseBy = jobPortInUseBy;
break;
default:
throw new InvalidDBOperationError(kind, "check if port is in use");
}
return inUseBy;
}
@Override
public boolean isServerVersionOk() {
return serverVersionOk;
}
@Override
public String getServerVersion() {
if(serverVersion == null) {
serverVersion = new MarkLogicVersion(hubConfig.getManageClient()).getVersionString();
}
return serverVersion;
}
@Override
public boolean upgradeHub() {
boolean isHubInstalled;
try {
isHubInstalled = this.isInstalled().isInstalled();
} catch (ResourceAccessException e) {
isHubInstalled = false;
}
if (isHubInstalled) {
final String minUpgradeVersion = "4.3.0";
final String installedVersion = versions.getInstalledVersion();
// Warn is used so this appears when using Gradle without "-i"
logger.warn("Currently installed DHF version: " + installedVersion);
if (Versions.compare(installedVersion, minUpgradeVersion) == -1) {
throw new RuntimeException("Cannot upgrade installed Data Hub; its version is " + installedVersion + ", and must be at least version " + minUpgradeVersion + " or higher");
}
}
verifyLocalProjectIs430OrGreater();
boolean result = false;
try {
if (hubConfig.getHubProject().isInitialized()) {
prepareProjectBeforeUpgrading(hubConfig.getHubProject(), hubConfig.getJarVersion());
hubConfig.getHubSecurityDir().resolve("roles").resolve("flow-operator.json").toFile().delete();
}
hubConfig.initHubProject();
hubConfig.getHubProject().upgradeProject(flowManager);
System.out.println("Starting in version 5.2.0, the default value of mlModulePermissions has been changed to \"data-hub-module-reader,read,data-hub-module-reader,execute,data-hub-module-writer,update,rest-extension-user,execute\". " +
"It is recommended to remove this property from gradle.properties unless you must customize the value." );
result = true;
} catch (IOException e) {
logger.error("Unable to upgrade project, cause: " + e.getMessage(), e);
}
return result;
}
/**
* Per DHFPROD-4912, instead of depending on mlDHFVersion, this logic takes advantage of the fact that the
* hub-internal-config triggers - including ml-dh-entity-create.json - had their permissions modified in the 4.3.0
* release.
*/
protected void verifyLocalProjectIs430OrGreater() {
File triggersDir = hubConfig.getHubProject().getHubTriggersDir().toFile();
if (!triggersDir.exists()) {
// If the internal triggers dir doesn't exist, then the project hasn't been initialized yet. That means
// we aren't trying to upgrade a pre-4.3.0 project, so the "upgrade" is safe to proceed.
return;
}
File triggerFile = new File(triggersDir, "ml-dh-entity-create.json");
boolean canBeUpgraded = true;
try {
JsonNode trigger = new ObjectMapper().readTree(triggerFile);
final String roleName = trigger.get("permission").get(0).get("role-name").asText();
if ("%%mlHubAdminRole%%".equals(roleName) || "%%mlHubUserRole%%".equals(roleName)) {
canBeUpgraded = false;
}
} catch (Exception ex) {
throw new RuntimeException("Unable to upgrade project; while trying to verify that the local project is of " +
"version 4.3.0 or greater, was unable to read JSON from ml-dh-entity-create.json file; cause: " + ex.getMessage(), ex);
}
if (!canBeUpgraded) {
throw new RuntimeException("Unable to upgrade current project, as its version is less than 4.3.0. Please " +
"first upgrade this project to 4.3.0. Consult the Data Hub documentation on performing this upgrade.");
}
}
/**
* The expectation is that a user has upgraded build.gradle to use a newer version of DHF but has not yet updated
* mlDHFVersion in gradle.properties. Thus, the value of mlDHFVersion is expected to be passed in here so that the
* backup path of hub-internal-config has the current version of DHF in its name.
*
* @param hubProject
* @param newDataHubVersion
* @throws IOException
*/
protected void prepareProjectBeforeUpgrading(HubProject hubProject, String newDataHubVersion) throws IOException {
final String backupPath = HubProject.HUB_CONFIG_DIR + "-pre-" + newDataHubVersion;
FileUtils.moveDirectory(hubProject.getHubConfigDir().toFile(), hubProject.getProjectDir().resolve(backupPath).toFile());
logger.warn("The " + HubProject.HUB_CONFIG_DIR + " directory has been moved to " + backupPath + " so that it can be re-initialized using the new version of Data Hub");
}
// only used in test
public void setHubConfig(HubConfigImpl hubConfig) {
this.hubConfig = hubConfig;
if (this.loadUserModulesCommand != null) {
this.loadUserModulesCommand.setHubConfig(hubConfig);
}
if (this.loadHubModulesCommand != null) {
this.loadHubModulesCommand.setHubConfig(hubConfig);
}
if (this.loadHubArtifactsCommand != null) {
this.loadHubArtifactsCommand.setHubConfig(hubConfig);
}
if (this.loadUserArtifactsCommand != null) {
this.loadUserArtifactsCommand.setHubConfig(hubConfig);
}
if (this.generateFunctionMetadataCommand != null) {
this.generateFunctionMetadataCommand.setHubConfig(hubConfig);
}
}
// only used in test
public HubConfig getHubConfig() {
return this.hubConfig;
}
// only used in test
public void setVersions(Versions versions) {
this.versions = versions;
}
/**
* Clear "user data", which is anything that's not a user or hub artifact. Depends on the ability to clear the
* staging, final, and job databases. An error will be thrown if a user doesn't have the privilege to perform any
* of those operations.
*
* This is intentionally not exposed in the DataHub interface, as there's no use case yet for a client of this
* library to perform this operation. It can instead be accessed via Hub Central and Gradle.
*/
public void clearUserData() {
clearUserData(null, null);
}
public void clearUserData(String targetDatabase, String sourceCollection) {
final HubClient hubClientToUse = hubClient != null ? hubClient : hubConfig.newHubClient();
long start = System.currentTimeMillis();
logger.info("Clearing user data as user: " + hubClientToUse.getUsername());
final List userAndHubArtifacts = readUserAndHubArtifacts(hubClientToUse);
logger.info("Count of user and hub artifacts read into memory: " + userAndHubArtifacts.size());
final DatabaseManager databaseManager = new DatabaseManager(hubClientToUse.getManageClient());
if (StringUtils.isEmpty(targetDatabase)) {
// If we clear a database Jobs is cleared first; in case this fails, there's no chance of staging/final having been cleared and their
// artifacts not being reloaded
clearDatabase(databaseManager, hubClientToUse.getDbName(DatabaseKind.JOB));
}
final String stagingDbName = hubClientToUse.getDbName(DatabaseKind.STAGING);
if (StringUtils.isEmpty(targetDatabase) || targetDatabase.equals(stagingDbName)) {
try {
if (StringUtils.isEmpty(sourceCollection)) {
clearDatabase(databaseManager, stagingDbName);
} else {
clearDatabaseCollection(sourceCollection, hubClientToUse.getStagingClient());
}
} finally {
// Still attempt to write artifacts in case the ML error still resulted in the database (or most of it) being cleared
writeUserAndHubArtifacts(hubClientToUse.getStagingClient().newJSONDocumentManager(), userAndHubArtifacts, stagingDbName);
}
}
final String finalDbName = hubClientToUse.getDbName(DatabaseKind.FINAL);
if (StringUtils.isEmpty(targetDatabase) || targetDatabase.equals(finalDbName)) {
try {
if (StringUtils.isEmpty(sourceCollection)) {
clearDatabase(databaseManager, finalDbName);
} else {
clearDatabaseCollection(sourceCollection, hubClientToUse.getFinalClient());
}
} finally {
// Still attempt to write artifacts in case the ML error still resulted in the database (or most of it) being cleared
writeUserAndHubArtifacts(hubClientToUse.getFinalClient().newJSONDocumentManager(), userAndHubArtifacts, finalDbName);
}
}
logger.info("Finished clearing user data; time elapsed: " + (System.currentTimeMillis() - start));
}
private static void clearDatabaseCollection(String sourceName, DatabaseClient databaseClientToUse) {
QueryManager qm = databaseClientToUse.newQueryManager();
DeleteQueryDefinition def=qm.newDeleteDefinition();
def.setCollections(sourceName);
qm.delete(def);
}
private void clearDatabase(DatabaseManager databaseManager, String databaseName) {
long start = System.currentTimeMillis();
logger.info("Clearing database: " + databaseName);
final boolean catchException = false;
databaseManager.clearDatabase(databaseName, catchException);
logger.info("Finished clearing database: " + databaseName + "; time elapsed: " + (System.currentTimeMillis() - start));
}
public void clearUserSchemas() {
final HubClient hubClientToUse = hubClient != null ? hubClient : hubConfig.newHubClient();
long start = System.currentTimeMillis();
logger.info("Clearing user schemas as user: " + hubClientToUse.getUsername());
String xquery = "cts:not-query(" +
"cts:collection-query((" +
"'http://marklogic.com/xdmp/temporal/axis', " +
"'http://marklogic.com/xdmp/temporal/collection', 'http://marklogic.com/xdmp/view'" +
"))" +
")";
String fullQuery = "cts:uris((), (), " + xquery + ") ! xdmp:document-delete(.)";
DatabaseClient stagingSchemasClient = hubConfig.newStagingClient(hubConfig.getDbName(DatabaseKind.STAGING_SCHEMAS));
DatabaseClient finalSchemasClient = hubConfig.newFinalClient(hubConfig.getDbName(DatabaseKind.FINAL_SCHEMAS));
Stream.of(stagingSchemasClient, finalSchemasClient).forEach(databaseClient -> {
try {
logger.info("Deleting user schemas in database '" + databaseClient.getDatabase() + "' via : " + fullQuery);
databaseClient.newServerEval().xquery(fullQuery).eval().close();
}
finally {
databaseClient.release();
}
});
logger.info("Finished clearing user schemas; time elapsed: " + (System.currentTimeMillis() - start));
}
/**
* Uses the REST API to determine the URIs of the artifacts that need to be read into memory, as the REST API will
* also be used to actually read the documents and their metadata into memory.
*
* @param hubClientToUse
* @return
*/
private static List readUserAndHubArtifacts(HubClient hubClientToUse) {
List docs = new ArrayList<>();
JSONDocumentManager mgr = hubClientToUse.getStagingClient().newJSONDocumentManager();
final String script = "import consts from '/data-hub/5/impl/consts.mjs';\n" +
"cts.uris(null, null, " +
" cts.collectionQuery(consts.USER_ARTIFACT_COLLECTIONS.concat(consts.HUB_ARTIFACT_COLLECTION).concat('http://marklogic.com/data-hub/mappings'))" +
")";
EvalResultIterator resultIterator = hubClientToUse.getStagingClient().newServerEval().javascript(script).eval();
resultIterator.iterator().forEachRemaining(item -> {
final String uri = item.getString();
DocumentMetadataHandle metadata = new DocumentMetadataHandle();
JacksonHandle content = new JacksonHandle();
mgr.read(uri, metadata, content);
docs.add(new DocumentWriteOperationImpl(DocumentWriteOperation.OperationType.DOCUMENT_WRITE, uri, metadata, content));
});
resultIterator.close();
return docs;
}
private void writeUserAndHubArtifacts(JSONDocumentManager mgr, List userAndHubArtifacts, String databaseName) {
DocumentWriteSet writeSet = mgr.newWriteSet();
writeSet.addAll(userAndHubArtifacts);
logger.info("Writing user and hub artifacts to " + databaseName + "; count: " + writeSet.size());
mgr.write(writeSet);
logger.info("Finished writing user and hub artifacts to " + databaseName);
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy