org.apache.jackrabbit.vault.fs.io.Importer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of aem-sdk-api Show documentation
Show all versions of aem-sdk-api Show documentation
The Adobe Experience Manager SDK
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.jackrabbit.vault.fs.io;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import javax.jcr.Node;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.security.AccessControlPolicy;
import javax.jcr.version.Version;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.api.security.authorization.PrincipalSetPolicy;
import org.apache.jackrabbit.spi.commons.namespace.NamespaceMapping;
import org.apache.jackrabbit.spi.commons.namespace.NamespaceResolver;
import org.apache.jackrabbit.spi.commons.namespace.SessionNamespaceResolver;
import org.apache.jackrabbit.util.Text;
import org.apache.jackrabbit.vault.fs.api.Artifact;
import org.apache.jackrabbit.vault.fs.api.ArtifactType;
import org.apache.jackrabbit.vault.fs.api.IdConflictPolicy;
import org.apache.jackrabbit.vault.fs.api.ImportInfo;
import org.apache.jackrabbit.vault.fs.api.ImportMode;
import org.apache.jackrabbit.vault.fs.api.NodeNameList;
import org.apache.jackrabbit.vault.fs.api.PathFilterSet;
import org.apache.jackrabbit.vault.fs.api.PathMapping;
import org.apache.jackrabbit.vault.fs.api.ProgressTrackerListener;
import org.apache.jackrabbit.vault.fs.api.SerializationType;
import org.apache.jackrabbit.vault.fs.api.VaultInputSource;
import org.apache.jackrabbit.vault.fs.api.WorkspaceFilter;
import org.apache.jackrabbit.vault.fs.config.ConfigurationException;
import org.apache.jackrabbit.vault.fs.config.DefaultWorkspaceFilter;
import org.apache.jackrabbit.vault.fs.config.MetaInf;
import org.apache.jackrabbit.vault.fs.config.VaultSettings;
import org.apache.jackrabbit.vault.fs.impl.ArtifactSetImpl;
import org.apache.jackrabbit.vault.fs.impl.DirectoryArtifact;
import org.apache.jackrabbit.vault.fs.impl.HintArtifact;
import org.apache.jackrabbit.vault.fs.impl.io.FileArtifactHandler;
import org.apache.jackrabbit.vault.fs.impl.io.FolderArtifactHandler;
import org.apache.jackrabbit.vault.fs.impl.io.GenericArtifactHandler;
import org.apache.jackrabbit.vault.fs.impl.io.ImportInfoImpl;
import org.apache.jackrabbit.vault.fs.impl.io.InputSourceArtifact;
import org.apache.jackrabbit.vault.fs.spi.ACLManagement;
import org.apache.jackrabbit.vault.fs.spi.CNDReader;
import org.apache.jackrabbit.vault.fs.spi.DefaultNodeTypeSet;
import org.apache.jackrabbit.vault.fs.spi.NodeTypeInstaller;
import org.apache.jackrabbit.vault.fs.spi.NodeTypeSet;
import org.apache.jackrabbit.vault.fs.spi.PrivilegeDefinitions;
import org.apache.jackrabbit.vault.fs.spi.PrivilegeInstaller;
import org.apache.jackrabbit.vault.fs.spi.ProgressTracker;
import org.apache.jackrabbit.vault.fs.spi.ServiceProviderFactory;
import org.apache.jackrabbit.vault.fs.spi.UserManagement;
import org.apache.jackrabbit.vault.fs.spi.impl.jcr20.accesscontrol.JackrabbitAccessControlPolicy;
import org.apache.jackrabbit.vault.packaging.PackageException;
import org.apache.jackrabbit.vault.packaging.impl.ActivityLog;
import org.apache.jackrabbit.vault.packaging.registry.impl.JcrPackageRegistry;
import org.apache.jackrabbit.vault.util.Constants;
import org.apache.jackrabbit.vault.util.PlatformNameFormat;
import org.apache.jackrabbit.vault.util.Tree;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Imports an {@link Archive} into a repository.
*
* file/directory combinations
*
* - plain file
*
* + foo
* - test.gif
*
* - plain files + special folder
*
* + foo
* - .content.xml
* + bar
* - test.gif
*
* - special file
*
* + foo
* - test.gif
* - test.gif.dir
* - .content.xml
*
* - special file + sub files
*
* + foo
* - test.gif
* - test.gif.dir
* - .content.xml
* + _jcr_content
* - thumbnail.gif
*
* - special file + sub special files
*
* + foo
* - test.gif
* - test.gif.dir
* - .content.xml
* + _jcr_content
* - thumbnail.gif
* + thumbnail.gif.dir
* - .content.xml
*
* - file/folder structure
*
* + foo
* + en
* - .content.xml
* + _cq_content
* - thumbnail.gif
* + company
* - .content.xml
*
*
*/
public class Importer {
/**
* default logger
*/
private static final Logger log = LoggerFactory.getLogger(Importer.class);
/**
* Activity Logger to track node level updates
*/
private static final Logger activityLog = LoggerFactory.getLogger(ActivityLog.class);
// not exported.
static final String FEATURE_STASH_PRINCIPAL_POLICIES = "vault.feature.stashPrincipalPolicies";
/**
* workspace filter to use during import
*/
private WorkspaceFilter filter;
/**
* tree constructed of the filter roots
*/
private final Tree filterTree = new Tree();
/**
* tracker to use for tracking messages
*/
private ProgressTracker tracker;
/**
* the node types detected during the prepare phase that needed to by registered
*/
private final DefaultNodeTypeSet nodeTypes = new DefaultNodeTypeSet("internal");
/**
* autosave structure that tracks the number of modified nodes
*/
private AutoSave autoSave = new AutoSave();
/**
* set of paths to versionable nodes that need to be checked in after import
*/
private final Set nodesToCheckin = new HashSet<>();
/**
* map of group memberships that need to be applied after import
*/
private final Map memberships = new HashMap<>();
private Map> deletedPrincipalAcls = new HashMap<>();
private List createdAuthorizableIds = new LinkedList<>();
/**
* general flag that indicates if the import had (recoverable) errors
*/
private boolean hasErrors = false;
/** If {@link #hasErrors} = {@code true} this one contains the first exception during package import */
private Exception firstException = null;
/**
* overall handler for importing folder artifacts
*/
private final FolderArtifactHandler folderHandler = new FolderArtifactHandler();
/**
* overall handler for importing generic artifacts
*/
private final GenericArtifactHandler genericHandler = new GenericArtifactHandler();
/**
* overall handler for importing file artifacts
*/
private final FileArtifactHandler fileHandler = new FileArtifactHandler();
/**
* list of archive entries that are detected as lowlevel patches and need to be copied to the
* filesystem after import.
*/
private final List patches = new LinkedList();
private Map intermediates = new LinkedHashMap();
private Archive archive;
/**
* list of paths to subpackages that were detected during the prepare phase
*/
private final List subPackages = new LinkedList();
/**
* overall acl management behavior
*/
private final ACLManagement aclManagement = ServiceProviderFactory.getProvider().getACLManagement();
/**
* overall user management behavior
*/
private final UserManagement userManagement = ServiceProviderFactory.getProvider().getUserManagement();
/**
* the import options
*/
private final ImportOptions opts;
/**
* the checkpoint state of the autosave. used for recovering from stale item errors during install.
*/
private AutoSave cpAutosave;
/**
* the checkpoint tx info. used for recovering from stale item errors during install.
*/
private TxInfo cpTxInfo;
/**
* the checkpoint import info.
*/
private ImportInfoImpl cpImportInfo;
/**
* retry counter for the batch auto recovery
*/
private int recoveryRetryCounter;
/**
* list of processed tx infos since the last auto save.
*/
private final List processedInfos = new ArrayList();
/**
* list of intermediate infos that were removed since the last auto save
*/
private Map removedIntermediates = new LinkedHashMap<>();
private final boolean isStrict;
private final boolean isStrictByDefault;
private final boolean overwritePrimaryTypesOfFoldersByDefault;
/**
* JCRVLT-683 feature flag. This variable is used to enable the new behavior of stashing principal policies when an
* Archive's package properties do not specify a {@code vault.feature.stashPrincipalPolicies} property.
* Initially false by default, with the ability to set a global override using the key as a System property.
* This default should be changed to true in a future release after sufficient testing across the ecosystem.
*/
private final boolean stashPrincipalPoliciesByDefault = Boolean.getBoolean(FEATURE_STASH_PRINCIPAL_POLICIES);
/**
* Default constructor neither setting specific import options nor defaults.
*/
public Importer() {
this(new ImportOptions(), false, true);
}
/**
* Constructor which is not setting default options.
* @param opts
* @see #Importer(ImportOptions, boolean, boolean, IdConflictPolicy)
*/
public Importer(ImportOptions opts) {
this(opts, false);
}
/**
* Shortcut for {@link Importer#Importer(ImportOptions, boolean, boolean, IdConflictPolicy)} with no default id conflict policy.
* Also primary types of existing nodes are always overwritten.
* @param opts the import options to use during {@link #run(Archive, Node)} or {@link #run(Archive, Session, String)}
* @param isStrictByDefault is true if packages should be installed in strict mode by default (if not set otherwise in {@code opts})
*/
public Importer(ImportOptions opts, boolean isStrictByDefault) {
this(opts, isStrictByDefault, true);
}
/**
* Shortcut for {@link Importer#Importer(ImportOptions, boolean, boolean, IdConflictPolicy)} with no default id conflict policy.
* @param opts the import options to use during {@link #run(Archive, Node)} or {@link #run(Archive, Session, String)}
* @param isStrictByDefault is true if packages should be installed in strict mode by default (if not set otherwise in {@code opts})
* @param overwritePrimaryTypesOfFoldersByDefault if folder aggregates' JCR primary type should be changed if the node is already existing or not
*/
public Importer(ImportOptions opts, boolean isStrictByDefault, boolean overwritePrimaryTypesOfFoldersByDefault) {
this(opts, isStrictByDefault, overwritePrimaryTypesOfFoldersByDefault, null);
}
/**
* Constructor setting both specific import options as well as some defaults for options not set.
* @param opts the import options to use during {@link #run(Archive, Node)} or {@link #run(Archive, Session, String)}
* @param isStrictByDefault is true if packages should be installed in strict mode by default (if not set otherwise in {@code opts})
* @param overwritePrimaryTypesOfFoldersByDefault if folder aggregates' JCR primary type should be changed if the node is already existing or not
* @param defaultIdConflictPolicy the default {@link IdConflictPolicy} to use if no policy is set in {@code opts}. May be {@code null}.
*/
public Importer(ImportOptions opts, boolean isStrictByDefault, boolean overwritePrimaryTypesOfFoldersByDefault, IdConflictPolicy defaultIdConflictPolicy) {
this.opts = opts;
this.isStrict = opts.isStrict(isStrictByDefault);
this.isStrictByDefault = isStrictByDefault;
this.overwritePrimaryTypesOfFoldersByDefault = overwritePrimaryTypesOfFoldersByDefault;
if (!this.opts.hasIdConflictPolicyBeenSet() && defaultIdConflictPolicy != null) {
this.opts.setIdConflictPolicy(defaultIdConflictPolicy);
}
}
public ImportOptions getOptions() {
return opts;
}
public List getSubPackages() {
return subPackages;
}
public boolean isStrictByDefault() {
return isStrictByDefault;
}
/**
* Debug settings to allows to produce failures after each {@code failAfterEach} save.
* @param failAfterEach cardinal indicating when to fail
*/
public void setDebugFailAfterSave(int failAfterEach) {
autoSave.setDebugFailEach(failAfterEach);
}
protected void track(String action, String path) {
if ("E".equals(action)) {
log.error("{} {}", action, path);
} else {
log.debug("{} {}", action, path);
}
activityLog.debug("{} {}", action, path);
if (tracker != null) {
tracker.track(action, path);
}
}
protected void track(Exception e, String path) {
log.error("E {} ({})", path, e.toString());
if (tracker != null) {
tracker.track(e, path);
}
}
/**
* Runs the importer from the given root node.
* Shortcut for {@link #run(Archive, Session, String)} with the session and path from the given node.
* @param archive the archive to import
* @param importRoot the root node to import
*
* @throws org.apache.jackrabbit.vault.fs.config.ConfigurationException if the importer is not properly configured
* @throws java.io.IOException if an I/O error occurs
* @throws javax.jcr.RepositoryException if an repository error occurs
*
* @since 2.3.20
*/
public void run(Archive archive, Node importRoot)
throws IOException, RepositoryException, ConfigurationException {
run(archive, importRoot.getSession(), importRoot.getPath());
}
/**
* Runs the importer with the given session.
* {@link Session#save()} and potentially {@link Session#refresh(boolean)} are automatically called during the import
* in the given session, except when {@link ImportOptions#setAutoSaveThreshold(int)} with value {@link Integer#MAX_VALUE} has been set.
* In all other cases the changes are automatically persisted (potentially in batches) potentially after advanced retry mechanisms.
*
* @param archive the archive to import
* @param session the session importing the archive
* @param parentPath the repository parent path where the archive will be imported
* @throws IOException if an I/O error occurs
* @throws RepositoryException if a repository error occurs
* @throws ConfigurationException if the importer is not properly configured
*
* @since 2.7.0
*/
public void run(Archive archive, Session session, String parentPath)
throws IOException, RepositoryException, ConfigurationException {
this.archive = archive;
// init tracker
if (opts.getListener() == null) {
tracker = null;
} else {
if (tracker == null) {
tracker = new ProgressTracker();
}
tracker.setListener(opts.getListener());
}
// check format version
int version = archive.getMetaInf().getPackageFormatVersion();
if (version > MetaInf.FORMAT_VERSION_2) {
String msg = "Content format version not supported (" + version + " > " + MetaInf.FORMAT_VERSION_2 + ")";
log.warn(msg);
throw new IOException(msg);
}
// init autosave
if (opts.getAutoSaveThreshold() >= 0) {
autoSave.setThreshold(opts.getAutoSaveThreshold());
}
autoSave.setDryRun(opts.isDryRun());
autoSave.setTracker(tracker);
// enable this to test auto-recovery of batch saves
// autoSave.setDebugFailEach(1);
// autoSave.setThreshold(4);
// propagate access control handling
if (opts.getAccessControlHandling() == null) {
opts.setAccessControlHandling(AccessControlHandling.IGNORE);
}
fileHandler.setAcHandling(opts.getAccessControlHandling());
fileHandler.setCugHandling(opts.getCugHandling());
genericHandler.setAcHandling(opts.getAccessControlHandling());
genericHandler.setCugHandling(opts.getCugHandling());
folderHandler.setAcHandling(opts.getAccessControlHandling());
folderHandler.setCugHandling(opts.getCugHandling());
folderHandler.setOverwritePrimaryTypesOfFolders(opts.overwritePrimaryTypesOfFolders(overwritePrimaryTypesOfFoldersByDefault));
filter = opts.getFilter();
if (filter == null) {
filter = archive.getMetaInf().getFilter();
}
if (filter == null) {
filter = new DefaultWorkspaceFilter();
}
// check path remapping
PathMapping pathMapping = opts.getPathMapping();
if (pathMapping != null) {
filter = filter.translate(pathMapping);
this.archive = archive = new MappedArchive(archive, pathMapping);
this.archive.open(true);
}
// set import mode if possible
if (opts.getImportMode() != null) {
if (filter instanceof DefaultWorkspaceFilter) {
((DefaultWorkspaceFilter) filter).setImportMode(opts.getImportMode());
} else {
log.warn("Unable to override import mode, incompatible filter: {}", filter.getClass().getName());
}
}
// build filter tree
for (PathFilterSet set: filter.getFilterSets()) {
filterTree.put(set.getRoot(), set);
}
if ("/".equals(parentPath)) {
parentPath = "";
}
track("Collecting import information...", "");
TxInfo root = prepare(archive.getJcrRoot(), parentPath, new SessionNamespaceResolver(session));
if (filter!=null && filter.getFilterSets() != null && filter.getFilterSets().size() > 0 ) {
root = postFilter(root);
}
log.debug("Access control handling set to {}", opts.getAccessControlHandling());
log.debug("CUG handling set to {}", opts.getCugHandling());
if (opts.isDryRun()) {
track("Dry Run: Skipping node types installation (might lead to errors).", "");
track("Simulating content import...", "");
} else {
track("Installing node types...", "");
installNodeTypes(session);
track("Installing privileges...", "");
registerPrivileges(session);
log.debug("Starting content import. autosave is {}", autoSave);
track("Importing content...", "");
if (tracker != null) {
tracker.setMode(ProgressTrackerListener.Mode.PATHS);
}
}
cpAutosave = autoSave.copy();
LinkedList skipList = new LinkedList<>();
while (recoveryRetryCounter++ < 10) {
try {
commit(session, root, skipList);
autoSave.save(session, false);
break;
} catch (RepositoryException e) {
if (autoSave.isDisabled() || recoveryRetryCounter == 10) {
log.error("Error while committing changes. Aborting.");
throw e;
} else {
log.warn("Error while committing changes: Retrying import from checkpoint at {}. Retries {}/10. {}",
cpTxInfo == null ? "/" : cpTxInfo.path, recoveryRetryCounter, getExtendedThrowableMessage(e));
autoSave = cpAutosave.copy();
// build skip list
skipList.clear();
TxInfo info = cpTxInfo;
while (info != null && info.parent != null) {
skipList.addFirst(info);
info = info.parent;
}
// reset any intermediate changes in this run
intermediates.putAll(removedIntermediates);
for (TxInfo i: removedIntermediates.values()) {
i.isIntermediate = 1;
}
removedIntermediates.clear();
processedInfos.clear();
session.refresh(false);
}
}
}
if (tracker != null) {
tracker.setMode(ProgressTrackerListener.Mode.TEXT);
}
restorePrincipalAcls(session, shouldStashPrincipalPoliciesForArchive(archive));
checkinNodes(session);
applyMemberships(session);
applyPatches();
if (opts.isDryRun()) {
if (hasErrors) {
track("Package import simulation finished. (with errors, check logs!)", "");
log.error("There were errors during package install simulation. Please check the logs for details.");
track("First error was " + getExtendedThrowableMessage(firstException), "");
} else {
track("Package import simulation finished.", "");
}
} else {
if (hasErrors) {
track("Package imported (with errors, check logs!)", "");
if (isStrict) {
throw new RepositoryException("Some errors occurred while installing packages. Please check the logs for details. First exception is logged as cause.", firstException);
}
log.error("There were errors during package install. Please check the logs for details.");
track("First error was " + getExtendedThrowableMessage(firstException), "");
} else {
track("Package imported.", "");
}
}
}
// remove featureStashPrincipalPolicies argument before making this method public
private void restorePrincipalAcls(Session session, boolean featureStashPrincipalPolicies) throws RepositoryException {
for (String authorizableId : createdAuthorizableIds) {
String principalName = userManagement.getPrincipalName(session, authorizableId);
if (deletedPrincipalAcls.containsKey(principalName)) {
if (opts.isDryRun()) {
track("Dry run: Would potentially restore principal ACLs of " + principalName + " ...", "");
} else if (!featureStashPrincipalPolicies) {
track(FEATURE_STASH_PRINCIPAL_POLICIES + " disabled: Would potentially restore principal ACLs of " + principalName + " ...", "");
} else {
for (AccessControlPolicy policy : deletedPrincipalAcls.get(principalName)) {
// CUG or ACL handling relevant?
AccessControlHandling aclHandling;
if (policy instanceof PrincipalSetPolicy) {
aclHandling = opts.getCugHandling();
} else {
aclHandling = opts.getAccessControlHandling();
}
// convert aclHandling (as this was set for the imported ACLs, not the existing ones)
final AccessControlHandling aclHandlingForRestoredPolicy;
switch (aclHandling) {
case OVERWRITE:
aclHandlingForRestoredPolicy = AccessControlHandling.IGNORE;
break;
case IGNORE:
aclHandlingForRestoredPolicy = AccessControlHandling.OVERWRITE;
break;
case CLEAR:
aclHandlingForRestoredPolicy = AccessControlHandling.IGNORE;
break;
case MERGE:
aclHandlingForRestoredPolicy = AccessControlHandling.MERGE;
break;
default:
aclHandlingForRestoredPolicy = AccessControlHandling.MERGE;
}
String accessControlledPath = userManagement.getAuthorizablePath(session, authorizableId);
List paths = JackrabbitAccessControlPolicy.fromAccessControlPolicy(policy).apply(session, aclHandlingForRestoredPolicy, accessControlledPath);
for (String path: paths) {
track("Restored principal ACLs of " + principalName, path);
}
}
}
}
}
}
/**
* Returns a human-readable error message from the throwable including all its causes till the root.
* Also the throwable class names are included in the message
* @param throwable from which to construct an error message
* @return the enhanced error message derived from the throwable
*/
static String getExtendedThrowableMessage(Throwable throwable) {
StringBuilder messageBuilder = new StringBuilder();
if (throwable == null) {
return "";
}
messageBuilder.append(throwable.getClass().getName()).append(": ");
messageBuilder.append(throwable.getMessage());
Throwable cause = throwable.getCause();
while (cause != null) {
if (!isDelimiter(messageBuilder.charAt(messageBuilder.length()-1))) {
messageBuilder.append(".");
}
messageBuilder.append(" Caused by ")
.append(cause.getClass().getName())
.append(": ")
.append(cause.getMessage());
cause = cause.getCause();
}
return messageBuilder.toString();
}
/** all punctuation delimiters between sentences in English */
private static final List DELIMITERS = Arrays.asList('.', '?', '!');
static boolean isDelimiter(char character) {
return DELIMITERS.contains(character);
}
private TxInfo postFilter(TxInfo root) {
TxInfo modifierRoot = root;
if (filter.contains(modifierRoot.path)){
return modifierRoot;
}
if (filter.isAncestor(modifierRoot.path)) {
for (String k : modifierRoot.children().keySet()) {
TxInfo child = modifierRoot.children().get(k);
modifierRoot.children().put(k, postFilter(child));
}
}
else {
modifierRoot.discard();
}
return modifierRoot;
}
public boolean hasErrors() {
return hasErrors;
}
protected VaultSettings getSettings() {
return archive.getMetaInf().getSettings();
}
private void installNodeTypes(Session session)
throws IOException, RepositoryException {
Collection metaTypes = archive.getMetaInf().getNodeTypes();
if (metaTypes != null) {
for (NodeTypeSet cnd: metaTypes) {
nodeTypes.add(cnd);
}
}
if (!nodeTypes.getNodeTypes().isEmpty()) {
NodeTypeInstaller installer = ServiceProviderFactory.getProvider().getDefaultNodeTypeInstaller(session);
try {
log.debug("Installing node types...");
installer.install(tracker, nodeTypes);
} catch (RepositoryException e) {
if (isStrict) {
throw e;
}
track(e, "Packaged node types");
}
} else {
log.debug("No node types provided.");
}
}
private void registerPrivileges(Session session) throws IOException, RepositoryException {
PrivilegeDefinitions privileges = archive.getMetaInf().getPrivileges();
if (privileges != null && !privileges.getDefinitions().isEmpty()) {
PrivilegeInstaller installer = ServiceProviderFactory.getProvider().getDefaultPrivilegeInstaller(session);
try {
log.debug("Registering privileges...");
installer.install(tracker, privileges);
} catch (RepositoryException e) {
if (isStrict) {
throw e;
}
track(e, "Packaged privileges");
}
} else {
log.debug("No privileges provided.");
}
}
/**
* Checks if the given file name is excluded
* @param fileName the file name
* @return {@code true} if excluded
*/
protected boolean isExcluded(String fileName) {
// hard coded exclusion of .vlt files/directories
return getSettings().getIgnoredNames().contains(fileName)
|| fileName.equals(".vlt")
|| fileName.startsWith(".vlt-");
}
private TxInfo prepare(Archive.Entry jcrRoot, String parentPath, NamespaceResolver resolver)
throws IOException, RepositoryException {
TxInfo root = new TxInfo(null, parentPath);
// special check for .content.xml in root directory
Archive.Entry contentXml = jcrRoot.getChild(Constants.DOT_CONTENT_XML);
if (contentXml != null) {
if (contentXml.isDirectory()) {
throw new IllegalArgumentException(Constants.DOT_CONTENT_XML + " is not a file");
}
// in this case, create a new info
root.artifacts.add(new InputSourceArtifact(
null,
"",
"",
ArtifactType.PRIMARY,
archive.getInputSource(contentXml),
SerializationType.XML_DOCVIEW
));
}
root.artifacts.add(new DirectoryArtifact(Text.getName(parentPath)));
prepare(jcrRoot, root, resolver);
// go over the filter roots and create intermediates for the parents if needed (bug #25370)
for (PathFilterSet sets: filter.getFilterSets()) {
String rootPath = sets.getRoot();
// make filter root relative to import root
if (parentPath.length() > 0 && rootPath.startsWith(parentPath)) {
rootPath = rootPath.substring(parentPath.length());
}
String[] segments = Text.explode(rootPath, '/');
TxInfo current = root;
StringBuilder path = new StringBuilder();
for (final String name : segments) {
path.append('/').append(name);
TxInfo child = current.children().get(name);
if (child == null) {
log.trace("Creating missing intermediate directory artifact for {}", name);
child = current.addChild(new TxInfo(current, path.toString()));
child.isIntermediate = 1;
intermediates.put(path.toString(), child);
}
current = child;
}
}
return root;
}
private void prepare(Archive.Entry directory, TxInfo parentInfo, NamespaceResolver resolver)
throws IOException, RepositoryException {
Collection files = directory.getChildren();
// first process the directories
for (Archive.Entry file: files) {
if (file.isDirectory()) {
String fileName = file.getName();
if (isExcluded(fileName)) {
continue;
}
String repoName = PlatformNameFormat.getRepositoryName(fileName);
String repoPath = parentInfo.path + "/" + repoName;
if (repoName.endsWith(".dir")) {
// fix repo path
repoName = repoName.substring(0, repoName.length() - 4);
repoPath = parentInfo.path + "/" + repoName;
}
TxInfo info = parentInfo.addChild(new TxInfo(parentInfo, repoPath));
log.trace("Creating directory artifact for {}", repoName);
Artifact parent = new DirectoryArtifact(repoName);
info.artifacts.add(parent);
Archive.Entry contentXml = file.getChild(Constants.DOT_CONTENT_XML);
if (contentXml != null) {
if (contentXml.isDirectory()) {
throw new IllegalArgumentException(Constants.DOT_CONTENT_XML + " is not a file");
}
// in this case, create a new info
info.artifacts.add(new InputSourceArtifact(
parent,
Constants.DOT_CONTENT_XML,
"",
ArtifactType.PRIMARY,
archive.getInputSource(contentXml),
SerializationType.XML_DOCVIEW
));
} else {
// this is an empty directory and potential intermediate
info.isIntermediate = 1;
intermediates.put(repoPath, info);
log.trace("Detecting intermediate directory {}", repoName);
}
prepare(file, info, resolver);
}
}
// second the files
for (Archive.Entry file: files) {
if (!file.isDirectory()) {
String fileName = file.getName();
if (isExcluded(fileName)) {
continue;
}
String repoName = PlatformNameFormat.getRepositoryName(fileName);
String repoPath = parentInfo.path + "/" + repoName;
if (file.getName().equals(Constants.DOT_CONTENT_XML)) {
continue;
}
if (opts.getPatchDirectory() != null && repoPath.startsWith(opts.getPatchParentPath())) {
patches.add(file);
if (!opts.isPatchKeepInRepo()) {
continue;
}
}
// todo: find better way to detect sub-packages
if (repoPath.startsWith(JcrPackageRegistry.DEFAULT_PACKAGE_ROOT_PATH_PREFIX) && (repoPath.endsWith(".jar") || repoPath.endsWith(".zip"))) {
subPackages.add(repoPath);
}
String repoBase = repoName;
String ext = "";
int idx = repoName.lastIndexOf('.');
if (idx > 0) {
repoBase = repoName.substring(0, idx);
ext = repoName.substring(idx);
}
SerializationType serType = SerializationType.GENERIC;
ArtifactType type = ArtifactType.PRIMARY;
VaultInputSource is = archive.getInputSource(file);
if (".xml".equals(ext)) {
// this can either be an generic exported docview or a 'user-xml' that is imported as file
// btw: this only works for input sources that can refetch their input stream
if (DocViewParser.isDocView(is)) {
// in this case, the extension was added by the exporter.
repoName = repoBase;
serType = SerializationType.XML_DOCVIEW;
} else {
ext = "";
serType = SerializationType.GENERIC;
type = ArtifactType.FILE;
}
} else if (".cnd".equals(ext)) {
if (opts.getCndPattern().matcher(repoPath).matches()) {
InputStream in = is.getByteStream();
try (Reader r = new InputStreamReader(in, "utf8")) {
CNDReader reader = ServiceProviderFactory.getProvider().getCNDReader();
// provide session namespaces
reader.read(r, is.getSystemId(), new NamespaceMapping(resolver));
nodeTypes.add(reader);
log.debug("Loaded nodetypes from {}.", repoPath);
} catch (IOException e1) {
log.error("Error while reading CND.", e1);
}
}
ext = "";
type = ArtifactType.FILE;
} else if (".binary".equals(ext)) {
serType = SerializationType.GENERIC;
type = ArtifactType.BINARY;
repoName = repoBase;
} else {
ext = "";
type = ArtifactType.FILE;
}
if (type != ArtifactType.PRIMARY) {
// check if info already exists (in case of .dir artifacts)
TxInfo parent = parentInfo.children().get(repoName);
if (parent == null) {
if (type == ArtifactType.BINARY) {
// search next parent for binary artifacts
parent = parentInfo;
while (parent != null && parent.isIntermediate > 0) {
parent = parent.parent;
}
if (parent == null) {
log.warn("No parent info found {}. using direct.");
parent = parentInfo;
}
} else {
// "normal" file
TxInfo tx = new TxInfo(parentInfo, parentInfo.path + "/" + repoName);
log.trace("Creating file artifact for {}", repoName);
tx.artifacts.add(new InputSourceArtifact(null,
repoName, ext, type, is, serType
));
parentInfo.addChild(tx);
}
}
if (parent != null) {
String path = parentInfo.path + "/" + repoName;
String relPath = parent.name + path.substring(parent.path.length());
log.trace("Attaching {} artifact {}", type, path);
parent.artifacts.add(new InputSourceArtifact(null,
relPath, ext, type, is, serType
));
}
}
if (type == ArtifactType.PRIMARY) {
// if primary artifact, add new tx info
TxInfo tx = new TxInfo(parentInfo, parentInfo.path + "/" + repoName);
log.trace("Creating primary artifact for {}", repoName);
tx.artifacts.add(new InputSourceArtifact(null,
repoName, ext, type, is, serType
));
parentInfo.addChild(tx);
}
}
}
// sort the child infos according to the workspace filter rules if possible
Tree.Node filterNode = filterTree.getNode(parentInfo.path);
if (filterNode != null) {
parentInfo.sort(filterNode.getChildren().keySet());
}
}
private void commit(Session session, TxInfo info, LinkedList skipList) throws RepositoryException, IOException {
try {
ImportInfoImpl imp = null;
if (skipList.isEmpty()) {
if (info == cpTxInfo) {
// don't need to import again, just set import info
log.trace("skipping last checkpoint info {}", info.path);
imp = cpImportInfo;
} else {
imp = commit(session, info);
if (imp != null) {
nodesToCheckin.addAll(imp.getToVersion());
memberships.putAll(imp.getMemberships());
autoSave.modified(imp.numModified());
deletedPrincipalAcls.putAll(imp.getDeletedPrincipalAcls());
createdAuthorizableIds.addAll(imp.getCreatedAuthorizableIds());
}
}
} else if (log.isDebugEnabled()) {
StringBuilder skips = new StringBuilder();
for (TxInfo i: skipList) {
skips.append(i.path).append(',');
}
log.trace("skip list: {}", skips);
}
if (autoSave.needsSave()) {
autoSave.save(session, true); // this is only intermediate
// save checkpoint
cpTxInfo = info;
cpAutosave = autoSave.copy();
cpImportInfo = imp;
recoveryRetryCounter = 0;
/*
todo: check retry logic if it's ok to discard all processed infos or if some ancestors should be excluded
// discard processed infos to free some memory
for (TxInfo i: processedInfos) {
i.discard();
}
*/
removedIntermediates.clear();
processedInfos.clear();
}
// copy the children collection since children could be removed during remapping
List children = new ArrayList(info.children().values());
// traverse children but skip the ones not in the skip list
TxInfo next = skipList.isEmpty() ? null : skipList.removeFirst();
for (TxInfo child: children) {
if (next == null || next == child) {
commit(session, child, skipList);
// continue normally after lng child was found
next = null;
} else {
log.trace("skipping {}", child.path);
}
}
// see if any child nodes need to be reordered
if (info.nameList != null) {
Node node = info.getNode(session);
if (node == null) {
log.warn("Unable to restore order of {}. Node does not exist.", info.path);
} else if (info.nameList.needsReorder(node)) {
log.trace("Restoring order of {}.", info.path);
info.nameList.restoreOrder(node);
}
}
processedInfos.add(info);
} catch (RepositoryException e) {
log.error("Error while committing {}: {}", info.path, e.toString());
throw e;
}
}
private ImportInfoImpl commit(Session session, TxInfo info) throws RepositoryException, IOException {
log.trace("committing {}", info.path);
ImportInfoImpl imp = null;
if (info.artifacts == null) {
log.debug("S {}", info.path);
} else if (info.artifacts.isEmpty()) {
// intermediate directory, check if node exists and filter
// matches. in this case remove the node (bug #25370)
// but only if intermediate is not processed yet (bug #42562)
if (filter.contains(info.path) && session.nodeExists(info.path) && info.isIntermediate < 2) {
Node node = session.getNode(info.path);
imp = new ImportInfoImpl();
if (aclManagement.isACLNode(node)) {
// Judging from isACLNode behavior, this part only applies
// to "rep:Policy" nodes so no need for special handling of CUG case.
if (opts.getAccessControlHandling() == AccessControlHandling.OVERWRITE
|| opts.getAccessControlHandling() == AccessControlHandling.CLEAR) {
imp.onDeleted(info.path);
aclManagement.clearACL(node.getParent());
}
} else {
if (filter.getImportMode(info.path) == ImportMode.REPLACE) {
imp.onDeleted(info.path);
node.remove();
} else {
imp.onNop(info.path);
}
}
}
} else if (info.artifacts.getPrimaryData() !=null && info.artifacts.size() == 1) {
// simple case, only 1 primary artifact
Node node = info.getParentNode(session);
if (node == null) {
imp = new ImportInfoImpl();
imp.onError(info.path, new IllegalStateException("Parent node not found."));
} else {
imp = genericHandler.accept(opts, isStrictByDefault, filter, node, info.artifacts.getPrimaryData().getRelativePath(), info.artifacts);
if (imp == null) {
throw new IllegalStateException("generic handler did not accept " + info.path);
}
}
} else if (info.artifacts.getDirectory() != null) {
String prefix = info.parent == null ? info.name : info.name + "/";
for (TxInfo child: info.children().values()) {
// add the directory artifacts as hint to this one.
if (child.artifacts == null) {
// in this case it's some deleted intermediate directory???
String path = prefix + child.name;
info.artifacts.add(new HintArtifact(path));
} else {
for (Artifact a: child.artifacts.values()) {
String path = prefix + a.getRelativePath();
info.artifacts.add(new HintArtifact(path));
}
}
}
Node node = info.getParentNode(session);
if (node == null) {
imp = new ImportInfoImpl();
imp.onError(info.path, new IllegalStateException("Parent node not found."));
} else {
if (info.isIntermediate == 2) {
// skip existing intermediate
log.trace("skipping intermediate node at {}", info.path);
} else if (info.artifacts.getPrimaryData() == null) {
// create nt:folder node if not exists
imp = folderHandler.accept(opts, isStrictByDefault, filter, node, info.name, info.artifacts);
if (imp == null) {
throw new IllegalStateException("folder handler did not accept " + info.path);
}
} else {
imp = genericHandler.accept(opts, isStrictByDefault, filter, node, info.artifacts.getDirectory().getRelativePath(), info.artifacts);
if (imp == null) {
throw new IllegalStateException("generic handler did not accept " + info.path);
}
}
}
} else if (info.artifacts.size(ArtifactType.FILE) > 0) {
Node node = info.getParentNode(session);
if (node == null) {
imp = new ImportInfoImpl();
imp.onError(info.path, new IllegalStateException("Parent node not found."));
} else {
imp = fileHandler.accept(opts, isStrictByDefault, filter, node, info.name, info.artifacts);
if (imp == null) {
throw new IllegalStateException("file handler did not accept " + info.path);
}
}
} else {
throw new UnsupportedOperationException("ArtifactSet not supported: " + info.artifacts);
}
if (imp != null) {
for (Map.Entry entry: imp.getInfos().entrySet()) {
String path = entry.getKey();
ImportInfo.Type type = entry.getValue().getType();
if (type != ImportInfoImpl.Type.DEL) {
// mark intermediates as processed
TxInfo im = intermediates.remove(path);
if (im != null) {
log.debug("P {}", path);
removedIntermediates.put(path, im);
im.isIntermediate = 2;
}
}
switch (type) {
case CRE:
track("A", path);
break;
case DEL:
track("D", path);
break;
case MOD:
track("U", path);
break;
case NOP:
track("-", path);
break;
case REP:
track("R", path);
break;
case MIS:
track("!", path);
break;
case ERR:
Exception error = entry.getValue().getError();
if (error == null) {
track("E", path);
} else {
track(error, path);
}
hasErrors = true;
if (firstException == null) {
firstException = new PackageException("Error creating/updating node " + path, error);
}
break;
}
// see if any child nodes need to be reordered and remember namelist.
// only restore order if in filter scope if freshly created
NodeNameList nameList = entry.getValue().getNameList();
if (nameList != null && (filter.contains(path) || type == ImportInfo.Type.CRE)) {
TxInfo subInfo = info.findChild(path);
if (subInfo != null) {
subInfo.nameList = nameList;
}
}
}
// remap the child tree in case some of the nodes where moved during import (e.g. authorizable)
// todo: this could be a problem during error recovery
info = info.remap(imp.getRemapped());
}
log.trace("committed {}", info.path);
return imp;
}
public void checkinNodes(Session session) {
if (nodesToCheckin.isEmpty()) {
return;
}
if (opts.isDryRun()) {
track("Dry run: Would commit versions...", "");
} else {
track("Committing versions...", "");
}
for (String path: nodesToCheckin) {
try {
Node node = session.getNode(path);
try {
if (opts.isDryRun()) {
track("V", String.format("%s (---)", path));
} else {
Version v = node.checkin();
track("V", String.format("%s (%s)", path, v.getName()));
}
} catch (RepositoryException e) {
log.error("Error while checkin node {}: {}",path, e.toString());
}
} catch (RepositoryException e) {
log.error("Error while retrieving node to be versioned at {}.", path, e);
}
}
nodesToCheckin.clear();
}
public void applyMemberships(Session session) {
if (memberships.isEmpty()) {
return;
}
if (opts.isDryRun()) {
track("Dry run: Would apply merged group memberships...", "");
} else {
track("Applying merged group memberships...", "");
}
for (String id: memberships.keySet()) {
String[] members = memberships.get(id);
String authPath = userManagement.getAuthorizablePath(session, id);
if (authPath != null) {
if (!opts.isDryRun()) {
userManagement.addMembers(session, id, members);
}
track("U", String.format("%s", authPath));
}
}
if (!autoSave.isDisabled()) {
try {
session.save();
} catch (RepositoryException e) {
log.error("Error while updating memberships.", e);
try {
session.refresh(false);
} catch (RepositoryException e1) {
// ignore
}
}
}
memberships.clear();
}
private void applyPatches() {
for (Archive.Entry e: patches) {
String name = e.getName();
File target = new File(opts.getPatchDirectory(), name);
if (opts.isDryRun()) {
log.debug("Dry run: Would copy patch {} to {}", name, target.getPath());
} else {
log.debug("Copying patch {} to {}", name, target.getPath());
try (InputStream in = archive.getInputSource(e).getByteStream();
OutputStream out = FileUtils.openOutputStream(target)) {
IOUtils.copy(in, out);
} catch (IOException e1) {
log.error("Error while copying patch.", e);
}
}
track("P", name);
}
}
private static class TxInfo {
private TxInfo parent;
private final String path;
private final String name;
private ArtifactSetImpl artifacts = new ArtifactSetImpl();
private Map children;
private byte isIntermediate = 0;
private NodeNameList nameList;
public TxInfo(TxInfo parent, String path) {
log.trace("New TxInfo {}" , path);
this.parent = parent;
this.path = path;
this.name = Text.getName(path);
}
public TxInfo addChild(TxInfo child) {
if (children == null) {
children = new LinkedHashMap();
}
children.put(child.name, child);
return child;
}
public Map children() {
if (children == null) {
return Collections.emptyMap();
} else {
return children;
}
}
public void sort(Collection names) {
if (children == null || children.size() <=1 || names == null || names.isEmpty()) {
return;
}
Map ret = new LinkedHashMap();
Iterator iter = names.iterator();
while (iter.hasNext() && children.size() > 1) {
String name = iter.next();
TxInfo info = children.remove(name);
if (info != null) {
ret.put(name, info);
}
}
ret.putAll(children);
children = ret;
}
public Node getParentNode(Session s) throws RepositoryException {
String parentPath = emptyPathToRoot(Text.getRelativeParent(path, 1));
return s.nodeExists(parentPath)
? s.getNode(parentPath)
: null;
}
public Node getNode(Session s) throws RepositoryException {
String p = emptyPathToRoot(path);
return s.nodeExists(p)
? s.getNode(p)
: null;
}
public void discard() {
log.trace("discarding {}", path);
artifacts = null;
children = null;
}
public TxInfo findChild(String absPath) {
if (path.equals(absPath)) {
return this;
}
if (!absPath.startsWith(path + "/")) {
return null;
}
absPath = absPath.substring(path.length());
TxInfo root = this;
for (String name: Text.explode(absPath, '/')) {
root = root.children().get(name);
if (root == null) {
break;
}
}
return root;
}
public TxInfo remap(PathMapping mapping) {
String mappedPath = mapping.map(path, true);
if (mappedPath.equals(path)) {
return this;
}
TxInfo ret = new TxInfo(parent, mappedPath);
// todo: what should we do with the artifacts ?
ret.artifacts.addAll(artifacts);
// todo: do we need to remap the namelist, too?
ret.nameList = nameList;
ret.isIntermediate = isIntermediate;
if (children != null) {
for (TxInfo child: children.values()) {
child = child.remap(mapping);
child.parent = this;
ret.addChild(child);
}
}
// ensure that our parent links the new info
if (parent.children != null) {
parent.children.put(ret.name, ret);
}
return ret;
}
private static String emptyPathToRoot(String path) {
return path == null || path.length() == 0 ? "/" : path;
}
}
boolean shouldStashPrincipalPoliciesForArchive(Archive archive) {
return Optional.ofNullable(archive.getMetaInf().getProperties())
.filter(properties -> properties.containsKey(FEATURE_STASH_PRINCIPAL_POLICIES))
.map(properties -> Boolean.valueOf(properties.getProperty(FEATURE_STASH_PRINCIPAL_POLICIES)))
.orElse(stashPrincipalPoliciesByDefault);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy