com.adobe.acs.commons.mcp.impl.processes.asset.UrlAssetImport Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of acs-aem-commons-bundle Show documentation
Show all versions of acs-aem-commons-bundle Show documentation
Core ACS AEM Commons OSGi Bundle. Includes commons utilities.
/*
* #%L
* ACS AEM Commons Bundle
* %%
* Copyright (C) 2017 Adobe
* %%
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* #L%
*/
package com.adobe.acs.commons.mcp.impl.processes.asset;
import com.adobe.acs.commons.data.CompositeVariant;
import com.adobe.acs.commons.data.Spreadsheet;
import com.adobe.acs.commons.fam.ActionManager;
import com.adobe.acs.commons.fam.actions.Actions;
import com.adobe.acs.commons.functions.CheckedConsumer;
import com.adobe.acs.commons.mcp.ProcessInstance;
import com.adobe.acs.commons.mcp.form.FileUploadComponent;
import com.adobe.acs.commons.mcp.form.FormField;
import com.adobe.acs.commons.mcp.form.PasswordComponent;
import com.day.cq.commons.jcr.JcrUtil;
import com.day.cq.dam.api.Asset;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.SocketConfig;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.osgi.services.HttpClientBuilderFactory;
import org.apache.sling.api.request.RequestParameter;
import org.apache.sling.api.resource.LoginException;
import org.apache.sling.api.resource.ModifiableValueMap;
import org.apache.sling.api.resource.PersistenceException;
import org.apache.sling.api.resource.Resource;
import org.apache.sling.api.resource.ResourceNotFoundException;
import org.apache.sling.api.resource.ResourceResolver;
import org.apache.sling.commons.mime.MimeTypeService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* Import assets and metadata provided by a spreadsheet
*/
public class UrlAssetImport extends AssetIngestor {
private static final String ACTION_SKIPPED = "Skipped";
private static final String ACTION_UNMATCHED = "Unmatched";
private static final String ACTION_IMPORT = "Import";
public static final String SOURCE = "source";
public static final String TARGET_FOLDER = "target";
public static final String ORIGINAL_FILE_NAME = "original";
public static final String RENDITION_NAME = "rendition";
public static final String CONTENT_BASE = "/content";
public static final String UNKNOWN_TARGET_FOLDER = "/content/dam/unsorted";
private static final Logger LOG = LoggerFactory.getLogger(UrlAssetImport.class);
private HttpClientBuilderFactory httpFactory;
private HttpClient httpClient = null;
public UrlAssetImport(MimeTypeService mimeTypeService, HttpClientBuilderFactory httpFactory) {
super(mimeTypeService);
this.httpFactory = httpFactory;
}
@FormField(
name = "Import data file",
description = "Data file containing asset import data",
component = FileUploadComponent.class
)
transient RequestParameter importFile;
@FormField(
name = "Default prefix",
description = "Added to source if it starts with / e.g. file:/ | file:/C: | http://www.somewebsite",
required = true,
options = ("default=file:/")
)
private String defaultPrefix = "file:/";
@FormField(
name = "Connection timeout",
description = "HTTP Connection timeout (in milliseconds)",
required = true,
options = ("default=30000")
)
private int timeout = 30000;
@FormField(
name = "Username",
description = "Username for connections that require login",
required = false
)
private String username = null;
@FormField(
name = "Password",
description = "Password for connections that require login",
required = false,
component = PasswordComponent.class
)
private String password = null;
transient Set files;
transient Map folders = new TreeMap<>((a, b) -> b.compareTo(a));
private ClientProvider clientProvider = new ClientProvider();
Spreadsheet fileData;
EnumMap importedRenditions
= trackDetailedActivity("All Renditions", ACTION_IMPORT, "Count of all rendition imports", 0L);
@Override
public void init() throws RepositoryException {
super.init();
if (httpFactory != null) {
HttpClientBuilder clientBuilder = httpFactory.newBuilder();
clientBuilder.setDefaultSocketConfig(
SocketConfig.custom()
.setSoTimeout(timeout)
.build());
clientBuilder.setDefaultRequestConfig(
RequestConfig.custom()
.setConnectTimeout(timeout)
.build()
);
httpClient = clientBuilder.build();
clientProvider.setHttpClientSupplier(this::getHttpClient);
clientProvider.setUsername(username);
clientProvider.setPassword(password);
}
}
@Override
public void buildProcess(ProcessInstance instance, ResourceResolver rr) throws LoginException, RepositoryException {
try {
fileData = new Spreadsheet(importFile, Arrays.asList(SOURCE, RENDITION_NAME, TARGET_FOLDER, ORIGINAL_FILE_NAME))
.buildSpreadsheet();
files = extractFilesAndFolders(fileData.getDataRowsAsCompositeVariants());
instance.getInfo().setDescription(String.format("Import %s (%s rows)", fileData.getFileName(), fileData.getRowCount()));
} catch (IOException ex) {
LOG.error("Unable to process import", ex);
instance.getInfo().setDescription(String.format("Import %s (failed)", fileData.getFileName()));
throw new RepositoryException("Unable to parse input file", ex);
}
trackUnmatchedRenditions();
trackIgnoredFiles();
instance.defineCriticalAction("Create Folders", rr, this::createFolders);
instance.defineAction(String.format("Import %s Assets", files.size()), rr, this::importAssets);
int countOfRenditions = files.stream().map(FileOrRendition::getRenditions).mapToInt(Map::size).sum();
if (countOfRenditions > 0) {
instance.defineAction(String.format("Import %s Renditions", countOfRenditions), rr, this::importRenditions);
}
instance.defineAction("Update Metadata", rr, this::updateMetadata);
}
private void trackIgnoredFiles() {
files.stream().filter(f -> !canImportContainingFolder(f)).forEach(file -> {
trackDetailedActivity(file.getNodePath(preserveFileName), ACTION_SKIPPED, "Skipped file because its folder is also skipped", 0L);
incrementCount(skippedFiles, 1 + file.getRenditions().size());
file.getRenditions().forEach((renditionName, rendition)
-> trackDetailedActivity(rendition.getNodePath(preserveFileName), ACTION_SKIPPED, "Skipped rendition " + renditionName + " because its parent file is skipped", 0L));
file.getRenditions().clear();
});
}
private void trackUnmatchedRenditions() {
unmatchedRenditions.forEach(row -> {
long rowNumber = this.fileData.getRowNum(row);
trackDetailedActivity(row.get(SOURCE).toString(), ACTION_UNMATCHED, "Unable to track original asset for rendition, row " + rowNumber, 0L);
incrementCount(skippedFiles, 1);
});
}
Set