org.usergrid.tools.Import Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of usergrid-tools Show documentation
Show all versions of usergrid-tools Show documentation
Command line tools for Usergrid system.
/*******************************************************************************
* Copyright 2012 Apigee Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
package org.usergrid.tools;
import static org.usergrid.persistence.Schema.PROPERTY_TYPE;
import static org.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.io.filefilter.PrefixFileFilter;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonToken;
import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.usergrid.management.ApplicationInfo;
import org.usergrid.management.OrganizationInfo;
import org.usergrid.management.UserInfo;
import org.usergrid.persistence.Entity;
import org.usergrid.persistence.EntityManager;
import org.usergrid.persistence.EntityRef;
import org.usergrid.persistence.Schema;
import org.usergrid.persistence.entities.Application;
import org.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.usergrid.tools.bean.ExportOrg;
import org.usergrid.utils.JsonUtils;
public class Import extends ToolBase {
private static final Logger logger = LoggerFactory.getLogger(Import.class);
/** Input directory where the .json export files are */
static final String INPUT_DIR = "inputDir";
static File importDir;
static final String DEFAULT_INPUT_DIR = "export";
JsonFactory jsonFactory = new JsonFactory();
@Override
@SuppressWarnings("static-access")
public Options createOptions() {
Option hostOption = OptionBuilder.withArgName("host").hasArg()
.withDescription("Cassandra host").create("host");
Option inputDir = OptionBuilder.hasArg()
.withDescription("input directory -inputDir").create(INPUT_DIR);
Option verbose = OptionBuilder
.withDescription(
"Print on the console an echo of the content written to the file")
.create(VERBOSE);
Options options = new Options();
options.addOption(hostOption);
options.addOption(inputDir);
options.addOption(verbose);
return options;
}
@Override
public void runTool(CommandLine line) throws Exception {
startSpring();
setVerbose(line);
openImportDirectory(line);
importOrganizations();
importApplications();
importCollections();
//forces the counters to flush
logger.info("Sleeping 30 seconds for batcher");
Thread.sleep(35000);
}
/**
* Import applications
*/
private void importApplications() throws Exception {
String[] nanemspaceFileNames = importDir.list(new PrefixFileFilter(
"application."));
logger.info("Applications to read: " + nanemspaceFileNames.length);
for (String applicationName : nanemspaceFileNames) {
try {
importApplication(applicationName);
} catch (Exception e) {
logger.warn("Unable to import application: " + applicationName,
e);
}
}
}
/**
* Imports a application
*
* @param applicationName
* file name where the application was exported.
*/
private void importApplication(String applicationName) throws Exception {
// Open up application file.
File applicationFile = new File(importDir, applicationName);
logger.info("Loading application file: "
+ applicationFile.getAbsolutePath());
JsonParser jp = getJsonParserForFile(applicationFile);
JsonToken token = jp.nextToken();
validateStartArray(token);
// Move to next object (the application).
// The application object is the first object followed by all the
// objects in this application.
token = jp.nextValue();
Application application = jp.readValueAs(Application.class);
@SuppressWarnings("unchecked")
String orgName = ((Map) application
.getMetadata("organization")).get("value");
OrganizationInfo info = managementService.getOrganizationByName(orgName);
if(info == null){
logger.error("Unable to import application '{}' for organisation with name '{}'", application.getName(), orgName);
return;
}
UUID appId = null;
try{
appId = managementService.importApplication(info.getUuid(), application);
}catch(ApplicationAlreadyExistsException aaee){
ApplicationInfo appInfo = managementService.getApplicationInfo(orgName+"/"+application.getName());
if(appInfo != null){
appId = appInfo.getId();
}
}
echo(application);
EntityManager em = emf.getEntityManager(appId);
// we now need to remove all roles, they'll be imported again below
for (Entry entry : em.getRoles().entrySet()) {
em.deleteRole(entry.getKey());
}
//load all the dictionaries
@SuppressWarnings("unchecked")
Map dictionaries = (Map) application.getMetadata("dictionaries");
if(dictionaries != null){
EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
Entity appEntity = rootEm.get(appId);
for(Entry dictionary: dictionaries.entrySet()){
@SuppressWarnings("unchecked")
Map value = (Map) dictionary.getValue();
em.addMapToDictionary(appEntity, dictionary.getKey(), value);
}
}
//load all counts and stats
// @SuppressWarnings("unchecked")
// Map stats = (Map) application.getMetadata("counters");
//
// for(Entry stat: stats.entrySet()){
// String entryName = stat.getKey();
// long amount = Long.parseLong(stat.getValue().toString());
//
//
// //anything that deals with collections or entities, we set to 0 since they'll be incremented during import
// if(!entryName.startsWith("application.collection") && !entryName.equals("application.entities")){
// em.incrementApplicationCounter(entryName, amount);
// }
//
// }
//explicity import all collections
@SuppressWarnings("unchecked")
List collections = (List) application.getMetadata("collections");
for(String collectionName: collections){
em.createApplicationCollection(collectionName);
}
while (jp.nextValue() != JsonToken.END_ARRAY) {
@SuppressWarnings("unchecked")
Map entityProps = jp.readValueAs(HashMap.class);
// Import/create the entity
UUID uuid = getId(entityProps);
String type = getType(entityProps);
try {
em.create(uuid, type, entityProps);
} catch (DuplicateUniquePropertyExistsException de) {
logger.error(
"Unable to create entity. It appears to be a duplicate",
de);
continue;
}
if(em.get(uuid) == null){
logger.error("Holy hell, we wrote an entity and it's missing. Entity Id was {} and type is {}", uuid, type);
System.exit(1);
}
logger.info("Counts {}", JsonUtils.mapToFormattedJsonString(em.getApplicationCounters()));
echo(entityProps);
}
logger.info("----- End of application:" + application.getName());
jp.close();
}
private String getType(Map entityProps) {
return (String) entityProps.get(PROPERTY_TYPE);
}
private UUID getId(Map entityProps) {
return UUID.fromString((String) entityProps.get(PROPERTY_UUID));
}
private void validateStartArray(JsonToken token) {
if (token != JsonToken.START_ARRAY) {
throw new RuntimeException("Token should be START ARRAY but it is:"
+ token.asString());
}
}
/**
* Import organizations
*
*/
private void importOrganizations() throws Exception {
String[] organizationFileNames = importDir.list(new PrefixFileFilter(
"organization."));
logger.info("Organizations to read: " + organizationFileNames.length);
for (String organizationFileName : organizationFileNames) {
try {
importOrganization(organizationFileName);
} catch (Exception e) {
logger.warn("Unable to import organization:"
+ organizationFileName, e);
}
}
}
/**
* Import an organization.
*
* @param organizationFileName
* file where the organization was exported
*/
private void importOrganization(String organizationFileName)
throws Exception {
ExportOrg acc = null;
// Open up organization dir.
File organizationFile = new File(importDir, organizationFileName);
logger.info("Loading organization file: "
+ organizationFile.getAbsolutePath());
JsonParser jp = getJsonParserForFile(organizationFile);
// Get the organization object and the only one in the file.
acc = jp.readValueAs(ExportOrg.class);
Map properties = new LinkedHashMap();
// properties.put("email", acc.getEmail());
// properties.put("password", "password".getBytes("UTF-8"));
echo(acc);
//check if the org exists, if it does, what do we do
OrganizationInfo org = managementService.getOrganizationByName(acc.getName());
//only import if the org doesn't exist
if(org == null){
org = managementService.importOrganization(acc.getUuid(), acc, properties);
}
//now go through and add each admin from the original org to the newly imported
for(String exportedUser: acc.getAdmins()){
UserInfo existing = managementService.getAdminUserByUsername(exportedUser);
if(existing != null){
managementService.addAdminUserToOrganization(existing, org, false);
}
}
jp.close();
}
private JsonParser getJsonParserForFile(File organizationFile)
throws Exception {
JsonParser jp = jsonFactory.createJsonParser(organizationFile);
jp.setCodec(new ObjectMapper());
return jp;
}
/**
* Import collections. Collections files are named:
* collections..Timestamp.json
*/
private void importCollections() throws Exception {
String[] collectionsFileNames = importDir.list(new PrefixFileFilter(
"collections."));
logger.info("Collections to read: " + collectionsFileNames.length);
for (String collectionName : collectionsFileNames) {
try {
importCollection(collectionName);
} catch (Exception e) {
logger.warn("Unable to import collection: " + collectionName, e);
}
}
}
private void importCollection(String collectionFileName) throws Exception {
// Retrieve the namepsace for this collection. It's part of the name
String applicationName = getApplicationFromColllection(collectionFileName);
UUID appId = emf.lookupApplication(applicationName);
//no org in path, this is a pre public beta so we need to create the new path
if(appId == null && !applicationName.contains("/")){
String fileName = collectionFileName.replace("collections", "application");
File applicationFile = new File(importDir, fileName);
if(!applicationFile.exists()){
logger.error("Could not load application file {} to search for org information", applicationFile.getAbsolutePath());
return;
}
logger.info("Loading application file: "
+ applicationFile.getAbsolutePath());
JsonParser jp = getJsonParserForFile(applicationFile);
JsonToken token = jp.nextToken();
validateStartArray(token);
// Move to next object (the application).
// The application object is the first object followed by all the
// objects in this application.
token = jp.nextValue();
Application application = jp.readValueAs(Application.class);
jp.close();
@SuppressWarnings("unchecked")
String orgName = ((Map) application
.getMetadata("organization")).get("value");
OrganizationInfo info = managementService.getOrganizationByName(orgName);
if(info == null){
logger.error("Could not find org with name {}", orgName);
return;
}
applicationName = orgName + "/" + applicationName;
appId = emf.lookupApplication(applicationName);
}
if(appId == null){
logger.error("Unable to find application with name {}. Skipping collections", appId);
return;
}
File collectionFile = new File(importDir, collectionFileName);
logger.info("Loading collections file: "
+ collectionFile.getAbsolutePath());
JsonParser jp = getJsonParserForFile(collectionFile);
jp.nextToken(); // START_OBJECT this is the outter hashmap
EntityManager em = emf.getEntityManager(appId);
while (jp.nextToken() != JsonToken.END_OBJECT) {
importEntitysStuff(jp, em);
}
logger.info("----- End of collections -----");
jp.close();
}
/**
* Imports the entity's connecting references (collections and connections)
*
* @param jp
* JsonPrser pointing to the beginning of the object.
* @param em
*/
private void importEntitysStuff(JsonParser jp, EntityManager em)
throws Exception {
// The entity that owns the collections
String entityOwnerId = jp.getCurrentName();
EntityRef ownerEntityRef = em.getRef(UUID.fromString(entityOwnerId));
jp.nextToken(); // start object
// Go inside the value after getting the owner entity id.
while (jp.nextToken() != JsonToken.END_OBJECT) {
String collectionName = jp.getCurrentName();
if (collectionName.equals("connections")) {
jp.nextToken(); // START_OBJECT
while (jp.nextToken() != JsonToken.END_OBJECT) {
String connectionType = jp.getCurrentName();
jp.nextToken(); // START_ARRAY
while (jp.nextToken() != JsonToken.END_ARRAY) {
String entryId = jp.getText();
EntityRef entryRef = em
.getRef(UUID.fromString(entryId));
// Store in DB
em.createConnection(ownerEntityRef, connectionType,
entryRef);
}
}
} else if (collectionName.equals("dictionaries")) {
jp.nextToken(); // START_OBJECT
while (jp.nextToken() != JsonToken.END_OBJECT) {
String dictionaryName = jp.getCurrentName();
jp.nextToken();
@SuppressWarnings("unchecked")
Map dictionary = jp
.readValueAs(HashMap.class);
em.addMapToDictionary(ownerEntityRef, dictionaryName,
dictionary);
}
}
else {
// Regular collections
jp.nextToken(); // START_ARRAY
while (jp.nextToken() != JsonToken.END_ARRAY) {
String entryId = jp.getText();
EntityRef entryRef = em.getRef(UUID.fromString(entryId));
// store it
em.addToCollection(ownerEntityRef, collectionName, entryRef);
}
}
}
}
/**
* Extract a application name from a collectionsFileName in the way:
* collections..TIMESTAMP.json
*
* @param collectionFileName
* a collection file name
* @return the application name for this collections file name
*/
/**
* Extract a application name from a collectionsFileName in the way:
* collections..TIMESTAMP.json
*
* @param collectionFileName
* a collection file name
* @return the application name for this collections file name
*/
private String getApplicationFromColllection(String collectionFileName) {
int firstDot = collectionFileName.indexOf(".");
int secondDot = collectionFileName.indexOf(".", firstDot + 1);
// The application will be in the subString between the dots.
String appName = collectionFileName.substring(firstDot + 1, secondDot);
return appName.replace(PATH_REPLACEMENT, "/");
}
/**
* Open up the import directory based on importDir
*/
private void openImportDirectory(CommandLine line) {
boolean hasInputDir = line.hasOption(INPUT_DIR);
if (hasInputDir) {
importDir = new File(line.getOptionValue(INPUT_DIR));
} else {
importDir = new File(DEFAULT_INPUT_DIR);
}
logger.info("Importing from:" + importDir.getAbsolutePath());
logger.info("Status. Exists: " + importDir.exists() + " - Readable: "
+ importDir.canRead());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy