org.usergrid.tools.Export Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of usergrid-tools Show documentation
Show all versions of usergrid-tools Show documentation
Command line tools for Usergrid system.
/*******************************************************************************
* Copyright 2012 Apigee Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
package org.usergrid.tools;
import java.io.File;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
import org.apache.commons.cli.CommandLine;
import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.usergrid.management.OrganizationInfo;
import org.usergrid.management.UserInfo;
import org.usergrid.persistence.ConnectionRef;
import org.usergrid.persistence.Entity;
import org.usergrid.persistence.EntityManager;
import org.usergrid.persistence.Query;
import org.usergrid.persistence.Results;
import org.usergrid.persistence.Results.Level;
import org.usergrid.persistence.cassandra.CassandraService;
import org.usergrid.tools.bean.ExportOrg;
import org.usergrid.utils.JsonUtils;
import com.google.common.collect.BiMap;
public class Export extends ExportingToolBase {
static final Logger logger = LoggerFactory.getLogger(Export.class);
JsonFactory jsonFactory = new JsonFactory();
@Override
public void runTool(CommandLine line) throws Exception {
startSpring();
setVerbose(line);
// ExportDataCreator dataCreator = new ExportDataCreator(emf,
// managementService);
// dataCreator.createTestData();
applyOrgId(line);
prepareBaseOutputFileName(line);
outputDir = createOutputParentDir();
logger.info("Export directory: " + outputDir.getAbsolutePath());
// Export organizations separately.
exportOrganizations();
// Loop through the organizations
Map organizations = getOrgs();
for (Entry organization : organizations.entrySet()) {
if (organization.equals(properties.getProperty("usergrid.test-account.organization"))) {
// Skip test data from being exported.
continue;
}
exportApplicationsForOrg(organization);
}
}
private Map getOrgs() throws Exception{
// Loop through the organizations
Map organizationNames = null;
if(orgId == null){
organizationNames = managementService.getOrganizations();
}
else{
OrganizationInfo info = managementService.getOrganizationByUuid(orgId);
if(info == null){
logger.error("Organization info is null!");
System.exit(1);
}
organizationNames = new HashMap();
organizationNames.put(orgId, info.getName());
}
return organizationNames;
}
private void exportApplicationsForOrg(Entry organization) throws Exception {
logger.info("" + organization);
// Loop through the applications per organization
BiMap applications = managementService.getApplicationsForOrganization(organization.getKey());
for (Entry application : applications.entrySet()) {
logger.info(application.getValue() + " : " + application.getKey());
// Get the JSon serializer.
JsonGenerator jg = getJsonGenerator(createOutputFile("application", application.getValue()));
// load the dictionary
EntityManager rootEm = emf.getEntityManager(CassandraService.MANAGEMENT_APPLICATION_ID);
Entity appEntity = rootEm.get(application.getKey());
Map dictionaries = new HashMap();
for (String dictionary : rootEm.getDictionaries(appEntity)) {
Map dict = rootEm.getDictionaryAsMap(appEntity, dictionary);
// nothing to do
if (dict.isEmpty()) {
continue;
}
dictionaries.put(dictionary, dict);
}
EntityManager em = emf.getEntityManager(application.getKey());
// Get application
Entity nsEntity = em.get(application.getKey());
Set collections = em.getApplicationCollections();
// load app counters
Map entityCounters = em.getApplicationCounters();
nsEntity.setMetadata("organization", organization);
nsEntity.setMetadata("dictionaries", dictionaries);
// counters for collections
nsEntity.setMetadata("counters", entityCounters);
nsEntity.setMetadata("collections", collections);
jg.writeStartArray();
jg.writeObject(nsEntity);
// Create a generator for the application collections.
JsonGenerator collectionsJg = getJsonGenerator(createOutputFile("collections", application.getValue()));
collectionsJg.writeStartObject();
Map metadata = em.getApplicationCollectionMetadata();
echo(JsonUtils.mapToFormattedJsonString(metadata));
// Loop through the collections. This is the only way to loop
// through the entities in the application (former namespace).
for (String collectionName : metadata.keySet()) {
Query query = new Query();
query.setLimit(MAX_ENTITY_FETCH);
query.setResultsLevel(Results.Level.ALL_PROPERTIES);
Results entities = em.searchCollection(em.getApplicationRef(), collectionName, query);
while (entities.size() > 0) {
for (Entity entity : entities) {
// Export the entity first and later the collections for
// this entity.
jg.writeObject(entity);
echo(entity);
saveCollectionMembers(collectionsJg, em, application.getValue(), entity);
}
//we're done
if(entities.getCursor() == null){
break;
}
query.setCursor(entities.getCursor());
entities = em.searchCollection(em.getApplicationRef(), collectionName, query);
}
}
// Close writer for the collections for this application.
collectionsJg.writeEndObject();
collectionsJg.close();
// Close writer and file for this application.
jg.writeEndArray();
jg.close();
}
}
/**
* Serialize and save the collection members of this entity
*
* @param em
* Entity Manager
* @param application
* Application name
* @param entity
* entity
* @throws Exception
*/
private void saveCollectionMembers(JsonGenerator jg, EntityManager em, String application, Entity entity)
throws Exception {
Set collections = em.getCollections(entity);
// Only create entry for Entities that have collections
if ((collections == null) || collections.isEmpty()) {
return;
}
jg.writeFieldName(entity.getUuid().toString());
jg.writeStartObject();
for (String collectionName : collections) {
jg.writeFieldName(collectionName);
// Start collection array.
jg.writeStartArray();
Results collectionMembers = em.getCollection(entity, collectionName, null, 100000, Level.IDS, false);
List entityIds = collectionMembers.getIds();
if ((entityIds != null) && !entityIds.isEmpty()) {
for (UUID childEntityUUID : entityIds) {
jg.writeObject(childEntityUUID.toString());
}
}
// End collection array.
jg.writeEndArray();
}
// Write connections
saveConnections(entity, em, jg);
// Write dictionaries
saveDictionaries(entity, em, jg);
// End the object if it was Started
jg.writeEndObject();
}
/**
* Persists the connection for this entity.
*/
private void saveDictionaries(Entity entity, EntityManager em, JsonGenerator jg) throws Exception {
jg.writeFieldName("dictionaries");
jg.writeStartObject();
Set dictionaries = em.getDictionaries(entity);
for (String dictionary : dictionaries) {
Map dict = em.getDictionaryAsMap(entity, dictionary);
// nothing to do
if (dict.isEmpty()) {
continue;
}
jg.writeFieldName(dictionary);
jg.writeStartObject();
for (Entry entry : dict.entrySet()) {
jg.writeFieldName(entry.getKey().toString());
jg.writeObject(entry.getValue());
}
jg.writeEndObject();
}
jg.writeEndObject();
}
/**
* Persists the connection for this entity.
*/
private void saveConnections(Entity entity, EntityManager em, JsonGenerator jg) throws Exception {
jg.writeFieldName("connections");
jg.writeStartObject();
Set connectionTypes = em.getConnectionTypes(entity);
for (String connectionType : connectionTypes) {
jg.writeFieldName(connectionType);
jg.writeStartArray();
Results results = em.getConnectedEntities(entity.getUuid(), connectionType, null, Level.IDS);
List connections = results.getConnections();
for (ConnectionRef connectionRef : connections) {
jg.writeObject(connectionRef.getConnectedEntity().getUuid());
}
jg.writeEndArray();
}
jg.writeEndObject();
}
/*-
* Set collections = em.getCollections(entity);
* for (String collection : collections) {
* Results collectionMembers = em.getCollection(
* entity, collection, null,
* MAX_ENTITY_FETCH, Level.IDS, false);
* write entity_id : { "collectionName" : [ids]
* }
* }
*
*
* {
* entity_id :
* { collection_name :
* [
* collected_entity_id,
* collected_entity_id
* ]
* },
* f47ac10b-58cc-4372-a567-0e02b2c3d479 :
* { "activtites" :
* [
* f47ac10b-58cc-4372-a567-0e02b2c3d47A,
* f47ac10b-58cc-4372-a567-0e02b2c3d47B
* ]
* }
* }
*
* http://jackson.codehaus.org/1.8.0/javadoc/org/codehaus/jackson/JsonGenerator.html
*
*
*-
* List connections = em.getConnections(entityId, query);
*/
private void exportOrganizations() throws Exception, UnsupportedEncodingException {
for (Entry organizationName : getOrgs().entrySet()) {
// Let's skip the test entities.
if (organizationName.equals(properties.getProperty("usergrid.test-account.organization"))) {
continue;
}
OrganizationInfo acc = managementService.getOrganizationByUuid(organizationName.getKey());
logger.info("Exporting Organization: " + acc.getName());
ExportOrg exportOrg = new ExportOrg(acc);
List users = managementService.getAdminUsersForOrganization(organizationName.getKey());
for (UserInfo user : users) {
exportOrg.addAdmin(user.getUsername());
}
// One file per Organization.
saveOrganizationInFile(exportOrg);
}
}
/**
* Serialize an Organization into a json file.
*
* @param acc
* OrganizationInfo
*/
private void saveOrganizationInFile(ExportOrg acc) {
try {
File outFile = createOutputFile("organization", acc.getName());
JsonGenerator jg = getJsonGenerator(outFile);
jg.writeObject(acc);
jg.close();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void streamOutput(File file, List entities) throws Exception {
JsonFactory jsonFactory = new JsonFactory();
// or, for data binding,
// org.codehaus.jackson.mapper.MappingJsonFactory
JsonGenerator jg = jsonFactory.createJsonGenerator(file, JsonEncoding.UTF8);
// or Stream, Reader
jg.writeStartArray();
for (Entity entity : entities) {
jg.writeObject(entity);
}
jg.writeEndArray();
jg.close();
}
// to generate the activities and user relationship, follow this:
// write field name (id)
// write start object
// write field name (collection name)
// write start array
// write object/string
// write another object
// write end array
// write end object
// ...... more objects
//
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy