All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • org.cloudgraph.hbase.mutation.GraphMutationCollector Maven / Gradle / Ivy

    Go to download

    CloudGraph(tm) is a suite of Service Data Object (SDO) 2.1 services designed for relational and big-table style "cloud" databases, such as HBase and others.

    There is a newer version: 2.0.4
    Show newest version
    /**
 *        CloudGraph Community Edition (CE) License
 * 
 * This is a community release of CloudGraph, a dual-license suite of
 * Service Data Object (SDO) 2.1 services designed for relational and 
 * big-table style "cloud" databases, such as HBase and others. 
 * This particular copy of the software is released under the 
 * version 2 of the GNU General Public License. CloudGraph was developed by 
 * TerraMeta Software, Inc.
 * 
 * Copyright (c) 2013, TerraMeta Software, Inc. All rights reserved.
 * 
 * General License information can be found below.
 * 
 * This distribution may include materials developed by third
 * parties. For license and attribution notices for these
 * materials, please refer to the documentation that accompanies
 * this distribution (see the "Licenses for Third-Party Components"
 * appendix) or view the online documentation at 
 * . 
 */
package org.cloudgraph.hbase.mutation;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.Row;
import org.apache.hadoop.hbase.util.Bytes;
import org.cloudgraph.hbase.io.DistributedGraphWriter;
import org.cloudgraph.hbase.io.DistributedWriter;
import org.cloudgraph.hbase.io.RowWriter;
import org.cloudgraph.hbase.io.TableWriter;
import org.cloudgraph.hbase.io.TableWriterCollector;
import org.cloudgraph.hbase.service.ServiceContext;
import org.cloudgraph.state.GraphState;
import org.cloudgraph.store.service.CreatedCommitComparator;
import org.plasma.sdo.AssociationPath;
import org.plasma.sdo.PlasmaChangeSummary;
import org.plasma.sdo.PlasmaDataObject;
import org.plasma.sdo.PlasmaType;
import org.plasma.sdo.access.provider.common.DeletedObjectCollector;
import org.plasma.sdo.access.provider.common.ModifiedObjectCollector;
import org.plasma.sdo.core.CoreDataObject;
import org.plasma.sdo.core.SnapshotMap;
import org.plasma.sdo.repository.Class_;
import org.plasma.sdo.repository.Classifier;
import org.plasma.sdo.repository.PlasmaRepository;

import commonj.sdo.DataGraph;
import commonj.sdo.DataObject;
import commonj.sdo.Property;
import sorts.InsertionSort;

/**
 * Traverses the change summary for one or more data graphs and collects changes in the form
 * of HBase row mutations.
 * 
    
 * For each graph: 
 * - performs any change summary ordering
 * - collects table writers based on graph metadata and configuration information
 * - assembles table writers for the graph into a single graph writer composed of table writers, which are
 * composed of row writers
 * - passes each changed object (created, modified, deleted) along with the graph writer to logic
 * within this class.    
 * - marshals out the state for each changed row after all changes complete in to the state column
 * - for each row, detects if the root object is deleted, and then adds a toumbsone column  
 * 
    
 *  
 * @author Scott Cinnamond
 * @since 0.5.8
 */
public class GraphMutationCollector extends DefaultMutation implements MutationCollector {
    private static Log log = LogFactory.getLog(GraphMutationCollector.class);
    protected Create create;
    protected Update update;
    protected Delete delete;
	public GraphMutationCollector(ServiceContext context,
			SnapshotMap snapshotMap, String username) {
		super(context, snapshotMap, username);
		this.create = new Create(context, snapshotMap, username);
		this.update = new Update(context, snapshotMap, username);
		this.delete = new Delete(context, snapshotMap, username);
	}
	
	/* (non-Javadoc)
	 * @see org.cloudgraph.hbase.mutation.MutationCollector#close()
	 */
	@Override
	public void close () {
		this.context.close();
	}
	
	/* (non-Javadoc)
	 * @see org.cloudgraph.hbase.mutation.MutationCollector#collectChanges(commonj.sdo.DataGraph)
	 */
	@Override
	public Map> collectChanges(DataGraph dataGraph) throws IOException, IllegalAccessException {
		Map> mutations = new HashMap>(); 

        PlasmaChangeSummary changeSummary = (PlasmaChangeSummary)dataGraph.getChangeSummary();
        if (log.isDebugEnabled())
            log.debug(changeSummary.toString());
            
        if (changeSummary.getChangedDataObjects().size() == 0)
        {
        	log.warn("no changes detected");
        	return mutations;
        }
    
        for (DataObject changed : changeSummary.getChangedDataObjects()) 
        	this.checkConcurrency(dataGraph, (PlasmaDataObject)changed);                   	        
        
        PlasmaDataObject[] created = sortCreated(changeSummary);
        
        ModifiedObjectCollector modified = new ModifiedObjectCollector(dataGraph);
        DeletedObjectCollector deleted = new DeletedObjectCollector(dataGraph);

		TableWriterCollector collector = 
			new TableWriterCollector(dataGraph, created, modified, deleted);
		
		DistributedWriter graphWriter = new DistributedGraphWriter( 
        	dataGraph, collector, this.context.getMarshallingContext());
        
		this.create(dataGraph, created, graphWriter);    		    		
		this.modify(dataGraph, modified, graphWriter);
		this.delete(dataGraph, deleted, graphWriter);
        
		for (TableWriter tableWriter : graphWriter.getTableWriters()) {
			List rowMutations = mutations.get(tableWriter);
			if (rowMutations == null) {
				rowMutations = new ArrayList();
				mutations.put(tableWriter, rowMutations);
			}
			
			for (RowWriter rowWriter : tableWriter.getAllRowWriters()) {
    			addRemoveStateColumns(tableWriter, rowWriter);
				collectRowMutations(tableWriter, rowWriter, rowMutations);
			}
		}
        return mutations;
	}
	
	/* (non-Javadoc)
	 * @see org.cloudgraph.hbase.mutation.MutationCollector#collectChanges(commonj.sdo.DataGraph[])
	 */
	@Override
	public Map> collectChanges(DataGraph[] dataGraphs) throws IOException, IllegalAccessException {
		Map> mutations = new HashMap>(); 
        
        boolean hasChanges = false;
    	for (DataGraph dataGraph : dataGraphs) {
            PlasmaChangeSummary changeSummary = (PlasmaChangeSummary)dataGraph.getChangeSummary();
            if (log.isDebugEnabled())
                log.debug(changeSummary.toString());
            if (changeSummary.getChangedDataObjects().size() > 0)
            	hasChanges = true;
    	}
    	
    	if (!hasChanges) {
        	log.warn("no changes detected");
        	return mutations;
        }
    
    	for (DataGraph dataGraph : dataGraphs) {
            PlasmaChangeSummary changeSummary = (PlasmaChangeSummary)dataGraph.getChangeSummary();
            for (DataObject changed : changeSummary.getChangedDataObjects()) 
        	    this.checkConcurrency(dataGraph, (PlasmaDataObject)changed);
    	}
        	    
    	List graphWriters = new ArrayList();
    	for (DataGraph dataGraph : dataGraphs) {
            PlasmaChangeSummary changeSummary = (PlasmaChangeSummary)dataGraph.getChangeSummary();

            PlasmaDataObject[] created = sortCreated(changeSummary);
	        
	        ModifiedObjectCollector modified = new ModifiedObjectCollector(dataGraph);
	        DeletedObjectCollector deleted = new DeletedObjectCollector(dataGraph);

			TableWriterCollector collector = 
				new TableWriterCollector(dataGraph, 
					created, modified, deleted);
			
			DistributedWriter graphWriter = new DistributedGraphWriter( 
	        	dataGraph, collector,
	        	this.context.getMarshallingContext());
			graphWriters.add(graphWriter);
	        
    		this.create(dataGraph, created, graphWriter);    		    		
    		this.modify(dataGraph, modified, graphWriter);
    		this.delete(dataGraph, deleted, graphWriter);
            
    	}    	
    	
    	for (DistributedWriter graphWriter : graphWriters) {
    		
    		for (TableWriter tableWriter : graphWriter.getTableWriters()) {
    			List rowMutations = mutations.get(tableWriter);
    			if (rowMutations == null) {
    				rowMutations = new ArrayList();
    				mutations.put(tableWriter, rowMutations);
    			}
    			
    			for (RowWriter rowWriter : tableWriter.getAllRowWriters()) {
        			if (log.isDebugEnabled())
        				log.debug("commiting data object: " + rowWriter.getRootDataObject().toString()); 
        			addRemoveStateColumns(tableWriter, rowWriter);
    				collectRowMutations(tableWriter, rowWriter, rowMutations);
    			}
    		}
    	}
    	
        return mutations;
    }

    private void modify(
    		DataGraph dataGraph,
    		ModifiedObjectCollector modified,
    		DistributedWriter graphWriter) throws IllegalAccessException, IOException 
    {
    	for (PlasmaDataObject dataObject : modified.getResult()) {
        	RowWriter rowWriter = graphWriter.getRowWriter(dataObject);
            TableWriter tableWriter = rowWriter.getTableWriter();
        	if (log.isDebugEnabled())
        		log.debug("validating modifications: " + dataObject.getType().getURI() 
        			+ "#" + dataObject.getType().getName());
            this.validateModifications(dataGraph, dataObject, rowWriter);
            
        	if (log.isDebugEnabled())
        		log.debug("modifying: " + dataObject.getType().getURI() 
        			+ "#" + dataObject.getType().getName());
            this.update.collect(dataGraph, dataObject, 
            	graphWriter, tableWriter, rowWriter);
    	}
    }
    
    private void delete(
    		DataGraph dataGraph,
    		DeletedObjectCollector deleted,
    		DistributedWriter graphWriter) throws IllegalAccessException, IOException 
    {
        for (PlasmaDataObject dataObject : deleted.getResult()) {

        	RowWriter rowWriter = graphWriter.getRowWriter(dataObject);
            TableWriter tableWriter = rowWriter.getTableWriter();        	
        	if (log.isDebugEnabled())
        		log.debug("deleting: " + dataObject.getType().getURI() 
        			+ "#" + dataObject.getType().getName());
        	
        	this.delete.collect(dataGraph, dataObject, 
        		 graphWriter, tableWriter, rowWriter);
            rowWriter.getGraphState().removeSequence(dataObject);
        }
    }    
   	
    private void create(DataGraph dataGraph,
    		PlasmaDataObject[] created,
    		DistributedWriter graphWriter) throws IOException, IllegalAccessException {

		for (PlasmaDataObject dataObject : created) {
        	RowWriter rowWriter = graphWriter.getRowWriter(dataObject);        	
        	rowWriter.getGraphState().addSequence(dataObject);
        }
        
        for (PlasmaDataObject dataObject : created) {
        	RowWriter rowWriter = graphWriter.getRowWriter(dataObject);
            TableWriter tableWriter = rowWriter.getTableWriter();        	
        	if (log.isDebugEnabled())
        		log.debug("creating: " + dataObject.getType().getURI() 
        			+ "#" + dataObject.getType().getName());
            this.create.collect(dataGraph, dataObject, 
            	graphWriter,  tableWriter,  rowWriter);
        }       
    }    
 	
	private PlasmaDataObject[] sortCreated(PlasmaChangeSummary changeSummary)
	{
        List createdList = new ArrayList();
        for (DataObject changed : changeSummary.getChangedDataObjects()) {
            if (changeSummary.isCreated(changed))
            	createdList.add((CoreDataObject)changed);
        }
        CoreDataObject[] createdArray = new CoreDataObject[createdList.size()];
        createdList.toArray(createdArray);
        Comparator comparator = new CreatedCommitComparator();
        InsertionSort sort = new InsertionSort();
        sort.sort(createdArray, comparator);
        PlasmaDataObject[] created = new PlasmaDataObject[createdArray.length];
        for (int i = 0; i < createdArray.length; i++)
        	created[i] = createdArray[i];
		return created;
	}
	
	private void collectRowMutations(TableWriter tableWriter, RowWriter rowWriter, List rowMutations) throws IOException
	{
		if (rowWriter.getRow().size() > 0)
			rowMutations.add(rowWriter.getRow());
		
		if (rowWriter.hasRowDelete()) // may have row delete for entire row or some columns
			rowMutations.add(rowWriter.getRowDelete());
	}
	
	private void addRemoveStateColumns(TableWriter tableWriter, RowWriter rowWriter) throws IOException
	{
		String rootUUID = ((PlasmaDataObject)rowWriter.getRootDataObject()).getUUIDAsString();
		if (rowWriter.isRootCreated()) {
		    rowWriter.getRow().addColumn(tableWriter.getTableConfig().getDataColumnFamilyNameBytes(), 
                Bytes.toBytes(GraphState.ROOT_UUID_COLUMN_NAME),
                Bytes.toBytes(rootUUID)); 
		    
		    if (tableWriter.getTableConfig().tombstoneRowsOverwriteable()) {
				rowWriter.getRowDelete().addColumns( // deletes all versions
						tableWriter.getTableConfig().getDataColumnFamilyNameBytes(), 
						Bytes.toBytes(GraphState.TOUMBSTONE_COLUMN_NAME));		    	
		    }
		}

		if (!rowWriter.isRootDeleted()) {
			String xml = rowWriter.getGraphState().marshal();
	    	if (log.isDebugEnabled())
	    		log.debug("writing state ("+rootUUID+"): " + xml);
			rowWriter.getRow().addColumn(Bytes.toBytes(tableWriter.getTableConfig().getDataColumnFamilyName()), 
	                Bytes.toBytes(GraphState.STATE_COLUMN_NAME),
	                Bytes.toBytes(xml));    		        					
		}
		else { // root is deleted
	    	if (log.isDebugEnabled())
	    		log.debug("processing deleted root ("+rootUUID+")");
			if (tableWriter.getTableConfig().tombstoneRows()) {
				// since this is the root, and someone could be pointing at us, 
				// add a tombstone column
				if (log.isDebugEnabled())
					log.debug("adding toumbstone for root "
						+ rowWriter.getRootDataObject().toString());
				rowWriter.getRow().addColumn(
					tableWriter.getTableConfig().getDataColumnFamilyNameBytes(), 
					Bytes.toBytes(GraphState.TOUMBSTONE_COLUMN_NAME), 
					Bytes.toBytes(this.snapshotMap.getSnapshotDate().getTime()));						
				rowWriter.getRowDelete().addColumns( // deletes all version
						tableWriter.getTableConfig().getDataColumnFamilyNameBytes(), 
						Bytes.toBytes(GraphState.STATE_COLUMN_NAME));
			}
		}		
	}
	
	private boolean parentLinksDeleted(PlasmaChangeSummary changeSummary, DataObject root) {
		boolean allDeleted = true;
		PlasmaType rootType = (PlasmaType)root.getType();
		List uris = PlasmaRepository.getInstance().getAllNamespaceUris();
		for (String uri : uris) {
		    List classifiers = PlasmaRepository.getInstance().getClassifiers(uri);
		    for (Classifier c : classifiers) {
		    	if (c instanceof Class_) {
		    		((Class_)rootType.getClassifier()).isRelation(((Class_)c), AssociationPath.singular_binary);
		    	}
		    }
		}
		
		return allDeleted;
	}
	
	/**
	 * Determines if all data object linked to every property where the root 
     * type is its source have been deleted
	 * @param changeSummary the change summary
	 * @param root the root data object
	 * @return whether all data object linked to every property where the root 
     * type is its source have been deleted
	 */
	private boolean childLinksDeleted(PlasmaChangeSummary changeSummary, DataObject root) {
		boolean allDeleted = true;
		PlasmaType rootType = (PlasmaType)root.getType();
		for (Property rootProp : rootType.getProperties()) {
			if (rootProp.getType().isDataType())
				continue;
			if (rootProp.isMany()) {
				@SuppressWarnings("unchecked")
				List list = root.getList(rootProp);
        	    for (DataObject other : list)
					if (!changeSummary.isDeleted(other))
						allDeleted = false;
			}
			else {
				DataObject other = (DataObject)root.getDataObject(rootProp);
				if (other != null) {
				    if (!changeSummary.isDeleted(other))
					    allDeleted = false;
				}
				else {
					// root has a property, but how to tell if its used even							
					log.warn("potentially incomplete delete graph detected for type, "
							+ rootProp.getType() + " within root graph type " + rootType
							+ " - ");	
				}
			}
		}
		return allDeleted;
	}

}




    © 2015 - 2025 Weber Informatics LLC | Privacy Policy