com.netflix.zeno.diff.TypeDiffOperation Maven / Gradle / Ivy

/*
 *
 *  Copyright 2013 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.zeno.diff;

import com.netflix.zeno.diff.TypeDiff.FieldDiffScore;
import com.netflix.zeno.serializer.NFTypeSerializer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import org.apache.commons.lang.mutable.MutableInt;

/**
 * This is the main interface for performing a diff between two arbitrary data states.
 *
 * @author dkoszewnik
 *
 */
public class TypeDiffOperation {

    private final TypeDiffInstruction instruction;

    public TypeDiffOperation(TypeDiffInstruction instruction) {
        this.instruction = instruction;
    }

    @SuppressWarnings("unchecked")
    public TypeDiff performDiff(DiffSerializationFramework framework, Iterable fromState, Iterable toState) {
        return performDiff(framework, fromState, toState, Runtime.getRuntime().availableProcessors());
    }

    @SuppressWarnings("unchecked")
    public TypeDiff performDiff(DiffSerializationFramework framework, Iterable fromState, Iterable toState, int numThreads) {
        Map fromStateObjects = new HashMap();

        for(T obj : fromState) {
            fromStateObjects.put(instruction.getKey(obj), obj);
        }

        ArrayList> perProcessorWorkList = new ArrayList>(numThreads); // each entry is a job
        for (int i =0; i < numThreads; ++i) {
            perProcessorWorkList.add(new ArrayList());
        }

        Map toStateKeys = new ConcurrentHashMap();

        int toIncrCount = 0;
        for(T toObject : toState) {
            perProcessorWorkList.get(toIncrCount % numThreads).add(toObject);
            toIncrCount++;
        }

        ExecutorService executor = Executors.newFixedThreadPool(numThreads, new ThreadFactory() {
            @Override
            public Thread newThread(Runnable r) {
                final Thread thread = new Thread(r, "TypeDiff_" + instruction.getTypeIdentifier());
                thread.setDaemon(true);
                return thread;
            }
        });

        try {
            ArrayList>> workResultList = new ArrayList>>(perProcessorWorkList.size());
            for (final List workList : perProcessorWorkList) {
                if (workList != null && !workList.isEmpty()) {
                    workResultList.add(executor.submit(new TypeDiffCallable(framework, instruction, fromStateObjects, toStateKeys, workList)));
                }
            }

            TypeDiff mergedDiff = new TypeDiff(instruction.getTypeIdentifier());
            for (final Future> future : workResultList) {
                try {
                    TypeDiff typeDiff = future.get();
                    mergeTypeDiff(mergedDiff, typeDiff);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }

            for(Map.Entry entry : fromStateObjects.entrySet()) {
                mergedDiff.incrementFrom();
                if(!toStateKeys.containsKey(entry.getKey()))
                    mergedDiff.addExtraInFrom(entry.getValue());
            }

            return mergedDiff;

        } finally {
            executor.shutdownNow();
        }
    }

    private void mergeTypeDiff(TypeDiff mergedDiff, TypeDiff typeDiff) {

        mergedDiff.getExtraInFrom().addAll(typeDiff.getExtraInFrom());
        mergedDiff.getExtraInTo().addAll(typeDiff.getExtraInTo());
        mergedDiff.getDiffObjects().addAll(typeDiff.getDiffObjects());
        mergedDiff.incrementFrom(typeDiff.getItemCountFrom());
        mergedDiff.incrementTo(typeDiff.getItemCountTo());

        Map> mergedFieldDifferences = mergedDiff.getFieldDifferences();
        Map> fieldDifferences = typeDiff.getFieldDifferences();
        for (final DiffPropertyPath path : fieldDifferences.keySet()) {
            FieldDiffScore fieldDiffScore = fieldDifferences.get(path);

            FieldDiffScore mergedFieldDiffScore = mergedFieldDifferences.get(path);
            if (mergedFieldDiffScore != null) {
                mergedFieldDiffScore.incrementDiffCountBy(fieldDiffScore.getDiffCount());
                mergedFieldDiffScore.incrementTotalCountBy(fieldDiffScore.getTotalCount());
                mergedFieldDiffScore.getDiffScores().addAll(fieldDiffScore.getDiffScores());
            } else {
                mergedFieldDifferences.put(path, fieldDiffScore);
            }
        }
    }

    class TypeDiffCallable implements Callable> {

        private final TypeDiffInstruction instruction;
        private final List workList;
        private final Map toStateKeys;
        private final Map fromStateObjects;
        private final DiffSerializationFramework framework;

        public TypeDiffCallable(DiffSerializationFramework framework, TypeDiffInstruction instruction, Map fromStateObjects, Map toStateKeys, List workList) {
            this.framework = framework;
            this.instruction = instruction;
            this.fromStateObjects = fromStateObjects;
            this.toStateKeys = toStateKeys;
            this.workList = workList;
        }

        @Override
        public TypeDiff call() throws Exception {
            TypeDiff diff = new TypeDiff(instruction.getTypeIdentifier());
            NFTypeSerializer typeSerializer = (NFTypeSerializer) framework.getSerializer(instruction.getSerializerName());

            DiffRecord fromRec = new DiffRecord();
            fromRec.setSchema(typeSerializer.getFastBlobSchema());
            DiffRecord toRec = new DiffRecord();
            toRec.setSchema(typeSerializer.getFastBlobSchema());
            fromRec.setTopLevelSerializerName(instruction.getSerializerName());
            toRec.setTopLevelSerializerName(instruction.getSerializerName());

            for(Z toObject : workList) {
                diff.incrementTo();
                Object toStateKey = instruction.getKey(toObject);
                toStateKeys.put(toStateKey, Boolean.TRUE);
                Z fromObject = fromStateObjects.get(toStateKey);

                if(fromObject == null) {
                    diff.addExtraInTo(toObject);
                } else {
                    int diffScore = diffFields(diff, fromRec, toRec, typeSerializer, toObject, fromObject);
                    if(diffScore > 0)
                        diff.addDiffObject(fromObject, toObject, diffScore);
                }
            }

            return diff;
        }

        private int diffFields(TypeDiff diff, DiffRecord fromRec, DiffRecord toRec, NFTypeSerializer typeSerializer, Z toObject, Z fromObject) {
            typeSerializer.serialize(toObject, toRec);
            typeSerializer.serialize(fromObject, fromRec);

            int diffScore = incrementDiffFields(diff, toRec, fromRec, toObject, fromObject);

            toRec.clear();
            fromRec.clear();

            return diffScore;
        }

        private int incrementDiffFields(TypeDiff diff, DiffRecord toRecord, DiffRecord fromRecord, Z toObject, Z fromObject) {
            int objectDiffScore = 0;

            for(DiffPropertyPath key : toRecord.getFieldValues().keySet()) {
                List