org.janusgraph.olap.OLAPTest Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of janusgraph-test Show documentation

There is a newer version: 100.3.2.1

// Copyright 2017 JanusGraph Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.janusgraph.olap;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.janusgraph.core.*;
import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJob;
import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics;
import org.janusgraph.graphdb.JanusGraphBaseTest;
import org.janusgraph.graphdb.olap.*;
import org.janusgraph.graphdb.olap.computer.FulgoraGraphComputer;
import org.janusgraph.graphdb.olap.job.GhostVertexRemover;
import org.apache.tinkerpop.gremlin.process.computer.*;
import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
import org.apache.tinkerpop.gremlin.process.computer.util.StaticVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.concurrent.ExecutionException;

import static org.janusgraph.testutil.JanusGraphAssert.assertCount;
import static org.junit.jupiter.api.Assertions.*;

/**
 * @author Matthias Broecheler ([email protected])
 */
public abstract class OLAPTest extends JanusGraphBaseTest {

    private static final Random random = new Random();

    private static final Logger log =
            LoggerFactory.getLogger(OLAPTest.class);

    @Override
    @BeforeEach
    public void setUp(TestInfo testInfo) throws Exception {
        super.setUp(testInfo);
    }

    private ScanMetrics executeScanJob(VertexScanJob job) throws Exception {
        return executeScanJob(VertexJobConverter.convert(graph,job));
    }

    private ScanMetrics executeScanJob(ScanJob job) throws Exception {
        return graph.getBackend().buildEdgeScanJob()
                .setNumProcessingThreads(2)
                .setWorkBlockSize(100)
                .setJob(job)
                .execute().get();
    }

    private int generateRandomGraph(int numV) {
        mgmt.makePropertyKey("uid").dataType(Integer.class).cardinality(Cardinality.SINGLE).make();
        mgmt.makeEdgeLabel("knows").multiplicity(Multiplicity.MULTI).make();
        mgmt.makePropertyKey("values").cardinality(Cardinality.LIST).dataType(Integer.class).make();
        mgmt.makePropertyKey("numvals").dataType(Integer.class).make();
        finishSchema();
        int numE = 0;
        JanusGraphVertex[] vs = new JanusGraphVertex[numV];
        for (int i=0;iproperty("uid").orElse(0) > 0);
                metrics.incrementCustom(DEGREE_COUNT,outDegree);
                metrics.incrementCustom(VERTEX_COUNT);
            }

            @Override
            public void getQueries(QueryContainer queries) {
                queries.addQuery().labels("knows").direction(Direction.OUT).edges();
                queries.addQuery().keys("uid").properties();
            }

            @Override
            public VertexScanJob clone() { return this; }
        });
        assertEquals(numV,result1.getCustom(VERTEX_COUNT));
        assertEquals(numE,result1.getCustom(DEGREE_COUNT));

        ScanMetrics result2 = executeScanJob(new VertexScanJob() {

            @Override
            public void process(JanusGraphVertex vertex, ScanMetrics metrics) {
                metrics.incrementCustom(VERTEX_COUNT);
                assertEquals(1 ,vertex.query().labels("numvals").propertyCount());
                int numvals = vertex.value("numvals");
                assertEquals(numvals, vertex.query().labels("values").propertyCount());
            }

            @Override
            public void getQueries(QueryContainer queries) {
                queries.addQuery().keys("values").properties();
                queries.addQuery().keys("numvals").properties();
            }

            @Override
            public VertexScanJob clone() { return this; }
        });
        assertEquals(numV,result2.getCustom(VERTEX_COUNT));
    }

    @Test
    public void removeGhostVertices() throws Exception {
        JanusGraphVertex v1 = tx.addVertex("person");
        v1.property("name","stephen");
        JanusGraphVertex v2 = tx.addVertex("person");
        v1.property("name","marko");
        JanusGraphVertex v3 = tx.addVertex("person");
        v1.property("name","dan");
        v2.addEdge("knows",v3);
        v1.addEdge("knows",v2);
        newTx();
        long v3id = getId(v3);
        long v1id = getId(v1);
        assertTrue(v3id>0);

        v3 = getV(tx, v3id);
        assertNotNull(v3);
        v3.remove();
        tx.commit();

        JanusGraphTransaction xx = graph.buildTransaction().checkExternalVertexExistence(false).start();
        v3 = getV(xx, v3id);
        assertNotNull(v3);
        v1 = getV(xx, v1id);
        assertNotNull(v1);
        v3.property("name", "deleted");
        v3.addEdge("knows", v1);
        xx.commit();

        newTx();
        assertNull(getV(tx,v3id));
        v1 = getV(tx, v1id);
        assertNotNull(v1);
        assertEquals(v3id, v1.query().direction(Direction.IN).labels("knows").vertices().iterator().next().longId());
        tx.commit();
        mgmt.commit();

        ScanMetrics result = executeScanJob(new GhostVertexRemover(graph));
        assertEquals(1,result.getCustom(GhostVertexRemover.REMOVED_VERTEX_COUNT));
        assertEquals(2,result.getCustom(GhostVertexRemover.REMOVED_RELATION_COUNT));
        assertEquals(0,result.getCustom(GhostVertexRemover.SKIPPED_GHOST_LIMIT_COUNT));
    }

    @Test
    public void testBasicComputeJob() {
        GraphTraversalSource g = graph.traversal().withComputer(FulgoraGraphComputer.class);
        System.out.println(g.V().count().next());
    }

    @Test
    public void degreeCounting() throws Exception {
        int numV = 200;
        int numE = generateRandomGraph(numV);
        clopen();

        final JanusGraphComputer computer = graph.compute();
        computer.resultMode(JanusGraphComputer.ResultMode.NONE);
        computer.workers(4);
        computer.program(new DegreeCounter());
        computer.mapReduce(new DegreeMapper());
        ComputerResult result = computer.submit().get();
        System.out.println("Execution time (ms) ["+numV+"|"+numE+"]: " + result.memory().getRuntime());
        assertTrue(result.memory().exists(DegreeMapper.DEGREE_RESULT));
        Map degrees = result.memory().get(DegreeMapper.DEGREE_RESULT);
        assertNotNull(degrees);
        assertEquals(numV,degrees.size());
        int totalCount = 0;
        for (Map.Entry entry : degrees.entrySet()) {
            int degree = entry.getValue();
            final JanusGraphVertex v = getV(tx, entry.getKey());
            int count = v.value("uid");
            assertEquals(count,degree);
            totalCount+= degree;
        }
        assertEquals(numV*(numV+1)/2,totalCount);
        assertEquals(1,result.memory().getIteration());
    }

    @Test
    public void vertexProgramExceptionPropagatesToCaller() throws InterruptedException
    {
        int numV = 100;
        generateRandomGraph(numV);
        clopen();

        final JanusGraphComputer computer = graph.compute();
        computer.resultMode(JanusGraphComputer.ResultMode.NONE);
        computer.workers(1);
        computer.program(new ExceptionProgram());

        try {
            computer.submit().get();
            fail();
        } catch (ExecutionException ignored) {
        }
    }

    @Test
    public void degreeCountingDistance() throws Exception {
        int numV = 100;
        int numE = generateRandomGraph(numV);
        clopen();

        // TODO does this iteration over JanusGraphComputer.ResultMode values imply that DegreeVariation's ResultGraph/Persist should also change?
        for (JanusGraphComputer.ResultMode mode : JanusGraphComputer.ResultMode.values()) {
            final JanusGraphComputer computer = graph.compute();
            computer.resultMode(mode);
            computer.workers(1);
            computer.program(new DegreeCounter(2));
            ComputerResult result = computer.submit().get();
            System.out.println("Execution time (ms) ["+numV+"|"+numE+"]: " + result.memory().getRuntime());
            assertEquals(2,result.memory().getIteration());

            Transaction gview = null;
            switch (mode) {
                case LOCALTX: gview = (Transaction) result.graph(); break;
                case PERSIST: newTx(); gview = tx; break;
                case NONE: break;
                default: throw new AssertionError(mode);
            }
            if (gview == null) continue;

            for (JanusGraphVertex v : gview.query().vertices()) {
                long degree2 = ((Integer)v.value(DegreeCounter.DEGREE)).longValue();
                long actualDegree2 = 0;
                for (Object w : v.query().direction(Direction.OUT).vertices()) {
                    actualDegree2 += Iterables.size(((JanusGraphVertex) w).query().direction(Direction.OUT).vertices());
                }
                assertEquals(actualDegree2,degree2);
            }
            if (mode== JanusGraphComputer.ResultMode.LOCALTX) {
                assertTrue(gview instanceof JanusGraphTransaction);
                ((JanusGraphTransaction) gview).rollback();
            }
        }
    }

    public static class ExceptionProgram extends StaticVertexProgram
    {

        @Override
        public void setup(Memory memory)
        {

        }

        @Override
        public void execute(Vertex vertex, Messenger messenger, Memory memory)
        {
            throw new NullPointerException();
        }

        @Override
        public boolean terminate(Memory memory)
        {
            return memory.getIteration() > 1;
        }

        @Override
        public Set getMessageScopes(Memory memory)
        {
            return ImmutableSet.of();
        }

        @Override
        public GraphComputer.ResultGraph getPreferredResultGraph() {
            return GraphComputer.ResultGraph.NEW;
        }

        @Override
        public GraphComputer.Persist getPreferredPersist() {
            return GraphComputer.Persist.VERTEX_PROPERTIES;
        }

        @Override
        public Features getFeatures() {
            return new Features() {
                @Override
                public boolean requiresLocalMessageScopes() {
                    return true;
                }

                @Override
                public boolean requiresVertexPropertyAddition() {
                    return true;
                }
            };
        }
    }

    public static class DegreeCounter extends StaticVertexProgram {

        public static final String DEGREE = "degree";
        public static final MessageCombiner ADDITION = (a,b) -> a+b;
        public static final MessageScope.Local DEG_MSG = MessageScope.Local.of(__::inE);

        private final int length;

        public DegreeCounter() {
            this(1);
        }

        public DegreeCounter(int length) {
            Preconditions.checkArgument(length>0);
            this.length = length;
        }

        @Override
        public void setup(Memory memory) {
        }

        @Override
        public void execute(Vertex vertex, Messenger messenger, Memory memory) {
            if (memory.isInitialIteration()) {
                messenger.sendMessage(DEG_MSG, 1);
            } else {
                int degree = IteratorUtils.stream(messenger.receiveMessages()).reduce(0, (a, b) -> a + b);
                vertex.property(VertexProperty.Cardinality.single, DEGREE, degree);
                if (memory.getIteration()=length;
        }

        @Override
        public Set getVertexComputeKeys() {
            return new HashSet<>(Collections.singletonList(VertexComputeKey.of(DEGREE, false)));
        }

        @Override
        public Set getMemoryComputeKeys() {
            return new HashSet<>(Collections.singletonList(MemoryComputeKey.of(DEGREE, Operator.assign, true, false)));
        }

        @Override
        public Optional> getMessageCombiner() {
            return Optional.of(ADDITION);
        }

        @Override
        public Set getMessageScopes(Memory memory) {
            if (memory.getIteration()> {

        public static final String DEGREE_RESULT = "degrees";

        @Override
        public boolean doStage(Stage stage) {
            return stage==Stage.MAP;
        }

        @Override
        public void map(Vertex vertex, MapEmitter emitter) {
            emitter.emit((Long)vertex.id(),vertex.value(DegreeCounter.DEGREE));
        }

        @Override
        public Map generateFinalResult(Iterator> keyValues) {
            Map result = new HashMap<>();
            for (; keyValues.hasNext(); ) {
                KeyValue r =  keyValues.next();
                result.put(r.getKey(),r.getValue());
            }
            return result;
        }

        @Override
        public String getMemoryKey() {
            return DEGREE_RESULT;
        }

    }

    public static class Degree {
        public int in;
        public int out;
        public int both;
        public int prop;

        public Degree(int in, int out,int prop) {
            this.in=in;
            this.out=out;
            both=in+out;
            this.prop = prop;
        }

        public Degree() {
            this(0,0,0);
        }

        public void add(Degree d) {
            in+=d.in;
            out+=d.out;
            both+=d.both;
            prop+=d.prop;
        }

    }


    private void expand(Vertex v, final int distance, final int diameter, final int branch) {
        v.property(VertexProperty.Cardinality.single, "distance", distance);
        if (distance{}", u.id(), v.id());
                // Commented since the PageRank implementation does not discriminate by label
//                if (previous!=null) {
//                    u.addEdge("knows",previous);
//                    log.error("knows {}->{}", u.id(), v.id());
//                }
//              previous=u;
                expand(u,distance+1,diameter,branch);
            }
        }
    }

    @Test
    public void testPageRank() throws ExecutionException, InterruptedException {
        mgmt.makePropertyKey("distance").dataType(Integer.class).cardinality(Cardinality.SINGLE).make();
        mgmt.makeEdgeLabel("knows").multiplicity(Multiplicity.MULTI).make();
        mgmt.makeEdgeLabel("likes").multiplicity(Multiplicity.MULTI).make();
        finishSchema();
        final int branch = 6;
        final int diameter = 5;
        final double alpha = 0.85d;
        int numV = (int)((Math.pow(branch,diameter+1)-1)/(branch-1));
        JanusGraphVertex v = tx.addVertex();
        expand(v,0,diameter,branch);
        clopen();
        assertCount(numV, tx.query().vertices());
        log.debug("PR test numV: {}", numV);
        newTx();

        //Precompute correct PR results:
        double[] correctPR = new double[diameter+1];
        for (int i=diameter;i>=0;i--) {
            double pr = (1.0D - alpha)/numV;
            if (ivalue("distance")];
        }

        final JanusGraphComputer computer = graph.compute();
        computer.resultMode(JanusGraphComputer.ResultMode.NONE);
        computer.workers(4);
        computer.program(PageRankVertexProgram.build().iterations(10).vertexCount(numV).dampingFactor(alpha).create(graph));
        computer.mapReduce(PageRankMapReduce.build().create());
        ComputerResult result = computer.submit().get();

        Iterator> ranks = result.memory().get(PageRankMapReduce.DEFAULT_MEMORY_KEY);
        assertNotNull(ranks);
        int vertexCounter = 0;
        double computedPRSum = 0;
        correctPRSum = 0;
        final Set vertexIDs = new HashSet<>(numV);
        while (ranks.hasNext()) {
            final KeyValue rank = ranks.next();
            final Long vertexID = rank.getKey();
            final Double computedPR = rank.getValue();
            assertNotNull(vertexID);
            assertNotNull(computedPR);
            final JanusGraphVertex u = getV(tx, vertexID);
            final int distance = u.value("distance");
            vertexCounter++;

            //assertEquals("Incorrect PR on vertex #" + vertexCounter, correctPR[distance], computedPR, EPSILON);
            computedPRSum += computedPR;
            correctPRSum += correctPR[distance];

            assertFalse(vertexIDs.contains(vertexID));
            vertexIDs.add(vertexID);

            log.debug("vertexID={} computedPR={}", vertexID, computedPR);
        }

        assertEquals(numV, vertexCounter);
        assertEquals(correctPRSum, computedPRSum, 0.001);
    }

    @Test
    public void testShortestDistance() throws Exception {
        PropertyKey distance = mgmt.makePropertyKey("distance").dataType(Integer.class).cardinality(Cardinality.SINGLE).make();
        mgmt.makeEdgeLabel("connect").signature(distance).multiplicity(Multiplicity.MULTI).make();
        finishSchema();

        int maxDepth = 16;
        int maxBranch = 5;
        JanusGraphVertex vertex = tx.addVertex();
        //Grow a star-shaped graph around vertex which will be the single-source for this shortest path computation
        final int numV = growVertex(vertex,0,maxDepth, maxBranch);
        final int numE = numV-1;
        assertCount(numV,tx.query().vertices());
        assertCount(numE,tx.query().edges());

        log.debug("seed inE count: {}", vertex.query().direction(Direction.IN).edgeCount());
        log.debug("seed outE count: {}", vertex.query().direction(Direction.OUT).edgeCount());

        clopen();

        final JanusGraphComputer computer = graph.compute();
        computer.resultMode(JanusGraphComputer.ResultMode.NONE);
        computer.workers(4);
        computer.program(ShortestDistanceVertexProgram.build().seed((long)vertex.id()).maxDepth(maxDepth + 4).create(graph));
        computer.mapReduce(ShortestDistanceMapReduce.build().create());
        ComputerResult result = computer.submit().get();

        Iterator> distances =
                result.memory().get(ShortestDistanceMapReduce.DEFAULT_MEMORY_KEY);

        int vertexCount = 0;

        while (distances.hasNext()) {
            final KeyValue kv = distances.next();
            final long dist = kv.getValue();
            assertTrue(dist >= 0 && dist < Integer.MAX_VALUE, "Invalid distance: " + dist);
            JanusGraphVertex v = getV(tx, kv.getKey());
            assertEquals(v.value("distance").intValue(), dist);
            vertexCount++;
        }

        assertEquals(numV, vertexCount);
        assertTrue(0 < vertexCount);
    }

    private int growVertex(Vertex vertex, int depth, int maxDepth, int maxBranch) {
        vertex.property(VertexProperty.Cardinality.single, "distance", depth);
        int total=1;
        if (depth>=maxDepth) return total;

        for (int i=0;i

    

    

    
            
    
            

    
        
            
                Related Artifacts
                
                     mysql-connector-java mysql
 facebook-messenger com.github.codedrinker
 selenium-java org.seleniumhq.selenium
 instagram-java com.github.sola92
 gson com.google.code.gson
 poi org.apache.poi
 httpclient org.apache.httpcomponents
 json org.json
 facebook-java-api com.google.code.facebook-java-api
 poi-ooxml org.apache.poi
 jackson-databind com.fasterxml.jackson.core
 junit junit
 primefaces org.primefaces
 ojdbc7 com.github.noraui
 jfoenix com.jfoenix
 testng org.testng
 json-simple com.googlecode.json-simple
 selenium-server org.seleniumhq.selenium
 itextpdf com.itextpdf
 spring-core org.springframework
                
            
        
        
            
                Related Groups
                
                     org.springframework
 org.apache.poi
 org.hibernate
 org.springframework.boot
 com.fasterxml.jackson.core
 com.itextpdf
 org.seleniumhq.selenium
 mysql
 org.finos.legend.engine
 org.apache.httpcomponents
 org.apache.logging.log4j
 org.openjfx
 org.apache.commons
 org.json
 com.google.guava
 com.google.zxing
 net.sf.jasperreports
 javax.xml.bind
 ojdbc
 com.google.code.facebook-java-api