de.bwaldvogel.mongo.backend.AbstractOplogTest Maven / Gradle / Ivy

Go to download
package de.bwaldvogel.mongo.backend;

import static com.mongodb.client.model.Aggregates.match;
import static com.mongodb.client.model.Filters.eq;
import static com.mongodb.client.model.Updates.set;
import static com.mongodb.client.model.Updates.unset;
import static de.bwaldvogel.mongo.backend.TestUtils.json;
import static de.bwaldvogel.mongo.backend.TestUtils.toArray;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;

import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.UUID;

import org.bson.BsonDocument;
import org.bson.BsonInt32;
import org.bson.BsonTimestamp;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

import com.mongodb.client.ChangeStreamIterable;
import com.mongodb.client.MongoChangeStreamCursor;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.ReplaceOptions;
import com.mongodb.client.model.changestream.ChangeStreamDocument;
import com.mongodb.client.model.changestream.FullDocument;
import com.mongodb.client.result.InsertOneResult;
import com.mongodb.client.result.UpdateResult;

import de.bwaldvogel.mongo.oplog.OperationType;

public abstract class AbstractOplogTest extends AbstractTest {

    protected static final String LOCAL_DATABASE = "local";
    protected static final String OPLOG_COLLECTION_NAME = "oplog.rs";

    @BeforeEach
    void beforeEach() {
        backend.enableOplog();
    }

    @Override
    protected void dropAllDatabases() {
        super.dropAllDatabases();
        clearOplog();
    }

    protected void clearOplog() {
        getOplogCollection().deleteMany(json(""));
    }

    protected MongoCollection getOplogCollection() {
        MongoDatabase localDb = syncClient.getDatabase(LOCAL_DATABASE);
        return localDb.getCollection(OPLOG_COLLECTION_NAME);
    }

    @Test
    void testListDatabaseNames() throws Exception {
        assertThat(listDatabaseNames()).contains(LOCAL_DATABASE);
        collection.insertOne(json(""));
        assertThat(listDatabaseNames()).containsExactlyInAnyOrder(db.getName(), LOCAL_DATABASE);
        syncClient.getDatabase("bar").getCollection("some-collection").insertOne(json(""));
        assertThat(listDatabaseNames()).containsExactlyInAnyOrder("bar", db.getName(), LOCAL_DATABASE);
    }

    @Test
    void testOplogInsertUpdateAndDelete() {
        Document document = json("_id: 1, name: 'testUser1'");

        collection.insertOne(document);
        clock.windForward(Duration.ofSeconds(1));
        collection.updateOne(json("_id: 1"), json("$set: {name: 'user 2'}"));
        clock.windForward(Duration.ofSeconds(1));
        collection.deleteOne(json("_id: 1"));

        List oplogDocuments = toArray(getOplogCollection().find().sort(json("ts: 1")));
        assertThat(oplogDocuments).hasSize(3);

        Document insertOplogDocument = oplogDocuments.get(0);
        assertThat(insertOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o");
        assertThat(insertOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
        assertThat(insertOplogDocument.get("t")).isEqualTo(1L);
        assertThat(insertOplogDocument.get("h")).isEqualTo(0L);
        assertThat(insertOplogDocument.get("v")).isEqualTo(2L);
        assertThat(insertOplogDocument.get("op")).isEqualTo(OperationType.INSERT.getCode());
        assertThat(insertOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
        assertThat(insertOplogDocument.get("ui")).isInstanceOf(UUID.class);
        assertThat(insertOplogDocument.get("wall")).isEqualTo(Date.from(Instant.parse("2019-05-23T12:00:00.123Z")));
        assertThat(insertOplogDocument.get("o")).isEqualTo(document);

        Document updateOplogDocument = oplogDocuments.get(1);
        assertThat(updateOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o", "o2");
        assertThat(updateOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
        assertThat(updateOplogDocument.get("t")).isEqualTo(1L);
        assertThat(updateOplogDocument.get("h")).isEqualTo(0L);
        assertThat(updateOplogDocument.get("v")).isEqualTo(2L);
        assertThat(updateOplogDocument.get("op")).isEqualTo(OperationType.UPDATE.getCode());
        assertThat(updateOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
        assertThat(updateOplogDocument.get("ui")).isInstanceOf(UUID.class);
        assertThat(updateOplogDocument.get("wall")).isEqualTo(Date.from(Instant.parse("2019-05-23T12:00:01.123Z")));
        assertThat(updateOplogDocument.get("o2")).isEqualTo(json("_id: 1"));
        assertThat(updateOplogDocument.get("o")).isEqualTo(json("$set: {name: 'user 2'}"));

        Document deleteOplogDocument = oplogDocuments.get(2);
        assertThat(deleteOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o");
        assertThat(deleteOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
        assertThat(deleteOplogDocument.get("t")).isEqualTo(1L);
        assertThat(deleteOplogDocument.get("h")).isEqualTo(0L);
        assertThat(deleteOplogDocument.get("v")).isEqualTo(2L);
        assertThat(deleteOplogDocument.get("op")).isEqualTo(OperationType.DELETE.getCode());
        assertThat(deleteOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
        assertThat(deleteOplogDocument.get("ui")).isInstanceOf(UUID.class);
        assertThat(deleteOplogDocument.get("wall")).isEqualTo(Date.from(Instant.parse("2019-05-23T12:00:02.123Z")));
        assertThat(deleteOplogDocument.get("o")).isEqualTo(json("_id: 1"));
    }

    @Test
    void testQueryOplogWhenOplogIsDisabled() throws Exception {
        backend.disableOplog();
        collection.insertOne(json("_id: 1"));
        assertThat(getOplogCollection().find()).isEmpty();
    }

    @Test
    void testSetOplogReplaceOneById() {
        collection.insertOne(json("_id: 1, b: 6"));
        collection.replaceOne(json("_id: 1"), json("a: 5, b: 7"));
        List oplogDocuments = toArray(getOplogCollection().find().sort(json("ts: 1")));
        Document updateOplogEntry = oplogDocuments.get(1);
        assertThat(updateOplogEntry.get("op")).isEqualTo(OperationType.UPDATE.getCode());
        assertThat(updateOplogEntry.get("ns")).isEqualTo(collection.getNamespace().toString());
        assertThat(updateOplogEntry.get("o")).isEqualTo(json("_id: 1, a: 5, b: 7"));
        assertThat(updateOplogEntry.get("o2")).isEqualTo(json("_id: 1"));
    }

    @Test
    void testSetOplogUpdateOneById() {
        collection.insertOne(json("_id: 34, b: 6"));
        collection.updateOne(eq("_id", 34), set("a", 6));
        List oplogDocuments = toArray(getOplogCollection().find(json("op: 'u'")).sort(json("ts: 1")));

        Document updateOplogDocument = CollectionUtils.getSingleElement(oplogDocuments);
        assertThat(updateOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o", "o2");
        assertThat(updateOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
        assertThat(updateOplogDocument.get("t")).isEqualTo(1L);
        assertThat(updateOplogDocument.get("h")).isEqualTo(0L);
        assertThat(updateOplogDocument.get("v")).isEqualTo(2L);
        assertThat(updateOplogDocument.get("op")).isEqualTo(OperationType.UPDATE.getCode());
        assertThat(updateOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
        assertThat(updateOplogDocument.get("ui")).isInstanceOf(UUID.class);
        assertThat(updateOplogDocument.get("o2")).isEqualTo(json("_id: 34"));
        assertThat(updateOplogDocument.get("o")).isEqualTo(json("$set: {a: 6}"));
    }

    @Test
    @Disabled("This test represents a missing feature")
    void testSetOplogUpdateOneByIdMultipleFields() {
        collection.insertOne(json("_id: 1, b: 6"));
        collection.updateOne(eq("_id", 1), List.of(set("a", 7), set("b", 7)));
        List oplogDocuments = toArray(getOplogCollection().find().sort(json("ts: 1")));

        Document updateOplogDocument = oplogDocuments.get(1);
        assertThat(updateOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o", "o2");
        assertThat(updateOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
        assertThat(updateOplogDocument.get("t")).isEqualTo(1L);
        assertThat(updateOplogDocument.get("h")).isEqualTo(0L);
        assertThat(updateOplogDocument.get("v")).isEqualTo(2L);
        assertThat(updateOplogDocument.get("op")).isEqualTo(OperationType.UPDATE.getCode());
        assertThat(updateOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
        assertThat(updateOplogDocument.get("ui")).isInstanceOf(UUID.class);
        assertThat(updateOplogDocument.get("o2")).isEqualTo(json("_id: 1"));
        assertThat(updateOplogDocument.get("o")).isEqualTo(json("$set: {a: 7, b: 7}"));
    }

    @Test
    void testSetOplogUpdateMany() {
        collection.insertMany(List.of(json("_id: 1, b: 6"), json("_id: 2, b: 6")));
        collection.updateMany(eq("b", 6), set("a", 7));

        List oplogDocuments = toArray(getOplogCollection().find(json("op: 'u'")).sort(json("ts: 1, 'o2._id': 1")));
        assertThat(oplogDocuments).hasSize(2);

        for (int i = 0; i < 2; i++) {
            Document updateOplogDocument = oplogDocuments.get(i);
            assertThat(updateOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o", "o2");
            assertThat(updateOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
            assertThat(updateOplogDocument.get("t")).isEqualTo(1L);
            assertThat(updateOplogDocument.get("h")).isEqualTo(0L);
            assertThat(updateOplogDocument.get("v")).isEqualTo(2L);
            assertThat(updateOplogDocument.get("op")).isEqualTo(OperationType.UPDATE.getCode());
            assertThat(updateOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
            assertThat(updateOplogDocument.get("ui")).isInstanceOf(UUID.class);
            assertThat(updateOplogDocument.get("o2")).isEqualTo(json(String.format("_id: %d", i + 1)));
            assertThat(updateOplogDocument.get("o")).isEqualTo(json("$set: {a: 7}"));
        }
    }

    @Test
    void testSetOplogDeleteMany() {
        collection.insertMany(List.of(json("_id: 1, b: 6"), json("_id: 2, b: 6")));
        collection.deleteMany(eq("b", 6));

        List oplogDocuments = toArray(getOplogCollection().find(json("op: 'd'")).sort(json("ts: 1, 'o._id': 1")));
        assertThat(oplogDocuments).hasSize(2);

        for (int i = 0; i < 2; i++) {
            Document updateOplogDocument = oplogDocuments.get(i);
            assertThat(updateOplogDocument).containsKeys("ts", "t", "h", "v", "op", "ns", "ui", "wall", "o");
            assertThat(updateOplogDocument.get("ts")).isInstanceOf(BsonTimestamp.class);
            assertThat(updateOplogDocument.get("t")).isEqualTo(1L);
            assertThat(updateOplogDocument.get("h")).isEqualTo(0L);
            assertThat(updateOplogDocument.get("v")).isEqualTo(2L);
            assertThat(updateOplogDocument.get("op")).isEqualTo(OperationType.DELETE.getCode());
            assertThat(updateOplogDocument.get("ns")).isEqualTo(collection.getNamespace().getFullName());
            assertThat(updateOplogDocument.get("ui")).isInstanceOf(UUID.class);
            assertThat(updateOplogDocument.get("o")).isEqualTo(json(String.format("_id: %d", i + 1)));
        }
    }

    @Test
    void testChangeStreamInsertAndUpdateFullDocumentLookup() {
        collection.insertOne(json("b: 1"));
        int numberOfDocs = 10;
        List insert = new ArrayList<>();
        List update = new ArrayList<>();
        List> changeStreamsResult = new ArrayList<>();
        List pipeline = List.of(match(Filters.or(
            Document.parse("{'fullDocument.b': 1}")))
        );

        try (MongoChangeStreamCursor> cursor =
                 collection.watch(pipeline).fullDocument(FullDocument.UPDATE_LOOKUP).cursor()) {

            final long cursorId = cursor.getServerCursor().getId();

            for (int i = 1; i < numberOfDocs + 1; i++) {
                Document doc = json(String.format("a: %d, b: 1", i));
                collection.insertOne(doc);
                collection.updateOne(eq("a", i), set("c", i * 10));

                assertThat(cursor.hasNext()).isTrue();
                ChangeStreamDocument insertDocument = cursor.next();
                assertThat(cursor.getServerCursor().getId()).isEqualTo(cursorId);

                assertThat(cursor.hasNext()).isTrue();
                ChangeStreamDocument updateDocument = cursor.next();
                assertThat(cursor.getServerCursor().getId()).isEqualTo(cursorId);

                assertThat(insertDocument.getFullDocument().get("a")).isEqualTo(i);
                insert.add(insertDocument.getFullDocument());

                assertThat(updateDocument.getFullDocument().get("a")).isEqualTo(i);
                update.add(updateDocument.getFullDocument());

                changeStreamsResult.addAll(List.of(insertDocument, updateDocument));
            }
        }

        assertThat(insert.size()).isEqualTo(numberOfDocs);
        assertThat(update.size()).isEqualTo(numberOfDocs);
        assertThat(changeStreamsResult.size()).isEqualTo(numberOfDocs * 2);
    }

    @Test
    void testChangeStreamUpdateDefault() {
        collection.insertOne(json("a: 1, b: 2, c: 3"));
        try (MongoChangeStreamCursor> cursor = collection.watch().cursor()) {
            collection.updateOne(eq("a", 1), json("$set: {b: 0, c: 10}"));
            ChangeStreamDocument updateDocument = cursor.next();
            Document fullDoc = updateDocument.getFullDocument();
            assertThat(fullDoc).isNotNull();
            assertThat(fullDoc.get("b")).isEqualTo(0);
            assertThat(fullDoc.get("c")).isEqualTo(10);

            collection.updateOne(eq("a", 1), unset("b"));
            updateDocument = cursor.next();
            fullDoc = updateDocument.getFullDocument();
            assertThat(fullDoc).isNotNull();
            assertThat(fullDoc.get("b")).isEqualTo("");
        }
    }

    @Test
    void testChangeStreamDelete() {
        collection.insertOne(json("_id: 1"));
        try (MongoChangeStreamCursor> cursor = collection.watch().cursor()) {
            collection.deleteOne(json("_id: 1"));
            ChangeStreamDocument deleteDocument = cursor.next();
            assertThat(deleteDocument.getDocumentKey().get("_id")).isEqualTo(new BsonInt32(1));
        }
    }

    @Test
    void testChangeStreamStartAfter() {
        collection.insertOne(json("a: 1")); // This is needed to initialize the collection in the server.
        try (MongoChangeStreamCursor> cursor = collection.watch().cursor()) {
            collection.insertOne(json("a: 2"));
            collection.insertOne(json("a: 3"));
            ChangeStreamDocument document = cursor.next();
            BsonDocument resumeToken = document.getResumeToken();

            try (MongoChangeStreamCursor> cursor2
                     = collection.watch().startAfter(resumeToken).cursor()) {
                ChangeStreamDocument document2 = cursor2.next();
                assertThat(document2.getFullDocument().get("a")).isEqualTo(3);
            }
        }
    }

    @Test
    void testChangeStreamResumeAfter() throws Exception {
        collection.insertOne(json("a: 1"));
        try (MongoChangeStreamCursor> cursor = collection.watch().cursor()) {
            awaitNumberOfOpenCursors(1);
            collection.insertOne(json("a: 2"));
            collection.insertOne(json("a: 3"));
            ChangeStreamDocument document = cursor.next();
            BsonDocument resumeToken = document.getResumeToken();

            try (MongoChangeStreamCursor> cursor2
                     = collection.watch().resumeAfter(resumeToken).cursor()) {
                awaitNumberOfOpenCursors(2);
                ChangeStreamDocument document2 = cursor2.next();
                assertThat(document2.getFullDocument().get("a")).isEqualTo(3);
            }
        }
    }

    @Test
    void testChangeStreamResumeAfterTerminalEvent() {
        MongoCollection col = db.getCollection("test-collection");
        ChangeStreamIterable watch = col.watch().fullDocument(FullDocument.UPDATE_LOOKUP).batchSize(1);
        try (MongoChangeStreamCursor> cursor = watch.cursor()) {
            col.insertOne(json("a: 1"));
            cursor.next();

            col.drop();

            ChangeStreamDocument document = cursor.next();
            BsonDocument resumeToken = document.getResumeToken();
            try (MongoChangeStreamCursor> resumeAfterCursor
                     = watch.resumeAfter(resumeToken).cursor();) {
                document = resumeAfterCursor.next();

                assertThat(document).isNotNull();
                assertThat(document.getOperationType())
                    .isEqualTo(com.mongodb.client.model.changestream.OperationType.INVALIDATE);

                assertThatExceptionOfType(NoSuchElementException.class)
                    .isThrownBy(resumeAfterCursor::next);
            }
        }
    }

    @Test
    void testChangeStreamStartAtOperationTime() {
        collection.insertOne(json("a: 1"));
        try (MongoChangeStreamCursor> cursor = collection.watch().cursor()) {
            collection.insertOne(json("a: 2"));
            collection.insertOne(json("a: 3"));
            ChangeStreamDocument document = cursor.next();
            BsonTimestamp startAtOperationTime = document.getClusterTime();

            try (MongoChangeStreamCursor> cursor2 = collection.watch().startAtOperationTime(startAtOperationTime).cursor()) {
                ChangeStreamDocument document2 = cursor2.next();
                assertThat(document2.getFullDocument().get("a")).isEqualTo(2);
                document2 = cursor2.next();
                assertThat(document2.getFullDocument().get("a")).isEqualTo(3);
            }
        }
    }

    @Test
    void testChangeStreamAndReplaceOneWithUpsertTrue() throws Exception {
        TestSubscriber> streamSubscriber = new TestSubscriber<>();
        asyncCollection.watch().fullDocument(FullDocument.UPDATE_LOOKUP).subscribe(streamSubscriber);
        awaitNumberOfOpenCursors(1);

        TestSubscriber replaceOneSubscriber = new TestSubscriber<>();
        asyncCollection.replaceOne(json("a: 1"), json("a: 1"), new ReplaceOptions().upsert(true))
            .subscribe(replaceOneSubscriber);

        replaceOneSubscriber.awaitSingleValue();

        TestSubscriber findSubscriber = new TestSubscriber<>();
        asyncCollection.find(json("a:1")).subscribe(findSubscriber);
        assertThat(findSubscriber.awaitSingleValue().get("a")).isEqualTo(1);

        ChangeStreamDocument value = streamSubscriber.awaitSingleValue();
        assertThat(value.getOperationType().getValue()).isEqualTo("insert");
        assertThat(value.getFullDocument()).isEqualTo(findSubscriber.awaitSingleValue());
    }

    @Test
    void testSimpleChangeStreamWithFilter() throws Exception {
        insertOne(asyncCollection, json("_id: 1"));

        Bson filter = match(Filters.eq("fullDocument.bu", "abc"));
        List pipeline = List.of(filter);

        super.assertNoOpenCursors();
        TestSubscriber> streamSubscriber = new TestSubscriber<>();
        asyncCollection.watch(pipeline).subscribe(streamSubscriber);
        awaitNumberOfOpenCursors(1);

        insertOne(asyncCollection, json("_id: 2, bu: 'abc'"));
        insertOne(asyncCollection, json("_id: 3, bu: 'xyz'"));

        ChangeStreamDocument changeStreamDocument = streamSubscriber.awaitSingleValue();
        assertThat(changeStreamDocument.getFullDocument().get("bu")).isEqualTo("abc");
    }

    @Test
    void testOplogSubscription() throws Exception {
        super.assertNoOpenCursors();
        TestSubscriber> streamSubscriber = new TestSubscriber<>();
        asyncCollection.watch().subscribe(streamSubscriber);
        awaitNumberOfOpenCursors(1);

        insertOne(asyncCollection, json("_id: 1"));

        ChangeStreamDocument changeStreamDocument = streamSubscriber.awaitSingleValue();
        assertThat(changeStreamDocument.getOperationType()).isEqualTo(com.mongodb.client.model.changestream.OperationType.INSERT);
        assertThat(changeStreamDocument.getFullDocument()).isEqualTo(json("_id: 1"));
    }

    @Test
    void testOplogShouldFilterNamespaceOnChangeStreams() throws Exception {
        com.mongodb.reactivestreams.client.MongoCollection asyncCollection1 =
            asyncDb.getCollection(asyncCollection.getNamespace().getCollectionName() + "1");

        insertOne(asyncCollection, json("_id: 1"));
        insertOne(asyncCollection1, json("_id: 1"));

        super.assertNoOpenCursors();
        TestSubscriber> streamSubscriber = new TestSubscriber<>();
        asyncCollection.watch().subscribe(streamSubscriber);
        awaitNumberOfOpenCursors(1);

        insertOne(asyncCollection1, json("_id: 2"));
        insertOne(asyncCollection, json("_id: 2"));

        streamSubscriber.awaitSingleValue();
    }

    private static void insertOne(com.mongodb.reactivestreams.client.MongoCollection collection, Document document) throws Exception {
        TestSubscriber insertSubscriber = new TestSubscriber<>();
        collection.insertOne(document).subscribe(insertSubscriber);
        insertSubscriber.awaitSingleValue();
    }

}