All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.projectnessie.versioned.persist.tests.AbstractCommitScenarios Maven / Gradle / Ivy

There is a newer version: 0.74.0
Show newest version
/*
 * Copyright (C) 2020 Dremio
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.projectnessie.versioned.persist.tests;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.projectnessie.versioned.store.DefaultStoreWorker.payloadForContent;
import static org.projectnessie.versioned.testworker.OnRefOnly.newOnRef;
import static org.projectnessie.versioned.testworker.OnRefOnly.onRef;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.function.IntFunction;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.projectnessie.model.Content;
import org.projectnessie.model.ContentKey;
import org.projectnessie.nessie.relocated.protobuf.ByteString;
import org.projectnessie.versioned.BranchName;
import org.projectnessie.versioned.Hash;
import org.projectnessie.versioned.ReferenceConflictException;
import org.projectnessie.versioned.ReferenceNotFoundException;
import org.projectnessie.versioned.persist.adapter.CommitLogEntry;
import org.projectnessie.versioned.persist.adapter.ContentAndState;
import org.projectnessie.versioned.persist.adapter.ContentId;
import org.projectnessie.versioned.persist.adapter.DatabaseAdapter;
import org.projectnessie.versioned.persist.adapter.ImmutableCommitParams;
import org.projectnessie.versioned.persist.adapter.KeyFilterPredicate;
import org.projectnessie.versioned.persist.adapter.KeyListEntry;
import org.projectnessie.versioned.persist.adapter.KeyWithBytes;
import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapter;
import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapterConfigItem;
import org.projectnessie.versioned.store.DefaultStoreWorker;
import org.projectnessie.versioned.testworker.OnRefOnly;

/** Tests performing commits. */
public abstract class AbstractCommitScenarios {

  private final DatabaseAdapter databaseAdapter;

  protected AbstractCommitScenarios(DatabaseAdapter databaseAdapter) {
    this.databaseAdapter = databaseAdapter;
  }

  static class RenameTable {
    final int setupCommits;
    final int afterInitialCommits;
    final int afterRenameCommits;
    final int afterDeleteCommits;

    RenameTable(
        int setupCommits, int afterInitialCommits, int afterRenameCommits, int afterDeleteCommits) {
      this.setupCommits = setupCommits;
      this.afterInitialCommits = afterInitialCommits;
      this.afterRenameCommits = afterRenameCommits;
      this.afterDeleteCommits = afterDeleteCommits;
    }

    @Override
    public String toString() {
      return "setupCommits="
          + setupCommits
          + ", afterInitialCommits="
          + afterInitialCommits
          + ", afterRenameCommits="
          + afterRenameCommits
          + ", afterDeleteCommits="
          + afterDeleteCommits;
    }
  }

  static Stream commitRenameTableParams() {
    Stream zero = Stream.of(new RenameTable(0, 0, 0, 0));

    Stream intervals =
        IntStream.of(19, 20, 21)
            .boxed()
            .flatMap(i -> Stream.of(new RenameTable(i, i, i, i), new RenameTable(0, 0, 0, 0)));

    return Stream.concat(zero, intervals);
  }

  /**
   * Test whether a "table rename operation" works as expected.
   *
   * 

A "table rename" is effectively a remove-operation plus a put-operation with a different key * but the same content-id. * *

Parameterized to force operations to cross/not-cross the number of commits between "full key * lists" and whether global-state shall be used. */ @ParameterizedTest @MethodSource("commitRenameTableParams") void commitRenameTable(RenameTable param) throws Exception { BranchName branch = BranchName.of("main"); ContentKey dummyKey = ContentKey.of("dummy"); ContentKey oldKey = ContentKey.of("hello-table"); ContentKey newKey = ContentKey.of("new-name"); ContentId contentId = ContentId.of("id-42"); IntFunction performDummyCommit = i -> { try { return databaseAdapter .commit( ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("dummy commit meta " + i)) .addUnchanged(dummyKey) .build()) .getCommitHash(); } catch (ReferenceNotFoundException | ReferenceConflictException e) { throw new RuntimeException(e); } }; List beforeInitial = IntStream.range(0, param.setupCommits) .mapToObj(performDummyCommit) .collect(Collectors.toList()); ImmutableCommitParams.Builder commit; Content initialContent = newOnRef("initial commit content"); Content renamContent = OnRefOnly.onRef("rename commit content", initialContent.getId()); byte payload = (byte) payloadForContent(initialContent); commit = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta")) .addPuts( KeyWithBytes.of( oldKey, contentId, payload, DefaultStoreWorker.instance().toStoreOnReferenceState(initialContent))); Hash hashInitial = databaseAdapter.commit(commit.build()).getCommitHash(); List beforeRename = IntStream.range(0, param.afterInitialCommits) .mapToObj(performDummyCommit) .collect(Collectors.toList()); commit = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("rename table")) .addDeletes(oldKey) .addPuts( KeyWithBytes.of( newKey, contentId, payload, DefaultStoreWorker.instance().toStoreOnReferenceState(renamContent))); Hash hashRename = databaseAdapter.commit(commit.build()).getCommitHash(); List beforeDelete = IntStream.range(0, param.afterRenameCommits) .mapToObj(performDummyCommit) .collect(Collectors.toList()); commit = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("delete table")) .addDeletes(newKey); Hash hashDelete = databaseAdapter.commit(commit.build()).getCommitHash(); List afterDelete = IntStream.range(0, param.afterDeleteCommits) .mapToObj(performDummyCommit) .collect(Collectors.toList()); int expectedCommitCount = 1; // Verify that the commits before the initial put return _no_ keys expectedCommitCount = renameCommitVerify( beforeInitial.stream(), expectedCommitCount, keys -> assertThat(keys).isEmpty()); // Verify that the commits since the initial put and before the rename-operation return the // _old_ key expectedCommitCount = renameCommitVerify( Stream.concat(Stream.of(hashInitial), beforeRename.stream()), expectedCommitCount, keys -> assertThat(keys) .containsExactly(KeyListEntry.of(oldKey, contentId, payload, hashInitial))); // Verify that the commits since the rename-operation and before the delete-operation return the // _new_ key expectedCommitCount = renameCommitVerify( Stream.concat(Stream.of(hashRename), beforeDelete.stream()), expectedCommitCount, keys -> assertThat(keys) .containsExactly(KeyListEntry.of(newKey, contentId, payload, hashRename))); // Verify that the commits since the delete-operation return _no_ keys expectedCommitCount = renameCommitVerify( Stream.concat(Stream.of(hashDelete), afterDelete.stream()), expectedCommitCount, keys -> assertThat(keys).isEmpty()); assertThat(expectedCommitCount - 1) .isEqualTo( param.setupCommits + 1 + param.afterInitialCommits + 1 + param.afterRenameCommits + 1 + param.afterDeleteCommits); } private int renameCommitVerify( Stream hashes, int expectedCommitCount, Consumer> keysStreamAssert) throws Exception { for (Hash hash : hashes.collect(Collectors.toList())) { try (Stream keys = databaseAdapter.keys(hash, KeyFilterPredicate.ALLOW_ALL)) { keysStreamAssert.accept(keys); } try (Stream log = databaseAdapter.commitLog(hash)) { assertThat(log).hasSize(expectedCommitCount++); } } return expectedCommitCount; } @ParameterizedTest @ValueSource(ints = {1, 3, 5}) void commit(int tablesPerCommit) throws Exception { BranchName branch = BranchName.of("main"); ArrayList keys = new ArrayList<>(tablesPerCommit); ImmutableCommitParams.Builder commit = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta")); for (int i = 0; i < tablesPerCommit; i++) { ContentKey key = ContentKey.of("my-table-num" + i); keys.add(key); String cid = "id-" + i; OnRefOnly c = OnRefOnly.onRef("initial commit content", cid); commit.addPuts( KeyWithBytes.of( key, ContentId.of(cid), (byte) payloadForContent(c), DefaultStoreWorker.instance().toStoreOnReferenceState(c))); } Hash head = databaseAdapter.commit(commit.build()).getCommitHash(); for (int commitNum = 0; commitNum < 3; commitNum++) { commit = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta")); for (int i = 0; i < tablesPerCommit; i++) { String cid = "id-" + i; OnRefOnly newContent = OnRefOnly.onRef("branch value", cid); commit.addPuts( KeyWithBytes.of( keys.get(i), ContentId.of(cid), (byte) payloadForContent(newContent), DefaultStoreWorker.instance().toStoreOnReferenceState(newContent))); } Hash newHead = databaseAdapter.commit(commit.build()).getCommitHash(); assertThat(newHead).isNotEqualTo(head); head = newHead; } } @Test void duplicateKeys() { BranchName branch = BranchName.of("main"); ContentKey key = ContentKey.of("my.awesome.table"); ContentId contentsId = ContentId.of("cid"); OnRefOnly tableRef = newOnRef("table ref state"); ByteString tableRefState = tableRef.serialized(); OnRefOnly noNo = onRef("no no", contentsId.getId()); KeyWithBytes createPut1 = KeyWithBytes.of(key, contentsId, (byte) payloadForContent(noNo), noNo.serialized()); KeyWithBytes createPut2 = KeyWithBytes.of(key, contentsId, (byte) payloadForContent(tableRef), tableRefState); ImmutableCommitParams.Builder commit1 = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial")) .addPuts(createPut1) .addPuts(createPut2); assertThatThrownBy(() -> databaseAdapter.commit(commit1.build())) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining(key.toString()); ImmutableCommitParams.Builder commit2 = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial")) .addDeletes(key) .addPuts(createPut2); assertThatThrownBy(() -> databaseAdapter.commit(commit2.build())) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining(key.toString()); ImmutableCommitParams.Builder commit3 = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial")) .addDeletes(key) .addUnchanged(key); assertThatThrownBy(() -> databaseAdapter.commit(commit3.build())) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining(key.toString()); ImmutableCommitParams.Builder commit4 = ImmutableCommitParams.builder() .toBranch(branch) .commitMetaSerialized(ByteString.copyFromUtf8("initial")) .addUnchanged(key) .addPuts(createPut2); assertThatThrownBy(() -> databaseAdapter.commit(commit4.build())) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining(key.toString()); } @Test public void smallEmbeddedKeyLists( @NessieDbAdapterConfigItem(name = "key.list.distance", value = "1") @NessieDbAdapter DatabaseAdapter mine) throws Exception { BranchName branchName = BranchName.of("foo"); Hash head = mine.noAncestorHash(); mine.create(branchName, head); ContentKey keyNation = ContentKey.of("tpch-nation"); ContentKey keyRegion = ContentKey.of("tpch-region"); ContentId idNation = ContentId.of("id-nation"); ContentId idRegion = ContentId.of("id-region"); OnRefOnly onRefNation = OnRefOnly.onRef("nation", idNation.getId()); OnRefOnly onRefRegion = OnRefOnly.onRef("region", idRegion.getId()); ByteString stateNation = DefaultStoreWorker.instance().toStoreOnReferenceState(onRefNation); ByteString stateRegion = DefaultStoreWorker.instance().toStoreOnReferenceState(onRefRegion); Hash commitNation = mine.commit( ImmutableCommitParams.builder() .toBranch(branchName) .expectedHead(Optional.of(head)) .commitMetaSerialized(ByteString.copyFromUtf8("commit nation")) .addPuts( KeyWithBytes.of( keyNation, idNation, (byte) payloadForContent(onRefNation), stateNation)) .build()) .getCommitHash(); Hash commitRegion = mine.commit( ImmutableCommitParams.builder() .toBranch(branchName) .expectedHead(Optional.of(commitNation)) .commitMetaSerialized(ByteString.copyFromUtf8("commit region")) .addPuts( KeyWithBytes.of( keyRegion, idRegion, (byte) payloadForContent(onRefRegion), stateRegion)) .build()) .getCommitHash(); List nonExistentKey = Collections.singletonList(ContentKey.of("non_existent")); assertThat(mine.values(commitNation, nonExistentKey, KeyFilterPredicate.ALLOW_ALL)).isEmpty(); assertThat( mine.values( commitNation, Collections.singletonList(keyNation), KeyFilterPredicate.ALLOW_ALL)) .containsEntry( keyNation, ContentAndState.of((byte) payloadForContent(onRefNation), stateNation)); assertThat( mine.values( commitNation, Collections.singletonList(keyRegion), KeyFilterPredicate.ALLOW_ALL)) .isEmpty(); assertThat( mine.values( commitRegion, Collections.singletonList(keyNation), KeyFilterPredicate.ALLOW_ALL)) .containsEntry( keyNation, ContentAndState.of((byte) payloadForContent(onRefNation), stateNation)); assertThat(mine.values(commitNation, nonExistentKey, KeyFilterPredicate.ALLOW_ALL)).isEmpty(); assertThat(mine.values(commitRegion, nonExistentKey, KeyFilterPredicate.ALLOW_ALL)).isEmpty(); assertThat( mine.values( commitRegion, Arrays.asList(keyNation, keyRegion), KeyFilterPredicate.ALLOW_ALL)) .containsEntry( keyNation, ContentAndState.of((byte) payloadForContent(onRefNation), stateNation)) .containsEntry( keyRegion, ContentAndState.of((byte) payloadForContent(onRefRegion), stateRegion)); } }





© 2015 - 2024 Weber Informatics LLC | Privacy Policy