org.dinky.shaded.paimon.mergetree.compact.LookupChangelogMergeFunctionWrapper Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of dinky-shaded-paimon Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.dinky.shaded.paimon.mergetree.compact;

import org.dinky.shaded.paimon.KeyValue;
import org.dinky.shaded.paimon.codegen.RecordEqualiser;
import org.dinky.shaded.paimon.data.InternalRow;
import org.dinky.shaded.paimon.types.RowKind;

import java.util.function.Function;

import static org.dinky.shaded.paimon.utils.Preconditions.checkArgument;

/**
 * Wrapper for {@link MergeFunction}s to produce changelog by lookup during the compaction involving
 * level 0 files.
 *
 * Changelog records are generated in the process of the level-0 file participating in the
 * compaction, if during the compaction processing:
 *
 * 

 *   Without level-0 records, no changelog.
 *   
With level-0 record, with level-x (x > 0) record, level-x record should be BEFORE, level-0
 *       should be AFTER.
 *   
With level-0 record, without level-x record, need to lookup the history value of the upper
 *       level as BEFORE.
 * 
 */
public class LookupChangelogMergeFunctionWrapper implements MergeFunctionWrapper {

    private final LookupMergeFunction mergeFunction;
    private final MergeFunction mergeFunction2;
    private final Function lookup;

    private final ChangelogResult reusedResult = new ChangelogResult();
    private final KeyValue reusedBefore = new KeyValue();
    private final KeyValue reusedAfter = new KeyValue();
    private final RecordEqualiser valueEqualiser;
    private final boolean changelogRowDeduplicate;

    public LookupChangelogMergeFunctionWrapper(
            MergeFunctionFactory mergeFunctionFactory,
            Function lookup,
            RecordEqualiser valueEqualiser,
            boolean changelogRowDeduplicate) {
        MergeFunction mergeFunction = mergeFunctionFactory.create();
        checkArgument(
                mergeFunction instanceof LookupMergeFunction,
                "Merge function should be a LookupMergeFunction, but is %s, there is a bug.",
                mergeFunction.getClass().getName());
        this.mergeFunction = (LookupMergeFunction) mergeFunction;
        this.mergeFunction2 = mergeFunctionFactory.create();
        this.lookup = lookup;
        this.valueEqualiser = valueEqualiser;
        this.changelogRowDeduplicate = changelogRowDeduplicate;
    }

    @Override
    public void reset() {
        mergeFunction.reset();
    }

    @Override
    public void add(KeyValue kv) {
        mergeFunction.add(kv);
    }

    @Override
    public ChangelogResult getResult() {
        reusedResult.reset();

        KeyValue result = mergeFunction.getResult();
        if (result == null) {
            return reusedResult;
        }

        KeyValue highLevel = mergeFunction.highLevel;
        boolean containLevel0 = mergeFunction.containLevel0;

        // 1. No level 0, just return
        if (!containLevel0) {
            return reusedResult.setResult(result);
        }

        // 2. With level 0, with the latest high level, return changelog
        if (highLevel != null) {
            // For first row, we should just return old value. And produce no changelog.
            setChangelog(highLevel, result);
            return reusedResult.setResult(result);
        }

        // 3. Lookup to find the latest high level record
        highLevel = lookup.apply(result.key());

        if (highLevel != null) {
            mergeFunction2.reset();
            mergeFunction2.add(highLevel);
            mergeFunction2.add(result);
            result = mergeFunction2.getResult();
            setChangelog(highLevel, result);
        } else {
            setChangelog(null, result);
        }
        return reusedResult.setResult(result);
    }

    private void setChangelog(KeyValue before, KeyValue after) {
        if (before == null || !isAdd(before)) {
            if (isAdd(after)) {
                reusedResult.addChangelog(replaceAfter(RowKind.INSERT, after));
            }
        } else {
            if (!isAdd(after)) {
                reusedResult.addChangelog(replaceBefore(RowKind.DELETE, before));
            } else if (!changelogRowDeduplicate
                    || !valueEqualiser.equals(before.value(), after.value())) {
                reusedResult
                        .addChangelog(replaceBefore(RowKind.UPDATE_BEFORE, before))
                        .addChangelog(replaceAfter(RowKind.UPDATE_AFTER, after));
            }
        }
    }

    private KeyValue replaceBefore(RowKind valueKind, KeyValue from) {
        return replace(reusedBefore, valueKind, from);
    }

    private KeyValue replaceAfter(RowKind valueKind, KeyValue from) {
        return replace(reusedAfter, valueKind, from);
    }

    private KeyValue replace(KeyValue reused, RowKind valueKind, KeyValue from) {
        return reused.replace(from.key(), from.sequenceNumber(), valueKind, from.value());
    }

    private boolean isAdd(KeyValue kv) {
        return kv.valueKind() == RowKind.INSERT || kv.valueKind() == RowKind.UPDATE_AFTER;
    }
}