org.dinky.shaded.paimon.mergetree.compact.FirstRowMergeTreeCompactRewriter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of dinky-shaded-paimon Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.dinky.shaded.paimon.mergetree.compact;

import org.dinky.shaded.paimon.KeyValue;
import org.dinky.shaded.paimon.annotation.VisibleForTesting;
import org.dinky.shaded.paimon.codegen.RecordEqualiser;
import org.dinky.shaded.paimon.data.InternalRow;
import org.dinky.shaded.paimon.io.DataFileMeta;
import org.dinky.shaded.paimon.io.KeyValueFileReaderFactory;
import org.dinky.shaded.paimon.io.KeyValueFileWriterFactory;
import org.dinky.shaded.paimon.mergetree.ContainsLevels;
import org.dinky.shaded.paimon.mergetree.MergeSorter;
import org.dinky.shaded.paimon.mergetree.SortedRun;
import org.dinky.shaded.paimon.utils.Filter;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Comparator;
import java.util.List;

import static org.dinky.shaded.paimon.utils.Preconditions.checkArgument;

/**
 * A {@link MergeTreeCompactRewriter} for first row merge engine which produces changelog files by
 * contains for the compaction involving level 0 files.
 */
public class FirstRowMergeTreeCompactRewriter extends ChangelogMergeTreeRewriter {

    private final ContainsLevels containsLevels;

    public FirstRowMergeTreeCompactRewriter(
            ContainsLevels containsLevels,
            KeyValueFileReaderFactory readerFactory,
            KeyValueFileWriterFactory writerFactory,
            Comparator keyComparator,
            MergeFunctionFactory mfFactory,
            MergeSorter mergeSorter,
            RecordEqualiser valueEqualiser,
            boolean changelogRowDeduplicate) {
        super(
                readerFactory,
                writerFactory,
                keyComparator,
                mfFactory,
                mergeSorter,
                valueEqualiser,
                changelogRowDeduplicate);
        this.containsLevels = containsLevels;
    }

    @Override
    protected boolean rewriteChangelog(
            int outputLevel, boolean dropDelete, List> sections) {
        return rewriteLookupChangelog(outputLevel, sections);
    }

    @Override
    protected boolean upgradeChangelog(int outputLevel, DataFileMeta file) {
        return file.level() == 0;
    }

    @Override
    protected MergeFunctionWrapper createMergeWrapper(int outputLevel) {
        return new FistRowMergeFunctionWrapper(
                mfFactory,
                key -> {
                    try {
                        return containsLevels.contains(key, outputLevel + 1);
                    } catch (IOException e) {
                        throw new UncheckedIOException(e);
                    }
                });
    }

    @Override
    public void close() throws IOException {
        containsLevels.close();
    }

    @VisibleForTesting
    static class FistRowMergeFunctionWrapper implements MergeFunctionWrapper {

        private final Filter contains;
        private final FirstRowMergeFunction mergeFunction;
        private final ChangelogResult reusedResult = new ChangelogResult();

        public FistRowMergeFunctionWrapper(
                MergeFunctionFactory mergeFunctionFactory, Filter contains) {
            this.contains = contains;
            MergeFunction mergeFunction = mergeFunctionFactory.create();
            checkArgument(
                    mergeFunction instanceof FirstRowMergeFunction,
                    "Merge function should be a FirstRowMergeFunction, but is %s, there is a bug.",
                    mergeFunction.getClass().getName());
            this.mergeFunction = (FirstRowMergeFunction) mergeFunction;
        }

        @Override
        public void reset() {
            mergeFunction.reset();
        }

        @Override
        public void add(KeyValue kv) {
            mergeFunction.add(kv);
        }

        @Override
        public ChangelogResult getResult() {
            reusedResult.reset();
            KeyValue result = mergeFunction.getResult();
            if (result == null) {
                return reusedResult;
            }

            if (contains.test(result.key())) {
                // empty
                return reusedResult;
            }

            reusedResult.setResult(result);
            if (result.level() == 0) {
                // new record, output changelog
                return reusedResult.addChangelog(result);
            }
            return reusedResult;
        }
    }
}