org.apache.hudi.common.table.log.HoodieFileSliceReader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hudi-flink1.16-bundle Show documentation
The newest version!
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.hudi.common.table.log;

import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodiePayloadProps;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordMerger;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.storage.HoodieFileReader;

import org.apache.avro.Schema;

import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;

public class HoodieFileSliceReader extends LogFileIterator {
  private Option> baseFileIterator;
  private HoodieMergedLogRecordScanner scanner;
  private Schema schema;
  private Properties props;

  private TypedProperties payloadProps = new TypedProperties();
  private Option> simpleKeyGenFieldsOpt;
  Map records;
  HoodieRecordMerger merger;

  public HoodieFileSliceReader(Option baseFileReader,
                                   HoodieMergedLogRecordScanner scanner, Schema schema, String preCombineField, HoodieRecordMerger merger,
                               Properties props, Option> simpleKeyGenFieldsOpt) throws IOException {
    super(scanner);
    if (baseFileReader.isPresent()) {
      this.baseFileIterator = Option.of(baseFileReader.get().getRecordIterator(schema));
    } else {
      this.baseFileIterator = Option.empty();
    }
    this.scanner = scanner;
    this.schema = schema;
    this.merger = merger;
    if (preCombineField != null) {
      payloadProps.setProperty(HoodiePayloadProps.PAYLOAD_ORDERING_FIELD_PROP_KEY, preCombineField);
    }
    this.props = props;
    this.simpleKeyGenFieldsOpt = simpleKeyGenFieldsOpt;
    this.records = scanner.getRecords();
  }

  private boolean hasNextInternal() {
    while (baseFileIterator.isPresent() && baseFileIterator.get().hasNext()) {
      try {
        HoodieRecord currentRecord = baseFileIterator.get().next().wrapIntoHoodieRecordPayloadWithParams(schema, props,
            simpleKeyGenFieldsOpt, scanner.isWithOperationField(), scanner.getPartitionNameOverride(), false, Option.empty());
        Option logRecord = removeLogRecord(currentRecord.getRecordKey());
        if (!logRecord.isPresent()) {
          nextRecord = currentRecord;
          return true;
        }
        Option> mergedRecordOpt =  merger.merge(currentRecord, schema, logRecord.get(), schema, payloadProps);
        if (mergedRecordOpt.isPresent()) {
          HoodieRecord mergedRecord = (HoodieRecord) mergedRecordOpt.get().getLeft();
          nextRecord = mergedRecord.wrapIntoHoodieRecordPayloadWithParams(schema, props, simpleKeyGenFieldsOpt, scanner.isWithOperationField(),
              scanner.getPartitionNameOverride(), false, Option.empty());
          return true;
        }
      } catch (IOException e) {
        throw new HoodieIOException("Failed to wrapIntoHoodieRecordPayloadWithParams: " + e.getMessage());
      }
    }
    return super.doHasNext();
  }

  @Override
  protected boolean doHasNext() {
    return hasNextInternal();
  }

}