com.uber.hoodie.common.table.view.PriorityBasedFileSystemView Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of hoodie-common Show documentation
The newest version!
/*
 * Copyright (c) 2019 Uber Technologies, Inc. ([email protected])
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *          http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.uber.hoodie.common.table.view;

import com.uber.hoodie.common.model.CompactionOperation;
import com.uber.hoodie.common.model.FileSlice;
import com.uber.hoodie.common.model.HoodieDataFile;
import com.uber.hoodie.common.model.HoodieFileGroup;
import com.uber.hoodie.common.table.HoodieTimeline;
import com.uber.hoodie.common.table.SyncableFileSystemView;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.util.Functions.Function0;
import com.uber.hoodie.common.util.Functions.Function1;
import com.uber.hoodie.common.util.Functions.Function2;
import com.uber.hoodie.common.util.Functions.Function3;
import com.uber.hoodie.common.util.Option;
import com.uber.hoodie.common.util.collection.Pair;
import java.io.Serializable;
import java.util.List;
import java.util.stream.Stream;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

/**
 * A file system view which proxies request to a preferred File System View implementation. In case of error,
 * flip all subsequent calls to a backup file-system view implementation.
 */
public class PriorityBasedFileSystemView implements SyncableFileSystemView, Serializable {

  private static Logger log = LogManager.getLogger(PriorityBasedFileSystemView.class);

  private final SyncableFileSystemView preferredView;
  private final SyncableFileSystemView secondaryView;
  private boolean errorOnPreferredView;

  public PriorityBasedFileSystemView(SyncableFileSystemView preferredView, SyncableFileSystemView secondaryView) {
    this.preferredView = preferredView;
    this.secondaryView = secondaryView;
    this.errorOnPreferredView = false;
  }

  private  R execute(Function0 preferredFunction, Function0 secondaryFunction) {
    if (errorOnPreferredView) {
      log.warn("Routing request to secondary file-system view");
      return secondaryFunction.apply();
    } else {
      try {
        return preferredFunction.apply();
      } catch (RuntimeException re) {
        log.error("Got error running preferred function. Trying secondary", re);
        errorOnPreferredView = true;
        return secondaryFunction.apply();
      }
    }
  }

  private  R execute(T1 val, Function1 preferredFunction, Function1 secondaryFunction) {
    if (errorOnPreferredView) {
      log.warn("Routing request to secondary file-system view");
      return secondaryFunction.apply(val);
    } else {
      try {
        return preferredFunction.apply(val);
      } catch (RuntimeException re) {
        log.error("Got error running preferred function. Trying secondary", re);
        errorOnPreferredView = true;
        return secondaryFunction.apply(val);
      }
    }
  }

  private  R execute(T1 val, T2 val2, Function2 preferredFunction,
      Function2 secondaryFunction) {
    if (errorOnPreferredView) {
      log.warn("Routing request to secondary file-system view");
      return secondaryFunction.apply(val, val2);
    } else {
      try {
        return preferredFunction.apply(val, val2);
      } catch (RuntimeException re) {
        log.error("Got error running preferred function. Trying secondary", re);
        errorOnPreferredView = true;
        return secondaryFunction.apply(val, val2);
      }
    }
  }

  private  R execute(T1 val, T2 val2, T3 val3, Function3 preferredFunction,
      Function3 secondaryFunction) {
    if (errorOnPreferredView) {
      log.warn("Routing request to secondary file-system view");
      return secondaryFunction.apply(val, val2, val3);
    } else {
      try {
        return preferredFunction.apply(val, val2, val3);
      } catch (RuntimeException re) {
        log.error("Got error running preferred function. Trying secondary", re);
        errorOnPreferredView = true;
        return secondaryFunction.apply(val, val2, val3);
      }
    }
  }

  @Override
  public Stream getLatestDataFiles(String partitionPath) {
    return execute(partitionPath, preferredView::getLatestDataFiles, secondaryView::getLatestDataFiles);
  }

  @Override
  public Stream getLatestDataFiles() {
    return execute(preferredView::getLatestDataFiles, secondaryView::getLatestDataFiles);
  }

  @Override
  public Stream getLatestDataFilesBeforeOrOn(String partitionPath, String maxCommitTime) {
    return execute(partitionPath, maxCommitTime, preferredView::getLatestDataFilesBeforeOrOn,
        secondaryView::getLatestDataFilesBeforeOrOn);
  }

  @Override
  public Option getLatestDataFile(String partitionPath, String fileId) {
    return execute(partitionPath, fileId, preferredView::getLatestDataFile, secondaryView::getLatestDataFile);
  }

  @Override
  public Option getDataFileOn(String partitionPath, String instantTime, String fileId) {
    return execute(partitionPath, instantTime, fileId, preferredView::getDataFileOn,
        secondaryView::getDataFileOn);
  }

  @Override
  public Stream getLatestDataFilesInRange(List commitsToReturn) {
    return execute(commitsToReturn, preferredView::getLatestDataFilesInRange, secondaryView::getLatestDataFilesInRange);
  }

  @Override
  public Stream getAllDataFiles(String partitionPath) {
    return execute(partitionPath, preferredView::getAllDataFiles, secondaryView::getAllDataFiles);
  }

  @Override
  public Stream getLatestFileSlices(String partitionPath) {
    return execute(partitionPath, preferredView::getLatestFileSlices, secondaryView::getLatestFileSlices);
  }

  @Override
  public Stream getLatestUnCompactedFileSlices(String partitionPath) {
    return execute(partitionPath, preferredView::getLatestUnCompactedFileSlices,
        secondaryView::getLatestUnCompactedFileSlices);
  }

  @Override
  public Stream getLatestFileSlicesBeforeOrOn(String partitionPath, String maxCommitTime) {
    return execute(partitionPath, maxCommitTime, preferredView::getLatestFileSlicesBeforeOrOn,
        secondaryView::getLatestFileSlicesBeforeOrOn);
  }

  @Override
  public Stream getLatestMergedFileSlicesBeforeOrOn(String partitionPath, String maxInstantTime) {
    return execute(partitionPath, maxInstantTime, preferredView::getLatestMergedFileSlicesBeforeOrOn,
        secondaryView::getLatestMergedFileSlicesBeforeOrOn);
  }

  @Override
  public Stream getLatestFileSliceInRange(List commitsToReturn) {
    return execute(commitsToReturn, preferredView::getLatestFileSliceInRange, secondaryView::getLatestFileSliceInRange);
  }

  @Override
  public Stream getAllFileSlices(String partitionPath) {
    return execute(partitionPath, preferredView::getAllFileSlices, secondaryView::getAllFileSlices);
  }

  @Override
  public Stream getAllFileGroups(String partitionPath) {
    return execute(partitionPath, preferredView::getAllFileGroups, secondaryView::getAllFileGroups);
  }

  @Override
  public Stream> getPendingCompactionOperations() {
    return execute(preferredView::getPendingCompactionOperations, secondaryView::getPendingCompactionOperations);
  }

  @Override
  public void close() {
    preferredView.close();
    secondaryView.close();
  }

  @Override
  public void reset() {
    preferredView.reset();
    secondaryView.reset();
  }

  @Override
  public Option getLastInstant() {
    return execute(preferredView::getLastInstant, secondaryView::getLastInstant);
  }

  @Override
  public HoodieTimeline getTimeline() {
    return execute(preferredView::getTimeline, secondaryView::getTimeline);
  }

  @Override
  public void sync() {
    preferredView.reset();
    secondaryView.reset();
  }

  @Override
  public Option getLatestFileSlice(String partitionPath, String fileId) {
    return execute(partitionPath, fileId, preferredView::getLatestFileSlice, secondaryView::getLatestFileSlice);
  }

  public SyncableFileSystemView getPreferredView() {
    return preferredView;
  }

  public SyncableFileSystemView getSecondaryView() {
    return secondaryView;
  }
}