Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.common.table.timeline;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* A helper class used to diff timeline.
*/
public class TimelineDiffHelper {
private static final Logger LOG = LoggerFactory.getLogger(TimelineDiffHelper.class);
public static TimelineDiffResult getNewInstantsForIncrementalSync(HoodieTimeline oldTimeline,
HoodieTimeline newTimeline) {
HoodieTimeline oldT = oldTimeline.filterCompletedAndCompactionInstants();
HoodieTimeline newT = newTimeline.filterCompletedAndCompactionInstants();
Option lastSeenInstant = oldT.lastInstant();
Option firstInstantInNewTimeline = newT.firstInstant();
if (lastSeenInstant.isPresent() && firstInstantInNewTimeline.isPresent()) {
if (HoodieTimeline.compareTimestamps(lastSeenInstant.get().getTimestamp(),
HoodieTimeline.LESSER_THAN, firstInstantInNewTimeline.get().getTimestamp())) {
// The last seen instant is no longer in the timeline. Do not incrementally Sync.
return TimelineDiffResult.UNSAFE_SYNC_RESULT;
}
Set oldTimelineInstants = oldT.getInstantsAsStream().collect(Collectors.toSet());
List newInstants = new ArrayList<>();
// Check If any pending compaction is lost. If so, do not allow incremental timeline sync
List> compactionInstants = getPendingCompactionTransitions(oldT, newT);
List lostPendingCompactions = compactionInstants.stream()
.filter(instantPair -> instantPair.getValue() == null).map(Pair::getKey).collect(Collectors.toList());
if (!lostPendingCompactions.isEmpty()) {
// If a compaction is unscheduled, fall back to complete refresh of fs view since some log files could have been
// moved. Its unsafe to incrementally sync in that case.
LOG.warn("Some pending compactions are no longer in new timeline (unscheduled ?). They are :"
+ lostPendingCompactions);
return TimelineDiffResult.UNSAFE_SYNC_RESULT;
}
List finishedCompactionInstants = compactionInstants.stream()
.filter(instantPair -> instantPair.getValue().getAction().equals(HoodieTimeline.COMMIT_ACTION)
&& instantPair.getValue().isCompleted())
.map(Pair::getKey).collect(Collectors.toList());
newTimeline.getInstantsAsStream().filter(instant -> !oldTimelineInstants.contains(instant)).forEach(newInstants::add);
List> logCompactionInstants = getPendingLogCompactionTransitions(oldTimeline, newTimeline);
List finishedOrRemovedLogCompactionInstants = logCompactionInstants.stream()
.filter(instantPair -> !instantPair.getKey().isCompleted()
&& (instantPair.getValue() == null || instantPair.getValue().isCompleted()))
.map(Pair::getKey).collect(Collectors.toList());
return new TimelineDiffResult(newInstants, finishedCompactionInstants, finishedOrRemovedLogCompactionInstants, true);
} else {
// One or more timelines is empty
LOG.warn("One or more timelines is empty");
return TimelineDiffResult.UNSAFE_SYNC_RESULT;
}
}
/**
* Getting pending log compaction transitions.
*/
private static List> getPendingLogCompactionTransitions(HoodieTimeline oldTimeline,
HoodieTimeline newTimeline) {
Set newTimelineInstants = newTimeline.getInstantsAsStream().collect(Collectors.toSet());
return oldTimeline.filterPendingLogCompactionTimeline().getInstantsAsStream().map(instant -> {
if (newTimelineInstants.contains(instant)) {
return Pair.of(instant, instant);
} else {
HoodieInstant logCompacted =
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, instant.getTimestamp());
if (newTimelineInstants.contains(logCompacted)) {
return Pair.of(instant, logCompacted);
}
HoodieInstant inflightLogCompacted =
new HoodieInstant(State.INFLIGHT, HoodieTimeline.LOG_COMPACTION_ACTION, instant.getTimestamp());
if (newTimelineInstants.contains(inflightLogCompacted)) {
return Pair.of(instant, inflightLogCompacted);
}
return Pair.of(instant, null);
}
}).collect(Collectors.toList());
}
/**
* Getting pending compaction transitions.
*/
private static List> getPendingCompactionTransitions(HoodieTimeline oldTimeline,
HoodieTimeline newTimeline) {
Set newTimelineInstants = newTimeline.getInstantsAsStream().collect(Collectors.toSet());
return oldTimeline.filterPendingCompactionTimeline().getInstantsAsStream().map(instant -> {
if (newTimelineInstants.contains(instant)) {
return Pair.of(instant, instant);
} else {
HoodieInstant compacted =
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, instant.getTimestamp());
if (newTimelineInstants.contains(compacted)) {
return Pair.of(instant, compacted);
}
HoodieInstant inflightCompacted =
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, instant.getTimestamp());
if (newTimelineInstants.contains(inflightCompacted)) {
return Pair.of(instant, inflightCompacted);
}
return Pair.of(instant, null);
}
}).collect(Collectors.toList());
}
/**
* A diff result of timeline.
*/
public static class TimelineDiffResult {
private final List newlySeenInstants;
private final List finishedCompactionInstants;
private final List finishedOrRemovedLogCompactionInstants;
private final boolean canSyncIncrementally;
public static final TimelineDiffResult UNSAFE_SYNC_RESULT =
new TimelineDiffResult(null, null, null, false);
public TimelineDiffResult(List newlySeenInstants, List finishedCompactionInstants,
List finishedOrRemovedLogCompactionInstants, boolean canSyncIncrementally) {
this.newlySeenInstants = newlySeenInstants;
this.finishedCompactionInstants = finishedCompactionInstants;
this.finishedOrRemovedLogCompactionInstants = finishedOrRemovedLogCompactionInstants;
this.canSyncIncrementally = canSyncIncrementally;
}
public List getNewlySeenInstants() {
return newlySeenInstants;
}
public List getFinishedCompactionInstants() {
return finishedCompactionInstants;
}
public List getFinishedOrRemovedLogCompactionInstants() {
return finishedOrRemovedLogCompactionInstants;
}
public boolean canSyncIncrementally() {
return canSyncIncrementally;
}
@Override
public String toString() {
return "TimelineDiffResult{"
+ "newlySeenInstants=" + newlySeenInstants
+ ", finishedCompactionInstants=" + finishedCompactionInstants
+ ", finishedOrRemovedLogCompactionInstants=" + finishedOrRemovedLogCompactionInstants
+ ", canSyncIncrementally=" + canSyncIncrementally
+ '}';
}
}
}
