All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.geode.internal.cache.BucketRegionQueue Maven / Gradle / Ivy

Go to download

Apache Geode provides a database-like consistency model, reliable transaction processing and a shared-nothing architecture to maintain very low latency performance with high concurrency processing

There is a newer version: 1.15.1
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
/**
 * 
 */
package org.apache.geode.internal.cache;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.logging.log4j.Logger;

import org.apache.geode.cache.CacheWriterException;
import org.apache.geode.cache.EntryNotFoundException;
import org.apache.geode.cache.RegionAttributes;
import org.apache.geode.cache.RegionDestroyedException;
import org.apache.geode.cache.TimeoutException;
import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
import org.apache.geode.internal.cache.persistence.query.mock.ByteComparator;
import org.apache.geode.internal.cache.versions.RegionVersionVector;
import org.apache.geode.internal.cache.versions.VersionSource;
import org.apache.geode.internal.cache.wan.AbstractGatewaySenderEventProcessor;
import org.apache.geode.internal.cache.wan.GatewaySenderEventImpl;
import org.apache.geode.internal.cache.wan.parallel.BucketRegionQueueUnavailableException;
import org.apache.geode.internal.cache.wan.parallel.ConcurrentParallelGatewaySenderQueue;
import org.apache.geode.internal.concurrent.Atomics;
import org.apache.geode.internal.i18n.LocalizedStrings;
import org.apache.geode.internal.logging.LogService;
import org.apache.geode.internal.offheap.OffHeapRegionEntryHelper;
import org.apache.geode.internal.offheap.annotations.Released;

/**
 * 
 */
public class BucketRegionQueue extends AbstractBucketRegionQueue {

  private static final Logger logger = LogService.getLogger();

  /**
   * The Map mapping the regionName->key to the queue key. This index allows fast
   * updating of entries in the queue for conflation. This is necesaary for Colocated regions and if
   * any of the regions use same key for data.
   */
  private final Map indexes;

  /**
   * A transient queue to maintain the eventSeqNum of the events that are to be sent to remote site.
   * It is cleared when the queue is cleared.
   */
  private final BlockingQueue eventSeqNumQueue = new LinkedBlockingQueue();

  // private final BlockingQueue eventSeqNumQueueWithEventId = new
  // LinkedBlockingQueue();

  private long lastKeyRecovered;

  private AtomicLong latestQueuedKey = new AtomicLong();

  private AtomicLong latestAcknowledgedKey = new AtomicLong();

  /**
   * @param regionName
   * @param attrs
   * @param parentRegion
   * @param cache
   * @param internalRegionArgs
   */
  public BucketRegionQueue(String regionName, RegionAttributes attrs, LocalRegion parentRegion,
      GemFireCacheImpl cache, InternalRegionArguments internalRegionArgs) {
    super(regionName, attrs, parentRegion, cache, internalRegionArgs);
    this.keySet();
    indexes = new ConcurrentHashMap();
  }

  protected void cleanUpDestroyedTokensAndMarkGIIComplete(
      InitialImageOperation.GIIStatus giiStatus) {
    // Load events from temp queued events
    loadEventsFromTempQueue();

    // Initialize the eventSeqNumQueue
    initializeEventSeqNumQueue();

    // Clean up destroyed tokens
    super.cleanUpDestroyedTokensAndMarkGIIComplete(giiStatus);
  }

  private void initializeEventSeqNumQueue() {
    getInitializationLock().writeLock().lock();
    try {
      if (!this.keySet().isEmpty()) {
        if (getPartitionedRegion().getColocatedWith() == null) {
          List keys = new ArrayList(this.keySet());
          Collections.sort(keys, new Comparator() {
            @Override
            public int compare(EventID o1, EventID o2) {
              int compareMem =
                  new ByteComparator().compare(o1.getMembershipID(), o2.getMembershipID());
              if (compareMem == 1) {
                return 1;
              } else if (compareMem == -1) {
                return -1;
              } else {
                if (o1.getThreadID() > o2.getThreadID()) {
                  return 1;
                } else if (o1.getThreadID() < o2.getThreadID()) {
                  return -1;
                } else {
                  return o1.getSequenceID() < o2.getSequenceID() ? -1
                      : o1.getSequenceID() == o2.getSequenceID() ? 0 : 1;
                }
              }
            }
          });
          for (EventID eventID : keys) {
            eventSeqNumQueue.add(eventID);
          }
        } else {
          TreeSet sortedKeys = new TreeSet(this.keySet());
          // although the empty check for this.keySet() is done above,
          // do the same for sortedKeys as well because the keySet() might have become
          // empty since the above check was made (keys might have been destroyed through
          // BatchRemoval)
          // fix for #49679 NoSuchElementException thrown from BucketRegionQueue.initialize
          if (!sortedKeys.isEmpty()) {
            for (Long key : sortedKeys) {
              eventSeqNumQueue.add(key);
            }
            lastKeyRecovered = sortedKeys.last();
            if (this.getEventSeqNum() != null) {
              Atomics.setIfGreater(getEventSeqNum(), lastKeyRecovered);
            }
          }
        }

        if (logger.isDebugEnabled()) {
          logger.debug(
              "For bucket {} ,total keys recovered are : {} last key recovered is : {} and the seqNo is ",
              getId(), eventSeqNumQueue.size(), lastKeyRecovered, getEventSeqNum());
        }
      }
      this.initialized = true;
      // Now, the bucket is initialized. Destroy the failedBatchRemovalKeys.
      destroyFailedBatchRemovalMessageKeys();
    } finally {
      notifyEventProcessor();
      getInitializationLock().writeLock().unlock();
    }
  }

  /**
   * When the GII was going, some of BatchRemoval messages (especially for events destroyed due to
   * conflation) might have failed since the event might not be found in the BucketRegionQueue.
   * Those messages are added to failedBatchRemovalMessageKeys map in ParallelQueueRemovalMessage.
   * Events found in the map need to be destroyed once GII is done. Fix for #47431. This method is
   * invoked deliberately after this.initialized is set to true to fix defect #50316.
   */
  private void destroyFailedBatchRemovalMessageKeys() {
    final boolean isDebugEnabled = logger.isDebugEnabled();

    Iterator itr = getFailedBatchRemovalMessageKeys().iterator();
    while (itr.hasNext()) {
      // at this point, failedBatchRemovalMessageKeys contains messages failed during bucket
      // initialization only. Iterate over failedBatchRemovalMessageKeys and clear it completely.
      Object key = itr.next();
      itr.remove();
      if (isDebugEnabled) {
        logger.debug("key from failedBatchRemovalMessageKeys is: {}", key);
      }
      if (containsKey(key)) {
        try {
          destroyKey(key);
          if (isDebugEnabled) {
            logger.debug("Destroyed {} from bucket: ", key, getId());
          }
        } catch (ForceReattemptException fe) {
          if (isDebugEnabled) {
            logger.debug("Bucket :{} moved to other member", getId());
          }
        }
      }
    }
  }

  @Override
  public void beforeAcquiringPrimaryState() {
    int batchSize = this.getPartitionedRegion().getParallelGatewaySender().getBatchSize();
    Iterator itr = eventSeqNumQueue.iterator();
    markEventsAsDuplicate(batchSize, itr);
  }

  @Override
  public void closeEntries() {
    OffHeapRegionEntryHelper.doWithOffHeapClear(new Runnable() {
      @Override
      public void run() {
        BucketRegionQueue.super.closeEntries();
      }
    });
    this.indexes.clear();
    this.eventSeqNumQueue.clear();
  }

  @Override
  public Set clearEntries(final RegionVersionVector rvv) {
    final AtomicReference> result = new AtomicReference>();
    OffHeapRegionEntryHelper.doWithOffHeapClear(new Runnable() {
      @Override
      public void run() {
        result.set(BucketRegionQueue.super.clearEntries(rvv));
      }
    });
    this.eventSeqNumQueue.clear();
    return result.get();
  }


  @Override
  public void forceSerialized(EntryEventImpl event) {
    // NOOP since we want the value in the region queue to stay in object form.
  }

  protected void clearQueues() {
    getInitializationLock().writeLock().lock();
    try {
      this.indexes.clear();
      this.eventSeqNumQueue.clear();
    } finally {
      getInitializationLock().writeLock().unlock();
    }
  }

  @Override
  protected boolean virtualPut(EntryEventImpl event, boolean ifNew, boolean ifOld,
      Object expectedOldValue, boolean requireOldValue, long lastModified,
      boolean overwriteDestroyed) throws TimeoutException, CacheWriterException {
    try {
      boolean success = super.virtualPut(event, ifNew, ifOld, expectedOldValue, requireOldValue,
          lastModified, overwriteDestroyed);

      if (success) {
        if (getPartitionedRegion().getColocatedWith() == null) {
          return success;
        }

        if (getPartitionedRegion().isConflationEnabled() && this.getBucketAdvisor().isPrimary()) {
          Object object = event.getNewValue();
          Long key = (Long) event.getKey();
          if (object instanceof Conflatable) {
            if (logger.isDebugEnabled()) {
              logger.debug("Key :{} , Object : {} is conflatable", key, object);
            }
            // TODO: TO optimize by destroying on primary and secondary separately
            // in case of conflation
            conflateOldEntry((Conflatable) object, key);
          } else {
            if (logger.isDebugEnabled()) {
              logger.debug("Object : {} is not conflatable", object);
            }
          }
        }
      } else {
        GatewaySenderEventImpl.release(event.getRawNewValue());
      }
      return success;
    } finally {
      GatewaySenderEventImpl.release(event.getRawOldValue());
    }
  }

  private void conflateOldEntry(Conflatable object, Long tailKey) {
    PartitionedRegion region = this.getPartitionedRegion();
    Conflatable conflatableObject = object;
    if (region.isConflationEnabled() && conflatableObject.shouldBeConflated()) {
      Object keyToConflate = conflatableObject.getKeyToConflate();
      String rName = object.getRegionToConflate();
      if (logger.isDebugEnabled()) {
        logger.debug(" The region name is : {}", rName);
      }
      Map latestIndexesForRegion = (Map) this.indexes.get(rName);
      if (latestIndexesForRegion == null) {
        latestIndexesForRegion = new ConcurrentHashMap();
        this.indexes.put(rName, latestIndexesForRegion);
      }
      Long previousTailKey = (Long) latestIndexesForRegion.put(keyToConflate, tailKey);
      if (previousTailKey != null) {
        if (logger.isDebugEnabled()) {
          logger.debug("{}: Conflating {} at queue index={} and previousTailKey: ", this, object,
              tailKey, previousTailKey);
        }
        AbstractGatewaySenderEventProcessor ep =
            region.getParallelGatewaySender().getEventProcessor();
        if (ep == null)
          return;
        ConcurrentParallelGatewaySenderQueue queue =
            (ConcurrentParallelGatewaySenderQueue) ep.getQueue();
        // Give the actual conflation work to another thread.
        // ParallelGatewaySenderQueue takes care of maintaining a thread pool.
        queue.conflateEvent(conflatableObject, getId(), previousTailKey);
      } else {
        region.getParallelGatewaySender().getStatistics().incConflationIndexesMapSize();
      }
    } else {
      if (logger.isDebugEnabled()) {
        logger.debug("{}: Not conflating {}", this, object);
      }
    }
  }

  // No need to synchronize because it is called from a synchronized method
  private void removeIndex(Long qkey) {
    // Determine whether conflation is enabled for this queue and object
    Object o = getNoLRU(qkey, true, false, false);
    if (o instanceof Conflatable) {
      Conflatable object = (Conflatable) o;
      if (object.shouldBeConflated()) {
        // Otherwise, remove the index from the indexes map.
        String rName = object.getRegionToConflate();
        Object key = object.getKeyToConflate();
        Map latestIndexesForRegion = (Map) this.indexes.get(rName);
        if (latestIndexesForRegion != null) {
          // Remove the index.
          Long index = (Long) latestIndexesForRegion.remove(key);
          if (index != null) {
            this.getPartitionedRegion().getParallelGatewaySender().getStatistics()
                .decConflationIndexesMapSize();
            if (logger.isDebugEnabled()) {
              logger.debug("{}: Removed index {} for {}", this, index, object);
            }
          }
        }
      }
    }
  }

  @Override
  protected void basicDestroy(final EntryEventImpl event, final boolean cacheWrite,
      Object expectedOldValue)
      throws EntryNotFoundException, CacheWriterException, TimeoutException {
    if (getPartitionedRegion().isConflationEnabled()) {
      removeIndex((Long) event.getKey());
    }
    try {
      super.basicDestroy(event, cacheWrite, expectedOldValue);
    } finally {
      GatewaySenderEventImpl.release(event.getRawOldValue());
    }

    // Primary buckets should already remove the key while peeking
    if (!this.getBucketAdvisor().isPrimary()) {
      if (logger.isDebugEnabled()) {
        logger.debug(" removing the key {} from eventSeqNumQueue", event.getKey());
      }
      this.eventSeqNumQueue.remove(event.getKey());
    }
  }

  /**
   * Does a get that gets the value without fault values in from disk.
   */
  private Object optimalGet(Object k) {
    // Get the object at that key (to remove the index).
    Object object = null;
    try {
      object = getValueInVMOrDiskWithoutFaultIn(k);
      if (object != null && object instanceof CachedDeserializable) {
        object = ((CachedDeserializable) object).getDeserializedValue(this, this.getRegionEntry(k));
      }
    } catch (EntryNotFoundException ok) {
      // just return null;
    }
    if (object == Token.TOMBSTONE) {
      object = null;
    }
    return object; // OFFHEAP: ok since callers are careful to do destroys on region queue after
                   // finished with peeked object.
  }

  public Object peek() {
    Object key = null;
    Object object = null;
    // doing peek in initializationLock because during region destroy, the clearQueues
    // clears the eventSeqNumQueue and can cause data inconsistency (defect #48984)
    getInitializationLock().readLock().lock();
    try {
      if (this.getPartitionedRegion().isDestroyed()) {
        throw new BucketRegionQueueUnavailableException();
      }
      key = this.eventSeqNumQueue.peek();
      if (key != null) {
        object = optimalGet(key);
        if (object == null && !this.getPartitionedRegion().isConflationEnabled()) {
          if (logger.isDebugEnabled()) {
            logger.debug(
                "The value against key {} in the bucket region queue with id {} is NULL for the GatewaySender {}",
                key, getId(), this.getPartitionedRegion().getParallelGatewaySender());
          }
        }
        // In case of conflation and a race where bucket recovers
        // key-value from other bucket while put has come to this bucket.
        // if (object != null) {
        // ParallelGatewaySenderQueue queue =
        // (ParallelGatewaySenderQueue)getPartitionedRegion()
        // .getParallelGatewaySender().getQueues().toArray(new
        // RegionQueue[1])[0];
        // //queue.addToPeekedKeys(key);
        // }
        this.eventSeqNumQueue.remove(key);
      }
      return object; // OFFHEAP: ok since callers are careful to do destroys on
                     // region queue after finished with peeked object.
    } finally {
      getInitializationLock().readLock().unlock();
    }
  }

  protected void addToEventQueue(Object key, boolean didPut, EntryEventImpl event) {
    if (didPut) {
      if (this.initialized) {
        this.eventSeqNumQueue.add(key);
        updateLargestQueuedKey((Long) key);
      }
      if (logger.isDebugEnabled()) {
        logger.debug("Put successfully in the queue : {} was initialized: {}",
            event.getRawNewValue(), this.initialized);
      }
    }
    if (this.getBucketAdvisor().isPrimary()) {
      incQueueSize(1);
    }
  }

  private void updateLargestQueuedKey(Long key) {
    Atomics.setIfGreater(this.latestQueuedKey, key);
  }

  private void setLatestAcknowledgedKey(Long key) {
    this.latestAcknowledgedKey.set(key);
  }

  public boolean waitUntilFlushed(long timeout, TimeUnit unit) throws InterruptedException {
    if (logger.isDebugEnabled()) {
      logger.debug("BucketRegionQueue: waitUntilFlushed bucket=" + getId() + "; time="
          + System.currentTimeMillis() + "; timeout=" + timeout + "; unit=" + unit);
    }
    boolean result = false;
    // Wait until latestAcknowledgedKey > latestQueuedKey or the queue is empty
    if (this.initialized) {
      long latestQueuedKeyToCheck = this.latestQueuedKey.get();
      long nanosRemaining = unit.toNanos(timeout);
      long endTime = System.nanoTime() + nanosRemaining;
      while (nanosRemaining > 0) {
        if (latestAcknowledgedKey.get() > latestQueuedKeyToCheck || isEmpty()) {
          result = true;
          break;
        }
        Thread.sleep(Math.min(TimeUnit.NANOSECONDS.toMillis(nanosRemaining) + 1, 100));
        nanosRemaining = endTime - System.nanoTime();
      }
    }
    if (logger.isDebugEnabled()) {
      logger.debug("BucketRegionQueue: waitUntilFlushed completed bucket=" + getId() + "; time="
          + System.currentTimeMillis() + "; result=" + result);
    }
    return result;
  }

  /**
   * It removes the first key from the queue.
   * 
   * @return Returns the key for which value was destroyed.
   * @throws ForceReattemptException
   */
  public Object remove() throws ForceReattemptException {
    Object key = this.eventSeqNumQueue.remove();
    if (key != null) {
      destroyKey(key);
    }
    return key;
  }

  /**
   * It removes the first key from the queue.
   * 
   * @return Returns the value.
   * @throws InterruptedException
   * @throws ForceReattemptException
   */
  public Object take() throws InterruptedException, ForceReattemptException {
    throw new UnsupportedOperationException();
    // Currently has no callers.
    // To support this callers need to call release on the returned GatewaySenderEventImpl.
    // Object key = this.eventSeqNumQueue.remove();
    // Object object = null;
    // if (key != null) {
    // object = PartitionRegionHelper
    // .getLocalPrimaryData(getPartitionedRegion()).get(key);
    // /**
    // * TODO: For the time being this is same as peek. To do a batch peek we
    // * need to remove the head key. We will destroy the key once the event is
    // * delivered to the GatewayReceiver.
    // */
    // destroyKey(key);
    // }
    // return object;
  }

  /**
   * Overriding this method from AbstractBucketRegionQueue in order to remove the event from
   * eventSeqNumQueue if EntryNotFoundException is encountered during basicDestroy. This change is
   * done during selective merge from r41425 from gemfire701X_maint.
   */
  public void destroyKey(Object key) throws ForceReattemptException {
    if (logger.isDebugEnabled()) {
      logger.debug(" destroying primary key {}", key);
    }
    @Released
    EntryEventImpl event = getPartitionedRegion().newDestroyEntryEvent(key, null);
    try {
      event.setEventId(new EventID(cache.getSystem()));
      event.setRegion(this);
      basicDestroy(event, true, null);
      setLatestAcknowledgedKey((Long) key);
      checkReadiness();
    } catch (EntryNotFoundException enf) {
      if (getPartitionedRegion().isDestroyed()) {
        getPartitionedRegion().checkReadiness();
        if (isBucketDestroyed()) {
          throw new ForceReattemptException("Bucket moved",
              new RegionDestroyedException(
                  LocalizedStrings.PartitionedRegionDataStore_REGION_HAS_BEEN_DESTROYED
                      .toLocalizedString(),
                  getPartitionedRegion().getFullPath()));
        }
      }
      throw enf;
    } catch (RegionDestroyedException rde) {
      getPartitionedRegion().checkReadiness();
      if (isBucketDestroyed()) {
        throw new ForceReattemptException("Bucket moved while destroying key " + key, rde);
      }
    } finally {
      event.release();
    }

    this.notifyEntriesRemoved();
  }

  public boolean isReadyForPeek() {
    return !this.getPartitionedRegion().isDestroyed() && !this.isEmpty()
        && !this.eventSeqNumQueue.isEmpty() && getBucketAdvisor().isPrimary();
  }

}
 















© 2015 - 2024 Weber Informatics LLC | Privacy Policy