com.gemstone.gemfire.internal.cache.snapshot.WindowedExporter Maven / Gradle / Ivy

/*
 * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */
package com.gemstone.gemfire.internal.cache.snapshot;

import static com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.getLoggerI18n;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import com.gemstone.gemfire.cache.EntryDestroyedException;
import com.gemstone.gemfire.cache.Region;
import com.gemstone.gemfire.cache.execute.Function;
import com.gemstone.gemfire.cache.execute.FunctionContext;
import com.gemstone.gemfire.cache.execute.FunctionException;
import com.gemstone.gemfire.cache.execute.FunctionService;
import com.gemstone.gemfire.cache.execute.RegionFunctionContext;
import com.gemstone.gemfire.cache.execute.ResultCollector;
import com.gemstone.gemfire.cache.execute.ResultSender;
import com.gemstone.gemfire.cache.partition.PartitionRegionHelper;
import com.gemstone.gemfire.cache.snapshot.SnapshotOptions;
import com.gemstone.gemfire.distributed.DistributedMember;
import com.gemstone.gemfire.distributed.internal.ReplyProcessor21;
import com.gemstone.gemfire.internal.cache.LocalRegion;
import com.gemstone.gemfire.internal.cache.execute.InternalExecution;
import com.gemstone.gemfire.internal.cache.execute.LocalResultCollector;
import com.gemstone.gemfire.internal.cache.snapshot.FlowController.Window;
import com.gemstone.gemfire.internal.cache.snapshot.RegionSnapshotServiceImpl.ExportSink;
import com.gemstone.gemfire.internal.cache.snapshot.RegionSnapshotServiceImpl.Exporter;
import com.gemstone.gemfire.internal.cache.snapshot.SnapshotPacket.SnapshotRecord;

/**
 * Exports snapshot data using a sliding window to prevent the nodes in a 
 * partitioned region from overrunning the exporter.  When a {@link SnapshotPacket}
 * is written to the {@link ExportSink}, an ACK is sent back to the source
 * node.  The source node will continue to send data until it runs out of permits;
 * it must then wait for ACK's to resume.
 * 
 * @author bakera
 *
 * @param  the key type
 * @param  the value type
 */
public class WindowedExporter implements Exporter {
  private static final int WINDOW_SIZE = Integer.getInteger("gemfire.WindowedExporter.WINDOW_SIZE", 10);

  @Override
  public long export(Region region, ExportSink sink, SnapshotOptions options) throws IOException {
    long count = 0;
    boolean error = true;
    LocalRegion local = RegionSnapshotServiceImpl.getLocalRegion(region);
    
    SnapshotPacket last = new SnapshotPacket();
    DistributedMember me = region.getCache().getDistributedSystem().getDistributedMember();
    
    WindowedArgs args = new WindowedArgs(me, options);
    WindowedExportCollector results = new WindowedExportCollector(local, last);
    try {
      // Since the ExportCollector already is a LocalResultsCollector it's ok not
      // to keep the reference to the ResultsCollector returned from execute().  
      // Normally discarding the reference can cause issues if GC causes the 
      // weak ref in ProcessorKeeper21 to be collected!!
      InternalExecution exec = (InternalExecution) FunctionService.onRegion(region)
          .withArgs(args)
          .withCollector(results);
      
      // Ensure that our collector gets all exceptions so we can shut down the
      // queue properly.
      exec.setForwardExceptions(true);
      exec.execute(new WindowedExportFunction());
      
      BlockingQueue queue = results.getResult();
      
      SnapshotPacket packet;
      while ((packet = queue.take()) != last) {
        results.ack(packet);
        sink.write(packet.getRecords());
        count += packet.getRecords().length;
      }
      
      error = false;
      FunctionException ex = results.getException();
      if (ex != null) {
        throw new IOException(ex);
      }
    } catch (FunctionException e) {
      throw new IOException(e);
      
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
      throw (IOException) new InterruptedIOException().initCause(e);
      
    } finally {
      if (error) {
        results.abort();
      }
    }
    return count;
  }

  /**
   * Carries the arguments to the export function.
   *
   * @param  the key type
   * @param  the value type
   */
  private static class WindowedArgs implements Serializable {
    private static final long serialVersionUID = 1;
    
    private final DistributedMember exporter;
    private final SnapshotOptions options;
    
    public WindowedArgs(DistributedMember exporter, SnapshotOptions options) {
      this.exporter = exporter;
      this.options = options;
    }
    
    public DistributedMember getExporter() {
      return exporter;
    }
    
    public SnapshotOptions getOptions() {
      return options;
    }
  }
  
  /**
   * Gathers the local data on the region and sends it back to the
   * {@link ResultCollector} in serialized form as {@link SnapshotPacket}s.  Uses
   * a sliding window provided by the {@link FlowController} to avoid over-running
   * the exporting member.
   * 
   * @param  the key type
   * @param  the value type
   * 
   * @see FlowController
   */
  private static class WindowedExportFunction implements Function {
    private static final long serialVersionUID = 1L;

    // We must keep a ref here since the ProcessorKeeper only has a weak ref. If
    // this object is GC'd it could cause a hang since we will no longer receive
    // ACK's for every packet.
    private transient volatile Window window;
    
    @Override
    public boolean hasResult() {
      return true;
    }

    @Override
    public void execute(FunctionContext context) {
      RegionFunctionContext ctx = (RegionFunctionContext) context;

      final WindowedArgs args = (WindowedArgs) ctx.getArguments();
      ResultSender rs = ctx.getResultSender();

      Region region = ctx.getDataSet();
      if (PartitionRegionHelper.isPartitionedRegion(region)) {
        region = PartitionRegionHelper.getLocalDataForContext(ctx);
      }

      LocalRegion local = RegionSnapshotServiceImpl.getLocalRegion(region);
      window = FlowController.getInstance().create(region, args.getExporter(), WINDOW_SIZE);

      try {
        int bufferSize = 0;
        List buffer = new ArrayList();
        DistributedMember me = region.getCache().getDistributedSystem().getDistributedMember();
        for (Iterator> iter = region.entrySet().iterator(); iter.hasNext() && !window.isAborted(); ) {
          Entry entry = iter.next();
          try {
            SnapshotOptions options = args.getOptions();
            if (options.getFilter() == null || options.getFilter().accept(entry)) {
              SnapshotRecord rec = new SnapshotRecord(local, entry);
              buffer.add(rec);
              bufferSize += rec.getSize();
            }
          } catch (EntryDestroyedException e) {
            // continue to next entry
            
          } catch (IOException e) {
            throw new FunctionException(e);
          }

          if (bufferSize > RegionSnapshotServiceImpl.BUFFER_SIZE) {
            window.waitForOpening();
            rs.sendResult(new SnapshotPacket(window.getWindowId(), me, buffer));
            buffer.clear();
            bufferSize = 0;
          }
        }
        
        window.waitForOpening();
        rs.lastResult(new SnapshotPacket(window.getWindowId(), me, buffer));
        if (getLoggerI18n().fineEnabled())
          getLoggerI18n().fine("SNP: Sent all entries in region " + region.getName());
        
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new FunctionException(e);
        
      } finally {
        window.close();
      }
    }

    @Override
    public String getId() {
      return "com.gemstone.gemfire.cache.snapshot.WindowedExport";
    }

    @Override
    public boolean optimizeForWrite() {
      return false;
    }

    @Override
    public boolean isHA() {
      return false;
    }
  }

  /**
   * Collects export results and places them in a queue for processing by the
   * function invoker.
   */
  private static class WindowedExportCollector implements LocalResultCollector> {
    /** the region being exported */
    private final LocalRegion region;
    
    /** marks the end of the queue */
    private final SnapshotPacket end;
    
    /** queue used to stream the snapshot entries */
    private final BlockingQueue entries;

    /** true if no more results are expected */
    private final AtomicBoolean done;
    
    /** the members involved in the export */
    private final Map members;
    
    /** set if there is an error during execution */
    private volatile FunctionException exception;

    /** store a ref to the processor to prevent the processor from being GC'd */
    private volatile ReplyProcessor21 processor;
    
    public WindowedExportCollector(LocalRegion region, SnapshotPacket end) {
      this.region = region;
      this.end = end;
      
      done = new AtomicBoolean(false);
      members = new ConcurrentHashMap();
      
      // cannot bound queue to exert back pressure
      entries = new LinkedBlockingQueue();
    }
    
    @Override
    public BlockingQueue getResult() throws FunctionException {
      return entries;
    }

    @Override
    public BlockingQueue getResult(long timeout, TimeUnit unit)
        throws FunctionException, InterruptedException {
      return getResult();
    }

    /**
     * Returns an exception that occurred during function exception.
     * @return the exception, or null
     */
    public FunctionException getException() {
      return exception;
    }
    
    /**
     * Aborts any further collection of results and forwards the cancellation to 
     * the members involved in the export.
     */
    public void abort() {
      try {
        if (done.compareAndSet(false, true)) {
          if (getLoggerI18n().fineEnabled())
            getLoggerI18n().fine("SNP: Aborting export of region");
          
          entries.clear();
          entries.put(end);
          
          for (Entry entry : members.entrySet()) {
            sendAbort(entry.getKey(), entry.getValue());
          }
        }
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
      }
    }
    
    public void ack(SnapshotPacket packet) {
      FlowController.getInstance().sendAck(region.getDistributionManager(), 
          packet.getSender(), packet.getWindowId(), packet.getPacketId());
    }
    
    @Override
    public void addResult(DistributedMember memberID, Object result) {
      // need to track participants so we can send acks and aborts
      if (!(result instanceof Throwable)) {
        int flowId = ((SnapshotPacket) result).getWindowId();
        if (done.get()) {
          sendAbort(memberID, flowId);
          
        } else {
          members.put(memberID, flowId);
        }
      }
      
      if (!done.get()) {
        try {
          if (result instanceof Throwable) {
            setException((Throwable) result);
            endResults();
            
          } else {
            SnapshotPacket sp = (SnapshotPacket) result;
            entries.put(sp);
          }
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
        }
      }
    }

    @Override
    public void endResults() {
      try {
        if (done.compareAndSet(false, true)) {
          if (getLoggerI18n().fineEnabled())
            getLoggerI18n().fine("SNP: All results received for export of region " + region.getName());
          
          entries.put(end);
        }
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
      }
    }

    @Override
    public void clearResults() {
      entries.clear();
      done.set(false);
      exception = null;
    }

    @Override
    public void setException(Throwable ex) {
      exception = (ex instanceof FunctionException) ?
          (FunctionException) ex : new FunctionException(ex);
    }

    @Override
    public void setProcessor(ReplyProcessor21 processor) {
      this.processor = processor;
    }

    @Override
    public ReplyProcessor21 getProcessor() {
      return processor;
    }
    
    private void sendAbort(DistributedMember member, int flowId) {
      FlowController.getInstance().sendAbort(region.getDistributionManager(), flowId, member);
    }
  }
}