overflowdb.storage.NodesWriter Maven / Gradle / Ivy

Go to download
package overflowdb.storage;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import overflowdb.Node;
import overflowdb.NodeDb;
import overflowdb.NodeRef;

import java.io.IOException;
import java.util.Spliterator;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.StreamSupport;

/**
 * Persists collections of nodes in bulk to disk. Used either by ReferenceManager (if overflow to disk is enabled),
 * or alternatively when closing the graph (if storage to disk is enabled).
 */
public class NodesWriter {
  private final Logger logger = LoggerFactory.getLogger(getClass());
  private final NodeSerializer nodeSerializer;
  private final OdbStorage storage;

  public NodesWriter(NodeSerializer nodeSerializer, OdbStorage storage) {
    this.nodeSerializer = nodeSerializer;
    this.storage = storage;
  }

  /**
   * Writes all references to storage, blocks until complete.
   * Serialization happens in parallel, however writing to storage happens sequentially, to avoid lock contention in mvstore.
   */
  public void writeAndClearBatched(Spliterator nodes, int estimatedTotalCount) {
    if (estimatedTotalCount > 0)
      logger.info(String.format("serializing and persisting %d nodes (this may take a while)", estimatedTotalCount));

    AtomicInteger count = new AtomicInteger(0);

    StreamSupport.stream(nodes, true)
        .map(this::serializeIfDirty)
        .sequential()
        .forEach(serializedNode -> {
          if (serializedNode != null) {
            storage.persist(serializedNode.id, serializedNode.data);

            /** counting only for printing statistics - this is rafher slow, but since persisting to disk is much slower
             * and also disk-bound, it doesn't really matter... */
            int currCount = count.incrementAndGet();
            if (currCount % 100_000 == 0) {
               float progressPercent = 100f * currCount / estimatedTotalCount;
               logger.debug(String.format("progress of writing nodes to storage: %.2f%s", Float.min(100f, progressPercent), "%"));
            }
          }
        });

    if (estimatedTotalCount > 0)
      logger.info(String.format("finished serializing and persisting %d nodes", estimatedTotalCount));
  }
  private SerializedNode serializeIfDirty(Node node) {
    NodeDb nodeDb = null;
    NodeRef ref = null;
    if (node instanceof NodeDb) {
      nodeDb = (NodeDb) node;
      ref = nodeDb.ref;
    } else if (node instanceof NodeRef) {
      ref = (NodeRef) node;
      if (ref.isSet()) nodeDb = ref.get();
    }

    if (nodeDb != null && nodeDb.isDirty()) {
      try {
        byte[] data = nodeSerializer.serialize(nodeDb);
        NodeRef.clear(ref);
        return new SerializedNode(ref.id(), data);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
    return null;
  }

  private static class SerializedNode {
    private final long id;
    private final byte[] data;

    private SerializedNode(long id, byte[] data) {
      this.id = id;
      this.data = data;
    }
  }

}