All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.github.dakusui.processstreamer.utils.StreamUtils Maven / Gradle / Ivy

Go to download 

package com.github.dakusui.processstreamer.utils;

import com.github.dakusui.processstreamer.exceptions.Exceptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static com.github.dakusui.processstreamer.utils.Checks.greaterThan;
import static com.github.dakusui.processstreamer.utils.Checks.requireArgument;
import static com.github.dakusui.processstreamer.utils.ConcurrencyUtils.updateAndNotifyAll;
import static com.github.dakusui.processstreamer.utils.ConcurrencyUtils.waitWhile;
import static java.lang.Math.abs;
import static java.util.Collections.singletonList;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;

/**
 * = A Stream Utility class
 * This is a stream utility class.
 */
public enum StreamUtils {
  ;
  private static final Logger LOGGER = LoggerFactory.getLogger(StreamUtils.class);

  /**
   * Returns a consumer which writes given string objects to an {@code OutputStream}
   * {@code os} using a {@code Charset} {@code charset}.
   * 

   * If {@code null} is given to the consumer returned by this method, the output
   * to {@code os} will be closed and the {@code null} will not be passed to it.
   *
   * @param os      OutputStream to which string objects given to returned consumer written.
   * @param charset A {@code Charset} object that specifies encoding by which
   * @return A closeable strinmg consumer object.
   */
  public static CloseableStringConsumer toCloseableStringConsumer(OutputStream os, Charset charset) {
    try {
      return CloseableStringConsumer.create(os, charset);
    } catch (UnsupportedEncodingException e) {
      throw Exceptions.wrap(e);
    }
  }

  public interface CloseableStringConsumer extends Consumer, Closeable {
    @Override
    default void accept(String s) {
      if (s != null)
        this.writeLine(s);
      else
        this.close();
    }

    default void writeLine(String s) {
      this.printStream().println(s);
    }

    @Override
    default void close() {
      printStream().flush();
      printStream().close();
    }

    PrintStream printStream();

    static CloseableStringConsumer create(OutputStream os, Charset charset) throws UnsupportedEncodingException {
      PrintStream ps = new PrintStream(os, true, charset.name());
      return () -> ps;
    }
  }

  /**
   * Returns a consumer that does nothing.
   *
   * @param  Type of values to be consumed by returned object.
   * @return A consumer that doesn't do anything.
   */
  public static  Consumer nop() {
    return e -> {
    };
  }

  public static  Stream closeOnFinish(Stream in) {
    return onFinish(in, Stream::close).onClose(in::close);
  }

  @SuppressWarnings("unchecked")
  public static  Stream onFinish(Stream in, Consumer> action) {
    Object sentinel = new Object() {
      @Override
      public String toString() {
        return "(sentinel)";
      }
    };
    return Stream.concat(requireNonNull(in), Stream.of(sentinel))
        .filter(o -> {
          if (o != sentinel)
            return true;
          else {
            action.accept(in);
            return false;
          }
        })
        .map(each -> (T) each);
  }

  public static  List> partition(
      ExecutorService threadPool,
      Consumer threadPoolCloser,
      Stream in,
      int numQueues,
      int eachQueueSize,
      Function partitioner) {
    return split(
        threadPool, threadPoolCloser, in, numQueues, eachQueueSize,
        (blockingQueues, each) ->
            singletonList(blockingQueues.get(abs(partitioner.apply(each)) % numQueues)));
  }

  public static  List> tee(
      ExecutorService threadPool,
      Consumer threadPoolCloser,
      Stream in,
      int numQueues,
      int queueSize) {
    return split(threadPool, threadPoolCloser, in, numQueues, queueSize, (blockingQueues, t) -> blockingQueues);
  }

  private static  List> split(
      ExecutorService threadPool,
      Consumer threadPoolCloser,
      Stream in,
      int numQueues,
      int eachQueueSize,
      BiFunction>, T, List>> selector) {
    requireArgument(numQueues, greaterThan(0));
    if (numQueues == 1)
      return singletonList(in);
    List> queues = IntStream.range(0, numQueues)
        .mapToObj(i -> new ArrayBlockingQueue<>(eachQueueSize))
        .collect(toList());

    Object sentinel = initializeSplit(threadPool, threadPoolCloser, in, selector, queues);

    return IntStream.range(0, numQueues)
        .mapToObj(
            c -> StreamSupport.stream(
                ((Iterable) () -> iteratorFinishingOnSentinel(
                    e -> e == sentinel,
                    blockingDataReader(queues.get(c)))).spliterator(),
                false))
        .collect(toList());
  }

  private static  Object initializeSplit(
      ExecutorService threadPool,
      Consumer threadPoolCloser,
      Stream in,
      BiFunction>, T, List>> selector,
      List> queues) {
    class TaskSubmitter implements Runnable {
      private final AtomicBoolean initialized = new AtomicBoolean(false);
      private final Object        sentinel;

      private TaskSubmitter(Object sentinel) {
        this.sentinel = sentinel;
      }

      @SuppressWarnings("unchecked")
      public void run() {
        threadPool.submit(
            () -> StreamUtils.closeOnFinish(
                Stream.concat(in, Stream.of(sentinel))
                    .onClose(() -> threadPoolCloser.accept(threadPool)))
                .forEach(e -> {
                      if (e == sentinel)
                        queues.forEach(q -> {
                          initializeIfNecessaryAndNotifyAll();
                          putElement(q, e);
                        });
                      else
                        selector.apply(queues, (T) e).forEach(q -> {
                          initializeIfNecessaryAndNotifyAll();
                          putElement(q, e);
                        });
                    }
                )
        );
        synchronized (initialized) {
          waitWhile(initialized, i -> !i.get());
        }
      }

      private void initializeIfNecessaryAndNotifyAll() {
        synchronized (initialized) {
          if (!initialized.get())
            updateAndNotifyAll(initialized, v -> v.set(true));
        }
      }
    }
    Object sentinel = createSentinel(0);
    new TaskSubmitter(sentinel).run();
    return sentinel;
  }

  /**
   * = Merging function
   * Merges given streams possibly block into one keeping orders where elements
   * appear in original streams.
   *
   * [ditaa]
   * ----
   *
   * +-----+
   * |Queue|
   * +-----+
   *
   * ----
   *
   * @param threadPool       A thread pool that gives threads by which data in {@code streams}
   *                         drained to the returned stream.
   * @param threadPoolCloser A consumer that closes {@code threadPool}.
   * @param queueSize        The size of queue
   * @param streams          input streams
   * @param               Type of elements that given streams contain.
   * @return merged stream
   */
  @SafeVarargs
  public static  Stream merge(
      ExecutorService threadPool,
      Consumer threadPoolCloser,
      int queueSize,
      Stream... streams) {
    if (streams.length == 0)
      return Stream.empty();
    if (streams.length == 1)
      return streams[0];
    BlockingQueue queue = new ArrayBlockingQueue<>(queueSize);
    Set sentinels = new HashSet<>();

    AtomicInteger remainingStreams = new AtomicInteger(streams.length);
    int i = 0;
    for (Stream eachStream : streams) {
      Object sentinel = createSentinel(i++);
      sentinels.add(sentinel);
      LOGGER.trace("Submitting task for:{}", sentinel);
      Consumer action = new Consumer() {
        boolean started = false;

        @Override
        public void accept(Object elementOrSentinel) {
          LOGGER.trace("{}, is trying to put:{}", sentinel, elementOrSentinel);
          if (!started) {
            LOGGER.trace("task:stream:sentinel={} starting", sentinel);
            synchronized (remainingStreams) {
              updateAndNotifyAll(remainingStreams, AtomicInteger::decrementAndGet);
              started = true;
            }
          }
          putElement(queue, elementOrSentinel);
        }
      };
      threadPool.execute(
          () -> Stream.concat(eachStream, Stream.of(sentinel)).forEach(action)
      );
      LOGGER.trace("Submitted task for:{}", eachStream);
    }
    synchronized (remainingStreams) {
      boolean succeeded = false;
      try {
        waitWhile(remainingStreams, v -> v.get() > 0);
        succeeded = true;
      } finally {
        if (!succeeded)
          LOGGER.error("remainingStreams={}", remainingStreams);
      }
    }
    Supplier reader = blockingDataReader(queue);
    Set remainingSentinels = new HashSet<>(sentinels);
    Predicate isSentinel = sentinels::contains;
    return StreamSupport.stream(new Iterable() {
      final Supplier readNext = () -> {
        Object nextElementOrSentinel = reader.get();
        if (isSentinel.test(nextElementOrSentinel)) {
          remainingSentinels.remove(nextElementOrSentinel);
          if (remainingSentinels.isEmpty())
            return nextElementOrSentinel;
          else
            return this.readNext.get();
        }
        return nextElementOrSentinel;
      };

      @Override
      public Iterator iterator() {
        return iteratorFinishingOnSentinel(isSentinel, readNext);
      }

    }.spliterator(), false).onClose(() -> threadPoolCloser.accept(threadPool));
  }

  private static  Iterator iteratorFinishingOnSentinel(
      Predicate isSentinel, Supplier readNext) {
    return new Iterator() {
      /**
       * An object to let this iterator know that the {@code next} field
       * is not valid anymore and it needs to read the next value from the
       * source {@code i}.
       * This is different from  a sentinel.
       */
      private Object invalid = new Object();
      Object next = invalid;

      @Override
      public boolean hasNext() {
        if (this.next == this.invalid)
          this.next = readNext.get();
        return !isSentinel.test(this.next);
      }

      @SuppressWarnings("unchecked")
      @Override
      public T next() {
        if (this.next == this.invalid)
          this.next = readNext.get();
        if (isSentinel.test(this.next))
          throw new NoSuchElementException();
        try {
          return (T) this.next;
        } finally {
          this.next = this.invalid;
        }
      }
    };
  }

  private static Supplier blockingDataReader(BlockingQueue queue) {
    return () -> {
      while (true) {
        try {
          return queue.take();
        } catch (InterruptedException ignored) {
        }
      }
    };
  }

  private static Object createSentinel(int i) {
    return new Object() {
      @Override
      public String toString() {
        return String.format("SENTINEL:%s", i);
      }
    };
  }

  private static void putElement(BlockingQueue queue, Object e) {
    try {
      queue.put(e);
    } catch (InterruptedException ignored) {
    }
  }

  public static Stream stream(InputStream is, Charset charset) {
    return IoUtils.bufferedReader(is, charset).lines();
  }

  public interface RingBuffer {
    void write(E elem);

    Stream stream();

    static  RingBuffer create(int size) {
      return new RingBuffer() {
        int cur = 0;
        List buffer = new ArrayList<>(size);

        @Override
        public void write(E elem) {
          this.buffer.add(cur++, elem);
          cur %= size;
        }

        @Override
        public synchronized Stream stream() {
          return Stream.concat(
              this.buffer.subList(cur, this.buffer.size()).stream(),
              this.buffer.subList(0, cur).stream());
        }
      };
    }
  }
}
    

    

    

            

    

            



    
    






    
© 2015 - 2024 Weber Informatics LLC | Privacy Policy