collector.Loader.scala Maven / Gradle / Ivy
// Copyright 2015-2022 by Carnegie Mellon University
// See license information in LICENSE.txt
package org.cert.netsa.mothra.tools.collector
import com.typesafe.scalalogging.StrictLogging
import java.util.concurrent.{ThreadPoolExecutor, TimeUnit, LinkedBlockingDeque}
import org.apache.hadoop.conf.Configuration
class Loader private(concurrentCopies: Int)
extends Runnable with StrictLogging
{
val POLL_INTERVAL = 30 * 1000 // 30 seconds
// Hadoop configuration to be passed to functions that need it
implicit val conf = new Configuration()
private var mappings: Seq[DirMapping] = Seq.empty
private val pool: ThreadPoolExecutor =
new ThreadPoolExecutor(
concurrentCopies, concurrentCopies,
0, TimeUnit.SECONDS,
new LinkedBlockingDeque())
// Add load mappings
def addMappings(newMappings: Iterable[DirMapping]): Unit = {
synchronized {
mappings ++= newMappings
}
logger.info("Added mappings:")
for ( m <- newMappings ) {
logger.info(s" $m")
}
}
// Main run loop
def run(): Unit = {
while ( true ) {
if ( pool.getActiveCount() == 0 ) {
for ( m <- mappings; job <- m.jobs ) {
pool.execute(job)
}
}
Thread.sleep(POLL_INTERVAL)
}
}
}
object Loader {
def apply(concurrentCopies: Int = 2): Loader =
new Loader(concurrentCopies)
}
// @LICENSE_FOOTER@
//
// Copyright 2015-2022 Carnegie Mellon University. All Rights Reserved.
//
// This material is based upon work funded and supported by the
// Department of Defense and Department of Homeland Security under
// Contract No. FA8702-15-D-0002 with Carnegie Mellon University for the
// operation of the Software Engineering Institute, a federally funded
// research and development center sponsored by the United States
// Department of Defense. The U.S. Government has license rights in this
// software pursuant to DFARS 252.227.7014.
//
// NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING
// INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON
// UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR
// IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
// FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
// OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT
// MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT,
// TRADEMARK, OR COPYRIGHT INFRINGEMENT.
//
// Released under a GNU GPL 2.0-style license, please see LICENSE.txt or
// contact [email protected] for full terms.
//
// [DISTRIBUTION STATEMENT A] This material has been approved for public
// release and unlimited distribution. Please see Copyright notice for
// non-US Government use and distribution.
//
// Carnegie Mellon(R) and CERT(R) are registered in the U.S. Patent and
// Trademark Office by Carnegie Mellon University.
//
// This software includes and/or makes use of third party software each
// subject to its own license as detailed in LICENSE-thirdparty.tx
//
// DM20-1143