collector.DirMapping.scala Maven / Gradle / Ivy
// Copyright 2015-2022 by Carnegie Mellon University
// See license information in LICENSE.txt
package org.cert.netsa.mothra.tools.collector
import com.typesafe.scalalogging.StrictLogging
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{Path, PathFilter}
import scala.util.Try
case class DirMapping(sourceDir: Path, targetDir: Path) extends StrictLogging {
def clearStaging()(implicit conf: Configuration): Unit = {
val fs = targetDir.getFileSystem(conf)
for ( status <- fs.listStatus(targetDir, DirMapping.onlyDotFiles) ) {
fs.delete(status.getPath(), false)
}
}
def jobs(implicit conf: Configuration): Iterable[CopyJob] =
try {
val sFs = sourceDir.getFileSystem(conf)
val tFs = targetDir.getFileSystem(conf)
for {
sourceStatus <- sFs.listStatus(
sourceDir, DirMapping.excludeDotFiles)
if Try(sourceStatus.isFile()).getOrElse(false)
if sourceStatus.getLen() > 0
sourcePath = sourceStatus.getPath()
targetPath = new Path(targetDir, sourcePath.getName())
targetStatusOpt = Try(tFs.getFileStatus(targetPath)).toOption
} yield CopyJob(sourcePath, targetPath)
} catch {
case ex: Exception =>
logger.error(s"Failure reading source direcotry '${sourceDir}': $ex")
Iterable()
}
}
object DirMapping {
private object onlyDotFiles extends PathFilter {
def accept(p: Path): Boolean = p.getName().startsWith(".")
}
private object excludeDotFiles extends PathFilter {
def accept(p: Path): Boolean = !p.getName().startsWith(".")
}
}
// @LICENSE_FOOTER@
//
// Copyright 2015-2022 Carnegie Mellon University. All Rights Reserved.
//
// This material is based upon work funded and supported by the
// Department of Defense and Department of Homeland Security under
// Contract No. FA8702-15-D-0002 with Carnegie Mellon University for the
// operation of the Software Engineering Institute, a federally funded
// research and development center sponsored by the United States
// Department of Defense. The U.S. Government has license rights in this
// software pursuant to DFARS 252.227.7014.
//
// NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING
// INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON
// UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR
// IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
// FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
// OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT
// MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT,
// TRADEMARK, OR COPYRIGHT INFRINGEMENT.
//
// Released under a GNU GPL 2.0-style license, please see LICENSE.txt or
// contact [email protected] for full terms.
//
// [DISTRIBUTION STATEMENT A] This material has been approved for public
// release and unlimited distribution. Please see Copyright notice for
// non-US Government use and distribution.
//
// Carnegie Mellon(R) and CERT(R) are registered in the U.S. Patent and
// Trademark Office by Carnegie Mellon University.
//
// This software includes and/or makes use of third party software each
// subject to its own license as detailed in LICENSE-thirdparty.tx
//
// DM20-1143