collector.CopyJob.scala Maven / Gradle / Ivy

Go to download
// Copyright 2015-2022 by Carnegie Mellon University
// See license information in LICENSE.txt

package org.cert.netsa.mothra.tools.collector

import com.typesafe.scalalogging.StrictLogging
import java.io.IOException
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.IOUtils
import resource.managed
import scala.util.{Failure, Try}

case class CopyJob(sourcePath: Path, targetPath: Path)
  (implicit conf: Configuration)
    extends Runnable with StrictLogging
{
  def run(): Unit = Try {
    val sourceFs = sourcePath.getFileSystem(conf)
    val targetFs = targetPath.getFileSystem(conf)
    val stagePath =
      new Path(targetPath.getParent(), "." + targetPath.getName())
    val sourceStatus =
      sourcePath.getFileSystem(conf).getFileStatus(sourcePath)
    // check if source exists and is newer/larger than target
    if ( targetFs.exists(targetPath) ) {
      // Use a Try because isFile can throw an exception D:
      if ( !Try(targetFs.getFileStatus(targetPath).isFile).getOrElse(false) ) {
        throw new IOException(
          s"Cannot overwrite '${targetPath}' which is not a file")
      }
      val targetStatus = targetFs.getFileStatus(targetPath)
      if ( sourceStatus.getLen() < targetStatus.getLen() ) {
        throw new IOException(
          s"Will not overwrite existing larger file '${targetPath}'")
      } else if ( sourceStatus.getModificationTime() < targetStatus.getModificationTime() ) {
        throw new IOException(
          s"Will not overwrite existing newer file '${targetPath}'")
      }
      // Try removing the target
      if ( ! targetFs.delete(targetPath, false) ) {
        throw new IOException(
          s"Cannot delete target file '${targetPath}'")
      }
    }
    // copy bytes from source to staging path
    for {
      sourceStream <- managed(sourceFs.open(sourcePath))
      stageStream <- managed(targetFs.create(stagePath, true))
    } {
      IOUtils.copyBytes(sourceStream, stageStream, conf)
    }
    // set modification time for staging file
    // The -1 means "don't set the atime".
    targetFs.setTimes(stagePath, sourceStatus.getModificationTime(), -1)
    // move staging path into place
    if ( ! targetFs.rename(stagePath, targetPath) ) {
      throw new IOException(
        s"Failed to move staging file '${stagePath}' into place at '${targetPath}'")
    }
    // finally, now that all that worked, remove the source file
    if ( ! sourceFs.delete(sourcePath, false) ) {
      throw new IOException(
        s"Failed to remove source file '${sourcePath}' after successful loading into '${targetPath}'")
    }
  } match {
    case Failure(e) => logger.error(e.toString)
    case _ => logger.info(s"'${sourcePath}' => '${targetPath}'")
  }
}

// @LICENSE_FOOTER@
//
// Copyright 2015-2022 Carnegie Mellon University. All Rights Reserved.
//
// This material is based upon work funded and supported by the
// Department of Defense and Department of Homeland Security under
// Contract No. FA8702-15-D-0002 with Carnegie Mellon University for the
// operation of the Software Engineering Institute, a federally funded
// research and development center sponsored by the United States
// Department of Defense. The U.S. Government has license rights in this
// software pursuant to DFARS 252.227.7014.
//
// NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING
// INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON
// UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR
// IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
// FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
// OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT
// MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT,
// TRADEMARK, OR COPYRIGHT INFRINGEMENT.
//
// Released under a GNU GPL 2.0-style license, please see LICENSE.txt or
// contact [email protected] for full terms.
//
// [DISTRIBUTION STATEMENT A] This material has been approved for public
// release and unlimited distribution. Please see Copyright notice for
// non-US Government use and distribution.
//
// Carnegie Mellon(R) and CERT(R) are registered in the U.S. Patent and
// Trademark Office by Carnegie Mellon University.
//
// This software includes and/or makes use of third party software each
// subject to its own license as detailed in LICENSE-thirdparty.tx
//
// DM20-1143