io.findify.s3mock.provider.FileProvider.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of s3mock_2.12 Show documentation
s3mock
The newest version!
package io.findify.s3mock.provider
import java.util.UUID
import java.io.{FileInputStream, File => JFile}

import akka.http.scaladsl.model.DateTime
import better.files.File
import better.files.File.OpenOptions
import com.amazonaws.services.s3.model.ObjectMetadata
import com.typesafe.scalalogging.LazyLogging
import io.findify.s3mock.error.{NoSuchBucketException, NoSuchKeyException}
import io.findify.s3mock.provider.metadata.{MapMetadataStore, MetadataStore}
import io.findify.s3mock.request.{CompleteMultipartUpload, CreateBucketConfiguration}
import io.findify.s3mock.response._
import org.apache.commons.codec.digest.DigestUtils

import scala.util.Random

/**
  * Created by shutty on 8/9/16.
  */
class FileProvider(dir:String) extends Provider with LazyLogging {
  val workDir = File(dir)
  if (!workDir.exists) workDir.createDirectories()

  private val meta = new MapMetadataStore(dir)

  override def metadataStore: MetadataStore = meta

  override def listBuckets: ListAllMyBuckets = {
    val buckets = File(dir).list.map(f => Bucket(fromOs(f.name), DateTime(f.lastModifiedTime.toEpochMilli))).toList
    logger.debug(s"listing buckets: ${buckets.map(_.name)}")
    ListAllMyBuckets("root", UUID.randomUUID().toString, buckets)
  }

  override def listBucket(bucket: String, prefix: Option[String], delimiter: Option[String], maxkeys: Option[Int]) = {
    def commonPrefix(dir: String, p: String, d: String): Option[String] = {
      dir.indexOf(d, p.length) match {
        case -1 => None
        case pos => Some(p + dir.substring(p.length, pos) + d)
      }
    }
    val prefixNoLeadingSlash = prefix.getOrElse("").dropWhile(_ == '/')
    val bucketFile = File(s"$dir/$bucket/")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    val bucketFileString = fromOs(bucketFile.toString)
    val bucketFiles = bucketFile.listRecursively(File.VisitOptions.follow).filter(f => {
        val fString = fromOs(f.toString).drop(bucketFileString.length).dropWhile(_ == '/')
        fString.startsWith(prefixNoLeadingSlash) && !f.isDirectory
      })
    val files = bucketFiles.map(f => {
      val stream = new FileInputStream(f.toJava)
      try {
        val md5 = DigestUtils.md5Hex(stream)
        Content(fromOs(f.toString).drop(bucketFileString.length+1).dropWhile(_ == '/'), DateTime(f.lastModifiedTime.toEpochMilli), md5, f.size, "STANDARD")
      } finally {
        stream.close()
      }
    }).toList
    logger.debug(s"listing bucket contents: ${files.map(_.key)}")
    val commonPrefixes = normalizeDelimiter(delimiter) match {
      case Some(del) => files.flatMap(f => commonPrefix(f.key, prefixNoLeadingSlash, del)).distinct.sorted
      case None => Nil
    }
    val filteredFiles = files.filterNot(f => commonPrefixes.exists(p => f.key.startsWith(p)))
    val count = maxkeys.getOrElse(Int.MaxValue)
    val result = filteredFiles.sortBy(_.key)
    ListBucket(bucket, prefix, delimiter, commonPrefixes, result.take(count), isTruncated = result.size>count)
  }

  override def createBucket(name:String, bucketConfig:CreateBucketConfiguration) = {
    val bucket = File(s"$dir/$name")
    if (!bucket.exists) bucket.createDirectory()
    logger.debug(s"creating bucket $name")
    CreateBucket(name)
  }
  override def putObject(bucket:String, key:String, data:Array[Byte], objectMetadata: ObjectMetadata): Unit = {
    val bucketFile = File(s"$dir/$bucket")
    val file = File(s"$dir/$bucket/$key")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    file.createIfNotExists(createParents = true)
    logger.debug(s"writing file for s3://$bucket/$key to $dir/$bucket/$key, bytes = ${data.length}")
    file.writeByteArray(data)(OpenOptions.default)
    objectMetadata.setLastModified(org.joda.time.DateTime.now().toDate)
    metadataStore.put(bucket, key, objectMetadata)
  }
  override def getObject(bucket:String, key:String): GetObjectData = {
    val bucketFile = File(s"$dir/$bucket")
    val file = File(s"$dir/$bucket/$key")
    logger.debug(s"reading object for s3://$bucket/$key")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    if (!file.exists) throw NoSuchKeyException(bucket, key)
    if (file.isDirectory) throw NoSuchKeyException(bucket, key)
    val meta = metadataStore.get(bucket, key)
    GetObjectData(file.byteArray, meta)
  }

  override def putObjectMultipartStart(bucket:String, key:String, metadata: ObjectMetadata):InitiateMultipartUploadResult = {
    val id = Math.abs(Random.nextLong()).toString
    val bucketFile = File(s"$dir/$bucket")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    File(s"$dir/.mp/$bucket/$key/$id/.keep").createIfNotExists(createParents = true)
    metadataStore.put(bucket, key, metadata)
    logger.debug(s"starting multipart upload for s3://$bucket/$key")
    InitiateMultipartUploadResult(bucket, key, id)
  }
  override def putObjectMultipartPart(bucket:String, key:String, partNumber:Int, uploadId:String, data:Array[Byte]) = {
    val bucketFile = File(s"$dir/$bucket")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    val file = File(s"$dir/.mp/$bucket/$key/$uploadId/$partNumber")
    logger.debug(s"uploading multipart chunk $partNumber for s3://$bucket/$key")
    file.writeByteArray(data)(OpenOptions.default)
  }

  override def putObjectMultipartComplete(bucket:String, key:String, uploadId:String, request:CompleteMultipartUpload): CompleteMultipartUploadResult = {
    val bucketFile = File(s"$dir/$bucket")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    val files = request.parts.map(part => File(s"$dir/.mp/$bucket/$key/$uploadId/${part.partNumber}"))
    val parts = files.map(f => f.byteArray)
    val file = File(s"$dir/$bucket/$key")
    file.createIfNotExists(createParents = true)
    val data = parts.fold(Array[Byte]())(_ ++ _)
    file.writeBytes(data.toIterator)
    File(s"$dir/.mp/$bucket/$key").delete()
    val hash = file.md5
    metadataStore.get(bucket, key).foreach {m =>
      m.setContentMD5(hash)
      m.setLastModified(org.joda.time.DateTime.now().toDate)
    }
    logger.debug(s"completed multipart upload for s3://$bucket/$key")
    CompleteMultipartUploadResult(bucket, key, hash)
  }

  override def copyObject(sourceBucket: String, sourceKey: String, destBucket: String, destKey: String, newMeta: Option[ObjectMetadata] = None): CopyObjectResult = {
    val sourceBucketFile = File(s"$dir/$sourceBucket")
    val destBucketFile = File(s"$dir/$destBucket")
    if (!sourceBucketFile.exists) throw NoSuchBucketException(sourceBucket)
    if (!destBucketFile.exists) throw NoSuchBucketException(destBucket)
    val sourceFile = File(s"$dir/$sourceBucket/$sourceKey")
    val destFile = File(s"$dir/$destBucket/$destKey")
    destFile.createIfNotExists(createParents = true)
    sourceFile.copyTo(destFile, overwrite = true)
    logger.debug(s"Copied s3://$sourceBucket/$sourceKey to s3://$destBucket/$destKey")
    val sourceMeta = newMeta.orElse(metadataStore.get(sourceBucket, sourceKey))
    sourceMeta.foreach(meta => metadataStore.put(destBucket, destKey, meta))
    CopyObjectResult(DateTime(sourceFile.lastModifiedTime.toEpochMilli), destFile.md5)
  }


  override def copyObjectMultipart(sourceBucket: String, sourceKey: String, destBucket: String, destKey: String, part: Int, uploadId:String, fromByte: Int, toByte: Int, newMeta: Option[ObjectMetadata] = None): CopyObjectResult = {
    val data = getObject(sourceBucket, sourceKey).bytes.slice(fromByte, toByte + 1)
    putObjectMultipartPart(destBucket, destKey, part, uploadId, data)
    new CopyObjectResult(DateTime.now, DigestUtils.md5Hex(data))
  }

  override def deleteObject(bucket:String, key:String): Unit = {
    val file = File(s"$dir/$bucket/$key")
    logger.debug(s"deleting object s://$bucket/$key")
    if (!file.exists) throw NoSuchKeyException(bucket, key)
    if (!file.isDirectory) {
      file.delete()
      metadataStore.delete(bucket, key)
    }
  }

  override def deleteBucket(bucket:String): Unit = {
    val bucketFile = File(s"$dir/$bucket")
    logger.debug(s"deleting bucket s://$bucket")
    if (!bucketFile.exists) throw NoSuchBucketException(bucket)
    bucketFile.delete()
    metadataStore.remove(bucket)
  }

  /** Replace the os separator with a '/' */
  private def fromOs(path: String): String = {
    path.replace(JFile.separatorChar, '/')
  }

}