packer.FlexiblePackLogic.scala Maven / Gradle / Ivy
// Copyright 2015-2022 by Carnegie Mellon University
// See license information in LICENSE.txt
package org.cert.netsa.mothra.packer
import java.lang.{Long => JLong}
import java.net.URLEncoder
import java.time.Instant
import java.util.Base64
import org.cert.netsa.data.net.{IPv4Address, IPv6Address}
import org.cert.netsa.io.ipfix.{
DataTypes, InfoModel, Record, SimpleFieldExtractor}
/**
* An implementation of the [[PackingLogic]] trait. The `pack()`
* method stores records into files based on a list of field names specified
* in this class.
*
*/
private[mothra] case class FlexiblePackLogic(infoModel: InfoModel) extends PackingLogic
{
/** The observationDomain to use for newly created files. */
val domain = 0
/** The version of the packing logic; this value becomes part of the
* path in the directory hierarchy. */
val version = 1
/** The names of the fields to use for packing each record; first
* value is primary split, second value is secondary, etc. */
val fieldNames = Seq(
"protocolIdentifier",
"sourceTransportPort" //,
//"flowStartMilliseconds",
//"sourceIPv4Address",
//"sourceIPv6Address"
)
// The URL-safe version is used because it's also filename-safe.
// It uses "-" and "_" instead of "+" and "/"
private[this] val base64Encoder = Base64.getUrlEncoder()
private[this] def makeExtractor(name: String): (Record => Option[String]) = {
def extract[T](f: T => String): (Record => Option[String]) = {
val ex = { SimpleFieldExtractor[T](name) }
{ (r: Record) => ex.extractFrom(r).map(f) }
}
infoModel.get(name) match {
case None => { _ => None }
case Some(ie) => ie.dataTypeId match {
// Unsigned values use the next larger number type
case DataTypes.Unsigned8 =>
extract[Short] { n => s"$name=$n" }
case DataTypes.Unsigned16 =>
extract[Int] { n => s"$name=$n" }
case DataTypes.Unsigned32 =>
extract[Long] { n => s"$name=$n" }
case DataTypes.Unsigned64 =>
extract[Long] { n => s"$name=${JLong.toUnsignedString(n)}"
}
// Signed values
case DataTypes.Signed8 =>
extract[Byte] { n => s"$name=$n" }
case DataTypes.Signed16 =>
extract[Short] { n => s"$name=$n" }
case DataTypes.Signed32 =>
extract[Int] { n => s"$name=$n" }
case DataTypes.Signed64 =>
extract[Long] { n => s"$name=$n" }
// IP Addresses
case DataTypes.IPv4Address =>
extract[IPv4Address] { n => s"$name=$n" }
case DataTypes.IPv6Address =>
extract[IPv6Address] { n =>
s"$name=${URLEncoder.encode(n.toString, "utf-8")}"
}
// Boolean
case DataTypes.Boolean =>
extract[Boolean] { n => s"$name=$n" }
// Floating point values
case DataTypes.Float32 =>
extract[Float] { n => s"$name=$n" }
case DataTypes.Float64 =>
extract[Double] { n => s"$name=$n" }
// Timestaamps
case DataTypes.DateTimeSeconds =>
extract[Instant] { n => s"$name=$n" }
case DataTypes.DateTimeMilliseconds =>
extract[Instant] { n => s"$name=$n" }
case DataTypes.DateTimeMicroseconds =>
extract[Instant] { n => s"$name=$n" }
case DataTypes.DateTimeNanoseconds =>
extract[Instant] { n => s"$name=$n" }
// Strings and Octet Arrays
case DataTypes.String =>
extract[String] { n =>
s"$name=${URLEncoder.encode(n, "utf-8")}"
}
case DataTypes.MacAddress =>
extract[Array[Byte]] { n =>
s"$name=${base64Encoder.encode(n)}"
}
case DataTypes.OctetArray =>
extract[Array[Byte]] { n =>
s"$name=${base64Encoder.encode(n)}"
}
// Structured data types
case DataTypes.BasicList => { _ => None }
case DataTypes.SubTemplateList => { _ => None }
case DataTypes.SubTemplateMultiList => { _ => None }
}
}
}
/** Extracts the protocol field's value from a Record */
private[this] val extractors: Seq[Record => Option[String]] =
for (name <- fieldNames) yield makeExtractor(name)
// Define the pack() method required by our trait.
// The `packStream()` method (on [[Packer]]) calls it for each
// IPFIX record to determines how the record is stored.
def pack(record: Record): Iterator[PackableRecord] = {
// extract the fields
val fields: Seq[Option[String]] = for (ex <- extractors) yield (ex(record))
// create the output path
val relPath = Seq[String](s"v${version}") ++ (
for { f <- fields ; f2 <- f } yield f2)
// write the record
Iterator(PackableRecord(record, relPath.mkString("/"), domain))
}
}
// @LICENSE_FOOTER@
//
// Copyright 2015-2022 Carnegie Mellon University. All Rights Reserved.
//
// This material is based upon work funded and supported by the
// Department of Defense and Department of Homeland Security under
// Contract No. FA8702-15-D-0002 with Carnegie Mellon University for the
// operation of the Software Engineering Institute, a federally funded
// research and development center sponsored by the United States
// Department of Defense. The U.S. Government has license rights in this
// software pursuant to DFARS 252.227.7014.
//
// NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING
// INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON
// UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR
// IMPLIED, AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF
// FITNESS FOR PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS
// OBTAINED FROM USE OF THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT
// MAKE ANY WARRANTY OF ANY KIND WITH RESPECT TO FREEDOM FROM PATENT,
// TRADEMARK, OR COPYRIGHT INFRINGEMENT.
//
// Released under a GNU GPL 2.0-style license, please see LICENSE.txt or
// contact [email protected] for full terms.
//
// [DISTRIBUTION STATEMENT A] This material has been approved for public
// release and unlimited distribution. Please see Copyright notice for
// non-US Government use and distribution.
//
// Carnegie Mellon(R) and CERT(R) are registered in the U.S. Patent and
// Trademark Office by Carnegie Mellon University.
//
// This software includes and/or makes use of third party software each
// subject to its own license as detailed in LICENSE-thirdparty.tx
//
// DM20-1143