io.cequence.openaiscala.task.examples.GenerateFineTuningDatasetExample.scala Maven / Gradle / Ivy
The newest version!
package io.cequence.openaiscala.task.examples
import akka.actor.ActorSystem
import akka.stream.scaladsl.{FileIO, Source}
import akka.stream.{ActorMaterializer, IOResult, Materializer}
import akka.util.ByteString
import io.cequence.openaiscala.task.domain.BinaryTaskCoreSettings
import io.cequence.openaiscala.task.FineTuningTrainingSetGenerator
import io.cequence.openaiscala.task.binary.ConvertBinaryToDecimalTask
import java.nio.file.Paths
import scala.concurrent.{ExecutionContext, Future}
object GenerateFineTuningDatasetExample extends App {
// change implicits if needed
private implicit val actorSystem = ActorSystem()
private implicit val materializer: Materializer = ActorMaterializer()
private implicit val ec: ExecutionContext = ExecutionContext.Implicits.global
private val generator = FineTuningTrainingSetGenerator(
new ConvertBinaryToDecimalTask(withAssistantSeedPrompt = false)
)
private val count = 5
// task specific settings
private val taskSpecificSettings = BinaryTaskCoreSettings(
minStringSize = 10,
maxStringSize = 10,
withSpaces = true,
withExamples = true,
useDensityUniformDistribution = true
)
private val source = generator.generate(count, taskSpecificSettings)
println("Enter output file name:")
val outputFileName = scala.io.StdIn.readLine()
writeLines(source, outputFileName).map(_ =>
System.exit(0)
).recover {
case e: Exception =>
e.printStackTrace()
System.exit(1)
}
private def writeLines(
source: Source[String, _],
fileName: String)(
implicit materializer: Materializer
): Future[IOResult] =
source.map(line => ByteString(line + "\n")).runWith(
FileIO.toPath(Paths.get(fileName))
)
}