Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/*
* Copyright 2017-2024 John Snow Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.johnsnowlabs.nlp.annotators.seq2seq
import com.johnsnowlabs.ml.ai.util.Generation.GenerationConfig
import com.johnsnowlabs.ml.ai.LLAMA3
import com.johnsnowlabs.ml.onnx.OnnxWrapper.DecoderWrappers
import com.johnsnowlabs.ml.onnx.{OnnxWrapper, ReadOnnxModel, WriteOnnxModel}
import com.johnsnowlabs.ml.openvino.{OpenvinoWrapper, ReadOpenvinoModel, WriteOpenvinoModel}
import com.johnsnowlabs.ml.util.LoadExternalModel.{
loadJsonStringAsset,
loadSentencePieceAsset,
loadTextAsset,
modelSanityCheck,
notSupportedEngineError
}
import com.johnsnowlabs.ml.util.{ONNX, Openvino}
import com.johnsnowlabs.nlp.AnnotatorType.DOCUMENT
import com.johnsnowlabs.nlp._
import com.johnsnowlabs.ml.tensorflow.sentencepiece.{
ReadSentencePieceModel,
SentencePieceWrapper,
WriteSentencePieceModel
}
import com.johnsnowlabs.nlp.serialization.MapFeature
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.ml.param._
import org.apache.spark.ml.util.Identifiable
import org.apache.spark.sql.SparkSession
import com.johnsnowlabs.nlp.serialization.{MapFeature, StructFeature}
import org.json4s._
import org.json4s.jackson.JsonMethods._
/** Llama 3: Cutting-Edge Foundation and Fine-Tuned Chat Models
*
* The Llama 3 release introduces a new family of large language models, ranging from 8B to 70B
* parameters. Llama 3 models are designed with a greater emphasis on efficiency, performance,
* and safety, achieving remarkable advancements in training and deployment processes. These
* models are trained on a diversified dataset that significantly enhances their capability to
* generate more accurate and contextually relevant outputs.
*
* The fine-tuned variants, known as Llama 3-instruct, are specifically optimized for
* dialogue-based applications, making use of Reinforcement Learning from Human Feedback (RLHF)
* with an advanced reward model. Llama 3-instruct models demonstrate state-of-the-art
* performance across multiple benchmarks and surpass the capabilities of Llama 2, particularly
* in conversational settings.
*
* Pretrained models can be loaded with `pretrained` of the companion object:
* {{{
* val llama3 = LLAMA3Transformer.pretrained()
* .setInputCols("document")
* .setOutputCol("generation")
* }}}
* The default model is `"llama_3_7b_chat_hf_int8"`, if no name is provided. For available
* pretrained models please see the [[https://sparknlp.org/models?q=llama3 Models Hub]].
*
* For extended examples of usage, see
* [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/seq2seq/LLAMA3TestSpec.scala LLAMA3TestSpec]].
*
* '''References:'''
* - [[https://ai.meta.com/blog/meta-llama-3/ Meta's Llama 3: Cutting-Edge Foundation and Fine-Tuned Chat Models]]
* - [[https://github.com/facebookresearch/llama]]
*
* '''Paper Abstract:'''
*
* ''Llama 3 represents Meta’s latest innovation in the development of large language models
* (LLMs), offering a series of models from 1 billion to 70 billion parameters. These models have
* been fine-tuned for dialogue applications under the Llama 3-Chat series, ensuring they are
* highly responsive and context-aware. Our Llama 3 models not only excel in various benchmarks
* but also incorporate enhanced safety and alignment features to address ethical concerns and
* ensure responsible AI deployment. We invite the community to explore the capabilities of Llama
* 3 and contribute to ongoing research in the field of natural language processing.''
*
* '''Note:'''
*
* This is a resource-intensive module, especially with larger models and sequences. Use of
* accelerators such as GPUs is strongly recommended.
*
* ==Example==
* {{{
* import spark.implicits._
* import com.johnsnowlabs.nlp.base.DocumentAssembler
* import com.johnsnowlabs.nlp.annotators.seq2seq.LLAMA3Transformer
* import org.apache.spark.ml.Pipeline
*
* val documentAssembler = new DocumentAssembler()
* .setInputCol("text")
* .setOutputCol("documents")
*
* val llama3 = LLAMA3Transformer.pretrained("llama_3_7b_chat_hf_int8")
* .setInputCols(Array("documents"))
* .setMinOutputLength(15)
* .setMaxOutputLength(60)
* .setDoSample(false)
* .setTopK(40)
* .setNoRepeatNgramSize(3)
* .setOutputCol("generation")
*
* val pipeline = new Pipeline().setStages(Array(documentAssembler, llama3))
*
* val data = Seq(
* (
* 1,
* """<|start_header_id|>system<|end_header_id|>
*
* You are a minion chatbot who always responds in minion speak!
*
* <|start_header_id|>user<|end_header_id|>
*
* Who are you?
*
* <|start_header_id|>assistant<|end_header_id|>
* """.stripMargin)
* ).toDF("id", "text")
*
* val result = pipeline.fit(data).transform(data)
*
* result.select("generation.result").show(truncate = false)
* +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
* |result |
* +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
* |[Oooh, me am Minion! Me help you with things! Me speak Minion language, yeah! Bana-na-na!] |
* +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
* }}}
*
* @param uid
* required uid for storing annotator to disk
* @groupname anno Annotator types
* @groupdesc anno
* Required input and expected output annotator types
* @groupname Ungrouped Members
* @groupname param Parameters
* @groupname setParam Parameter setters
* @groupname getParam Parameter getters
* @groupname Ungrouped Members
* @groupprio param 1
* @groupprio anno 2
* @groupprio Ungrouped 3
* @groupprio setParam 4
* @groupprio getParam 5
* @groupdesc param
* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
* parameter values through setters and getters, respectively.
*/
class LLAMA3Transformer(override val uid: String)
extends AnnotatorModel[LLAMA3Transformer]
with HasBatchedAnnotate[LLAMA3Transformer]
with ParamsAndFeaturesWritable
with WriteOnnxModel
with WriteOpenvinoModel
with HasGeneratorProperties
with HasEngine {
def this() = this(Identifiable.randomUID("LLAMA3TRANSFORMER"))
/** Input annotator type : DOCUMENT
*
* @group param
*/
override val inputAnnotatorTypes: Array[AnnotatorType] = Array(DOCUMENT)
/** Output annotator type : DOCUMENT
*
* @group param
*/
override val outputAnnotatorType: String = DOCUMENT
/** @group setParam */
def setRandomSeed(value: Int): LLAMA3Transformer.this.type = {
if (randomSeed.isEmpty) {
this.randomSeed = Some(value)
}
this
}
/** A list of token ids which are ignored in the decoder's output (Default: `Array()`)
*
* @group param
*/
var ignoreTokenIds = new IntArrayParam(
this,
"ignoreTokenIds",
"A list of token ids which are ignored in the decoder's output")
/** @group setParam */
def setIgnoreTokenIds(tokenIds: Array[Int]): LLAMA3Transformer.this.type = {
set(ignoreTokenIds, tokenIds)
}
/** @group getParam */
def getIgnoreTokenIds: Array[Int] = $(ignoreTokenIds)
/** Vocabulary used to encode the words to ids with bpeTokenizer.encode
*
* @group param
*/
val vocabulary: MapFeature[String, Int] = new MapFeature(this, "vocabulary").setProtected()
/** @group setParam */
def setVocabulary(value: Map[String, Int]): this.type = set(vocabulary, value)
/** Holding merges.txt coming from RoBERTa model
*
* @group param
*/
val merges: MapFeature[(String, String), Int] = new MapFeature(this, "merges").setProtected()
/** @group setParam */
def setMerges(value: Map[(String, String), Int]): this.type = set(merges, value)
/** Additional tokens to be added to the vocabulary
*
* @group param
*/
val addedTokens: MapFeature[String, Int] = new MapFeature(this, "addedTokens").setProtected()
/** @group setParam */
def setAddedTokens(value: Map[String, Int]): this.type = set(addedTokens, value)
/** Stop tokens to terminate the generation
*
* @group param
*/
override val stopTokenIds =
new IntArrayParam(this, "stopTokenIds", "Stop tokens to terminate the generation")
/** @group setParam */
override def setStopTokenIds(value: Array[Int]): this.type = {
set(stopTokenIds, value)
}
/** @group getParam */
override def getStopTokenIds: Array[Int] = $(stopTokenIds)
private var _model: Option[Broadcast[LLAMA3]] = None
val generationConfig: StructFeature[GenerationConfig] =
new StructFeature(this, "generationConfig").setProtected()
def setGenerationConfig(value: GenerationConfig): this.type =
set(generationConfig, value)
def getGenerationConfig: GenerationConfig = $$(generationConfig)
/** @group setParam */
def setModelIfNotSet(
spark: SparkSession,
onnxWrappers: Option[DecoderWrappers],
openvinoWrapper: Option[OpenvinoWrapper]): this.type = {
if (_model.isEmpty) {
_model = Some(
spark.sparkContext.broadcast(
new LLAMA3(
onnxWrappers,
openvinoWrapper,
$$(merges),
$$(vocabulary),
$$(addedTokens),
generationConfig = getGenerationConfig)))
}
this
}
/** @group getParam */
def getModelIfNotSet: LLAMA3 = _model.get.value
setDefault(
minOutputLength -> 0,
maxOutputLength -> 20,
doSample -> false,
temperature -> 0.6,
topK -> -1,
topP -> 0.9,
repetitionPenalty -> 1.0,
noRepeatNgramSize -> 3,
ignoreTokenIds -> Array(),
batchSize -> 1,
beamSize -> 1,
maxInputLength -> 4096,
stopTokenIds -> Array(128001))
/** takes a document and annotations and produces new annotations of this annotator's annotation
* type
*
* @param batchedAnnotations
* Annotations that correspond to inputAnnotationCols generated by previous annotators if any
* @return
* any number of annotations processed for every input annotation. Not necessary one to one
* relationship
*/
override def batchAnnotate(batchedAnnotations: Seq[Array[Annotation]]): Seq[Seq[Annotation]] = {
val allAnnotations = batchedAnnotations
.filter(_.nonEmpty)
.zipWithIndex
.flatMap { case (annotations, i) =>
annotations.filter(_.result.nonEmpty).map(x => (x, i))
}
val processedAnnotations = if (allAnnotations.nonEmpty) {
this.getModelIfNotSet.predict(
sentences = allAnnotations.map(_._1),
batchSize = $(batchSize),
minOutputLength = $(minOutputLength),
maxOutputLength = $(maxOutputLength),
doSample = $(doSample),
temperature = $(temperature),
topK = $(topK),
topP = $(topP),
repetitionPenalty = $(repetitionPenalty),
noRepeatNgramSize = $(noRepeatNgramSize),
randomSeed = this.randomSeed,
ignoreTokenIds = $(ignoreTokenIds),
beamSize = $(beamSize),
maxInputLength = $(maxInputLength),
stopTokenIds = $(stopTokenIds))
} else {
Seq()
}
Seq(processedAnnotations)
}
override def onWrite(path: String, spark: SparkSession): Unit = {
super.onWrite(path, spark)
getEngine match {
case ONNX.name =>
val wrappers = getModelIfNotSet.onnxWrappers
writeOnnxModels(
path,
spark,
Seq((wrappers.get.decoder, "decoder_model.onnx")),
LLAMA3Transformer.suffix)
case Openvino.name =>
val wrappers = getModelIfNotSet.openvinoWrapper
writeOpenvinoModel(
path,
spark,
wrappers.get,
LLAMA3Transformer.suffix,
LLAMA3Transformer.openvinoFile)
}
}
}
trait ReadablePretrainedLLAMA3TransformerModel
extends ParamsAndFeaturesReadable[LLAMA3Transformer]
with HasPretrained[LLAMA3Transformer] {
override val defaultModelName: Some[String] = Some("llama3")
/** Java compliant-overrides */
override def pretrained(): LLAMA3Transformer = super.pretrained()
override def pretrained(name: String): LLAMA3Transformer = super.pretrained(name)
override def pretrained(name: String, lang: String): LLAMA3Transformer =
super.pretrained(name, lang)
override def pretrained(name: String, lang: String, remoteLoc: String): LLAMA3Transformer =
super.pretrained(name, lang, remoteLoc)
}
trait ReadLLAMA3TransformerDLModel extends ReadOnnxModel with ReadOpenvinoModel {
this: ParamsAndFeaturesReadable[LLAMA3Transformer] =>
override val onnxFile: String = "llama3_onnx"
val suffix: String = "_llama3"
override val openvinoFile: String = "llama3_openvino"
def readModel(instance: LLAMA3Transformer, path: String, spark: SparkSession): Unit = {
instance.getEngine match {
case ONNX.name =>
val wrappers =
readOnnxModels(path, spark, Seq("decoder_model.onnx"), suffix)
val onnxWrappers =
DecoderWrappers(decoder = wrappers("decoder_model.onnx"))
instance.setModelIfNotSet(spark, Some(onnxWrappers), None)
case Openvino.name =>
val ovWrapper =
readOpenvinoModel(path, spark, "_llama3_ov")
instance.setModelIfNotSet(spark, None, Some(ovWrapper))
case _ =>
throw new Exception(notSupportedEngineError)
}
}
addReader(readModel)
def loadSavedModel(
modelPath: String,
spark: SparkSession,
useOpenvino: Boolean = false): LLAMA3Transformer = {
implicit val formats: DefaultFormats.type = DefaultFormats // for json4
val (localModelPath, detectedEngine) =
modelSanityCheck(modelPath, isDecoder = true)
val modelConfig: JValue =
parse(loadJsonStringAsset(localModelPath, "config.json"))
val beginSuppressTokens: Array[Int] =
(modelConfig \ "begin_suppress_tokens").extract[Array[Int]]
val suppressTokenIds: Array[Int] =
(modelConfig \ "suppress_tokens").extract[Array[Int]]
val forcedDecoderIds: Array[(Int, Int)] =
(modelConfig \ "forced_decoder_ids").extract[Array[Array[Int]]].map {
case idxWithTokenId: Array[Int] if idxWithTokenId.length == 2 =>
(idxWithTokenId(0), idxWithTokenId(1))
case _ =>
throw new Exception(
"Could not extract forced_decoder_ids. Should be a list of tuples with 2 entries.")
}
def arrayOrNone[T](array: Array[T]): Option[Array[T]] =
if (array.nonEmpty) Some(array) else None
val bosTokenId = (modelConfig \ "bos_token_id").extract[Int]
val eosTokenId = (modelConfig \ "eos_token_id").extract[Int]
val padTokenId = (modelConfig \ "eos_token_id").extract[Int]
val vocabSize = (modelConfig \ "vocab_size").extract[Int]
// Check if tokenizer.json exists
val tokenizerPath = s"$localModelPath/assets/tokenizer.json"
val tokenizerExists = new java.io.File(tokenizerPath).exists()
val (vocabs, addedTokens, bytePairs) = if (tokenizerExists) {
val tokenizerConfig: JValue = parse(loadJsonStringAsset(localModelPath, "tokenizer.json"))
// extract vocab from tokenizer.json ( model -> vocab)
var vocabs: Map[String, Int] =
(tokenizerConfig \ "model" \ "vocab").extract[Map[String, Int]]
// extract merges from tokenizer.json ( model -> merges)
val bytePairs = (tokenizerConfig \ "model" \ "merges")
.extract[List[String]]
.map(_.split(" "))
.filter(w => w.length == 2)
.map { case Array(c1, c2) => (c1, c2) }
.zipWithIndex
.toMap
// extract added_tokens from tokenizer.json (added_tokens)
// "added_tokens": [
// {
// "id": 128000,
// "content": "<|begin_of_text|>",
// "single_word": false,
// "lstrip": false,
// "rstrip": false,
// "normalized": false,
// "special": true
// }, ...
// ]
val addedTokens = (tokenizerConfig \ "added_tokens")
.extract[List[Map[String, Any]]]
.map { token =>
val id = token("id").asInstanceOf[BigInt].intValue()
val content = token("content").asInstanceOf[String]
(content, id)
}
.toMap
// update vocab with added tokens
addedTokens.foreach { case (content, id) =>
vocabs += (content -> id)
}
(vocabs, addedTokens, bytePairs)
} else {
val vocabs = loadTextAsset(localModelPath, "vocab.txt").zipWithIndex.toMap
val addedTokens = loadTextAsset(localModelPath, "added_tokens.txt").zipWithIndex.toMap
val bytePairs = loadTextAsset(localModelPath, "merges.txt")
.map(_.split(" "))
.filter(w => w.length == 2)
.map { case Array(c1, c2) => (c1, c2) }
.zipWithIndex
.toMap
(vocabs, addedTokens, bytePairs)
}
val annotatorModel = new LLAMA3Transformer()
.setGenerationConfig(
GenerationConfig(
bosTokenId,
padTokenId,
eosTokenId,
vocabSize,
arrayOrNone(beginSuppressTokens),
arrayOrNone(suppressTokenIds),
arrayOrNone(forcedDecoderIds)))
.setVocabulary(vocabs)
.setMerges(bytePairs)
.setAddedTokens(addedTokens)
val modelEngine =
if (useOpenvino)
Openvino.name
else
detectedEngine
annotatorModel.set(annotatorModel.engine, modelEngine)
detectedEngine match {
case ONNX.name =>
val onnxWrapperDecoder =
OnnxWrapper.read(
spark,
localModelPath,
zipped = false,
useBundle = true,
modelName = "decoder_model")
val onnxWrappers = DecoderWrappers(onnxWrapperDecoder)
annotatorModel
.setModelIfNotSet(spark, Some(onnxWrappers), None)
case Openvino.name =>
val openvinoWrapper =
OpenvinoWrapper.read(
spark,
localModelPath,
zipped = false,
useBundle = true,
detectedEngine = detectedEngine)
annotatorModel.setModelIfNotSet(spark, None, Some(openvinoWrapper))
case _ =>
throw new Exception(notSupportedEngineError)
}
annotatorModel
}
}
object LLAMA3Transformer
extends ReadablePretrainedLLAMA3TransformerModel
with ReadLLAMA3TransformerDLModel
