org.jetbrains.kotlin.incremental.IncrementalJvmCache.kt Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of kotlin-compiler-embeddable Show documentation
Show all versions of kotlin-compiler-embeddable Show documentation
the Kotlin compiler embeddable
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.incremental
import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName
import com.intellij.util.io.BooleanDataDescriptor
import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.inline.InlineFunction
import org.jetbrains.kotlin.inline.InlineFunctionOrAccessor
import org.jetbrains.kotlin.inline.InlinePropertyAccessor
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
import org.jetbrains.kotlin.metadata.jvm.deserialization.ModuleMapping
import org.jetbrains.kotlin.metadata.jvm.serialization.JvmStringTable
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import java.io.File
import java.security.MessageDigest
const val KOTLIN_CACHE_DIRECTORY_NAME = "kotlin"
open class IncrementalJvmCache(
targetDataRoot: File,
icContext: IncrementalCompilationContext,
targetOutputDir: File?,
) : AbstractIncrementalCache(
workingDir = File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME),
icContext,
), IncrementalCache {
companion object {
private const val PROTO_MAP = "proto"
private const val FE_PROTO_MAP = "fe-proto"
private const val CONSTANTS_MAP = "constants"
private const val PACKAGE_PARTS = "package-parts"
private const val MULTIFILE_CLASS_FACADES = "multifile-class-facades"
private const val MULTIFILE_CLASS_PARTS = "multifile-class-parts"
private const val INLINE_FUNCTIONS = "inline-functions"
private const val INTERNAL_NAME_TO_SOURCE = "internal-name-to-source"
private const val JAVA_SOURCES_PROTO_MAP = "java-sources-proto-map"
private const val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
}
override val sourceToClassesMap = registerMap(SourceToJvmNameMap(SOURCE_TO_CLASSES.storageFile, icContext))
override val dirtyOutputClassesMap = registerMap(DirtyClassesJvmNameMap(DIRTY_OUTPUT_CLASSES.storageFile, icContext))
private val protoMap = registerMap(ProtoMap(PROTO_MAP.storageFile, icContext))
private val feProtoMap = registerMap(ProtoMap(FE_PROTO_MAP.storageFile, icContext))
private val constantsMap = registerMap(ConstantsMap(CONSTANTS_MAP.storageFile, icContext))
private val packagePartMap = registerMap(PackagePartMap(PACKAGE_PARTS.storageFile, icContext))
private val multifileFacadeToParts = registerMap(MultifileClassFacadeMap(MULTIFILE_CLASS_FACADES.storageFile, icContext))
private val partToMultifileFacade = registerMap(MultifileClassPartMap(MULTIFILE_CLASS_PARTS.storageFile, icContext))
private val inlineFunctionsMap = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile, icContext))
// todo: try to use internal names only?
private val internalNameToSource = registerMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile, icContext))
// gradle only
private val javaSourcesProtoMap = registerMap(JavaSourcesProtoMap(JAVA_SOURCES_PROTO_MAP.storageFile, icContext))
private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory" } }
protected open fun debugLog(message: String) {}
fun isTrackedFile(file: File) = sourceToClassesMap.contains(file)
// used in gradle
@Suppress("unused")
fun classesBySources(sources: Iterable): Iterable =
sources.flatMap { sourceToClassesMap[it].orEmpty() }
fun sourcesByInternalName(internalName: String): Collection =
internalNameToSource[internalName].orEmpty()
fun getAllPartsOfMultifileFacade(facade: JvmClassName): Collection? {
return multifileFacadeToParts[facade]
}
fun isMultifileFacade(className: JvmClassName): Boolean =
className in multifileFacadeToParts
override fun getClassFilePath(internalClassName: String): String {
return toSystemIndependentName(File(outputDir, "$internalClassName.class").normalize().absolutePath)
}
override fun updateComplementaryFiles(dirtyFiles: Collection, expectActualTracker: ExpectActualTrackerImpl) {
if (icContext.useCompilerMapsOnly) return
super.updateComplementaryFiles(dirtyFiles, expectActualTracker)
}
fun saveModuleMappingToCache(sourceFiles: Collection, file: File) {
val jvmClassName = JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)
protoMap.storeModuleMapping(jvmClassName, file.readBytes())
dirtyOutputClassesMap.notDirty(jvmClassName)
sourceFiles.forEach { sourceToClassesMap.append(it, jvmClassName) }
}
open fun saveFileToCache(generatedClass: GeneratedJvmClass, changesCollector: ChangesCollector) {
saveClassToCache(KotlinClassInfo.createFrom(generatedClass.outputClass), generatedClass.sourceFiles, changesCollector)
}
/**
* Saves information about the given (Kotlin) class to this cache, and stores changes between this class and its previous version into
* the given [ChangesCollector].
*
* @param kotlinClassInfo Information about a Kotlin class
* @param sourceFiles The source files that the given class was generated from, or `null` if this information is not available
* @param changesCollector A [ChangesCollector]
*/
fun saveClassToCache(kotlinClassInfo: KotlinClassInfo, sourceFiles: List?, changesCollector: ChangesCollector) {
val className = kotlinClassInfo.className
dirtyOutputClassesMap.notDirty(className)
if (sourceFiles != null) {
sourceFiles.forEach {
sourceToClassesMap.append(it, className)
}
if (!icContext.useCompilerMapsOnly) internalNameToSource[className.internalName] = sourceFiles
}
if (kotlinClassInfo.classId.isLocal) return
when (kotlinClassInfo.classKind) {
KotlinClassHeader.Kind.FILE_FACADE -> {
if (sourceFiles != null) {
assert(sourceFiles.size == 1) { "Package part from several source files: $sourceFiles" }
}
packagePartMap.addPackagePart(className)
protoMap.process(kotlinClassInfo, changesCollector)
if (!icContext.useCompilerMapsOnly) {
constantsMap.process(kotlinClassInfo, changesCollector)
inlineFunctionsMap.process(kotlinClassInfo, changesCollector)
}
}
KotlinClassHeader.Kind.MULTIFILE_CLASS -> {
val partNames = kotlinClassInfo.classHeaderData.toList()
check(partNames.isNotEmpty()) { "Multifile class has no parts: $className" }
multifileFacadeToParts[className] = partNames
// When a class is replaced with a facade with the same name,
// the class' proto wouldn't ever be deleted,
// because we don't write proto for multifile facades.
// As a workaround we can remove proto values for multifile facades.
if (className in protoMap) {
changesCollector.collectSignature(className.fqNameForClassNameWithoutDollars, areSubclassesAffected = true)
}
protoMap.remove(className, changesCollector)
classFqNameToSourceMap.remove(className.fqNameForClassNameWithoutDollars)
if (!icContext.useCompilerMapsOnly) {
classAttributesMap.remove(className.fqNameForClassNameWithoutDollars)
internalNameToSource.remove(className.internalName)
// TODO NO_CHANGES? (delegates only)
constantsMap.process(kotlinClassInfo, changesCollector)
inlineFunctionsMap.process(kotlinClassInfo, changesCollector)
}
}
KotlinClassHeader.Kind.MULTIFILE_CLASS_PART -> {
if (sourceFiles != null) {
assert(sourceFiles.size == 1) { "Multifile class part from several source files: $sourceFiles" }
}
packagePartMap.addPackagePart(className)
partToMultifileFacade[className] = kotlinClassInfo.multifileClassName!!
protoMap.process(kotlinClassInfo, changesCollector)
if (!icContext.useCompilerMapsOnly) {
constantsMap.process(kotlinClassInfo, changesCollector)
inlineFunctionsMap.process(kotlinClassInfo, changesCollector)
}
}
KotlinClassHeader.Kind.CLASS -> {
if (!icContext.useCompilerMapsOnly) {
addToClassStorage(kotlinClassInfo.protoData as ClassProtoData, sourceFiles?.let { sourceFiles.single() })
}
protoMap.process(kotlinClassInfo, changesCollector)
if (!icContext.useCompilerMapsOnly) {
constantsMap.process(kotlinClassInfo, changesCollector)
inlineFunctionsMap.process(kotlinClassInfo, changesCollector)
}
}
KotlinClassHeader.Kind.UNKNOWN, KotlinClassHeader.Kind.SYNTHETIC_CLASS -> {
}
}
}
fun saveFrontendClassToCache(
classId: ClassId,
classProto: ProtoBuf.Class,
stringTable: JvmStringTable,
sourceFiles: List?,
changesCollector: ChangesCollector,
) {
val className = JvmClassName.byClassId(classId)
if (sourceFiles != null) {
internalNameToSource[className.internalName] = sourceFiles
}
if (classId.isLocal) return
val newProtoData = ClassProtoData(classProto, stringTable.toNameResolver())
addToClassStorage(newProtoData, sourceFiles?.let { sourceFiles.single() })
feProtoMap.putAndCollect(
className,
ProtoMapValue(
false,
JvmProtoBufUtil.writeDataBytes(stringTable, classProto),
stringTable.strings.toTypedArray()
),
newProtoData,
changesCollector
)
}
fun collectClassChangesByFeMetadata(
className: JvmClassName, classProto: ProtoBuf.Class, stringTable: JvmStringTable, changesCollector: ChangesCollector,
) {
//class
feProtoMap.check(className, classProto, stringTable, changesCollector)
}
fun saveJavaClassProto(source: File?, serializedJavaClass: SerializedJavaClass, collector: ChangesCollector) {
val jvmClassName = JvmClassName.byClassId(serializedJavaClass.classId)
if (!icContext.useCompilerMapsOnly) {
javaSourcesProtoMap.process(jvmClassName, serializedJavaClass, collector)
}
source?.let { sourceToClassesMap.append(source, jvmClassName) }
if (!icContext.useCompilerMapsOnly) {
addToClassStorage(serializedJavaClass.toProtoData(), source)
// collector.addJavaProto(ClassProtoData(proto, nameResolver))
}
dirtyOutputClassesMap.notDirty(jvmClassName)
}
fun getObsoleteJavaClasses(): Collection =
dirtyOutputClassesMap.getDirtyOutputClasses()
.mapNotNull {
javaSourcesProtoMap[it]?.classId
}
fun isJavaClassToTrack(classId: ClassId): Boolean {
val jvmClassName = JvmClassName.byClassId(classId)
return dirtyOutputClassesMap.isDirty(jvmClassName) ||
jvmClassName !in javaSourcesProtoMap
}
fun isJavaClassAlreadyInCache(classId: ClassId): Boolean {
val jvmClassName = JvmClassName.byClassId(classId)
return jvmClassName in javaSourcesProtoMap
}
override fun clearCacheForRemovedClasses(changesCollector: ChangesCollector) {
val dirtyClasses = dirtyOutputClassesMap.getDirtyOutputClasses()
val facadesWithRemovedParts = hashMapOf>()
for (dirtyClass in dirtyClasses) {
val facade = partToMultifileFacade[dirtyClass] ?: continue
val facadeClassName = JvmClassName.byInternalName(facade)
val removedParts = facadesWithRemovedParts.getOrPut(facadeClassName) { hashSetOf() }
removedParts.add(dirtyClass.internalName)
}
for ((facade, removedParts) in facadesWithRemovedParts.entries) {
val allParts = multifileFacadeToParts[facade] ?: continue
val notRemovedParts = allParts.filter { it !in removedParts }
if (notRemovedParts.isEmpty()) {
multifileFacadeToParts.remove(facade)
} else {
multifileFacadeToParts[facade] = notRemovedParts
}
}
dirtyClasses.forEach {
protoMap.remove(it, changesCollector)
feProtoMap.remove(it, changesCollector)
packagePartMap.remove(it)
multifileFacadeToParts.remove(it)
partToMultifileFacade.remove(it)
if (!icContext.useCompilerMapsOnly) {
constantsMap.remove(it)
inlineFunctionsMap.remove(it)
internalNameToSource.remove(it.internalName)
javaSourcesProtoMap.remove(it, changesCollector)
}
}
if (!icContext.useCompilerMapsOnly) {
removeAllFromClassStorage(dirtyClasses.map { it.fqNameForClassNameWithoutDollars }, changesCollector)
}
dirtyOutputClassesMap.clear()
}
override fun getObsoletePackageParts(): Collection {
val obsoletePackageParts = dirtyOutputClassesMap.getDirtyOutputClasses().filter(packagePartMap::isPackagePart)
debugLog("Obsolete package parts: $obsoletePackageParts")
return obsoletePackageParts.map { it.internalName }
}
override fun getPackagePartData(partInternalName: String): JvmPackagePartProto? {
return protoMap[JvmClassName.byInternalName(partInternalName)]?.let { value ->
JvmPackagePartProto(value.bytes, value.strings)
}
}
override fun getObsoleteMultifileClasses(): Collection {
val obsoleteMultifileClasses = linkedSetOf()
for (dirtyClass in dirtyOutputClassesMap.getDirtyOutputClasses()) {
val dirtyFacade = partToMultifileFacade[dirtyClass] ?: continue
obsoleteMultifileClasses.add(dirtyFacade)
}
debugLog("Obsolete multifile class facades: $obsoleteMultifileClasses")
return obsoleteMultifileClasses
}
override fun getStableMultifileFacadeParts(facadeInternalName: String): Collection? {
val jvmClassName = JvmClassName.byInternalName(facadeInternalName)
val partNames = multifileFacadeToParts[jvmClassName] ?: return null
return partNames.filter { !dirtyOutputClassesMap.isDirty(JvmClassName.byInternalName(it)) }
}
override fun getModuleMappingData(): ByteArray? {
return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]?.bytes
}
private inner class ProtoMap(
storageFile: File,
icContext: IncrementalCompilationContext,
) : BasicStringMap(storageFile, ProtoMapValueExternalizer, icContext) {
@Synchronized
fun process(kotlinClassInfo: KotlinClassInfo, changesCollector: ChangesCollector) {
return putAndCollect(
kotlinClassInfo.className,
kotlinClassInfo.protoMapValue,
kotlinClassInfo.protoData,
changesCollector
)
}
// A module mapping (.kotlin_module file) is stored in a cache,
// because a corresponding file will be deleted on each round
// (it is reported as output for each [package part?] source file).
// If a mapping is not preserved, a resulting file will only contain data
// from files compiled during last round.
// However there is no need to compare old and new data in this case
// (also that would fail with exception).
@Synchronized
fun storeModuleMapping(className: JvmClassName, bytes: ByteArray) {
storage[className.internalName] = ProtoMapValue(isPackageFacade = false, bytes = bytes, strings = emptyArray())
}
@Synchronized
fun putAndCollect(
className: JvmClassName,
newMapValue: ProtoMapValue,
newProtoData: ProtoData,
changesCollector: ChangesCollector,
) {
val key = className.internalName
val oldMapValue = storage[key]
storage[key] = newMapValue
changesCollector.collectProtoChanges(oldMapValue?.toProtoData(className.packageFqName), newProtoData, packageProtoKey = key)
}
fun check(
className: JvmClassName, classProto: ProtoBuf.Class, stringTable: JvmStringTable, changesCollector: ChangesCollector,
) {
val key = className.internalName
val oldProtoData = storage[key]?.toProtoData(className.packageFqName)
val newProtoData = ClassProtoData(classProto, stringTable.toNameResolver())
changesCollector.collectProtoChanges(oldProtoData, newProtoData, packageProtoKey = key)
}
operator fun contains(className: JvmClassName): Boolean =
className.internalName in storage
operator fun get(className: JvmClassName): ProtoMapValue? =
storage[className.internalName]
@Synchronized
fun remove(className: JvmClassName, changesCollector: ChangesCollector) {
val key = className.internalName
val oldValue = storage[key] ?: return
if (key != MODULE_MAPPING_FILE_NAME) {
changesCollector.collectProtoChanges(oldData = oldValue.toProtoData(className.packageFqName), newData = null)
}
storage.remove(key)
}
override fun dumpValue(value: ProtoMapValue): String {
return (if (value.isPackageFacade) "1" else "0") + java.lang.Long.toHexString(value.bytes.md5())
}
}
private inner class JavaSourcesProtoMap(
storageFile: File,
icContext: IncrementalCompilationContext,
) :
BasicStringMap(storageFile, JavaClassProtoMapValueExternalizer, icContext) {
@Synchronized
fun process(jvmClassName: JvmClassName, newData: SerializedJavaClass, changesCollector: ChangesCollector) {
val key = jvmClassName.internalName
val oldData = storage[key]
storage[key] = newData
changesCollector.collectProtoChanges(
oldData?.toProtoData(), newData.toProtoData(),
collectAllMembersForNewClass = true
)
}
@Synchronized
fun remove(className: JvmClassName, changesCollector: ChangesCollector) {
val key = className.internalName
val oldValue = storage[key] ?: return
storage.remove(key)
changesCollector.collectProtoChanges(oldValue.toProtoData(), newData = null)
}
operator fun get(className: JvmClassName): SerializedJavaClass? =
storage[className.internalName]
operator fun contains(className: JvmClassName): Boolean =
className.internalName in storage
override fun dumpValue(value: SerializedJavaClass): String =
java.lang.Long.toHexString(value.proto.toByteArray().md5())
}
// todo: reuse code with InlineFunctionsMap?
private inner class ConstantsMap(
storageFile: File,
icContext: IncrementalCompilationContext,
) :
BasicStringMap