main.name.remal.gradle_plugins.plugins.classes_processing.ClassesProcessingPlugin.kt Maven / Gradle / Ivy
package name.remal.gradle_plugins.plugins.classes_processing
import com.google.common.cache.CacheBuilder
import com.google.common.cache.CacheLoader
import com.google.common.cache.LoadingCache
import com.google.common.cache.RemovalCause.REPLACED
import com.google.common.cache.RemovalListener
import name.remal.CLASS_FILE_NAME_SUFFIX
import name.remal.Services
import name.remal.accept
import name.remal.arrayEquals
import name.remal.buildList
import name.remal.concurrentMapOf
import name.remal.concurrentSetOf
import name.remal.createParentDirectories
import name.remal.gradle_plugins.api.classes_processing.BytecodeModifier
import name.remal.gradle_plugins.api.classes_processing.ClassesProcessor
import name.remal.gradle_plugins.api.classes_processing.ClassesProcessorsGradleTaskFactory
import name.remal.gradle_plugins.api.classes_processing.ProcessContext
import name.remal.gradle_plugins.dsl.ApplyPluginClasses
import name.remal.gradle_plugins.dsl.BaseReflectiveProjectPlugin
import name.remal.gradle_plugins.dsl.Plugin
import name.remal.gradle_plugins.dsl.PluginAction
import name.remal.gradle_plugins.dsl.extensions.all
import name.remal.gradle_plugins.dsl.extensions.doFirstOrdered
import name.remal.gradle_plugins.dsl.extensions.doLastOrdered
import name.remal.gradle_plugins.dsl.extensions.doSetup
import name.remal.gradle_plugins.dsl.extensions.forClassLoader
import name.remal.gradle_plugins.dsl.extensions.forEachCreatedClassFile
import name.remal.gradle_plugins.dsl.extensions.isDebugLogEnabled
import name.remal.gradle_plugins.dsl.extensions.logDebug
import name.remal.gradle_plugins.dsl.extensions.readAll
import name.remal.gradle_plugins.plugins.common.CommonSettingsPlugin
import name.remal.inc
import name.remal.loadServices
import name.remal.loadServicesList
import name.remal.plusAssign
import name.remal.resourceNameToClassName
import name.remal.toList
import name.remal.use
import org.gradle.api.GradleException
import org.gradle.api.tasks.TaskContainer
import org.gradle.api.tasks.compile.AbstractCompile
import org.objectweb.asm.ClassReader
import org.objectweb.asm.ClassWriter
import org.objectweb.asm.util.CheckClassAdapter
import java.io.Closeable
import java.io.File
import java.lang.Math.abs
import java.lang.Math.max
import java.lang.Math.min
import java.lang.Math.round
import java.lang.System.currentTimeMillis
import java.net.URLClassLoader
import java.nio.charset.StandardCharsets.UTF_8
import java.util.concurrent.BlockingQueue
import java.util.concurrent.PriorityBlockingQueue
import java.util.concurrent.atomic.AtomicLong
@Plugin(
id = "name.remal.classes-processing",
description = "Plugin that adds ability to process *.class files. It executes all ClassesProcessor services for each compiled class file.",
tags = ["java"]
)
class ClassesProcessingPlugin : BaseReflectiveProjectPlugin() {
companion object {
private val classesProcessingFilters = loadServicesList(ClassesProcessingFilter::class.java)
private val classesProcessingFiltersFactories = loadServicesList(ClassesProcessingFiltersFactory::class.java)
}
@PluginAction(isHidden = true)
@Suppress("ComplexMethod", "LongMethod")
fun TaskContainer.setupClassesProcessing() {
all(AbstractCompile::class.java) { compileTask ->
compileTask.doSetup {
compileTask.classpath.forClassLoader { classLoader ->
arrayOf(
ClassesProcessor::class.java,
ClassesProcessorsGradleTaskFactory::class.java
).forEach { serviceType ->
compileTask.inputs.property(
"$$" + serviceType.name.replace('.', '$') + ".names",
Services.readServiceLines(serviceType.name, classLoader).toList()
)
}
}
}
val ioOperationsTiming = Timing()
fun ioOperation(operation: () -> R): R {
val startTime = currentTimeMillis()
val result = operation()
ioOperationsTiming.totalExecutionMillis += max(0, currentTimeMillis() - startTime)
++ioOperationsTiming.executionsCount
return result
}
var prevRunMaxLastModified = -1L
compileTask.doFirstOrdered {
ioOperation {
compileTask.forEachCreatedClassFile {
val lastModified = it.lastModified
if (prevRunMaxLastModified < lastModified) {
prevRunMaxLastModified = lastModified
}
}
}
}
compileTask.doLastOrdered { _ ->
compileTask.classpath.forClassLoader { classLoader ->
val classesProcessors = buildList {
loadServices(ClassesProcessor::class.java, classLoader)
.forEach { add(it) }
loadServices(ClassesProcessorsGradleTaskFactory::class.java, classLoader)
.forEach { addAll(it.createClassesProcessors(compileTask)) }
if (isEmpty()) return@forClassLoader
}.sorted()
compileTask.logDebug("Classes processors: {}", classesProcessors.map(Any::javaClass).map(Class<*>::getName))
val destinationDir = compileTask.destinationDirectory.asFile.get()
val locks = (0 until (1 + Runtime.getRuntime().availableProcessors() * 2)).map { Any() }
fun getLock(relativePath: String): Any {
return locks[abs(relativePath.hashCode()) % locks.size]
}
val staleClassRelativePaths = concurrentSetOf()
val bytecodeCache: LoadingCache = CacheBuilder.newBuilder()
.weigher { _, bytes -> bytes.size }
.maximumWeight(50 * 1024 * 1024)
.removalListener(RemovalListener { notification ->
val relativePath = notification.key ?: return@RemovalListener
synchronized(getLock(relativePath)) {
if (REPLACED == notification.cause) {
compileTask.logDebug("{}: marking bytecode as modified", relativePath)
staleClassRelativePaths.add(relativePath)
} else if (relativePath in staleClassRelativePaths) {
staleClassRelativePaths.remove(relativePath)
compileTask.logDebug("{}: writing modified bytecode to disk", relativePath)
ioOperation { File(destinationDir, relativePath).createParentDirectories().writeBytes(notification.value!!) }
}
Unit
}
})
.build(object : CacheLoader() {
override fun load(relativePath: String) = ioOperation { File(destinationDir, relativePath).readBytes() }
})
fun putBytecodeInCache(relativePath: String, bytecode: ByteArray) {
if (null == bytecodeCache.getIfPresent(relativePath)) {
compileTask.logDebug("{}: marking bytecode as modified", relativePath)
staleClassRelativePaths.add(relativePath)
}
bytecodeCache.put(relativePath, bytecode)
}
val excludingFiltersTimings = concurrentMapOf()
val bytecodeFilters = buildList {
addAll(classesProcessingFilters)
classesProcessingFiltersFactories.forEach { addAll(it.createClassesProcessingFilters(compileTask)) }
}.sorted()
fun isExcluded(relativePath: String, bytecode: ByteArray): Boolean {
if (bytecodeFilters.isNotEmpty()) {
val excludingFilter = bytecodeFilters.firstOrNull { filter ->
val startTime = currentTimeMillis()
val result = !filter.canBytecodeBeProcessed(bytecode)
excludingFiltersTimings.computeIfAbsent(filter.javaClass.name, { Timing() }).let {
it.totalExecutionMillis += max(0, currentTimeMillis() - startTime)
++it.executionsCount
}
return@firstOrNull result
}
if (excludingFilter != null) {
compileTask.logDebug("{}: skip processing because of {}", relativePath, excludingFilter.javaClass.name)
return true
}
}
return false
}
val processedRelativePaths = concurrentSetOf()
val processActionsQueue: BlockingQueue = run {
val relativePaths = buildList {
ioOperation {
compileTask.forEachCreatedClassFile {
val relativePath = it.relativePath.toString()
if (it.lastModified <= prevRunMaxLastModified) {
compileTask.logDebug("{}: skip processing because it hasn't been compiled in this build", relativePath)
} else {
if (!isExcluded(relativePath, bytecodeCache[relativePath])) {
add(relativePath)
}
}
}
}
}
return@run PriorityBlockingQueue(
min(round(1.2 * classesProcessors.size * max(1, relativePaths.size)), Int.MAX_VALUE.toLong()).toInt(),
Comparator { action1, action2 ->
action1.classesProcessor.compareTo(action2.classesProcessor).let { if (it != 0) return@Comparator it }
action1.relativePath.compareTo(action2.relativePath).let { if (it != 0) return@Comparator it }
return@Comparator 0
}
).apply {
relativePaths.forEach { relativePath ->
processedRelativePaths.add(relativePath)
classesProcessors.forEach { processor ->
add(ProcessAction(processor, relativePath))
}
}
}
}
val processContext = object : ProcessContext {
override fun getClassesDir() = destinationDir
private val _classpath = compileTask.classpath.toList()
override fun getClasspath(): List = _classpath
private val _classLoader = URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray())
override fun getClasspathClassLoader() = _classLoader
override fun doesResourceExist(relativePath: String): Boolean {
val file = File(destinationDir, relativePath)
return file.exists()
}
override fun readBinaryResource(relativePath: String): ByteArray? {
synchronized(getLock(relativePath)) {
if (relativePath.endsWith(CLASS_FILE_NAME_SUFFIX)) {
bytecodeCache.getIfPresent(relativePath)?.let { return it }
}
val file = File(destinationDir, relativePath)
if (!file.exists()) return null
return ioOperation { file.readBytes() }
.also {
if (relativePath.endsWith(CLASS_FILE_NAME_SUFFIX)) {
putBytecodeInCache(relativePath, it)
}
}
}
}
override fun readTextResource(relativePath: String): String? {
return readBinaryResource(relativePath)?.toString(textResourceCharset)
}
override fun writeBinaryResource(relativePath: String, content: ByteArray) {
synchronized(getLock(relativePath)) {
if (relativePath.endsWith(CLASS_FILE_NAME_SUFFIX)) {
putBytecodeInCache(relativePath, content)
if (processedRelativePaths.add(relativePath)) {
if (!isExcluded(relativePath, content)) {
classesProcessors.forEach { processor ->
processActionsQueue.add(ProcessAction(processor, relativePath))
}
}
}
} else {
val file = File(destinationDir, relativePath)
compileTask.logDebug("Writing binary file: {}", file)
ioOperation { file.createParentDirectories().writeBytes(content) }
}
Unit
}
}
override fun writeTextResource(relativePath: String, text: String) {
synchronized(getLock(relativePath)) {
val file = File(destinationDir, relativePath)
compileTask.logDebug("Writing text file: {}", file)
ioOperation { file.createParentDirectories().writeText(text, textResourceCharset) }
}
}
override fun appendTextResource(relativePath: String, text: String) {
synchronized(getLock(relativePath)) {
val file = File(destinationDir, relativePath)
compileTask.logDebug("Appending text file: {}", file)
ioOperation { file.createParentDirectories().apply { createNewFile() }.appendText(text, textResourceCharset) }
}
}
override fun doesClasspathResourceExist(relativePath: String): Boolean {
return null != classLoader.getResource(relativePath)
}
private val classpathBinaryResourcesCache: LoadingCache = CacheBuilder.newBuilder()
.maximumWeight(100 * 1024 * 1024)
.weigher { _, value -> value.content?.size ?: 0 }
.build(object : CacheLoader() {
override fun load(relativePath: String): BinaryContent {
return ioOperation {
BinaryContent(classLoader.getResourceAsStream(relativePath)?.readAll())
}
}
})
override fun readClasspathBinaryResource(relativePath: String): ByteArray? {
return classpathBinaryResourcesCache[relativePath].content
}
override fun readClasspathTextResource(relativePath: String): String? {
return readClasspathBinaryResource(relativePath)?.toString(textResourceCharset)
}
val textResourceCharset = UTF_8
}
val processorsTimings = concurrentMapOf()
try {
processContext.use {
while (true) {
val processAction = processActionsQueue.poll() ?: break
val classesProcessor = processAction.classesProcessor
val relativePath = processAction.relativePath
try {
val startTime = currentTimeMillis()
compileTask.logDebug("{}: executing processor: {}", relativePath, classesProcessor.javaClass.name)
val bytecode = bytecodeCache[relativePath]
val bytecodeModifier = BytecodeModifier { modifiedBytecode ->
if (!bytecode.arrayEquals(modifiedBytecode)) {
//ClassReader(modifiedBytecode).accept(CheckClassAdapter(ClassWriter(0)))
putBytecodeInCache(relativePath, modifiedBytecode)
}
}
val className = resourceNameToClassName(relativePath)
classesProcessor.process(bytecode, bytecodeModifier, className, relativePath, processContext)
processorsTimings.computeIfAbsent(classesProcessor.javaClass.name, { Timing() }).let {
it.totalExecutionMillis += max(0, currentTimeMillis() - startTime)
++it.executionsCount
}
} catch (e: Exception) {
throw GradleException("Error processing $relativePath by ${classesProcessor.javaClass.name}", e)
}
}
}
} finally {
bytecodeCache.invalidateAll()
bytecodeCache.cleanUp()
if (compileTask.isDebugLogEnabled) {
compileTask.logDebug(
"IO operations took {}ms ({}ms per class in average)",
ioOperationsTiming.totalExecutionMillis,
round(1.0 * ioOperationsTiming.totalExecutionMillis.get() / ioOperationsTiming.executionsCount.get())
)
Timing().let { timing ->
excludingFiltersTimings.values.forEach {
timing.totalExecutionMillis += it.totalExecutionMillis
timing.executionsCount = AtomicLong(max(timing.executionsCount.get(), it.executionsCount.get()))
}
compileTask.logDebug(
"Excluding filters took {}ms ({}ms per class in average)",
timing.totalExecutionMillis,
round(1.0 * timing.totalExecutionMillis.get() / timing.executionsCount.get())
)
}
excludingFiltersTimings.forEach { excludingFilterClassName, timing ->
compileTask.logDebug(
"Excluding filter {} took {}ms ({}ms per class in average)",
excludingFilterClassName,
timing.totalExecutionMillis,
round(1.0 * timing.totalExecutionMillis.get() / timing.executionsCount.get())
)
}
Timing().let { timing ->
processorsTimings.values.forEach {
timing.totalExecutionMillis += it.totalExecutionMillis
timing.executionsCount = AtomicLong(max(timing.executionsCount.get(), it.executionsCount.get()))
}
compileTask.logDebug(
"Class processors took {}ms ({}ms per class in average)",
timing.totalExecutionMillis,
round(1.0 * timing.totalExecutionMillis.get() / timing.executionsCount.get())
)
}
processorsTimings.forEach { processorClassName, timing ->
compileTask.logDebug(
"Class processor {} took {}ms ({}ms per class in average)",
processorClassName,
timing.totalExecutionMillis,
round(1.0 * timing.totalExecutionMillis.get() / timing.executionsCount.get())
)
}
}
classesProcessors.forEach {
if (it is Closeable) {
it.close()
}
}
}
}
}
Unit
}
}
private data class Timing(
var totalExecutionMillis: AtomicLong = AtomicLong(0),
var executionsCount: AtomicLong = AtomicLong(0)
)
private class BinaryContent(val content: ByteArray?)
private data class ProcessAction(
val classesProcessor: ClassesProcessor,
val relativePath: String
)
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy