asset.pipeline.AssetCompiler.groovy Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of asset-pipeline-core Show documentation
Show all versions of asset-pipeline-core Show documentation
JVM Asset Pipeline library for serving static web assets, bundling, minifying, and extensibility for transpiling.
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package asset.pipeline
import asset.pipeline.processors.ClosureCompilerProcessor
import asset.pipeline.utils.MultiOutputStream
import asset.pipeline.processors.CssMinifyPostProcessor
import groovy.util.logging.Slf4j
import java.util.zip.GZIPOutputStream
import java.util.concurrent.Callable
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.ExecutorCompletionService
import java.util.concurrent.CompletionService
/**
* Build time compiler for assets. This does a differential comparison of the source directory
* and the destination directory currently utilizing the manifest.properties file. This is primarily used
* during compilation. The gradle plugin uses this class to compile assets as does the grails gant plugin.
*
* @author David Estes
* @author Graeme Rocher
*/
@Slf4j
class AssetCompiler {
def includeRules = [:]
def excludeRules = [:]
Map options = [:]
def eventListener
def filesToProcess = []
Properties manifestProperties
def threadPool
/**
* Creates an instance of the compiler given passed input options
* @param options A Map of options that can be passed to the library
*
* - compileDir - String Location of where assets should be compiled into
* - excludesGzip - List of extensions of files that should be excluded from gzip compression. (Most image types included by default)
* - enableGzip - Whether or not we should generate gzip files (default true)
* - enableDigests - Turns on generation of digest named assets (default true)
* - skipNonDigests - If turned on will not generate non digest named files (default false)
* - maxThreads - Compiler can concurrently compile assets now and defaults to a max thread count of 4
*
* @param eventListener
*/
AssetCompiler(options = [:], eventListener = null) {
this.eventListener = eventListener
this.options = options
if(!options.compileDir) {
options.compileDir = "target/assets"
}
if(!options.excludesGzip) {
options.excludesGzip = ['png', 'jpg', 'jpeg', 'gif', 'zip', 'gz']
} else {
options.excludesGzip += ['png', 'jpg', 'jpeg', 'gif', 'zip', 'gz']
}
if(!options.containsKey('enableGzip')) {
options.enableGzip = true
}
if(!options.containsKey('enableDigests')) {
options.enableDigests = true
}
if(!options.containsKey('skipNonDigests')) {
options.skipNonDigests = false
}
// Load in additional assetSpecs
options.specs?.each { spec ->
def specClass = this.class.classLoader.loadClass(spec)
if(specClass) {
AssetHelper.assetSpecs << (Class) specClass
}
}
manifestProperties = new Properties()
}
/**
* Main Target Endpoint for Launching The AssetCompile in a Forked Execution Environment
* Arguments
*
* - -o compileDir
* - -i sourceDir (List of SourceDirs)
* - -j List of Source Jars (, delimited)
* - -d Digests
* - -z Compression
* - -m SourceMaps
* - -n Skip Non Digests
* - -c Config Location
* - command - compile,watch
*
* This is NOT YET IMPLEMENTED
*/
static void main(String[] args) {
def properties = new java.util.Properties()
System.properties.each { k, v ->
if(k.startsWith('asset.pipeline')) {
def newKey = k.substring('asset.pipeline'.size())
println "Key ${k} - ${v}"
}
}
// def properties = System.getProperties()
def assetCompiler = new AssetCompiler()
}
void compile() {
def assetDir = initializeWorkspace()
threadPool = Executors.newFixedThreadPool(options.maxThreads ?: Runtime.getRuntime().availableProcessors())
CompletionService completionService = new ExecutorCompletionService(threadPool);
try {
def minifyCssProcessor = new CssMinifyPostProcessor()
filesToProcess = this.getAllAssets()?.sort { a,b -> (a instanceof GenericAssetFile ? 1 : 0) <=> (b instanceof GenericAssetFile ? 1 : 0)}
// Lets clean up assets that are no longer being compiled
removeDeletedFiles(filesToProcess)
def futures = []
for(int index = 0; index < filesToProcess.size(); index++) {
def assetFile = filesToProcess[index]
def indexPosition = new Integer(index)
futures << completionService.submit({ ->
def fileName = assetFile.path
String futureResult = assetFile.path
def startTime = new Date().time
eventListener?.triggerEvent("StatusUpdate", "Processing File ${indexPosition + 1} of ${filesToProcess.size()} - ${fileName}")
def digestName
def isUnchanged = false
def extension = AssetHelper.extensionFromURI(fileName)
fileName = AssetHelper.nameWithoutExtension(fileName)
def fileSystemName = fileName.replace(AssetHelper.DIRECTIVE_FILE_SEPARATOR, File.separator)
if(assetFile) {
def fileData
if(!(assetFile instanceof GenericAssetFile)) {
if(assetFile.compiledExtension) {
extension = assetFile.compiledExtension
fileName = AssetHelper.fileNameWithoutExtensionFromArtefact(fileName, assetFile)
}
def contentType = (assetFile.contentType instanceof String) ? assetFile.contentType : assetFile.contentType[0]
def directiveProcessor = new DirectiveProcessor(contentType, this, options.classLoader)
fileData = directiveProcessor.compile(assetFile)
digestName = AssetHelper.getByteDigest(fileData.bytes)
def existingDigestFile = manifestProperties.getProperty("${fileName}${extension ? ('.' + extension) : ''}")
if(existingDigestFile && existingDigestFile == "${fileName}-${digestName}${extension ? ('.' + extension) : ''}") {
isUnchanged = true
}
if(fileName.indexOf(".min") == -1 && contentType == 'application/javascript' && options.minifyJs && !isUnchanged && !isMinifyExcluded(assetFile.path)) {
def newFileData = fileData
try {
def closureCompilerProcessor = new ClosureCompilerProcessor(this)
// eventListener?.triggerEvent("StatusUpdate", "- Minifying File")
newFileData = closureCompilerProcessor.process(fileName, fileData, options.minifyOptions ?: [:])
} catch(e) {
log.error("Closure uglify JS Exception", e)
throw(e)
newFileData = fileData
}
fileData = newFileData
} else if(fileName.indexOf(".min") == -1 && contentType == 'text/css' && options.minifyCss && !isUnchanged && !isMinifyExcluded(assetFile.path)) {
def newFileData = fileData
try {
// eventListener?.triggerEvent("StatusUpdate", "- Minifying File")
newFileData = minifyCssProcessor.process(fileData)
} catch(e) {
log.error("Minify CSS Exception", e)
newFileData = fileData
}
fileData = newFileData
}
if(assetFile.encoding) {
fileData = fileData.getBytes(assetFile.encoding)
} else {
fileData = fileData.bytes
}
} else {
digestName = assetFile.getByteDigest()
def existingDigestFile = manifestProperties.getProperty("${fileName}${extension ? ('.' + extension) : ''}")
if(existingDigestFile && existingDigestFile == "${fileName}-${digestName}${extension ? ('.' + extension) : ''}") {
isUnchanged = true
}
}
if(!isUnchanged) {
def outputFileName = fileSystemName
if(extension) {
outputFileName = "${fileSystemName}.${extension}"
}
def outputFile = new File(options.compileDir, "${outputFileName}")
def parentTree = new File(outputFile.parent)
parentTree.mkdirs()
byte[] outputBytes
InputStream writeInputStream;
if(fileData) {
writeInputStream = new ByteArrayInputStream(fileData)
// outputBytes = fileData
} else {
if(assetFile instanceof GenericAssetFile) {
writeInputStream = assetFile.inputStream
outputBytes = assetFile.bytes
} else {
writeInputStream = assetFile.inputStream
outputBytes = assetFile.inputStream.bytes
digestName = assetFile.getByteDigest()
}
}
// TODO: Streamify!
// eventListener?.triggerEvent("StatusUpdate","- Writing File")
byte[] buffer = new byte[8192]
int nRead
def outputFileStream
def digestFileStream
def gzipFileStream
def gzipStreamCollection = []
if(!options.skipNonDigests) {
outputFile.createNewFile()
outputFileStream = outputFile.newOutputStream()
if(options.enableGzip == true && !options.excludesGzip.find {
it.toLowerCase() == extension?.toLowerCase()
}) {
File zipFile = new File("${outputFile.getAbsolutePath()}.gz")
zipFile.createNewFile()
gzipStreamCollection << zipFile.newOutputStream()
}
}
if(extension) {
if(options.enableDigests) {
def digestedFile = new File(options.compileDir, "${fileSystemName}-${digestName}${extension ? ('.' + extension) : ''}")
digestedFile.createNewFile()
digestFileStream = digestedFile.newOutputStream()
if(options.enableGzip == true && !options.excludesGzip.find {
it.toLowerCase() == extension?.toLowerCase()
}) {
File zipFileDigest = new File("${digestedFile.getAbsolutePath()}.gz")
zipFileDigest.createNewFile()
gzipStreamCollection << zipFileDigest.newOutputStream()
}
manifestProperties.setProperty("${fileName}${extension ? ('.' + extension) : ''}", "${fileName}-${digestName}${extension ? ('.' + extension) : ''}")
} else {
manifestProperties.setProperty("${fileName}${extension ? ('.' + extension) : ''}", "${fileName}${extension ? ('.' + extension) : ''}")
}
}
if(gzipStreamCollection) {
MultiOutputStream targetStream = new MultiOutputStream(gzipStreamCollection)
gzipFileStream = new GZIPOutputStream(targetStream, true)
}
while((nRead = writeInputStream.read(buffer, 0, buffer.length)) != -1) {
// noop (just to complete the stream)
outputFileStream?.write(buffer, 0, nRead);
digestFileStream?.write(buffer, 0, nRead);
gzipFileStream?.write(buffer, 0, nRead);
}
if(gzipFileStream) {
gzipFileStream.finish()
gzipFileStream.flush()
gzipFileStream.close()
gzipStreamCollection.each { stream ->
stream.flush()
stream.close()
}
}
digestFileStream?.flush()
outputFileStream?.flush()
digestFileStream?.close()
outputFileStream?.close()
writeInputStream.close()
return futureResult
}
}
} as Callable)
}
int pending = futures.size()
while (pending > 0) {
// Wait for up to 100ms to see if anything has completed.
// The completed future is returned if one is found; otherwise null.
// (Tune 100ms as desired)
def completed = completionService.poll(100, TimeUnit.MILLISECONDS);
if (completed != null) {
completed.get() //need this to throw exceptions on main thread it seems
--pending;
}
}
// Integer futureCounter = 1
// for(future in futures) {
// while(!future.isDone()) { sleep(100)}
// eventListener?.triggerEvent("StatusUpdate", "Future Completed ${futureCounter++}")
// }
} finally {
// eventListener?.triggerEvent("StatusUpdate", "Shutting Down ThreadPool")
threadPool.shutdown()
}
// eventListener?.triggerEvent("StatusUpdate", "Saving Manifest")
saveManifest()
eventListener?.triggerEvent("StatusUpdate", "Finished Precompiling Assets")
}
private initializeWorkspace() {
// Check for existing Compiled Assets
def assetDir = new File(options.compileDir)
if(assetDir.exists()) {
def manifestFile = new File(options.compileDir, "manifest.properties")
if(manifestFile.exists())
manifestProperties.load(manifestFile.newDataInputStream())
} else {
assetDir.mkdirs()
}
return assetDir
}
/**
* Checks any user passed minification exclude patterns at (minifyOptions.excludes=['blah.js'])
* Exclude patterns can use glob patterns by default or regular expressions by prefixing the pattern with 'regex:'
* @param filePath the file path being tested against
* @return true if the file should be excluded from minification
*/
private boolean isMinifyExcluded(String filePath) {
if(options.minifyOptions?.excludes) {
return AssetHelper.isFileMatchingPatterns(filePath, options.minifyOptions.excludes)
}
return false
}
def getIncludesForPathKey(String key) {
def includes = []
def defaultIncludes = includeRules.default
if(defaultIncludes) {
includes += defaultIncludes
}
if(includeRules[key]) {
includes += includeRules[key]
}
return includes.unique()
}
def getExcludesForPathKey(String key) {
def excludes = ["**/.*", "**/.DS_Store", 'WEB-INF/**/*', '**/META-INF/*', '**/_*.*', '**/.svn/**']
def defaultExcludes = excludeRules.default
if(defaultExcludes) {
excludes += defaultExcludes
}
if(excludeRules[key]) {
excludes += excludeRules[key]
}
return excludes.unique()
}
def getAllAssets() {
def filesToProcess = []
AssetPipelineConfigHolder.resolvers.each { resolver ->
def files = resolver.scanForFiles(getExcludesForPathKey(resolver.name), getIncludesForPathKey(resolver.name))
filesToProcess += files
}
filesToProcess.unique { a, b -> a.path <=> b.path }
return filesToProcess //Make sure we have a unique set
}
private saveManifest() {
// Update Manifest
def manifestFile = new File(options.compileDir, 'manifest.properties')
manifestProperties.store(manifestFile.newWriter(), "")
}
private removeDeletedFiles(filesToProcess) {
def compiledFileNames = filesToProcess.collect { assetFile ->
def fileName = assetFile.path
def extension = AssetHelper.extensionFromURI(fileName)
fileName = AssetHelper.nameWithoutExtension(fileName)
if(assetFile && !(assetFile instanceof GenericAssetFile) && assetFile.compiledExtension) {
extension = assetFile.compiledExtension
fileName = AssetHelper.fileNameWithoutExtensionFromArtefact(fileName, assetFile)
}
return "${fileName}${extension ? ('.' + extension) : ''}"
}
def propertiesToRemove = []
manifestProperties.keySet().each { compiledUri ->
def compiledName = compiledUri//.replace(AssetHelper.DIRECTIVE_FILE_SEPARATOR,File.separator)
def fileFound = compiledFileNames.find { it == compiledName.toString() }
if(!fileFound) {
def digestedUri = manifestProperties.getProperty(compiledName)
def digestedName = digestedUri//.replace(AssetHelper.DIRECTIVE_FILE_SEPARATOR,File.separator)
def compiledFile = new File(options.compileDir, compiledName)
def digestedFile = new File(options.compileDir, digestedName)
def zippedFile = new File(options.compileDir, "${compiledName}.gz")
def zippedDigestFile = new File(options.compileDir, "${digestedName}.gz")
if(compiledFile.exists()) {
compiledFile.delete()
}
if(digestedFile.exists()) {
digestedFile.delete()
}
if(zippedFile.exists()) {
zippedFile.delete()
}
if(zippedDigestFile.exists()) {
zippedDigestFile.delete()
}
propertiesToRemove << compiledName
} else {
def compiledFile = new File(options.compileDir, compiledName)
def zippedFile = new File(options.compileDir, "${compiledName}.gz")
if(compiledFile.exists() && options.skipNonDigests == true) {
compiledFile.delete()
}
if(zippedFile.exists() && options.skipNonDigests == true) {
zippedFile.delete()
}
propertiesToRemove << compiledName
}
}
propertiesToRemove.each {
manifestProperties.remove(it)
}
}
}