src.org.python.indexer.AstCache Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython Show documentation
Show all versions of jython Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
/**
* Copyright 2009, Google Inc. All rights reserved.
* Licensed to PSF under a Contributor Agreement.
*/
package org.python.indexer;
import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.RecognitionException;
import org.python.antlr.AnalyzingParser;
import org.python.antlr.base.mod;
import org.python.indexer.ast.NModule;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Provides a factory for python source ASTs. Maintains configurable on-disk and
* in-memory caches to avoid re-parsing files during analysis.
*/
public class AstCache {
public static final String CACHE_DIR = Util.getSystemTempDir() + "jython/ast_cache/";
private static final Logger LOG = Logger.getLogger(AstCache.class.getCanonicalName());
private Map cache = new HashMap();
private static AstCache INSTANCE;
private AstCache() throws Exception {
File f = new File(CACHE_DIR);
if (!f.exists()) {
f.mkdirs();
}
}
public static AstCache get() throws Exception {
if (INSTANCE == null) {
INSTANCE = new AstCache();
}
return INSTANCE;
}
/**
* Clears the memory cache.
*/
public void clear() {
cache.clear();
}
/**
* Removes all serialized ASTs from the on-disk cache.
* @return {@code true} if all cached AST files were removed
*/
public boolean clearDiskCache() {
try {
File dir = new File(CACHE_DIR);
for (File f : dir.listFiles()) {
if (f.isFile()) {
f.delete();
}
}
return true;
} catch (Exception x) {
severe("Failed to clear disk cache: " + x);
return false;
}
}
/**
* Returns the syntax tree for {@code path}. May find and/or create a
* cached copy in the mem cache or the disk cache.
* @param path absolute path to a source file
* @return the AST, or {@code null} if the parse failed for any reason
* @throws Exception if anything unexpected occurs
*/
public NModule getAST(String path) throws Exception {
if (path == null) throw new IllegalArgumentException("null path");
return fetch(path);
}
/**
* Returns the syntax tree for {@code path} with {@code contents}.
* Uses the memory cache but not the disk cache.
* This method exists primarily for unit testing.
* @param path a name for the file. Can be relative.
* @param contents the source to parse
*/
public NModule getAST(String path, String contents) throws Exception {
if (path == null) throw new IllegalArgumentException("null path");
if (contents == null) throw new IllegalArgumentException("null contents");
// Cache stores null value if the parse failed.
if (cache.containsKey(path)) {
return cache.get(path);
}
NModule mod = null;
try {
mod = parse(path, contents);
if (mod != null) {
mod.setFileAndMD5(path, Util.getMD5(contents.getBytes("UTF-8")));
}
} finally {
cache.put(path, mod); // may be null
}
return mod;
}
/**
* Get or create an AST for {@code path}, checking and if necessary updating
* the disk and memory caches.
* @param path absolute source path
*/
private NModule fetch(String path) throws Exception {
// Cache stores null value if the parse failed.
if (cache.containsKey(path)) {
return cache.get(path);
}
// Might be cached on disk but not in memory.
NModule mod = getSerializedModule(path);
if (mod != null) {
fine("reusing " + path);
cache.put(path, mod);
return mod;
}
try {
mod = parse(path);
} finally {
cache.put(path, mod); // may be null
}
if (mod != null) {
serialize(mod);
}
return mod;
}
/**
* Parse a file. Does not look in the cache or cache the result.
*/
private NModule parse(String path) throws Exception {
fine("parsing " + path);
mod ast = invokeANTLR(path);
return generateAST(ast, path);
}
/**
* Parse a string. Does not look in the cache or cache the result.
*/
private NModule parse(String path, String contents) throws Exception {
fine("parsing " + path);
mod ast = invokeANTLR(path, contents);
return generateAST(ast, path);
}
private NModule generateAST(mod ast, String path) throws Exception {
if (ast == null) {
Indexer.idx.reportFailedAssertion("ANTLR returned NULL for " + path);
return null;
}
// Convert to indexer's AST. Type conversion warnings are harmless here.
@SuppressWarnings("unchecked")
Object obj = ast.accept(new AstConverter());
if (!(obj instanceof NModule)) {
warn("\n[warning] converted AST is not a module: " + obj);
return null;
}
NModule module = (NModule)obj;
if (new File(path).canRead()) {
module.setFile(path);
}
return module;
}
private mod invokeANTLR(String filename) {
CharStream text = null;
try {
text = new ANTLRFileStream(filename);
} catch (IOException iox) {
fine(filename + ": " + iox);
return null;
}
return invokeANTLR(text, filename);
}
private mod invokeANTLR(String filename, String contents) {
CharStream text = new ANTLRStringStream(contents);
return invokeANTLR(text, filename);
}
private mod invokeANTLR(CharStream text, String filename) {
AnalyzingParser p = new AnalyzingParser(text, filename, null);
mod ast = null;
try {
ast = p.parseModule();
} catch (Exception x) {
fine("parse for " + filename + " failed: " + x);
}
recordParseErrors(filename, p.getRecognitionErrors());
return ast;
}
private void recordParseErrors(String path, List errs) {
if (errs.isEmpty()) {
return;
}
List diags = Indexer.idx.getParseErrs(path);
for (RecognitionException rx : errs) {
String msg = rx.line + ":" + rx.charPositionInLine + ":" + rx;
diags.add(new Diagnostic(path, Diagnostic.Type.ERROR, -1, -1, msg));
}
}
/**
* Each source file's AST is saved in an object file named for the MD5
* checksum of the source file. All that is needed is the MD5, but the
* file's base name is included for ease of debugging.
*/
public String getCachePath(File sourcePath) throws Exception {
return getCachePath(Util.getMD5(sourcePath), sourcePath.getName());
}
public String getCachePath(String md5, String name) {
return CACHE_DIR + name + md5 + ".ast";
}
// package-private for testing
void serialize(NModule ast) throws Exception {
String path = getCachePath(ast.getMD5(), new File(ast.getFile()).getName());
ObjectOutputStream oos = null;
FileOutputStream fos = null;
try {
fos = new FileOutputStream(path);
oos = new ObjectOutputStream(fos);
oos.writeObject(ast);
} finally {
if (oos != null) {
oos.close();
} else if (fos != null) {
fos.close();
}
}
}
// package-private for testing
NModule getSerializedModule(String sourcePath) {
try {
File sourceFile = new File(sourcePath);
if (sourceFile == null || !sourceFile.canRead()) {
return null;
}
File cached = new File(getCachePath(sourceFile));
if (!cached.canRead()) {
return null;
}
return deserialize(sourceFile);
} catch (Exception x) {
severe("Failed to deserialize " + sourcePath + ": " + x);
return null;
}
}
// package-private for testing
NModule deserialize(File sourcePath) throws Exception {
String cachePath = getCachePath(sourcePath);
FileInputStream fis = null;
ObjectInputStream ois = null;
try {
fis = new FileInputStream(cachePath);
ois = new ObjectInputStream(fis);
NModule mod = (NModule)ois.readObject();
// Files in different dirs may have the same base name and contents.
mod.setFile(sourcePath);
return mod;
} finally {
if (ois != null) {
ois.close();
} else if (fis != null) {
fis.close();
}
}
}
private void log(Level level, String msg) {
if (LOG.isLoggable(level)) {
LOG.log(level, msg);
}
}
private void severe(String msg) {
log(Level.SEVERE, msg);
}
private void warn(String msg) {
log(Level.WARNING, msg);
}
private void info(String msg) {
log(Level.INFO, msg);
}
private void fine(String msg) {
log(Level.FINE, msg);
}
private void finer(String msg) {
log(Level.FINER, msg);
}
}