org.apache.xmlbeans.impl.tool.BaseSchemaResourceManager Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of commons-xmlbeans Show documentation
The Apache Commons Codec package contains simple encoder and decoders for various formats such as Base64 and Hexadecimal. In addition to these widely used encoders and decoders, the codec package also maintains a collection of phonetic encoding utilities.
The newest version!
/*   Copyright 2004 The Apache Software Foundation
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package org.apache.xmlbeans.impl.tool;

import org.apache.xmlbeans.impl.xb.xsdownload.DownloadedSchemasDocument;
import org.apache.xmlbeans.impl.xb.xsdownload.DownloadedSchemaEntry;
import org.apache.xmlbeans.impl.xb.xsdownload.DownloadedSchemasDocument.DownloadedSchemas;
import org.apache.xmlbeans.impl.util.HexBin;
import org.apache.xmlbeans.impl.common.IOUtil;
import org.apache.xmlbeans.XmlOptions;
import org.apache.xmlbeans.XmlBeans;

import java.io.IOException;
import java.io.InputStream;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.util.Set;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.HashMap;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.apache.xmlbeans.impl.xb.xsdschema.SchemaDocument.Schema;
import org.apache.xmlbeans.impl.xb.xsdschema.SchemaDocument;

public abstract class BaseSchemaResourceManager extends SchemaImportResolver
{
    private static final String USER_AGENT = "XMLBeans/" + XmlBeans.getVersion() + " (" + XmlBeans.getTitle() + ")";

    private String _defaultCopyDirectory;
    private DownloadedSchemasDocument _importsDoc;
    private Map _resourceForFilename = new HashMap();
    private Map _resourceForURL = new HashMap();
    private Map _resourceForNamespace = new HashMap();
    private Map _resourceForDigest = new HashMap();
    private Map _resourceForCacheEntry = new HashMap();
    private Set _redownloadSet = new HashSet();

    protected BaseSchemaResourceManager()
    {
        // concrete subclasses should call init in their constructors
    }

    protected final void init()
    {
        if (fileExists(getIndexFilename()))
        {
            try
            {
                _importsDoc = DownloadedSchemasDocument.Factory.parse( inputStreamForFile( getIndexFilename() ) );
            }
            catch (IOException e)
            {
                _importsDoc = null;
            }
            catch (Exception e)
            {
                throw (IllegalStateException)(new IllegalStateException("Problem reading xsdownload.xml: please fix or delete this file")).initCause(e);
            }
        }
        if (_importsDoc == null)
        {
            try
            {
                _importsDoc = DownloadedSchemasDocument.Factory.parse(
                    ""
                );
            }
            catch (Exception e)
            {
                throw (IllegalStateException)(new IllegalStateException()).initCause(e);
            }
        }

        String defaultDir = _importsDoc.getDownloadedSchemas().getDefaultDirectory();
        if (defaultDir == null)
            defaultDir = getDefaultSchemaDir();;
        _defaultCopyDirectory = defaultDir;

        // now initialize data structures
        DownloadedSchemaEntry[] entries = _importsDoc.getDownloadedSchemas().getEntryArray();
        for (int i = 0; i < entries.length; i++)
        {
            updateResource(entries[i]);
        }
    }

    public final void writeCache() throws IOException
    {
        InputStream input = _importsDoc.newInputStream(new XmlOptions().setSavePrettyPrint());
        writeInputStreamToFile(input, getIndexFilename());
    }

    public final void processAll(boolean sync, boolean refresh, boolean imports)
    {
        if (refresh)
        {
            _redownloadSet = new HashSet();
        }
        else
        {
            _redownloadSet = null;
        }

        String[] allFilenames = getAllXSDFilenames();

        if (sync)
            syncCacheWithLocalXsdFiles(allFilenames, false);

        SchemaResource[] starters = (SchemaResource[])
                _resourceForFilename.values().toArray(new SchemaResource[0]);

        if (refresh)
            redownloadEntries(starters);

        if (imports)
            resolveImports(starters);

        _redownloadSet = null;
    }

    public final void process(String[] uris, String[] filenames, boolean sync, boolean refresh, boolean imports)
    {
        if (refresh)
        {
            _redownloadSet = new HashSet();
        }
        else
        {
            _redownloadSet = null;
        }

        if (filenames.length > 0)
            syncCacheWithLocalXsdFiles(filenames, true);
        else if (sync)
            syncCacheWithLocalXsdFiles(getAllXSDFilenames(), false);

        Set starterset = new HashSet();

        for (int i = 0; i < uris.length; i++)
        {
            SchemaResource resource = (SchemaResource)lookupResource(null, uris[i]);
            if (resource != null)
                starterset.add(resource);
        }

        for (int i = 0; i < filenames.length; i++)
        {
            SchemaResource resource = (SchemaResource)_resourceForFilename.get(filenames);
            if (resource != null)
                starterset.add(resource);
        }

        SchemaResource[] starters = (SchemaResource[])
               starterset.toArray(new SchemaResource[0]);

        if (refresh)
            redownloadEntries(starters);

        if (imports)
            resolveImports(starters);

        _redownloadSet = null;
    }

    /**
     * Adds items to the cache that point to new files that aren't
     * described in the cache, and optionally deletes old entries.
     *
     * If an old file is gone and a new file is
     * found with exactly the same contents, the cache entry is moved
     * to point to the new file.
     */
    public final void syncCacheWithLocalXsdFiles(String[] filenames, boolean deleteOnlyMentioned)
    {
        Set seenResources = new HashSet();
        Set vanishedResources = new HashSet();

        for (int i = 0; i < filenames.length; i++)
        {
            String filename = filenames[i];

            // first, if the filename matches exactly, trust the filename
            SchemaResource resource = (SchemaResource)_resourceForFilename.get(filename);
            if (resource != null)
            {
                if (fileExists(filename))
                    seenResources.add(resource);
                else
                    vanishedResources.add(resource);
                continue;
            }

            // new file that is not in the index?
            // not if the digest is known to the index and the original file is gone - that's a rename!
            String digest = null;
            try
            {
                digest = shaDigestForFile(filename);
                resource = (SchemaResource)_resourceForDigest.get(digest);
                if (resource != null)
                {
                    String oldFilename = resource.getFilename();
                    if (!fileExists(oldFilename))
                    {
                        warning("File " + filename + " is a rename of " + oldFilename);
                        resource.setFilename(filename);
                        seenResources.add(resource);
                        if (_resourceForFilename.get(oldFilename) == resource)
                            _resourceForFilename.remove(oldFilename);
                        if (_resourceForFilename.containsKey(filename))
                            _resourceForFilename.put(filename, resource);
                        continue;
                    }
                }
            }
            catch (IOException e)
            {
                // unable to read digest... no problem, ignore then
            }

            // ok, this really is a new XSD file then, of unknown URL origin
            DownloadedSchemaEntry newEntry = addNewEntry();
            newEntry.setFilename(filename);
            warning("Caching information on new local file " + filename);
            if (digest != null)
                newEntry.setSha1(digest);

            seenResources.add(updateResource(newEntry));
        }

        if (deleteOnlyMentioned)
            deleteResourcesInSet(vanishedResources, true);
        else
            deleteResourcesInSet(seenResources, false);
    }

    /**
     * Iterates through every entry and refetches it from its primary URL,
     * if known.  Replaces the contents of the file if the data is different.
     */
    private void redownloadEntries(SchemaResource[] resources)
    {
        for (int i = 0; i < resources.length; i++)
        {
            redownloadResource(resources[i]);
        }
    }

    private void deleteResourcesInSet(Set seenResources, boolean setToDelete)
    {
        Set seenCacheEntries = new HashSet();
        for (Iterator i = seenResources.iterator(); i.hasNext(); )
        {
            SchemaResource resource = (SchemaResource)i.next();
            seenCacheEntries.add(resource._cacheEntry);
        }

        DownloadedSchemas downloadedSchemas = _importsDoc.getDownloadedSchemas();
        for (int i = 0; i < downloadedSchemas.sizeOfEntryArray(); i++)
        {
            DownloadedSchemaEntry cacheEntry = downloadedSchemas.getEntryArray(i);

            if (seenCacheEntries.contains(cacheEntry) == setToDelete)
            {
                SchemaResource resource = (SchemaResource)_resourceForCacheEntry.get(cacheEntry);
                warning("Removing obsolete cache entry for " + resource.getFilename());

                if (resource != null)
                {
                    _resourceForCacheEntry.remove(cacheEntry);

                    if (resource == _resourceForFilename.get(resource.getFilename()))
                        _resourceForFilename.remove(resource.getFilename());

                    if (resource == _resourceForDigest.get(resource.getSha1()))
                        _resourceForDigest.remove(resource.getSha1());

                    if (resource == _resourceForNamespace.get(resource.getNamespace()))
                        _resourceForNamespace.remove(resource.getNamespace());

                    // Finally, any or all URIs
                    String[] urls = resource.getSchemaLocationArray();
                    for (int j = 0; j < urls.length; j++)
                    {
                        if (resource == _resourceForURL.get(urls[j]))
                            _resourceForURL.remove(urls[j]);
                    }
                }

                downloadedSchemas.removeEntry(i);
                i -= 1;
            }
        }
    }

    private SchemaResource updateResource(DownloadedSchemaEntry entry)
    {
        // The file
        String filename = entry.getFilename();
        if (filename == null)
            return null;

        SchemaResource resource = new SchemaResource(entry);
        _resourceForCacheEntry.put(entry, resource);

        if (!_resourceForFilename.containsKey(filename))
            _resourceForFilename.put(filename, resource);

        // The digest
        String digest = resource.getSha1();
        if (digest != null)
        {
            if (!_resourceForDigest.containsKey(digest))
                _resourceForDigest.put(digest, resource);
        }

        // Next, the namespace
        String namespace = resource.getNamespace();
        if (namespace != null)
        {
            if (!_resourceForNamespace.containsKey(namespace))
                _resourceForNamespace.put(namespace, resource);
        }

        // Finally, any or all URIs
        String[] urls = resource.getSchemaLocationArray();
        for (int j = 0; j < urls.length; j++)
        {
            if (!_resourceForURL.containsKey(urls[j]))
                _resourceForURL.put(urls[j], resource);
        }

        return resource;
    }

    private static DigestInputStream digestInputStream(InputStream input)
    {
        MessageDigest sha;
        try
        {
            sha = MessageDigest.getInstance("SHA");
        }
        catch (NoSuchAlgorithmException e)
        {
            throw (IllegalStateException)(new IllegalStateException().initCause(e));
        }

        DigestInputStream str = new DigestInputStream(input, sha);

        return str;
    }

    private DownloadedSchemaEntry addNewEntry()
    {
        return _importsDoc.getDownloadedSchemas().addNewEntry();
    }

    private class SchemaResource implements SchemaImportResolver.SchemaResource
    {
        SchemaResource(DownloadedSchemaEntry entry)
        {
            _cacheEntry = entry;
        }

        DownloadedSchemaEntry _cacheEntry;

        public void setFilename(String filename)
        {
            _cacheEntry.setFilename(filename);
        }

        public String getFilename()
        {
            return _cacheEntry.getFilename();
        }

        public Schema getSchema()
        {
            if (!fileExists(getFilename()))
                redownloadResource(this);

            try
            {
                return SchemaDocument.Factory.parse(inputStreamForFile(getFilename())).getSchema();
            }
            catch (Exception e)
            {
                return null; // return null if _any_ problems reading schema file
            }
        }

        public String getSha1()
        {
            return _cacheEntry.getSha1();
        }

        public String getNamespace()
        {
            return _cacheEntry.getNamespace();
        }

        public void setNamespace(String namespace)
        {
            _cacheEntry.setNamespace(namespace);
        }

        public String getSchemaLocation()
        {
            if (_cacheEntry.sizeOfSchemaLocationArray() > 0)
                return _cacheEntry.getSchemaLocationArray(0);
            return null;
        }

        public String[] getSchemaLocationArray()
        {
            return _cacheEntry.getSchemaLocationArray();
        }

        public int hashCode()
        {
            return getFilename().hashCode();
        }

        public boolean equals(Object obj)
        {
            return this == obj || getFilename().equals(((SchemaResource)obj).getFilename());
        }

        public void addSchemaLocation(String schemaLocation)
        {
            _cacheEntry.addSchemaLocation(schemaLocation);
        }
    }

    /**
     * Called when the ImportLoader wishes to resolve the
     * given import.  Should return a SchemaResource whose
     * "equals" relationship reveals when a SchemaResource is
     * duplicated and shouldn't be examined again.
     *
     * Returns null if the resource reference should be ignored.
     */
    public SchemaImportResolver.SchemaResource lookupResource(String nsURI, String schemaLocation)
    {
        SchemaResource result = fetchFromCache(nsURI, schemaLocation);
        if (result != null)
        {
            if (_redownloadSet != null)
            {
                redownloadResource(result);
            }
            return result;
        }

        if (schemaLocation == null)
        {
            warning("No cached schema for namespace '" + nsURI + "', and no url specified");
            return null;
        }

        result = copyOrIdentifyDuplicateURL(schemaLocation, nsURI);
        if (_redownloadSet != null)
            _redownloadSet.add(result);
        return result;
    }

    private SchemaResource fetchFromCache(String nsURI, String schemaLocation)
    {
        SchemaResource result;

        if (schemaLocation != null)
        {
            result = (SchemaResource)_resourceForURL.get(schemaLocation);
            if (result != null)
                return result;
        }

        if (nsURI != null)
        {
            result = (SchemaResource)_resourceForNamespace.get(nsURI);
            if (result != null)
                return result;
        }

        return null;
    }

    private String uniqueFilenameForURI(String schemaLocation) throws IOException, URISyntaxException
    {
        String localFilename = new URI( schemaLocation ).getRawPath();
        int i = localFilename.lastIndexOf('/');
        if (i >= 0)
            localFilename = localFilename.substring(i + 1);
        if (localFilename.endsWith(".xsd"))
            localFilename = localFilename.substring(0, localFilename.length() - 4);
        if (localFilename.length() == 0)
            localFilename = "schema";

        // TODO: remove other unsafe characters for filenames?

        String candidateFilename = localFilename;
        int suffix = 1;
        while (suffix < 1000)
        {
            String candidate = _defaultCopyDirectory + "/" + candidateFilename + ".xsd";
            if (!fileExists(candidate))
                return candidate;
            suffix += 1;
            candidateFilename = localFilename + suffix;
        }

        throw new IOException("Problem with filename " + localFilename + ".xsd");
    }

    private void redownloadResource(SchemaResource resource)
    {
        if (_redownloadSet != null)
        {
            if (_redownloadSet.contains(resource))
                return;
            _redownloadSet.add(resource);
        }

        String filename = resource.getFilename();
        String schemaLocation = resource.getSchemaLocation();
        String digest = null;

        // nothing to do?
        if (schemaLocation == null || filename == null)
            return;

        ByteArrayOutputStream buffer = new ByteArrayOutputStream();

        try
        {
            URL url = new URL( schemaLocation );
            URLConnection conn = url.openConnection();
            conn.addRequestProperty("User-Agent", USER_AGENT);
            conn.addRequestProperty("Accept", "application/xml, text/xml, */*");
            DigestInputStream input = digestInputStream(conn.getInputStream());
            IOUtil.copyCompletely(input, buffer);
            digest = HexBin.bytesToString(input.getMessageDigest().digest());
        }
        catch (Exception e)
        {
            warning("Could not copy remote resource " + schemaLocation + ":" + e.getMessage());
            return;
        }

        if (digest.equals(resource.getSha1()) && fileExists(filename))
        {
            warning("Resource " + filename + " is unchanged from " + schemaLocation + ".");
            return;
        }

        try
        {
            InputStream source = new ByteArrayInputStream(buffer.toByteArray());
            writeInputStreamToFile(source, filename);
        }
        catch (IOException e)
        {
            warning("Could not write to file " + filename + " for " + schemaLocation + ":" + e.getMessage());
            return;
        }

        warning("Refreshed " + filename + " from " + schemaLocation);
    }

    private SchemaResource copyOrIdentifyDuplicateURL(String schemaLocation, String namespace)
    {
        String targetFilename;
        String digest;
        SchemaResource result;

        try
        {
            targetFilename = uniqueFilenameForURI(schemaLocation);
        }
        catch (URISyntaxException e)
        {
            warning("Invalid URI '" + schemaLocation + "':" + e.getMessage());
            return null;
        }
        catch (IOException e)
        {
            warning("Could not create local file for " + schemaLocation + ":" + e.getMessage());
            return null;
        }

        try
        {
            URL url = new URL( schemaLocation );
            DigestInputStream input = digestInputStream(url.openStream());
            writeInputStreamToFile(input, targetFilename);
            digest = HexBin.bytesToString(input.getMessageDigest().digest());
        }
        catch (Exception e)
        {
            warning("Could not copy remote resource " + schemaLocation + ":" + e.getMessage());
            return null;
        }

        result = (SchemaResource)_resourceForDigest.get(digest);
        if (result != null)
        {
            deleteFile(targetFilename);
            result.addSchemaLocation(schemaLocation);
            if (!_resourceForURL.containsKey(schemaLocation))
                _resourceForURL.put(schemaLocation, result);
            return result;
        }

        warning("Downloaded " + schemaLocation + " to " + targetFilename);

        DownloadedSchemaEntry newEntry = addNewEntry();
        newEntry.setFilename(targetFilename);
        newEntry.setSha1(digest);
        if (namespace != null)
            newEntry.setNamespace(namespace);
        newEntry.addSchemaLocation(schemaLocation);
        return updateResource(newEntry);
    }

    /**
     * Updates actual namespace in the table.
     */
    public void reportActualNamespace(SchemaImportResolver.SchemaResource rresource, String actualNamespace)
    {
        SchemaResource resource = (SchemaResource)rresource;
        String oldNamespace = resource.getNamespace();
        if (oldNamespace != null && _resourceForNamespace.get(oldNamespace) == resource)
            _resourceForNamespace.remove(oldNamespace);
        if (!_resourceForNamespace.containsKey(actualNamespace))
            _resourceForNamespace.put(actualNamespace, resource);
        resource.setNamespace(actualNamespace);
    }

    private String shaDigestForFile(String filename) throws IOException
    {
        DigestInputStream str = digestInputStream(inputStreamForFile(filename));

        byte[] dummy = new byte[4096];
        for (int i = 1; i > 0; i = str.read(dummy));

        str.close();

        return HexBin.bytesToString(str.getMessageDigest().digest());
    }

    // SOME METHODS TO OVERRIDE ============================

    protected String getIndexFilename()
    {
        return "./xsdownload.xml";
    }

    protected String getDefaultSchemaDir()
    {
        return "./schema";
    }

    /**
     * Produces diagnostic messages such as "downloading X to file Y".
     */
    abstract protected void warning(String msg);

    /**
     * Returns true if the given filename exists.  The filenames
     * are of the form "/foo/bar/zee.xsd" and should be construed
     * as rooted at the root of the project.
     */
    abstract protected boolean fileExists(String filename);

    /**
     * Gets the data in the given filename as an InputStream.
     */
    abstract protected InputStream inputStreamForFile(String filename) throws IOException;

    /**
     * Writes an entire file in one step.  An InputStream is passed and
     * copied to the file.
     */
    abstract protected void writeInputStreamToFile(InputStream input, String filename) throws IOException;

    /**
     * Deletes a file.  Sometimes immediately after writing a new file
     * we notice that it's exactly the same as an existing file and
     * we delete it. We never delete a file that was given to us
     * by the user.
     */
    abstract protected void deleteFile(String filename);

    /**
     * Returns a list of all the XSD filesnames in the project.
     */
    abstract protected String[] getAllXSDFilenames();
}