org.apache.xmlbeans.impl.tool.BaseSchemaResourceManager Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of com.liferay.dynamic.data.lists.service
Liferay Dynamic Data Lists Service
There is a newer version: 5.0.70
/*   Copyright 2004 The Apache Software Foundation
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package org.apache.xmlbeans.impl.tool;

import org.apache.xmlbeans.impl.xb.xsdownload.DownloadedSchemasDocument;
import org.apache.xmlbeans.impl.xb.xsdownload.DownloadedSchemaEntry;
import org.apache.xmlbeans.impl.xb.xsdownload.DownloadedSchemasDocument.DownloadedSchemas;
import org.apache.xmlbeans.impl.util.HexBin;
import org.apache.xmlbeans.impl.common.IOUtil;
import org.apache.xmlbeans.XmlOptions;
import org.apache.xmlbeans.XmlBeans;

import java.io.IOException;
import java.io.InputStream;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.util.Set;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.HashMap;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import org.apache.xmlbeans.impl.xb.xsdschema.SchemaDocument.Schema;
import org.apache.xmlbeans.impl.xb.xsdschema.SchemaDocument;

public abstract class BaseSchemaResourceManager extends SchemaImportResolver
{
    private static final String USER_AGENT = "XMLBeans/" + XmlBeans.getVersion() + " (" + XmlBeans.getTitle() + ")";

    private String _defaultCopyDirectory;
    private DownloadedSchemasDocument _importsDoc;
    private Map _resourceForFilename = new HashMap();
    private Map _resourceForURL = new HashMap();
    private Map _resourceForNamespace = new HashMap();
    private Map _resourceForDigest = new HashMap();
    private Map _resourceForCacheEntry = new HashMap();
    private Set _redownloadSet = new HashSet();

    protected BaseSchemaResourceManager()
    {
        // concrete subclasses should call init in their constructors
    }

    protected final void init()
    {
        if (fileExists(getIndexFilename()))
        {
            try
            {
                _importsDoc = DownloadedSchemasDocument.Factory.parse( inputStreamForFile( getIndexFilename() ) );
            }
            catch (IOException e)
            {
                _importsDoc = null;
            }
            catch (Exception e)
            {
                throw (IllegalStateException)(new IllegalStateException("Problem reading xsdownload.xml: please fix or delete this file")).initCause(e);
            }
        }
        if (_importsDoc == null)
        {
            try
            {
                _importsDoc = DownloadedSchemasDocument.Factory.parse(
                    ""
                );
            }
            catch (Exception e)
            {
                throw (IllegalStateException)(new IllegalStateException()).initCause(e);
            }
        }

        String defaultDir = _importsDoc.getDownloadedSchemas().getDefaultDirectory();
        if (defaultDir == null)
            defaultDir = getDefaultSchemaDir();;
        _defaultCopyDirectory = defaultDir;

        // now initialize data structures
        DownloadedSchemaEntry[] entries = _importsDoc.getDownloadedSchemas().getEntryArray();
        for (int i = 0; i < entries.length; i++)
        {
            updateResource(entries[i]);
        }
    }

    public final void writeCache() throws IOException
    {
        InputStream input = _importsDoc.newInputStream(new XmlOptions().setSavePrettyPrint());
        writeInputStreamToFile(input, getIndexFilename());
    }

    public final void processAll(boolean sync, boolean refresh, boolean imports)
    {
        if (refresh)
        {
            _redownloadSet = new HashSet();
        }
        else
        {
            _redownloadSet = null;
        }

        String[] allFilenames = getAllXSDFilenames();

        if (sync)
            syncCacheWithLocalXsdFiles(allFilenames, false);

        SchemaResource[] starters = (SchemaResource[])
                _resourceForFilename.values().toArray(new SchemaResource[0]);

        if (refresh)
            redownloadEntries(starters);

        if (imports)
            resolveImports(starters);

        _redownloadSet = null;
    }

    public final void process(String[] uris, String[] filenames, boolean sync, boolean refresh, boolean imports)
    {
        if (refresh)
        {
            _redownloadSet = new HashSet();
        }
        else
        {
            _redownloadSet = null;
        }

        if (filenames.length > 0)
            syncCacheWithLocalXsdFiles(filenames, true);
        else if (sync)
            syncCacheWithLocalXsdFiles(getAllXSDFilenames(), false);

        Set starterset = new HashSet();

        for (int i = 0; i < uris.length; i++)
        {
            SchemaResource resource = (SchemaResource)lookupResource(null, uris[i]);
            if (resource != null)
                starterset.add(resource);
        }

        for (int i = 0; i < filenames.length; i++)
        {
            SchemaResource resource = (SchemaResource)_resourceForFilename.get(filenames);
            if (resource != null)
                starterset.add(resource);
        }

        SchemaResource[] starters = (SchemaResource[])
               starterset.toArray(new SchemaResource[0]);

        if (refresh)
            redownloadEntries(starters);

        if (imports)
            resolveImports(starters);

        _redownloadSet = null;
    }

    /**
     * Adds items to the cache that point to new files that aren't
     * described in the cache, and optionally deletes old entries.
     *
     * If an old file is gone and a new file is
     * found with exactly the same contents, the cache entry is moved
     * to point to the new file.
     */
    public final void syncCacheWithLocalXsdFiles(String[] filenames, boolean deleteOnlyMentioned)
    {
        Set seenResources = new HashSet();
        Set vanishedResources = new HashSet();

        for (int i = 0; i < filenames.length; i++)
        {
            String filename = filenames[i];

            // first, if the filename matches exactly, trust the filename
            SchemaResource resource = (SchemaResource)_resourceForFilename.get(filename);
            if (resource != null)
            {
                if (fileExists(filename))
                    seenResources.add(resource);
                else
                    vanishedResources.add(resource);
                continue;
            }

            // new file that is not in the index?
            // not if the digest is known to the index and the original file is gone - that's a rename!
            String digest = null;
            try
            {
                digest = shaDigestForFile(filename);
                resource = (SchemaResource)_resourceForDigest.get(digest);
                if (resource != null)
                {
                    String oldFilename = resource.getFilename();
                    if (!fileExists(oldFilename))
                    {
                        warning("File " + filename + " is a rename of " + oldFilename);
                        resource.setFilename(filename);
                        seenResources.add(resource);
                        if (_resourceForFilename.get(oldFilename) == resource)
                            _resourceForFilename.remove(oldFilename);
                        if (_resourceForFilename.containsKey(filename))
                            _resourceForFilename.put(filename, resource);
                        continue;
                    }
                }
            }
            catch (IOException e)
            {
                // unable to read digest... no problem, ignore then
            }

            // ok, this really is a new XSD file then, of unknown URL origin
            DownloadedSchemaEntry newEntry = addNewEntry();
            newEntry.setFilename(filename);
            warning("Caching information on new local file " + filename);
            if (digest != null)
                newEntry.setSha1(digest);

            seenResources.add(updateResource(newEntry));
        }

        if (deleteOnlyMentioned)
            deleteResourcesInSet(vanishedResources, true);
        else
            deleteResourcesInSet(seenResources, false);
    }

    /**
     * Iterates through every entry and refetches it from its primary URL,
     * if known.  Replaces the contents of the file if the data is different.
     */
    private void redownloadEntries(SchemaResource[] resources)
    {
        for (int i = 0; i < resources.length; i++)
        {
            redownloadResource(resources[i]);
        }
    }

    private void deleteResourcesInSet(Set seenResources, boolean setToDelete)
    {
        Set seenCacheEntries = new HashSet();
        for (Iterator i = seenResources.iterator(); i.hasNext(); )
        {
            SchemaResource resource = (SchemaResource)i.next();
            seenCacheEntries.add(resource._cacheEntry);
        }

        DownloadedSchemas downloadedSchemas = _importsDoc.getDownloadedSchemas();
        for (int i = 0; i < downloadedSchemas.sizeOfEntryArray(); i++)
        {
            DownloadedSchemaEntry cacheEntry = downloadedSchemas.getEntryArray(i);

            if (seenCacheEntries.contains(cacheEntry) == setToDelete)
            {
                SchemaResource resource = (SchemaResource)_resourceForCacheEntry.get(cacheEntry);
                warning("Removing obsolete cache entry for " + resource.getFilename());

                if (resource != null)
                {
                    _resourceForCacheEntry.remove(cacheEntry);

                    if (resource == _resourceForFilename.get(resource.getFilename()))
                        _resourceForFilename.remove(resource.getFilename());

                    if (resource == _resourceForDigest.get(resource.getSha1()))
                        _resourceForDigest.remove(resource.getSha1());

                    if (resource == _resourceForNamespace.get(resource.getNamespace()))
                        _resourceForNamespace.remove(resource.getNamespace());

                    // Finally, any or all URIs
                    String[] urls = resource.getSchemaLocationArray();
                    for (int j = 0; j < urls.length; j++)
                    {
                        if (resource == _resourceForURL.get(urls[j]))
                            _resourceForURL.remove(urls[j]);
                    }
                }

                downloadedSchemas.removeEntry(i);
                i -= 1;
            }
        }
    }

    private SchemaResource updateResource(DownloadedSchemaEntry entry)
    {
        // The file
        String filename = entry.getFilename();
        if (filename == null)
            return null;

        SchemaResource resource = new SchemaResource(entry);
        _resourceForCacheEntry.put(entry, resource);

        if (!_resourceForFilename.containsKey(filename))
            _resourceForFilename.put(filename, resource);

        // The digest
        String digest = resource.getSha1();
        if (digest != null)
        {
            if (!_resourceForDigest.containsKey(digest))
                _resourceForDigest.put(digest, resource);
        }

        // Next, the namespace
        String namespace = resource.getNamespace();
        if (namespace != null)
        {
            if (!_resourceForNamespace.containsKey(namespace))
                _resourceForNamespace.put(namespace, resource);
        }

        // Finally, any or all URIs
        String[] urls = resource.getSchemaLocationArray();
        for (int j = 0; j < urls.length; j++)
        {
            if (!_resourceForURL.containsKey(urls[j]))
                _resourceForURL.put(urls[j], resource);
        }

        return resource;
    }

    private static DigestInputStream digestInputStream(InputStream input)
    {
        MessageDigest sha;
        try
        {
            sha = MessageDigest.getInstance("SHA");
        }
        catch (NoSuchAlgorithmException e)
        {
            throw (IllegalStateException)(new IllegalStateException().initCause(e));
        }

        DigestInputStream str = new DigestInputStream(input, sha);

        return str;
    }

    private DownloadedSchemaEntry addNewEntry()
    {
        return _importsDoc.getDownloadedSchemas().addNewEntry();
    }

    private class SchemaResource implements SchemaImportResolver.SchemaResource
    {
        SchemaResource(DownloadedSchemaEntry entry)
        {
            _cacheEntry = entry;
        }

        DownloadedSchemaEntry _cacheEntry;

        public void setFilename(String filename)
        {
            _cacheEntry.setFilename(filename);
        }

        public String getFilename()
        {
            return _cacheEntry.getFilename();
        }

        public Schema getSchema()
        {
            if (!fileExists(getFilename()))
                redownloadResource(this);

            try
            {
                return SchemaDocument.Factory.parse(inputStreamForFile(getFilename())).getSchema();
            }
            catch (Exception e)
            {
                return null; // return null if _any_ problems reading schema file
            }
        }

        public String getSha1()
        {
            return _cacheEntry.getSha1();
        }

        public String getNamespace()
        {
            return _cacheEntry.getNamespace();
        }

        public void setNamespace(String namespace)
        {
            _cacheEntry.setNamespace(namespace);
        }

        public String getSchemaLocation()
        {
            if (_cacheEntry.sizeOfSchemaLocationArray() > 0)
                return _cacheEntry.getSchemaLocationArray(0);
            return null;
        }

        public String[] getSchemaLocationArray()
        {
            return _cacheEntry.getSchemaLocationArray();
        }

        public int hashCode()
        {
            return getFilename().hashCode();
        }

        public boolean equals(Object obj)
        {
            return this == obj || getFilename().equals(((SchemaResource)obj).getFilename());
        }

        public void addSchemaLocation(String schemaLocation)
        {
            _cacheEntry.addSchemaLocation(schemaLocation);
        }
    }

    /**
     * Called when the ImportLoader wishes to resolve the
     * given import.  Should return a SchemaResource whose
     * "equals" relationship reveals when a SchemaResource is
     * duplicated and shouldn't be examined again.
     *
     * Returns null if the resource reference should be ignored.
     */
    public SchemaImportResolver.SchemaResource lookupResource(String nsURI, String schemaLocation)
    {
        SchemaResource result = fetchFromCache(nsURI, schemaLocation);
        if (result != null)
        {
            if (_redownloadSet != null)
            {
                redownloadResource(result);
            }
            return result;
        }

        if (schemaLocation == null)
        {
            warning("No cached schema for namespace '" + nsURI + "', and no url specified");
            return null;
        }

        result = copyOrIdentifyDuplicateURL(schemaLocation, nsURI);
        if (_redownloadSet != null)
            _redownloadSet.add(result);
        return result;
    }

    private SchemaResource fetchFromCache(String nsURI, String schemaLocation)
    {
        SchemaResource result;

        if (schemaLocation != null)
        {
            result = (SchemaResource)_resourceForURL.get(schemaLocation);
            if (result != null)
                return result;
        }

        if (nsURI != null)
        {
            result = (SchemaResource)_resourceForNamespace.get(nsURI);
            if (result != null)
                return result;
        }

        return null;
    }

    private String uniqueFilenameForURI(String schemaLocation) throws IOException, URISyntaxException
    {
        String localFilename = new URI( schemaLocation ).getRawPath();
        int i = localFilename.lastIndexOf('/');
        if (i >= 0)
            localFilename = localFilename.substring(i + 1);
        if (localFilename.endsWith(".xsd"))
            localFilename = localFilename.substring(0, localFilename.length() - 4);
        if (localFilename.length() == 0)
            localFilename = "schema";

        // TODO: remove other unsafe characters for filenames?

        String candidateFilename = localFilename;
        int suffix = 1;
        while (suffix < 1000)
        {
            String candidate = _defaultCopyDirectory + "/" + candidateFilename + ".xsd";
            if (!fileExists(candidate))
                return candidate;
            suffix += 1;
            candidateFilename = localFilename + suffix;
        }

        throw new IOException("Problem with filename " + localFilename + ".xsd");
    }

    private void redownloadResource(SchemaResource resource)
    {
        if (_redownloadSet != null)
        {
            if (_redownloadSet.contains(resource))
                return;
            _redownloadSet.add(resource);
        }

        String filename = resource.getFilename();
        String schemaLocation = resource.getSchemaLocation();
        String digest = null;

        // nothing to do?
        if (schemaLocation == null || filename == null)
            return;

        ByteArrayOutputStream buffer = new ByteArrayOutputStream();

        try
        {
            URL url = new URL( schemaLocation );
            URLConnection conn = url.openConnection();
            conn.addRequestProperty("User-Agent", USER_AGENT);
            conn.addRequestProperty("Accept", "application/xml, text/xml, */*");
            DigestInputStream input = digestInputStream(conn.getInputStream());
            IOUtil.copyCompletely(input, buffer);
            digest = HexBin.bytesToString(input.getMessageDigest().digest());
        }
        catch (Exception e)
        {
            warning("Could not copy remote resource " + schemaLocation + ":" + e.getMessage());
            return;
        }

        if (digest.equals(resource.getSha1()) && fileExists(filename))
        {
            warning("Resource " + filename + " is unchanged from " + schemaLocation + ".");
            return;
        }

        try
        {
            InputStream source = new ByteArrayInputStream(buffer.toByteArray());
            writeInputStreamToFile(source, filename);
        }
        catch (IOException e)
        {
            warning("Could not write to file " + filename + " for " + schemaLocation + ":" + e.getMessage());
            return;
        }

        warning("Refreshed " + filename + " from " + schemaLocation);
    }

    private SchemaResource copyOrIdentifyDuplicateURL(String schemaLocation, String namespace)
    {
        String targetFilename;
        String digest;
        SchemaResource result;

        try
        {
            targetFilename = uniqueFilenameForURI(schemaLocation);
        }
        catch (URISyntaxException e)
        {
            warning("Invalid URI '" + schemaLocation + "':" + e.getMessage());
            return null;
        }
        catch (IOException e)
        {
            warning("Could not create local file for " + schemaLocation + ":" + e.getMessage());
            return null;
        }

        try
        {
            URL url = new URL( schemaLocation );
            DigestInputStream input = digestInputStream(url.openStream());
            writeInputStreamToFile(input, targetFilename);
            digest = HexBin.bytesToString(input.getMessageDigest().digest());
        }
        catch (Exception e)
        {
            warning("Could not copy remote resource " + schemaLocation + ":" + e.getMessage());
            return null;
        }

        result = (SchemaResource)_resourceForDigest.get(digest);
        if (result != null)
        {
            deleteFile(targetFilename);
            result.addSchemaLocation(schemaLocation);
            if (!_resourceForURL.containsKey(schemaLocation))
                _resourceForURL.put(schemaLocation, result);
            return result;
        }

        warning("Downloaded " + schemaLocation + " to " + targetFilename);

        DownloadedSchemaEntry newEntry = addNewEntry();
        newEntry.setFilename(targetFilename);
        newEntry.setSha1(digest);
        if (namespace != null)
            newEntry.setNamespace(namespace);
        newEntry.addSchemaLocation(schemaLocation);
        return updateResource(newEntry);
    }

    /**
     * Updates actual namespace in the table.
     */
    public void reportActualNamespace(SchemaImportResolver.SchemaResource rresource, String actualNamespace)
    {
        SchemaResource resource = (SchemaResource)rresource;
        String oldNamespace = resource.getNamespace();
        if (oldNamespace != null && _resourceForNamespace.get(oldNamespace) == resource)
            _resourceForNamespace.remove(oldNamespace);
        if (!_resourceForNamespace.containsKey(actualNamespace))
            _resourceForNamespace.put(actualNamespace, resource);
        resource.setNamespace(actualNamespace);
    }

    private String shaDigestForFile(String filename) throws IOException
    {
        DigestInputStream str = digestInputStream(inputStreamForFile(filename));

        byte[] dummy = new byte[4096];
        for (int i = 1; i > 0; i = str.read(dummy));

        str.close();

        return HexBin.bytesToString(str.getMessageDigest().digest());
    }

    // SOME METHODS TO OVERRIDE ============================

    protected String getIndexFilename()
    {
        return "./xsdownload.xml";
    }

    protected String getDefaultSchemaDir()
    {
        return "./schema";
    }

    /**
     * Produces diagnostic messages such as "downloading X to file Y".
     */
    abstract protected void warning(String msg);

    /**
     * Returns true if the given filename exists.  The filenames
     * are of the form "/foo/bar/zee.xsd" and should be construed
     * as rooted at the root of the project.
     */
    abstract protected boolean fileExists(String filename);

    /**
     * Gets the data in the given filename as an InputStream.
     */
    abstract protected InputStream inputStreamForFile(String filename) throws IOException;

    /**
     * Writes an entire file in one step.  An InputStream is passed and
     * copied to the file.
     */
    abstract protected void writeInputStreamToFile(InputStream input, String filename) throws IOException;

    /**
     * Deletes a file.  Sometimes immediately after writing a new file
     * we notice that it's exactly the same as an existing file and
     * we delete it. We never delete a file that was given to us
     * by the user.
     */
    abstract protected void deleteFile(String filename);

    /**
     * Returns a list of all the XSD filesnames in the project.
     */
    abstract protected String[] getAllXSDFilenames();
}