com.bigdata.rdf.store.TestDataLoader Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of bigdata-rdf-test Show documentation
Blazegraph(TM) RDF Test Suites
There is a newer version: 2.1.4
/**

Copyright (C) SYSTAP, LLC DBA Blazegraph 2006-2016.  All rights reserved.

Contact:
     SYSTAP, LLC DBA Blazegraph
     2501 Calvert ST NW #106
     Washington, DC 20008
     [email protected]

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
package com.bigdata.rdf.store;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Properties;

import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;

import com.bigdata.rdf.store.DataLoader.ClosureEnum;
import com.bigdata.rdf.store.DataLoader.CommitEnum;
import com.bigdata.util.InnerCause;

/**
 * Test suite for the {@link DataLoader}.
 *
 * @author bryan
 * 
 *         TODO Tests to verify that data was actually loaded.... (read back
 *         from last commit point and verify statements found). Do this for each
 *         of the public {@link DataLoader} methods.
 * 
 *         TODO Test to verify that end-of-batch handling is correct (no
 *         additional commits).
 */
public class TestDataLoader extends AbstractTripleStoreTestCase {

	public TestDataLoader() {
	}

	public TestDataLoader(final String name) {
		super(name);
	}

	/**
	 * Simple test of ability to load a resource (from the classpath).
	 */
	public void test_DataLoader_loadResource01() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final DataLoader dataLoader = new DataLoader(store);

			final String resource = "com/bigdata/rdf/store/sample-data.ttl";

			final String baseURL = new File(resource).toURI().toString();

			dataLoader.loadData(new String[] { resource }, new String[] { baseURL },
					new RDFFormat[] { RDFFormat.TURTLE });

		} finally {

			store.__tearDownUnitTest();

		}

	}

	/**
	 * Test where an error in a source file SHOULD NOT be ignored because we
	 * have NOT specified {@link DataLoader.Options#IGNORE_INVALID_FILES}.
	 * 
	 * @see BLZG-1531 (Add option to make the DataLoader robust to files that
	 *      cause rio to throw a fatal exception)
	 */
	public void test_DataLoader_ignoreFailures01() throws IOException {

		boolean ok = false;
		
		final String resource = "com/bigdata/rdf/store/sample-data-bad.ttl";

		final AbstractTripleStore store = getStore();

		try {

			final DataLoader dataLoader = new DataLoader(store);

			final String baseURL = new File(resource).toURI().toString();

			dataLoader.loadData(new String[] { resource }, new String[] { baseURL },
					new RDFFormat[] { RDFFormat.TURTLE });

			ok = true;

		} catch (Throwable t) {

			if (!InnerCause.isInnerCause(t, RDFParseException.class)) {

				fail("Expected inner cause " + RDFParseException.class + " not found in " + t, t);

			}

		} finally {

			store.__tearDownUnitTest();

		}

		if (ok)
			fail("Error should have been reported for " + resource);
	
	}

	/**
	 * Test where an error in a source file SHOULD be ignored because we have
	 * specified {@link DataLoader.Options#IGNORE_INVALID_FILES}.
	 * 
	 * @see BLZG-1531 (Add option to make the DataLoader robust to files that
	 *      cause rio to throw a fatal exception)
	 */
	public void test_DataLoader_ignoreFailures02() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());
			
			properties.setProperty(DataLoader.Options.IGNORE_INVALID_FILES, "true");
			
			final DataLoader dataLoader = new DataLoader(properties, store);

			final String resource = "com/bigdata/rdf/store/sample-data-bad.ttl";

			final String baseURL = new File(resource).toURI().toString();

			dataLoader.loadData(new String[] { resource }, new String[] { baseURL },
					new RDFFormat[] { RDFFormat.TURTLE });

		} finally {

			store.__tearDownUnitTest();

		}

	}

	/**
	 * Test where the load should fail even though we specified
	 * {@link DataLoader.Options#IGNORE_INVALID_FILES} because it error was a
	 * "resource not found" problem rather than a parser error.
	 * 
	 * @see BLZG-1531 (Add option to make the DataLoader robust to files that
	 *      cause rio to throw a fatal exception)
	 */
	public void test_DataLoader_ignoreFailures03() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());
			
			properties.setProperty(DataLoader.Options.IGNORE_INVALID_FILES, "true");
			
			final DataLoader dataLoader = new DataLoader(properties, store);

			final String resource = "com/bigdata/rdf/store/sample-data-DOES-NOT-EXIST.ttl";

			final String baseURL = new File(resource).toURI().toString();

			try {

				dataLoader.loadData(new String[] { resource }, new String[] { baseURL },
						new RDFFormat[] { RDFFormat.TURTLE });
				
				fail();
				
			} catch (IOException ex) {
				
				if (log.isInfoEnabled())
					log.info("Ignoring expected exception: " + ex);
				
			}
			
		} finally {

			store.__tearDownUnitTest();

		}

	}

	/**
	 * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#None}
	 */
	public void test_durableQueues01_incrementalCommit_noClosure() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());

			// enable durable queues.
			properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true");

			// Incremental commit.
			properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name());

			properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name());

			final DataLoader dataLoader = new DataLoader(properties, store);

			doDurableQueueTest(dataLoader);
			
		} finally {

			store.__tearDownUnitTest();
		}

	}
	
	/**
	 * Test durable queues using {@link CommitEnum#Incremental} and {@link ClosureEnum#Batch}
	 */
	public void test_durableQueues01_incrementalCommit_batchClosure() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());

			// enable durable queues.
			properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true");

			// Incremental commit.
			properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Incremental.name());

			properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name());

			final DataLoader dataLoader = new DataLoader(properties, store);

			doDurableQueueTest(dataLoader);
			
		} finally {

			store.__tearDownUnitTest();
		}

	}
	
	/**
	 * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Batch}
	 */
	public void test_durableQueues02_batchCommit_batchClosure() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());

			// enable durable queues.
			properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true");

			// Batch commit.
			properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name());

			properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Batch.name());

			final DataLoader dataLoader = new DataLoader(properties, store);

			doDurableQueueTest(dataLoader);
			
		} finally {

			store.__tearDownUnitTest();
		}

	}
	
	/**
	 * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#None}
	 */
	public void test_durableQueues02_batchCommit_noClosure() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());

			// enable durable queues.
			properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true");

			// Batch commit.
			properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name());

			properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.name());

			final DataLoader dataLoader = new DataLoader(properties, store);

			doDurableQueueTest(dataLoader);
			
		} finally {

			store.__tearDownUnitTest();
		}

	}
	
	/**
	 * Test durable queues using {@link CommitEnum#Batch} and {@link ClosureEnum#Incremental}
	 */
	public void test_durableQueues02_batchCommit_incrementalClosure() throws IOException {

		final AbstractTripleStore store = getStore();

		try {

			final Properties properties = new Properties(store.getProperties());

			// enable durable queues.
			properties.setProperty(DataLoader.Options.DURABLE_QUEUES, "true");

			// Batch commit.
			properties.setProperty(DataLoader.Options.COMMIT, CommitEnum.Batch.name());

			properties.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.Incremental.name());

			final DataLoader dataLoader = new DataLoader(properties, store);

			doDurableQueueTest(dataLoader);
			
		} finally {

			store.__tearDownUnitTest();
		}

	}
	
	private void doDurableQueueTest(final DataLoader dataLoader) throws IOException {
		
		// temporary directory where we setup the test.
		final File tmpDir = File.createTempFile(getClass().getName(), ".tmp");
		try {
			
			tmpDir.delete(); // delete random file name.
			tmpDir.mkdir(); // recreate it as a directory.
		
			// Setup directory with files.
			final File goodFile = new File(tmpDir, "good.ttl");
			final File failFile = new File(tmpDir, "fail.ttl");
			{

				final String goodData = ""+//
				"@prefix bd:  .\n"+//
				"@prefix rdf:  .\n"+//
				"@prefix rdfs:  .\n"+//
				"@prefix foaf:  .\n"+//
				"bd:Mike rdf:type foaf:Person .\n"+//
				"bd:Bryan rdf:type foaf:Person .\n"+//
				"bd:Martyn rdf:type foaf:Person .\n"+//
				"bd:Mike rdfs:label \"Mike\" .\n"+//
				"bd:Bryan rdfs:label \"Bryan\" .\n"+//
				"bd:DC rdfs:label \"DC\" .\n"+//
				"bd:Mike foaf:knows bd:Bryan .\n"+//
				"bd:Bryan foaf:knows bd:Mike .\n"+//
				"bd:Bryan foaf:knows bd:Martyn .\n"+//
				"bd:Martyn foaf:knows bd:Bryan .\n"+//
				"";

				writeOnFile(goodFile, goodData);
				
				// Note: has a Literal in the Subject position.
				final String failData = ""+//
				"@prefix bd:  .\n"+//
				"@prefix rdf:  .\n"+//
				"@prefix rdfs:  .\n"+//
				"@prefix foaf:  .\n"+//
				"\"Mike\" rdf:type foaf:Person .\n"+//
				"bd:Bryan rdf:type foaf:Person .\n"+//
				"bd:Martyn rdf:type foaf:Person .\n"+//
				"bd:Mike rdfs:label \"Mike\" .\n"+//
				"bd:Bryan rdfs:label \"Bryan\" .\n"+//
				"bd:DC rdfs:label \"DC\" .\n"+//
				"bd:Mike foaf:knows bd:Bryan .\n"+//
				"bd:Bryan foaf:knows bd:Mike .\n"+//
				"bd:Bryan foaf:knows bd:Martyn .\n"+//
				"bd:Martyn foaf:knows bd:Bryan .\n"+//
				"";

				writeOnFile(failFile, failData);

			}

			// Run loader.
			{

				dataLoader.loadFiles(tmpDir, null/* baseURI */, RDFFormat.TURTLE, null/* defaultGraph */,
						new FilenameFilter() {
							@Override
							public boolean accept(File dir, String name) {
								return name.endsWith(".ttl");
							}
						});

			}
			
			// Verify post-conditions in the tmp directory.
			{
				
				if (goodFile.exists())
					fail("File was not renamed: " + goodFile);
				
				if (failFile.exists())
					fail("File was not renamed: " + failFile);

				{
					
					final File tmp = new File(goodFile.getPath() + ".good");

					if (!tmp.exists())
						fail("File not found: " + tmp);

				}
				
				{
			
					final File tmp = new File(failFile.getPath() + ".fail");

					if (!tmp.exists())
						fail("File not found: " + tmp);
					
				}

			}

		} finally {

			// destroy the temporary directory.
			recursiveDelete(tmpDir);
//			System.err.println("tmpDir=" + tmpDir);
			
		}

	}

	private void writeOnFile(final File file, final String data) throws IOException {
		
		file.createNewFile();
		
		try (final FileWriter fileWriter = new FileWriter(file)) {

			try (final BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) {

				bufferedWriter.write(data);

				bufferedWriter.flush();

			}

		}

	}

}