com.threatconnect.sdk.parser.AbstractPagedParser Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of threatconnect-parser Show documentation

There is a newer version: 2.9.0

package com.threatconnect.sdk.parser;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import com.threatconnect.sdk.model.Item;
import com.threatconnect.sdk.parser.result.PageResult;
import com.threatconnect.sdk.parser.source.DataSource;

/**
 * Represents a parser that can contain multiple pages of data to parse
 * 
 * @author Greg Marut
 */
public abstract class AbstractPagedParser extends AbstractParser
{
	public AbstractPagedParser(final DataSource dataSource)
	{
		super(dataSource);
	}
	
	@Override
	public List parseData() throws ParserException
	{
		// holds the list of items to return
		List items = new ArrayList();
		
		// holds the set of all of the datasources that have been parsed so that we can ensure that
		// an infinite loop does not occur
		Set parsedDataSources = new HashSet();
		
		// holds the next data source to parse
		DataSource nextDataSource = getDataSource();
		
		// while the next datasource is not null and if the set does not contain this datasource
		while (null != nextDataSource && !parsedDataSources.contains(nextDataSource))
		{
			// hold onto the current datasource and set the next datasource to null
			DataSource currentDataSource = nextDataSource;
			nextDataSource = null;
			
			// parse the page
			getLogger().info("Parsing: {}", currentDataSource.toString());
			PageResult pageResult = parsePage(currentDataSource);
			parsedDataSources.add(currentDataSource);
			
			// make sure the page is not null
			if (null != pageResult)
			{
				// add all the items to the list
				items.addAll(pageResult.getItems());
				
				// make sure the next data source is not null
				if (null != pageResult.getNextDataSource())
				{
					nextDataSource = pageResult.getNextDataSource();
				}
			}
		}
		
		return items;
	}
	
	protected abstract PageResult parsePage(final DataSource dataSource) throws ParserException;
}