com.opengamma.strata.loader.csv.PositionCsvLoader Maven / Gradle / Ivy
Show all versions of strata-loader Show documentation
/*
* Copyright (C) 2017 - present by OpenGamma Inc. and the OpenGamma group of companies
*
* Please see distribution for license.
*/
package com.opengamma.strata.loader.csv;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.ID_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.ID_SCHEME_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.POSITION_TYPE_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.TRADE_TYPE_FIELD;
import static java.util.stream.Collectors.toList;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.CharSource;
import com.opengamma.strata.basics.ReferenceData;
import com.opengamma.strata.basics.StandardId;
import com.opengamma.strata.basics.StandardSchemes;
import com.opengamma.strata.collect.ArgChecker;
import com.opengamma.strata.collect.Guavate;
import com.opengamma.strata.collect.MapStream;
import com.opengamma.strata.collect.Messages;
import com.opengamma.strata.collect.io.CharSources;
import com.opengamma.strata.collect.io.CsvIterator;
import com.opengamma.strata.collect.io.CsvRow;
import com.opengamma.strata.collect.io.ResourceLocator;
import com.opengamma.strata.collect.io.UnicodeBom;
import com.opengamma.strata.collect.named.ExtendedEnum;
import com.opengamma.strata.collect.result.FailureAttributeKeys;
import com.opengamma.strata.collect.result.FailureItem;
import com.opengamma.strata.collect.result.FailureReason;
import com.opengamma.strata.collect.result.ParseFailureException;
import com.opengamma.strata.collect.result.ValueWithFailures;
import com.opengamma.strata.product.Position;
import com.opengamma.strata.product.PositionInfo;
import com.opengamma.strata.product.PositionInfoBuilder;
import com.opengamma.strata.product.SecurityPosition;
import com.opengamma.strata.product.etd.EtdContractSpec;
import com.opengamma.strata.product.etd.EtdContractSpecId;
import com.opengamma.strata.product.etd.EtdFuturePosition;
import com.opengamma.strata.product.etd.EtdIdUtils;
import com.opengamma.strata.product.etd.EtdOptionPosition;
import com.opengamma.strata.product.etd.EtdOptionType;
import com.opengamma.strata.product.etd.EtdSettlementType;
/**
* Loads positions from CSV files.
*
* The positions are expected to be in a CSV format known to Strata.
* The parser is flexible, understanding a number of different ways to define each position.
* Columns may occur in any order.
*
*
Common
*
* The following standard columns are supported:
*
* - The 'Strata Position Type' column is optional, but mandatory when checking headers
* to see if the file is a known format. It defines the instrument type,
* 'SEC' or'Security' for standard securities,
* 'FUT' or 'Future' for ETD futures, and
* 'OPT' or 'Option' for ETD options.
* If absent, the type is derived based on the presence or absence of the 'Expiry' column.
*
- The 'Id Scheme' column is optional, and is the name of the scheme that the position
* identifier is unique within, such as 'OG-Position'.
*
- The 'Id' column is optional, and is the identifier of the position,
* such as 'POS12345'.
*
* Note that trades may be included in the same file as positions.
* The 'Strata Position Type' column must either be empty or have the value 'Position'.
*
* SEC/Security
*
* The following columns are supported:
*
* - 'Security Id Scheme' - optional, defaults to 'OG-Security'
*
- 'Security Id' - mandatory
*
- 'Quantity' - see below
*
- 'Long Quantity' - see below
*
- 'Short Quantity' - see below
*
*
* The quantity will normally be set from the 'Quantity' column.
* If that column is not found, the 'Long Quantity' and 'Short Quantity' columns will be used instead.
*
*
FUT/Future
*
* The following columns are supported:
*
* - 'Exchange' - mandatory, the MIC code of the exchange where the ETD is traded
*
- 'Contract Code' - mandatory, the contract code of the ETD at the exchange
*
- 'Quantity' - see below
*
- 'Long Quantity' - see below
*
- 'Short Quantity' - see below
*
- 'Expiry' - mandatory, the year-month of the expiry, in the format 'yyyy-MM'
*
- 'Expiry Week' - optional, only used to obtain a weekly-expiring ETD
*
- 'Expiry Day' - optional, only used to obtain a daily-expiring ETD, or Flex
*
- 'Settlement Type' - optional, only used for Flex, see {@link EtdSettlementType}
*
*
* The exchange and contract code are combined to form an {@link EtdContractSpecId} which is
* resolved in {@link ReferenceData} to find additional details about the ETD.
* This process can be changed by providing an alternative {@link PositionCsvInfoResolver}.
*
* The quantity will normally be set from the 'Quantity' column.
* If that column is not found, the 'Long Quantity' and 'Short Quantity' columns will be used instead.
*
* The expiry is normally controlled using just the 'Expiry' column.
* Flex options will also set the 'Expiry Day' and 'Settlement Type'.
*
*
OPT/Option
*
* The following columns are supported:
*
* - 'Exchange' - mandatory, the MIC code of the exchange where the ETD is traded
*
- 'Contract Code' - mandatory, the contract code of the ETD at the exchange
*
- 'Quantity' - see below
*
- 'Long Quantity' - see below
*
- 'Short Quantity' - see below
*
- 'Expiry' - mandatory, the year-month of the expiry, in the format 'yyyy-MM'
*
- 'Expiry Week' - optional, only used to obtain a weekly-expiring ETD
*
- 'Expiry Day' - optional, only used to obtain a daily-expiring ETD, or Flex
*
- 'Settlement Type' - optional, only used for Flex, see {@link EtdSettlementType}
*
- 'Exercise Style' - optional, only used for Flex, see {@link EtdOptionType}
*
- 'Put Call' - mandatory, 'Put', 'P', 'Call' or 'C'
*
- 'Exercise Price' - mandatory, the strike price, such as 1.23
*
- 'Version' - optional, the version of the contract, not widely used, defaults to zero
*
- 'Underlying Expiry' - optional, the expiry year-month of the underlying instrument if applicable, in the format 'yyyy-MM'
*
*
* The exchange and contract code are combined to form an {@link EtdContractSpecId} which is
* resolved in {@link ReferenceData} to find additional details about the ETD.
* This process can be changed by providing an alternative {@link PositionCsvInfoResolver}.
*
* The quantity will normally be set from the 'Quantity' column.
* If that column is not found, the 'Long Quantity' and 'Short Quantity' columns will be used instead.
*
* The expiry is normally controlled using just the 'Expiry' column.
* Flex options will also set the 'Expiry Day', 'Settlement Type' and 'Exercise Style'.
*/
public final class PositionCsvLoader {
// default schemes
static final String DEFAULT_POSITION_SCHEME = StandardSchemes.OG_POSITION_SCHEME;
static final String DEFAULT_SECURITY_SCHEME = StandardSchemes.OG_SECURITY_SCHEME;
/**
* The lookup of position parsers.
*/
static final ExtendedEnum ENUM_LOOKUP = ExtendedEnum.of(PositionCsvParserPlugin.class);
/**
* The lookup of position parsers.
*/
private static final ImmutableMap PLUGINS =
MapStream.of(PositionCsvParserPlugin.extendedEnum().lookupAllNormalized().values())
.flatMapKeys(plugin -> plugin.positionTypeNames().stream())
.toMap((a, b) -> {
System.err.println("Two plugins declare the same product type: " + a.positionTypeNames());
return a;
});
/**
* The resolver, providing additional information.
*/
private final PositionCsvInfoResolver resolver;
//-------------------------------------------------------------------------
/**
* Obtains an instance that uses the standard set of reference data.
*
* @return the loader
*/
public static PositionCsvLoader standard() {
return new PositionCsvLoader(PositionCsvInfoResolver.standard());
}
/**
* Obtains an instance that uses the specified set of reference data.
*
* @param refData the reference data
* @return the loader
*/
public static PositionCsvLoader of(ReferenceData refData) {
return new PositionCsvLoader(PositionCsvInfoResolver.of(refData));
}
/**
* Obtains an instance that uses the specified resolver for additional information.
*
* @param resolver the resolver used to parse additional information
* @return the loader
*/
public static PositionCsvLoader of(PositionCsvInfoResolver resolver) {
return new PositionCsvLoader(resolver);
}
// restricted constructor
private PositionCsvLoader(PositionCsvInfoResolver resolver) {
this.resolver = ArgChecker.notNull(resolver, "resolver");
}
//-------------------------------------------------------------------------
/**
* Loads one or more CSV format position files.
*
* CSV files sometimes contain a Unicode Byte Order Mark.
* This method uses {@link UnicodeBom} to interpret it.
*
* @param resources the CSV resources
* @return the loaded positions, position-level errors are captured in the result
*/
public ValueWithFailures> load(ResourceLocator... resources) {
return load(Arrays.asList(resources));
}
/**
* Loads one or more CSV format position files.
*
* CSV files sometimes contain a Unicode Byte Order Mark.
* This method uses {@link UnicodeBom} to interpret it.
*
* @param resources the CSV resources
* @return the loaded positions, all errors are captured in the result
*/
public ValueWithFailures> load(Collection resources) {
Collection charSources = resources.stream()
.map(r -> r.getByteSource().asCharSourceUtf8UsingBom())
.collect(toList());
return parse(charSources);
}
//-------------------------------------------------------------------------
/**
* Checks whether the source is a CSV format position file.
*
* This parses the headers as CSV and checks that mandatory headers are present.
* This is determined entirely from the 'Strata Position Type' column.
*
* @param charSource the CSV character source to check
* @return true if the source is a CSV file with known headers, false otherwise
*/
public boolean isKnownFormat(CharSource charSource) {
try (CsvIterator csv = CsvIterator.of(charSource, true)) {
return csv.containsHeader(POSITION_TYPE_FIELD);
} catch (RuntimeException ex) {
return false;
}
}
//-------------------------------------------------------------------------
/**
* Parses one or more CSV format position files, returning ETD futures and
* options using information from reference data.
*
* When an ETD row is found, reference data is used to find the correct security.
* This uses {@link EtdContractSpec} by default, although this can be overridden in the resolver.
* Futures and options will be returned as {@link EtdFuturePosition} and {@link EtdOptionPosition}.
*
* CSV files sometimes contain a Unicode Byte Order Mark.
* Callers are responsible for handling this, such as by using {@link UnicodeBom}.
*
* @param charSources the CSV character sources
* @return the loaded positions, all errors are captured in the result
*/
public ValueWithFailures> parse(Collection charSources) {
return parse(charSources, Position.class);
}
/**
* Parses one or more CSV format position files, returning ETD futures and
* options by identifier without using reference data.
*
* When an ETD row is found, {@link EtdIdUtils} is used to create an identifier.
* The identifier is used to create a {@link SecurityPosition}, with no call to reference data.
*
* CSV files sometimes contain a Unicode Byte Order Mark.
* Callers are responsible for handling this, such as by using {@link UnicodeBom}.
*
* @param charSources the CSV character sources
* @return the loaded positions, all errors are captured in the result
* @deprecated Use {@link LightweightPositionCsvInfoResolver} instead
*/
@Deprecated
public ValueWithFailures> parseLightweight(Collection charSources) {
return parse(charSources, SecurityPosition.class);
}
/**
* Parses one or more CSV format position files.
*
* A type is specified to filter the positions.
* If the type is {@link SecurityPosition}, then ETD parsing will proceed as per {@link #parseLightweight(Collection)}.
* Otherwise, ETD parsing will proceed as per {@link #parse(Collection)}.
*
* CSV files sometimes contain a Unicode Byte Order Mark.
* Callers are responsible for handling this, such as by using {@link UnicodeBom}.
*
* @param the position type
* @param charSources the CSV character sources
* @param positionType the position type to return
* @return the loaded positions, all errors are captured in the result
*/
public ValueWithFailures> parse(Collection charSources, Class positionType) {
try {
ValueWithFailures> result = ValueWithFailures.of(ImmutableList.of());
for (CharSource charSource : charSources) {
ValueWithFailures> singleResult = parseFile(charSource, positionType);
result = result.combinedWith(singleResult, Guavate::concatToList);
}
return result;
} catch (RuntimeException ex) {
return ValueWithFailures.of(ImmutableList.of(), FailureItem.of(FailureReason.ERROR, ex));
}
}
// loads a single CSV file, filtering by position type
private ValueWithFailures> parseFile(CharSource charSource, Class positionType) {
try (CsvIterator csv = CsvIterator.of(charSource, true)) {
if (!csv.headers().contains(POSITION_TYPE_FIELD)) {
return ValueWithFailures.of(
ImmutableList.of(),
FailureItem.of(
FailureReason.PARSING,
"CSV position file '{fileName}' does not contain '{header}' header",
CharSources.extractFileName(charSource),
POSITION_TYPE_FIELD));
}
return parseFile(csv, charSource, positionType);
} catch (RuntimeException ex) {
return ValueWithFailures.of(
ImmutableList.of(),
FailureItem.of(
FailureReason.PARSING,
ex,
"CSV position file '{fileName}' could not be parsed: {exceptionMessage}",
CharSources.extractFileName(charSource),
ex.getMessage()));
}
}
// loads a single CSV file
@SuppressWarnings("unchecked")
private ValueWithFailures> parseFile(CsvIterator csv, CharSource charSource, Class posType) {
List positions = new ArrayList<>();
List failures = new ArrayList<>();
for (CsvRow row : csv.asIterable()) {
// handle mixed trade/position files
Optional tradeTypeOpt = row.findValue(TRADE_TYPE_FIELD).filter(str -> !str.equalsIgnoreCase("POSITION"));
Optional positionTypeOpt = row.findValue(POSITION_TYPE_FIELD).filter(str -> !str.equalsIgnoreCase("TRADE"));
if (tradeTypeOpt.isPresent() && positionTypeOpt.isPresent()) {
failures.add(FailureItem.of(
FailureReason.PARSING,
"CSV position file '{fileName}' contained row with mixed trade/position type '{type}' at line {lineNumber}",
CharSources.extractFileName(charSource),
tradeTypeOpt.get() + "/" + positionTypeOpt.get(),
row.lineNumber()));
continue; // ignore bad row
} else if (tradeTypeOpt.isPresent()) {
continue; // quietly ignore a trade row
}
// handle position row
String typeRaw = positionTypeOpt.orElse("SMART");
String typeUpper = typeRaw.toUpperCase(Locale.ENGLISH);
try {
PositionInfo info = parsePositionInfo(row);
// type specified
PositionCsvParserPlugin plugin = PLUGINS.get(typeUpper);
if (plugin != null) {
plugin.parsePosition(posType, row, info, resolver)
.filter(parsed -> posType.isInstance(parsed))
.ifPresent(parsed -> positions.add((T) parsed));
} else {
// failed to find the type
FailureItem failureItem = FailureItem.of(
FailureReason.PARSING,
"CSV position file '{fileName}' contained unknown position type '{type}' at line {lineNumber}",
CharSources.extractFileName(charSource),
typeRaw,
row.lineNumber())
.withAttribute(
FailureAttributeKeys.SHORT_MESSAGE,
Messages.format("Unknown '{}', '{}'", POSITION_TYPE_FIELD, typeRaw))
.withAttribute(FailureAttributeKeys.TYPE, typeRaw)
.withAttribute(FailureAttributeKeys.ROOT_CAUSE, "inputData");
failures.add(failureItem);
}
} catch (ParseFailureException ex) {
String shortMessage = ex.getMessage();
String fileName = CharSources.extractFileName(charSource);
String lineNumber = Integer.toString(row.lineNumber());
FailureItem failureItem = ex.getFailureItem()
.withAttribute(FailureAttributeKeys.SHORT_MESSAGE, shortMessage)
.withAttribute(FailureAttributeKeys.LINE_NUMBER, lineNumber)
.withAttribute(FailureAttributeKeys.FILE_NAME, fileName)
.withAttribute(FailureAttributeKeys.TYPE, typeRaw)
.withAttribute(FailureAttributeKeys.ROOT_CAUSE, "inputData");
FailureItem updatedFailure = failureItem.mapMessage(ignored -> Messages.format(
"CSV position file '{}' type '{}' could not be parsed at line {}: {}",
fileName,
typeRaw,
lineNumber,
shortMessage));
failures.add(updatedFailure);
} catch (RuntimeException ex) {
failures.add(FailureItem.of(
FailureReason.PARSING,
ex,
"CSV position file '{fileName}' type '{type}' could not be parsed at line {lineNumber}: {exceptionMessage}",
CharSources.extractFileName(charSource),
typeRaw,
row.lineNumber(),
ex.getMessage()));
}
}
return ValueWithFailures.of(positions, failures);
}
// parse the position info
private PositionInfo parsePositionInfo(CsvRow row) {
PositionInfoBuilder infoBuilder = PositionInfo.builder();
String scheme = row.findField(ID_SCHEME_FIELD).orElse(DEFAULT_POSITION_SCHEME);
row.findValue(ID_FIELD).ifPresent(id -> infoBuilder.id(StandardId.of(scheme, id)));
resolver.parseStandardAttributes(row, infoBuilder);
resolver.parsePositionInfo(row, infoBuilder);
return infoBuilder.build();
}
}