All Downloads are FREE. Search and download functionalities are using the official Maven repository.

com.opengamma.strata.loader.csv.TradeCsvLoader Maven / Gradle / Ivy

There is a newer version: 2.12.46
Show newest version
/*
 * Copyright (C) 2017 - present by OpenGamma Inc. and the OpenGamma group of companies
 *
 * Please see distribution for license.
 */
package com.opengamma.strata.loader.csv;

import static com.opengamma.strata.loader.csv.CsvLoaderColumns.CPTY_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.CPTY_SCHEME_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.ID_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.ID_SCHEME_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.POSITION_TYPE_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.SETTLEMENT_DATE_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.TRADE_DATE_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.TRADE_TIME_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.TRADE_TYPE_FIELD;
import static com.opengamma.strata.loader.csv.CsvLoaderColumns.TRADE_ZONE_FIELD;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;

import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Optional;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.CharSource;
import com.opengamma.strata.basics.ReferenceData;
import com.opengamma.strata.basics.StandardId;
import com.opengamma.strata.basics.StandardSchemes;
import com.opengamma.strata.collect.ArgChecker;
import com.opengamma.strata.collect.Guavate;
import com.opengamma.strata.collect.MapStream;
import com.opengamma.strata.collect.io.CharSources;
import com.opengamma.strata.collect.io.CsvIterator;
import com.opengamma.strata.collect.io.CsvRow;
import com.opengamma.strata.collect.io.ResourceLocator;
import com.opengamma.strata.collect.io.UnicodeBom;
import com.opengamma.strata.collect.named.ExtendedEnum;
import com.opengamma.strata.collect.result.FailureItem;
import com.opengamma.strata.collect.result.FailureReason;
import com.opengamma.strata.collect.result.ValueWithFailures;
import com.opengamma.strata.loader.LoaderUtils;
import com.opengamma.strata.product.Trade;
import com.opengamma.strata.product.TradeInfo;
import com.opengamma.strata.product.TradeInfoBuilder;
import com.opengamma.strata.product.deposit.type.TermDepositConventions;
import com.opengamma.strata.product.fra.type.FraConventions;
import com.opengamma.strata.product.swap.type.SingleCurrencySwapConvention;

/**
 * Loads trades from CSV files.
 * 

* The trades are expected to be in a CSV format known to Strata. * The parser is flexible, understanding a number of different ways to define each trade. * Columns may occur in any order. * *

Common

*

* The following standard columns are supported:
*

    *
  • The 'Strata Trade Type' column is required, and must be the instrument type, * such as 'Fra' or 'Swap' *
  • The 'Id Scheme' column is optional, and is the name of the scheme that the trade * identifier is unique within, such as 'OG-Trade' *
  • The 'Id' column is optional, and is the identifier of the trade, * such as 'FRA12345' *
  • The 'Counterparty Scheme' column is optional, and is the name of the scheme that the trade * identifier is unique within, such as 'OG-Counterparty' *
  • The 'Counterparty' column is optional, and is the identifier of the trade's counterparty, * such as 'Bank-A' *
  • The 'Trade Date' column is optional, and is the date that the trade occurred, such as '2017-08-01' *
  • The 'Trade Time' column is optional, and is the time of day that the trade occurred, * such as '11:30' *
  • The 'Trade Zone' column is optional, and is the time-zone that the trade occurred, * such as 'Europe/London' *
  • The 'Settlement Date' column is optional, and is the date that the trade settles, such as '2017-08-01' *
* Note that positions may be included in the same file as trades. * The 'Strata Trade Type' column must either be empty or have the value 'Position'. * *

Fra

*

* The following columns are supported for 'Fra' trades: *

    *
  • 'Buy Sell' - mandatory *
  • 'Notional' - mandatory *
  • 'Fixed Rate' - mandatory, percentage *
  • 'Convention' - see below, see {@link FraConventions} *
  • 'Period To Start' - see below *
  • 'Start Date' - see below *
  • 'End Date' - see below *
  • 'Index' - see below *
  • 'Interpolated Index' - see below *
  • 'Day Count' - see below *
  • 'Date Convention' - optional *
  • 'Date Calendar' - optional *
*

* Valid combinations to define a FRA are: *

    *
  • 'Convention', 'Trade Date', 'Period To Start' *
  • 'Convention', 'Start Date', 'End Date' *
  • 'Index', 'Start Date', 'End Date' plus optionally 'Interpolated Index', 'Day Count' *
* *

Swap

*

* The following columns are supported for 'Swap' trades: *

    *
  • 'Buy Sell' - mandatory *
  • 'Notional' - mandatory *
  • 'Fixed Rate' - mandatory, percentage (treated as the spread for some swap types) *
  • 'Convention' - mandatory, see {@link SingleCurrencySwapConvention} implementations *
  • 'Period To Start'- see below *
  • 'Tenor'- see below *
  • 'Start Date'- see below *
  • 'End Date'- see below *
  • 'Roll Convention' - optional *
  • 'Stub Convention' - optional *
  • 'First Regular Start Date' - optional *
  • 'Last Regular End Date' - optional *
  • 'Date Convention' - optional *
  • 'Date Calendar' - optional *
*

* Valid combinations to define a Swap are: *

    *
  • 'Convention', 'Trade Date', 'Period To Start', 'Tenor' *
  • 'Convention', 'Start Date', 'End Date' *
  • 'Convention', 'Start Date', 'Tenor' *
  • Explicitly by defining each leg (not detailed here) *
* *

Term Deposit

*

* The following columns are supported for 'TermDeposit' trades: *

    *
  • 'Buy Sell' - mandatory *
  • 'Notional' - mandatory *
  • 'Fixed Rate' - mandatory, percentage *
  • 'Convention'- see below, see {@link TermDepositConventions} implementations *
  • 'Tenor'- see below *
  • 'Start Date'- see below *
  • 'End Date'- see below *
  • 'Currency'- see below *
  • 'Day Count'- see below *
  • 'Date Convention' - optional *
  • 'Date Calendar' - optional *
*

* Valid combinations to define a Term Deposit are: *

    *
  • 'Convention', 'Trade Date', 'Period To Start' *
  • 'Convention', 'Start Date', 'End Date' *
  • 'Start Date', 'End Date', 'Currency', 'Day Count' *
* *

FX Singles

*

* The following columns are supported for 'FX Singles' (FX Spots and FX Forwards) trades: *

    *
  • 'Buy Sell' - optional, if not present notional must be signed *
  • 'Currency' - mandatory *
  • 'Notional' - mandatory *
  • 'FX Rate' - mandatory *
  • 'Payment Date - mandatory' *
  • 'Payment Date Convention' - optional field. See {@link com.opengamma.strata.basics.date.BusinessDayConventions} for possible values. *
  • 'Payment Date Calendar' - optional field. See {@link com.opengamma.strata.basics.date.HolidayCalendarIds} for possible values. *
* *

FX Swaps

*

* The following columns are supported for 'FxSwap' trades: *

    *
  • 'Buy Sell' - optional, if not present notional must be signed *
  • 'Currency' - mandatory *
  • 'Notional' - mandatory *
  • 'FX Rate' - mandatory *
  • Payment Date - mandatory *
  • 'Far FX Rate' - mandatory *
  • 'Far Payment Date' - mandatory *
  • 'Payment Date Convention' - optional field. See {@link com.opengamma.strata.basics.date.BusinessDayConventions} for possible values. *
  • 'Payment Date Calendar' - optional field. See {@link com.opengamma.strata.basics.date.HolidayCalendarIds} for possible values. *
* *

Security

*

* The following columns are supported for 'Security' trades: *

    *
  • 'Security Id Scheme' - optional, defaults to 'OG-Security' *
  • 'Security Id' - mandatory *
  • 'Quantity' - see below *
  • 'Long Quantity' - see below *
  • 'Short Quantity' - see below *
  • 'Price' - optional *
*

* The quantity will normally be set from the 'Quantity' column. * If that column is not found, the 'Long Quantity' and 'Short Quantity' columns will be used instead. */ public final class TradeCsvLoader { // default schemes private static final String DEFAULT_TRADE_SCHEME = StandardSchemes.OG_TRADE_SCHEME; private static final String DEFAULT_CPTY_SCHEME = StandardSchemes.OG_COUNTERPARTY; /** * The lookup of trade parsers. */ static final ExtendedEnum ENUM_LOOKUP = ExtendedEnum.of(TradeCsvParserPlugin.class); /** * The lookup of trade parsers. */ private static final ImmutableMap PLUGINS = MapStream.of(TradeCsvParserPlugin.extendedEnum().lookupAllNormalized().values()) .flatMapKeys(plugin -> plugin.tradeTypeNames().stream()) .toMap((a, b) -> { System.err.println("Two plugins declare the same product type: " + a.tradeTypeNames()); return a; }); /** * The resolver, providing additional information. */ private final TradeCsvInfoResolver resolver; //------------------------------------------------------------------------- /** * Obtains an instance that uses the standard set of reference data. * * @return the loader */ public static TradeCsvLoader standard() { return new TradeCsvLoader(TradeCsvInfoResolver.standard()); } /** * Obtains an instance that uses the specified set of reference data. * * @param refData the reference data * @return the loader */ public static TradeCsvLoader of(ReferenceData refData) { return new TradeCsvLoader(TradeCsvInfoResolver.of(refData)); } /** * Obtains an instance that uses the specified resolver for additional information. * * @param resolver the resolver used to parse additional information * @return the loader */ public static TradeCsvLoader of(TradeCsvInfoResolver resolver) { return new TradeCsvLoader(resolver); } // restricted constructor private TradeCsvLoader(TradeCsvInfoResolver resolver) { this.resolver = ArgChecker.notNull(resolver, "resolver"); } //------------------------------------------------------------------------- /** * Loads one or more CSV format trade files. *

* CSV files sometimes contain a Unicode Byte Order Mark. * This method uses {@link UnicodeBom} to interpret it. * * @param resources the CSV resources * @return the loaded trades, trade-level errors are captured in the result */ public ValueWithFailures> load(ResourceLocator... resources) { return load(Arrays.asList(resources)); } /** * Loads one or more CSV format trade files. *

* CSV files sometimes contain a Unicode Byte Order Mark. * This method uses {@link UnicodeBom} to interpret it. * * @param resources the CSV resources * @return the loaded trades, all errors are captured in the result */ public ValueWithFailures> load(Collection resources) { Collection charSources = resources.stream() .map(r -> r.getByteSource().asCharSourceUtf8UsingBom()) .collect(toList()); return parse(charSources); } //------------------------------------------------------------------------- /** * Checks whether the source is a CSV format trade file. *

* This parses the headers as CSV and checks that mandatory headers are present. * This is determined entirely from the 'Strata Trade Type' column. * * @param charSource the CSV character source to check * @return true if the source is a CSV file with known headers, false otherwise */ public boolean isKnownFormat(CharSource charSource) { try (CsvIterator csv = CsvIterator.of(charSource, true)) { return csv.containsHeader(TRADE_TYPE_FIELD); } catch (RuntimeException ex) { return false; } } //------------------------------------------------------------------------- /** * Parses one or more CSV format trade files. *

* CSV files sometimes contain a Unicode Byte Order Mark. * Callers are responsible for handling this, such as by using {@link UnicodeBom}. * * @param charSources the CSV character sources * @return the loaded trades, all errors are captured in the result */ public ValueWithFailures> parse(Collection charSources) { return parse(charSources, Trade.class); } /** * Parses one or more CSV format trade files with an error-creating type filter. *

* A list of types is specified to filter the trades. * Trades that do not match the type will be included in the failure list. *

* CSV files sometimes contain a Unicode Byte Order Mark. * Callers are responsible for handling this, such as by using {@link UnicodeBom}. * * @param charSources the CSV character sources * @param tradeTypes the trade types to return * @return the loaded trades, all errors are captured in the result */ public ValueWithFailures> parse( Collection charSources, List> tradeTypes) { ValueWithFailures> parsed = parse(charSources, Trade.class); List valid = new ArrayList<>(); List failures = new ArrayList<>(parsed.getFailures()); for (Trade trade : parsed.getValue()) { if (tradeTypes.contains(trade.getClass())) { valid.add(trade); } else { failures.add(FailureItem.of( FailureReason.PARSING, "Trade type not allowed {type}, only these types are supported: {options}", trade.getClass().getName(), tradeTypes.stream().map(Class::getSimpleName).collect(joining(", ")))); } } return ValueWithFailures.of(valid, failures); } /** * Parses one or more CSV format trade files with a quiet type filter. *

* A type is specified to filter the trades. * Trades that do not match the type are silently dropped. *

* CSV files sometimes contain a Unicode Byte Order Mark. * Callers are responsible for handling this, such as by using {@link UnicodeBom}. * * @param the trade type * @param charSources the CSV character sources * @param tradeType the trade type to return * @return the loaded trades, all errors are captured in the result */ public ValueWithFailures> parse(Collection charSources, Class tradeType) { try { ValueWithFailures> result = ValueWithFailures.of(ImmutableList.of()); for (CharSource charSource : charSources) { ValueWithFailures> singleResult = parseFile(charSource, tradeType); result = result.combinedWith(singleResult, Guavate::concatToList); } return result; } catch (RuntimeException ex) { return ValueWithFailures.of(ImmutableList.of(), FailureItem.of(FailureReason.ERROR, ex)); } } // loads a single CSV file, filtering by trade type private ValueWithFailures> parseFile(CharSource charSource, Class tradeType) { try (CsvIterator csv = CsvIterator.of(charSource, true)) { if (!csv.headers().contains(TRADE_TYPE_FIELD)) { return ValueWithFailures.of( ImmutableList.of(), FailureItem.of( FailureReason.PARSING, "CSV trade file '{fileName}' does not contain '{header}' header", CharSources.extractFileName(charSource), TRADE_TYPE_FIELD)); } return parseFile(csv, charSource, tradeType); } catch (RuntimeException ex) { return ValueWithFailures.of( ImmutableList.of(), FailureItem.of( FailureReason.PARSING, ex, "CSV trade file '{fileName}' could not be parsed: {exceptionMessage}", CharSources.extractFileName(charSource), ex.getMessage())); } } // loads a single CSV file @SuppressWarnings("unchecked") private ValueWithFailures> parseFile(CsvIterator csv, CharSource charSource, Class tradeType) { List trades = new ArrayList<>(); List failures = new ArrayList<>(); for (CsvRow row : csv.asIterable()) { // handle mixed trade/position files Optional tradeTypeOpt = row.findValue(TRADE_TYPE_FIELD).filter(str -> !str.equalsIgnoreCase("POSITION")); Optional positionTypeOpt = row.findValue(POSITION_TYPE_FIELD).filter(str -> !str.equalsIgnoreCase("TRADE")); if (tradeTypeOpt.isPresent() && positionTypeOpt.isPresent()) { failures.add(FailureItem.of( FailureReason.PARSING, "CSV position file '{fileName}' contained row with mixed trade/position type '{type}' at line {lineNumber}", CharSources.extractFileName(charSource), tradeTypeOpt.get() + "/" + positionTypeOpt.get(), row.lineNumber())); continue; // ignore bad row } else if (positionTypeOpt.isPresent()) { continue; // quietly ignore a position row } // handle trade row String typeRaw = row.findField(TRADE_TYPE_FIELD).orElse(""); String typeUpper = typeRaw.toUpperCase(Locale.ENGLISH); try { TradeInfo info = parseTradeInfo(row); // allow type matching to be overridden Optional overrideOpt = resolver.overrideParseTrade(typeUpper, row, info); if (overrideOpt.isPresent()) { if (tradeType.isInstance(overrideOpt.get())) { trades.add(tradeType.cast(overrideOpt.get())); } continue; } // standard type matching TradeCsvParserPlugin plugin = PLUGINS.get(typeUpper); if (plugin != null) { List additionalRows = new ArrayList<>(); while (csv.hasNext() && plugin.isAdditionalRow(row, csv.peek())) { additionalRows.add(csv.next()); } plugin.parseTrade(tradeType, row, additionalRows, info, resolver) .filter(tradeType::isInstance) .ifPresent(parsed -> trades.add((T) parsed)); continue; } // match type using the resolver Optional parsedOpt = resolver.parseOtherTrade(typeUpper, row, info); if (parsedOpt.isPresent()) { if (tradeType.isInstance(parsedOpt.get())) { trades.add(tradeType.cast(parsedOpt.get())); } continue; } // better error for VARIABLE if (typeUpper.equals("VARIABLE")) { failures.add(FailureItem.of( FailureReason.PARSING, "CSV trade file '{fileName}' contained a 'Variable' type at line {lineNumber} " + "that was not preceeded by a 'Swap' or 'Swaption'", CharSources.extractFileName(charSource), row.lineNumber())); } else { // failed to find the type failures.add(FailureItem.of( FailureReason.PARSING, "CSV trade file '{fileName}' contained unknown trade type '{type}' at line {lineNumber}", CharSources.extractFileName(charSource), typeRaw, row.lineNumber())); } } catch (RuntimeException ex) { failures.add(FailureItem.of( FailureReason.PARSING, ex, "CSV trade file '{fileName}' type '{type}' could not be parsed at line {lineNumber}: {exceptionMessage}", CharSources.extractFileName(charSource), typeRaw, row.lineNumber(), ex.getMessage())); } } return ValueWithFailures.of(trades, failures); } // parse the trade info private TradeInfo parseTradeInfo(CsvRow row) { TradeInfoBuilder infoBuilder = TradeInfo.builder(); String scheme = row.findField(ID_SCHEME_FIELD).orElse(DEFAULT_TRADE_SCHEME); row.findValue(ID_FIELD).ifPresent(id -> infoBuilder.id(StandardId.of(scheme, id))); String schemeCpty = row.findValue(CPTY_SCHEME_FIELD).orElse(DEFAULT_CPTY_SCHEME); row.findValue(CPTY_FIELD).ifPresent(cpty -> infoBuilder.counterparty(StandardId.of(schemeCpty, cpty))); row.findValue(TRADE_DATE_FIELD).ifPresent(dateStr -> infoBuilder.tradeDate(LoaderUtils.parseDate(dateStr))); row.findValue(TRADE_TIME_FIELD).ifPresent(timeStr -> infoBuilder.tradeTime(LoaderUtils.parseTime(timeStr))); row.findValue(TRADE_ZONE_FIELD).ifPresent(zoneStr -> infoBuilder.zone(ZoneId.of(zoneStr))); row.findValue(SETTLEMENT_DATE_FIELD) .ifPresent(dateStr -> infoBuilder.settlementDate(LoaderUtils.parseDate(dateStr))); resolver.parseStandardAttributes(row, infoBuilder); resolver.parseTradeInfo(row, infoBuilder); return infoBuilder.build(); } }





© 2015 - 2025 Weber Informatics LLC | Privacy Policy