apoc.export.cypher.formatter.AbstractCypherFormatter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of apoc-common Show documentation
Data types package for Neo4j Procedures
There is a newer version: 5.25.1
package apoc.export.cypher.formatter;

import apoc.export.util.ExportConfig;
import apoc.export.util.ExportFormat;
import apoc.export.util.Reporter;
import apoc.util.Util;
import apoc.util.collection.Iterables;
import org.apache.commons.lang3.StringUtils;
import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.Transaction;

import java.io.PrintWriter;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import static apoc.export.cypher.formatter.CypherFormatterUtils.Q_UNIQUE_ID_LABEL;
import static apoc.export.cypher.formatter.CypherFormatterUtils.Q_UNIQUE_ID_REL;
import static apoc.export.cypher.formatter.CypherFormatterUtils.UNIQUE_ID_PROP;
import static apoc.export.cypher.formatter.CypherFormatterUtils.simpleKeyValue;

/**
 * @author AgileLARUS
 *
 * @since 16-06-2017
 */
abstract class AbstractCypherFormatter implements CypherFormatter {

	private static final String STATEMENT_CONSTRAINTS = "CREATE CONSTRAINT %s%s FOR (node:%s) REQUIRE (%s) %s;";
	private static final String STATEMENT_DROP_CONSTRAINTS = "DROP CONSTRAINT %s;";

	private static final String STATEMENT_NODE_FULLTEXT_IDX = "CREATE FULLTEXT INDEX %s FOR (n:%s) ON EACH [%s];";
	private static final String STATEMENT_REL_FULLTEXT_IDX = "CREATE FULLTEXT INDEX %s FOR ()-[rel:%s]-() ON EACH [%s];";
	public static final String PROPERTY_QUOTING_FORMAT = "%s.`%s`";
	private static final String ID_REL_KEY = "id";

	@Override
	public String statementForCleanUp(int batchSize) {
		return "MATCH (n:" + Q_UNIQUE_ID_LABEL + ") " +
				" WITH n LIMIT " + batchSize +
				" REMOVE n:" + Q_UNIQUE_ID_LABEL + " REMOVE n." + Util.quote(UNIQUE_ID_PROP) + ";";
	}

	@Override
	public String statementForNodeIndex(String indexType, String label, Iterable keys, boolean ifNotExists, String idxName) {
		return String.format("CREATE %s INDEX%s%s FOR (n:%s) ON (%s);", 
				indexType, idxName, getIfNotExists(ifNotExists), Util.quote(label), getPropertiesQuoted(keys, "n."));
	}
	
	@Override
	public String statementForIndexRelationship(String indexType, String type, Iterable keys, boolean ifNotExists, String idxName) {
		return String.format("CREATE %s INDEX%s%s FOR ()-[rel:%s]-() ON (%s);", 
				indexType, idxName, getIfNotExists(ifNotExists), Util.quote(type), getPropertiesQuoted(keys, "rel."));
	}

	@Override
	public String statementForNodeFullTextIndex(String name, Iterable labels, Iterable keys) {
		String label = StreamSupport.stream(labels.spliterator(), false)
				.map(Label::name)
				.map(Util::quote)
				.collect(Collectors.joining("|"));
		String key = StreamSupport.stream(keys.spliterator(), false)
				.map(s -> String.format(PROPERTY_QUOTING_FORMAT, "n", s))
				.collect(Collectors.joining(","));
		return String.format(STATEMENT_NODE_FULLTEXT_IDX, name, label, key);
	}

	@Override
	public String statementForRelationshipFullTextIndex(String name, Iterable types, Iterable keys) {
		String type = StreamSupport.stream(types.spliterator(), false)
				.map(RelationshipType::name)
				.map(Util::quote)
				.collect(Collectors.joining("|"));
		String key = StreamSupport.stream(keys.spliterator(), false)
				.map(s -> String.format(PROPERTY_QUOTING_FORMAT, "rel", s))
				.collect(Collectors.joining(","));
		return String.format(STATEMENT_REL_FULLTEXT_IDX, name, type, key);
	}

	@Override
	public String statementForCreateConstraint(String name, String label, Iterable keys, boolean ifNotExists) {
		String keysString = getPropertiesQuoted(keys, "node.");

		return String.format(STATEMENT_CONSTRAINTS, Util.quote(name), getIfNotExists(ifNotExists), Util.quote(label), keysString, Iterables.count(keys) > 1 ? "IS NODE KEY" : "IS UNIQUE");
	}

	@Override
	public String statementForDropConstraint(String name) {
		return String.format(STATEMENT_DROP_CONSTRAINTS, Util.quote(name));
	}

	private String getIfNotExists(boolean ifNotExists) {
		return ifNotExists ? " IF NOT EXISTS" : "";
	}

	private String getPropertiesQuoted(Iterable keys, String prefix) {
		String keysString = StreamSupport.stream(keys.spliterator(), false)
				.map(key -> prefix + Util.quote(key))
				.collect(Collectors.joining(", "));
		return keysString;
	}

	protected String mergeStatementForNode(CypherFormat cypherFormat, Node node, Map> uniqueConstraints, Set indexedProperties, Set indexNames) {
		StringBuilder result = new StringBuilder(1000);
		result.append("MERGE ");
		result.append(CypherFormatterUtils.formatNodeLookup("n", node, uniqueConstraints, indexNames));
		String notUniqueProperties = CypherFormatterUtils.formatNotUniqueProperties("n", node, uniqueConstraints, indexedProperties, false);
		String notUniqueLabels = CypherFormatterUtils.formatNotUniqueLabels("n", node, uniqueConstraints);
		if (!notUniqueProperties.isEmpty() || !notUniqueLabels.isEmpty()) {
			result.append(cypherFormat.equals(CypherFormat.ADD_STRUCTURE) ? " ON CREATE SET " : " SET ");
			result.append(notUniqueProperties);
			result.append(!"".equals(notUniqueProperties) && !"".equals(notUniqueLabels) ? ", " : "");
			result.append(notUniqueLabels);
		}
		result.append(";");
		return result.toString();
	}

	public String mergeStatementForRelationship(CypherFormat cypherFormat, Relationship relationship, Map> uniqueConstraints, Set indexedProperties, ExportConfig exportConfig) {
		StringBuilder result = new StringBuilder(1000);
		result.append("MATCH ");
		final Node startNode = relationship.getStartNode();
		result.append(CypherFormatterUtils.formatNodeLookup("n1", startNode, uniqueConstraints, indexedProperties));
		result.append(", ");
		final Node endNode = relationship.getEndNode();
		result.append(CypherFormatterUtils.formatNodeLookup("n2", endNode, uniqueConstraints, indexedProperties));
		final RelationshipType type = relationship.getType();
		final boolean withMultiRels = exportConfig.isMultipleRelationshipsWithType() &&
				StreamSupport.stream(startNode.getRelationships(Direction.OUTGOING, type).spliterator(), false)
						.anyMatch(r -> !r.equals(relationship) && r.getEndNode().equals(endNode));

		String mergeUniqueKey = withMultiRels
				? simpleKeyValue(Q_UNIQUE_ID_REL, relationship.getId())
				: "";
		result.append(" MERGE (n1)-[r:" + Util.quote(type.name()) + mergeUniqueKey + "]->(n2)");
		if (relationship.getPropertyKeys().iterator().hasNext()) {
			result.append(cypherFormat.equals(CypherFormat.UPDATE_STRUCTURE) ? " ON CREATE SET " : " SET ");
			result.append(CypherFormatterUtils.formatRelationshipProperties("r", relationship, false));
		}
		result.append(";");
		return result.toString();
	}

	public void buildStatementForNodes(String nodeClause, String setClause,
									   Iterable nodes, Map> uniqueConstraints,
									   ExportConfig exportConfig,
									   PrintWriter out, Reporter reporter,
									   GraphDatabaseService db) {
		AtomicInteger nodeCount = new AtomicInteger(0);
		Function, Set>> keyMapper = (node) -> {
			try (Transaction tx = db.beginTx()) {
				node = tx.getNodeByElementId(node.getElementId());
				Set idProperties = CypherFormatterUtils.getNodeIdProperties(node, uniqueConstraints).keySet();
				Set labels = getLabels(node);
				tx.commit();
				return new AbstractMap.SimpleImmutableEntry<>(labels, idProperties);
			}
		};
		Map, Set>, List> groupedData = StreamSupport.stream(nodes.spliterator(), true)
				.collect(Collectors.groupingByConcurrent(keyMapper));

		AtomicInteger propertiesCount = new AtomicInteger(0);

		AtomicInteger batchCount = new AtomicInteger(0);
		groupedData.forEach((key, nodeList) -> {
			AtomicInteger unwindCount = new AtomicInteger(0);
			final int nodeListSize = nodeList.size();
			final Node last = nodeList.get(nodeListSize - 1);
			nodeCount.addAndGet(nodeListSize);
			for (int index = 0; index < nodeList.size(); index++) {
				Node node = nodeList.get(index);
				writeBatchBegin(exportConfig, out, batchCount);
				writeUnwindStart(exportConfig, out, unwindCount);
				batchCount.incrementAndGet();
				unwindCount.incrementAndGet();
				Map props = node.getAllProperties();
				// start element
				out.append("{");

				// id
				Map idMap = CypherFormatterUtils.getNodeIdProperties(node, uniqueConstraints);
				writeNodeIds(out, idMap);

				// properties
				out.append(", ");
				out.append("properties:");

				propertiesCount.addAndGet(props.size());
				props.keySet().removeAll(idMap.keySet());
				writeProperties(out, props);

				// end element
				out.append("}");
				if (last.equals(node) || isBatchMatch(exportConfig, batchCount) || isUnwindBatchMatch(exportConfig, unwindCount)) {
					closeUnwindNodes(nodeClause, setClause, uniqueConstraints, exportConfig, out, key, last);
					writeBatchEnd(exportConfig, out, batchCount);
					unwindCount.set(0);
				} else {
					out.append(", ");
				}
			}
		});
		addCommitToEnd(exportConfig, out, batchCount);

		reporter.update(nodeCount.get(), 0, propertiesCount.longValue());
	}

	private void closeUnwindNodes(String nodeClause, String setClause, Map> uniqueConstraints, ExportConfig exportConfig, PrintWriter out, Map.Entry, Set> key, Node last) {
		writeUnwindEnd(exportConfig, out);
		out.append(StringUtils.LF);
		out.append(nodeClause);

		String label = getUniqueConstrainedLabel(last, uniqueConstraints);
		out.append("(n:");
		out.append(Util.quote(label));
		out.append("{");
		writeSetProperties(out, key.getValue());
		out.append("}) ");
		out.append(setClause);
		out.append("n += row.properties");
		String addLabels = key.getKey().stream()
				.filter(l -> !l.equals(label))
				.map(Util::quote)
				.collect(Collectors.joining(":"));
		if (!addLabels.isEmpty()) {
			out.append(" SET n:");
			out.append(addLabels);
		}
		out.append(";");
		out.append(StringUtils.LF);
	}

	private void writeSetProperties(PrintWriter out, Set value) {
		writeSetProperties(out, value, null);
	}

	private void writeSetProperties(PrintWriter out, Set value, String prefix) {
		if (prefix == null) prefix = "";
		int size = value.size();
		for (String s: value) {
			--size;
			out.append(Util.quote(s) + ": row." + prefix + formatNodeId(s));
			if (size > 0) {
				out.append(", ");
			}
		}
	}

	private boolean isBatchMatch(ExportConfig exportConfig, AtomicInteger batchCount) {
		return batchCount.get() % exportConfig.getBatchSize() == 0;
	}

	public void buildStatementForRelationships(String relationshipClause,
											   String setClause, Iterable relationship,
											   Map> uniqueConstraints, ExportConfig exportConfig,
											   PrintWriter out, Reporter reporter,
											   GraphDatabaseService db) {
		AtomicInteger relCount = new AtomicInteger(0);

		Function> keyMapper = (rel) -> {
			try (Transaction tx = db.beginTx()) {
				rel = tx.getRelationshipByElementId(rel.getElementId());
				Node start = rel.getStartNode();
				Set startLabels = getLabels(start);

				// define the end labels
				Node end = rel.getEndNode();
				Set endLabels = getLabels(end);

				// define the type
				String type = rel.getType().name();

				// create the path
				Map key = Util.map("type", type,
						"start", new AbstractMap.SimpleImmutableEntry<>(startLabels, CypherFormatterUtils.getNodeIdProperties(start, uniqueConstraints).keySet()),
						"end", new AbstractMap.SimpleImmutableEntry<>(endLabels, CypherFormatterUtils.getNodeIdProperties(end, uniqueConstraints).keySet()));

				tx.commit();
				return key;
			}
		};
		Map, List> groupedData = StreamSupport.stream(relationship.spliterator(), true)
				.collect(Collectors.groupingByConcurrent(keyMapper));

		AtomicInteger propertiesCount = new AtomicInteger(0);
		AtomicInteger batchCount = new AtomicInteger(0);

		String start = "start";
		String end = "end";
		groupedData.forEach((path, relationshipList) -> {
			AtomicInteger unwindCount = new AtomicInteger(0);
			final int relSize = relationshipList.size();
			relCount.addAndGet(relSize);
			final Relationship last = relationshipList.get(relSize - 1);
			for (int index = 0; index < relationshipList.size(); index++) {
				Relationship rel = relationshipList.get(index);
				writeBatchBegin(exportConfig, out, batchCount);
				writeUnwindStart(exportConfig, out, unwindCount);
				batchCount.incrementAndGet();
				unwindCount.incrementAndGet();
				Map props = rel.getAllProperties();
				// start element
				out.append("{");

				// start node
				Node startNode = rel.getStartNode();
				writeRelationshipNodeIds(uniqueConstraints, out, start, startNode);

				Node endNode = rel.getEndNode();
				final boolean withMultipleRels = exportConfig.isMultipleRelationshipsWithType() && 
						relationshipList.stream()
								.anyMatch(r -> !r.equals(rel) && r.getEndNode().equals(endNode) && r.getStartNode().equals(startNode));
				out.append(", ");
				if (withMultipleRels) {
					String uniqueId = String.format("%s: %s, ", ID_REL_KEY, rel.getId());
					out.append(uniqueId);
				}

				// end node
				writeRelationshipNodeIds(uniqueConstraints, out, end, endNode);

				// properties
				out.append(", ");
				out.append("properties:");
				writeProperties(out, props);
				propertiesCount.addAndGet(props.size());

				// end element
				out.append("}");

				if (last.equals(rel) || isBatchMatch(exportConfig, batchCount) || isUnwindBatchMatch(exportConfig, unwindCount)) {
					closeUnwindRelationships(relationshipClause, setClause, uniqueConstraints, exportConfig, out, start, end, path, last, withMultipleRels);
					writeBatchEnd(exportConfig, out, batchCount);
					unwindCount.set(0);
				} else {
					out.append(", ");
				}
			}
		});
		addCommitToEnd(exportConfig, out, batchCount);

		reporter.update(0, relCount.get(), propertiesCount.longValue());
	}

	private void closeUnwindRelationships(String relationshipClause, String setClause, Map> uniqueConstraints, ExportConfig exportConfig, PrintWriter out, String start, String end, Map path, Relationship last, boolean withMultipleRels) {
		writeUnwindEnd(exportConfig, out);
		// match start node
		writeRelationshipMatchAsciiNode(last.getStartNode(), out, start, uniqueConstraints);

		// match end node
		writeRelationshipMatchAsciiNode(last.getEndNode(), out, end, uniqueConstraints);

		out.append(StringUtils.LF);

		// create the relationship (depends on the strategy)
		out.append(relationshipClause);
		String mergeUniqueKey = withMultipleRels
				? simpleKeyValue(Q_UNIQUE_ID_REL, "row." + ID_REL_KEY)
				: "";
		out.append("(start)-[r:" + Util.quote(path.get("type").toString()) + mergeUniqueKey + "]->(end) ");
		out.append(setClause);
		out.append("r += row.properties;");
		out.append(StringUtils.LF);
	}

	private boolean isUnwindBatchMatch(ExportConfig exportConfig, AtomicInteger batchCount) {
		return batchCount.get() % exportConfig.getUnwindBatchSize() == 0;
	}

	private void writeBatchEnd(ExportConfig exportConfig, PrintWriter out, AtomicInteger batchCount) {
		if (isBatchMatch(exportConfig, batchCount)) {
			out.append(exportConfig.getFormat().commit());
		}
	}

	public void writeProperties(PrintWriter out, Map props) {
		out.append("{");
		if (!props.isEmpty()) {
			int size = props.size();
			for (Map.Entry es : props.entrySet()) {
				--size;
				out.append(Util.quote(es.getKey()));
				out.append(":");
				out.append(CypherFormatterUtils.toString(es.getValue()));
				if (size > 0) {
					out.append(", ");
				}
			}
		}
		out.append("}");
	}

	private String formatNodeId(String key) {
		if (CypherFormatterUtils.UNIQUE_ID_PROP.equals(key)) {
			key = "_id";
		}
		return Util.quote(key);
	}

	private void addCommitToEnd(ExportConfig exportConfig, PrintWriter out, AtomicInteger batchCount) {
		if (batchCount.get() % exportConfig.getBatchSize() != 0) {
			out.append(exportConfig.getFormat().commit());
		}
	}

	private void writeBatchBegin(ExportConfig exportConfig, PrintWriter out, AtomicInteger batchCount) {
		if (isBatchMatch(exportConfig, batchCount)) {
			out.append(exportConfig.getFormat().begin());
		}
	}

	private void writeUnwindStart(ExportConfig exportConfig, PrintWriter out, AtomicInteger batchCount) {
		if (isUnwindBatchMatch(exportConfig, batchCount)) {
			String start = (exportConfig.getFormat() == ExportFormat.CYPHER_SHELL
					&& exportConfig.getOptimizationType() == ExportConfig.OptimizationType.UNWIND_BATCH_PARAMS) ?
					":param rows => [" : "UNWIND [";
			out.append(start);
		}
	}

	private void writeUnwindEnd(ExportConfig exportConfig, PrintWriter out) {
		out.append("]");
		if (exportConfig.getFormat() == ExportFormat.CYPHER_SHELL
				&& exportConfig.getOptimizationType() == ExportConfig.OptimizationType.UNWIND_BATCH_PARAMS) {
			out.append(StringUtils.LF);
			out.append("UNWIND $rows");
		}
		out.append(" AS row");
	}

	private String getUniqueConstrainedLabel(Node node, Map> uniqueConstraints) {
		return uniqueConstraints.entrySet().stream()
				.filter(e -> node.hasLabel(Label.label(e.getKey())) && e.getValue().stream().anyMatch(k -> node.hasProperty(k)))
				.map(e -> e.getKey())
				.findFirst()
				.orElse(CypherFormatterUtils.UNIQUE_ID_LABEL);
	}

	private Set getUniqueConstrainedProperties(Map> uniqueConstraints, String uniqueConstrainedLabel) {
		Set props = uniqueConstraints.get(uniqueConstrainedLabel);
		if (props == null || props.isEmpty()) {
			props = Collections.singleton(UNIQUE_ID_PROP);
		}
		return props;
	}

	private Set getLabels(Node node) {
		Set labels = StreamSupport.stream(node.getLabels().spliterator(), false)
				.map(Label::name)
				.collect(Collectors.toSet());
		if (labels.isEmpty()) {
			labels.add(CypherFormatterUtils.UNIQUE_ID_LABEL);
		}
		return labels;
	}

	private void writeRelationshipMatchAsciiNode(Node node, PrintWriter out, String key, Map> uniqueConstraints) {
		String uniqueConstrainedLabel = getUniqueConstrainedLabel(node, uniqueConstraints);
		Set uniqueConstrainedProps = getUniqueConstrainedProperties(uniqueConstraints, uniqueConstrainedLabel);

		out.append(StringUtils.LF);
		out.append("MATCH ");
		out.append("(");
		out.append(key);
		out.append(":");
		out.append(Util.quote(uniqueConstrainedLabel));
		out.append("{");
		writeSetProperties(out, uniqueConstrainedProps, key + ".");
		out.append("})");
	}

	private void writeRelationshipNodeIds(Map> uniqueConstraints, PrintWriter out, String key, Node node) {
		String uniqueConstrainedLabel = getUniqueConstrainedLabel(node, uniqueConstraints);
		Set props = getUniqueConstrainedProperties(uniqueConstraints, uniqueConstrainedLabel);
		Map properties;
		if (!props.contains(UNIQUE_ID_PROP)) {
			String[] propsArray = props.toArray(new String[props.size()]);
			properties = node.getProperties(propsArray);
		} else {
			// UNIQUE_ID_PROP is always the only member of the Set
			properties = Util.map(UNIQUE_ID_PROP, node.getId());
		}

		out.append(key + ": ");
		out.append("{");
		writeNodeIds(out, properties);
		out.append("}");
	}

	private void writeNodeIds(PrintWriter out, Map properties) {
		int size = properties.size();
		for (Map.Entry es : properties.entrySet()) {
			--size;
			out.append(formatNodeId(es.getKey()));
			out.append(":");
			out.append(CypherFormatterUtils.toString(es.getValue()));
			if (size > 0) {
				out.append(", ");
			}
		}
	}
}