com.pushtechnology.diffusion.client.features.control.topics.views.TopicViews Maven / Gradle / Ivy
/*******************************************************************************
* Copyright (c) 2023 DiffusionData Ltd., All Rights Reserved.
*
* Use is subject to license terms.
*
* NOTICE: All information contained herein is, and remains the
* property of Push Technology. The intellectual and technical
* concepts contained herein are proprietary to Push Technology and
* may be covered by U.S. and Foreign Patents, patents in process, and
* are protected by trade secret or copyright law.
*******************************************************************************/
package com.pushtechnology.diffusion.client.features.control.topics.views;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import com.pushtechnology.diffusion.client.features.ClusterRoutingException;
import com.pushtechnology.diffusion.client.features.ScriptException;
import com.pushtechnology.diffusion.client.features.Topics;
import com.pushtechnology.diffusion.client.features.control.RemoteServers;
import com.pushtechnology.diffusion.client.session.Feature;
import com.pushtechnology.diffusion.client.session.PermissionsException;
import com.pushtechnology.diffusion.client.session.Session;
import com.pushtechnology.diffusion.client.session.SessionClosedException;
import com.pushtechnology.diffusion.client.topics.TopicSelectors;
import com.pushtechnology.diffusion.client.topics.details.TopicSpecification;
import com.pushtechnology.diffusion.client.topics.details.TopicType;
import com.pushtechnology.diffusion.client.types.GlobalPermission;
import com.pushtechnology.diffusion.client.types.PathPermission;
/**
* This feature allows a client session to manage topic views.
*
*
* A topic view maps one part of a server's topic tree to another. It
* dynamically creates a set of reference topics from a set of
* source topics, based on a declarative topic view
* specification. The capabilities of topic views range from simple
* mirroring of topics within the topic tree to advanced capabilities including
* publication of partial values, expanding a single topic value into many
* topics, changing topic values, inserting values from other topics, throttling
* the rate of publication, and applying a fixed delay to the publication.
*
*
* A topic view can also map topics from another server (in a different
* cluster). This capability is referred to as 'remote topic views'. The view
* can specify the server that the source topics are hosted on in terms
* of a remote server (see {@link RemoteServers} for details of how to
* create and maintain remote servers).
*
*
* Each reference topic has a single source topic and has the same topic type as
* its source topic. Reference topics are read-only (they cannot be updated),
* nor can they be created or removed directly. Otherwise, they behave just like
* standard topics. A client session can subscribe to a reference topic, and can
* fetch the reference topic's current value if it has one.
*
*
* The source topics of a topic view are defined by a topic selector. One or
* more reference topics are created for each source topic, according to the
* topic view. If a source topic is removed, reference topics that are derived
* from it will automatically be removed. If a topic is added that matches the
* source topic selector of a topic view, corresponding reference topics will be
* created. Removing a topic view will remove all of its reference topics.
*
*
Topic view specifications
*
*
* The following is a simple topic view specification that mirrors all topics
* below the path a to reference topics below the path
* b.
*
*
* map ?a// to b/<path(1)>
*
*
*
* A topic view with this specification will map a source topic at the path
* a/x/y/z to a reference topic at the path b/x/y/z.
* The specification is simple, so the reference topic will exactly mirror the
* source topic.
*
*
* A topic view specification comprises three main parts:
*
* - The mapping part which specifies the source topics to map from
* and the mappings to target reference topics.
*
- Optional transformations which transform the topic value in some
* way.
*
- Optional options which specify other changes that the view may
* apply.
*
*
* Mapping comprises:
*
* - The source topic clause identifying the source topics that the
* view can apply to.
*
- The optional from clause which may identify a remote server that
* hosts the source topics.
*
- The path mapping clause which determines how reference topic
* paths are derived from the source topic paths, and when expanding to more
* than one reference topic, from where the values are obtained.
*
* Transformations can be:
*
* - patch transformation(s) specifying that a JSON patch is applied
* to the reference topic value.
*
- process transformations that allow conditional processing and/or
* calculations to be applied to the reference topic value.
*
- insert transformation(s) specifying that values from other
* topics are inserted into the reference topic value.
*
* Options can be:
*
* - The topic property mapping clause determines how reference topic
* properties are derived from source topic properties.
*
- The value mapping clause determines how reference topic values
* are derived from source topic or expanded values.
*
- The throttle clause constrains the rate at which each reference
* topic is updated when its source topic is updated.
*
- The delay by clause causes a change to a view's source topic to
* be delayed by a fixed time before it is reflected in reference topics.
*
- The separator clause can define a replacement path separator for
* values extracted using the scalar or expand directives.
*
- The type clause can specify that the reference topic that is
* created is of a different {@link TopicType} from the selected source topic.
*
*
* Mapping
* Source topic clause
*
* The source topic clause begins with the {@code map} keyword and is followed
* by a topic selector. These topic selectors follow the same
* {@link TopicSelectors#parse parsing rules} as other topic selectors.
*
*
* When evaluating a topic view, all topics in the topic tree that match the
* source topic selector are considered (excluding {@link TopicType#ROUTING
* ROUTING} topics). However, if a view specification uses some feature that can
* only be applied to JSON topics then only JSON topics will be selected.
*
*
* Reference topics are valid source topics. In particular, chaining of topic
* views is supported; that is, a reference topic created by one topic view can
* be the source topic of another topic view. Additionally, a reference topic
* can be the source topic of a routing topic subscription.
*
*
From clause
*
* The 'from' clause optionally follows the source topic clause. It begins with
* the {@code from} keyword and is followed by a remote server name. The name
* refers to a remote server created using the {@link RemoteServers} feature.
*
*
* The presence of the clause indicates that the source topics will be selected
* from the specified server and not from the local server.
*
*
* Further details regarding the processing of remote topic views are given
* below.
*
*
Path mapping clause
*
* The paths of reference topics are derived from the source topic according to
* the path mapping clause. The path mapping allows the source topic path and
* the value of the source topic to determine the path of the reference topic.
* In addition the path mapping can include expand directives which
* allow objects and arrays in JSON source topic values to be expanded to
* produce many reference topics.
*
*
* A path mapping clause begins with the {@code to} keyword and is followed by a
* path mapping template. A path mapping template is a topic path with embedded
* directives. Directives are evaluated when creating the topic
* reference and substituted into the topic path. Directives are delimited by
* angle brackets ({@code <}, {@code >}) and consist of the name of the
* directive and a list of parameters. The parameter list is comma-separated and
* surrounded by parentheses ({@code (}, {@code )}).
*
*
* The following path mapping directives are supported:
*
*
* - Source path directives
* - Source path directives extract a portion of the source path and are
* parameterized by the index of the start part of the source path and the
* number of parts to include. The number of parts parameter is optional – if it
* is missing, the selection extends to the end of the source path. The syntax
* is
<path(start, number)>, or
* <path(start)> when the number of parts parameter
* is omitted.
*
* For example, given the source path {@code a/b/c/d}, the source path directive
* {@code } is mapped to the reference topic path {@code b/c}, and
* the source path directive {@code } is mapped to the reference topic
* path {@code c/d}.
*
* - Source value ("scalar") directives
* - Source value directives are only applied to {@link TopicType#JSON JSON}
* source topics or {@link TopicType#TIME_SERIES TIME_SERIES} source topics with
* a {@code JSON} event type; if the path mapping contains a source value
* directive, topics with other topic types matching the source topic selector
* are ignored.
*
* Source value directives use the keyword {@code scalar} and are parameterized
* by a single JSON pointer
* that extracts a scalar value from the source (or current) value. A scalar
* value is a string, a number, {@code true}, {@code false}, or {@code null},
* that is, anything other than an array or a object. If the JSON pointer does
* not refer to a scalar value in the source (or current) value, no reference
* topic will be created. This includes cases where the JSON pointer refers to
* an array or an object), or when no part of the source value is selected.
*
* Deriving the reference topic paths from part of the source topic value
* effectively creates a secondary index on the value. For source value
* directives to work efficiently, the selected scalar values should be
* relatively stable. If an update to the source topic changes the selected
* scalar value, the corresponding reference topic will be removed and a new
* reference topic will be created.
*
* For example, given a source value of
*
*
* {
* "account" : "1234",
* "balance" : { "amount" : 12.57, "currency" : "USD" }
* }
*
*
* and the source value directive
* {@code currency//account/}, the
* reference topic path will be {@code currency/USD/account/1234}.
*
*
* If the extracted value is a string, it is copied literally to the reference
* topic path. A value that contains path separators ({@code /}) will create a
* reference topic path with more levels than the path mapping template. Use the
* separator directive to replace path separators with an alternative string.
*
*
* An extracted value of {@code null} will be copied to the reference topic path
* as the string {@code "null"}.
*
* - Expand value directives
*
* - Expand value directives are only applied to {@link TopicType#JSON JSON}
* source topics; if the path mapping contains an expand value directive,
* non-JSON topics matching the source topic selector are ignored.
*
* Expand value directives use the keyword {@code expand} and are parameterized
* by one or two JSON
* pointers.
*
* The first pointer indicates the element within the value to be expanded, and
* if omitted, the value is expanded from the root. Expansion of a source topic
* indicates that every direct child of the element pointed to by the expand
* pointer will be used to create a new reference topic (or provide input to
* later expand or scalar directives). For example {@code } would
* expand every child item in the source value and {@code }
* would expand every child of the {@code account} value in the source value.
* The specified value could be an object, an array or even a scalar value, but
* a scalar value would expand to only a single new value.
*
* The optional second parameter of the expand directive specifies a pointer to
* a scalar value within the expanded value which will be used to derive the
* path fragment of the reference topic path. If the second pointer is not
* specified or no scalar value is found for the pointer, the path fragment is
* taken from the key (if the child value is an object) or the index (if the
* child value is an array). Scalar child values will expand to a reference
* topic but will not add anything to the generated path. For example
* {@code } would expand from the root of the source value and
* each child value path fragment would be obtained from the scalar value with
* the key {@code name}.
*
* So if a source topic had a value of
*
*
* {
* "values": [1, 5, 7]
* }
*
*
* a path mapping of {@code value} would expand the value to
* the following reference topics:-
*
* path {@code value0} with a value of {@code 1}
* path {@code value1} with a value of {@code 5}
* path {@code value2} with a value of {@code 7}
*
* Expand directives can be nested (i.e. there can be more than one expand
* directive in a path mapping). In this case a second expand directive will use
* the value from the previous expand as its source (root) value and not the
* value of the source topic. This also applies to scalar directives that follow
* an expand directive.
*
* If expansion causes more than one mapping to the same topic path, only the
* first encountered will be created and updated.
*
* Expanding source topic values effectively creates secondary indices on the
* value. For expanded value directives to work efficiently, the value selected
* for expansion should be relatively stable in terms of the children it
* contains. If an update to the source topic changes the children of the
* expanded value, then corresponding reference topics will be removed and
* created. Updates should generally be limited to changing values within the
* expanded values.
*
*
* Transformations
*
* Transformations are specified after the mapping and before any options.
* Transformations can only be applied to JSON topics.
*
* Transformations are applied to the value extracted from the source topic in
* the order specified. There can be any number of transformations interspersed
* with one another and the value from one will be that which is input to the
* next. The only restriction is that all insert transformations must
* occur at the end, after any others.
*
* A transformation is applied to the current value within a view processing
* chain, so if a transformation occurs after an expand then it will be applied
* to each expanded value.
*
*
Process transformations
*
* Process transformations may be used to apply conditional processing to a
* value (optionally determining whether a reference topic is created) and/or
* change the value in some way (for example, by applying some calculation to a
* field within the value).
*
* The format of a process transformation is:-
*
*
* process {statement}
*
*
* Where the statement can be:-
*
* - Operation(s).
* One or more operations separated by ';'.
* - A conditional statement.
* Comprising one or more conditions with operations to perform if they are
* satisfied.
*
*
* For example, the following topic view specification could be used to write a
* field into the value of the reference topic :
*
*
* map ?a// to b/<path(1)> process {set(/Name, 'John')};
*
*
* The following example shows a simple conditional statement which would only
* generate reference topics if the value of field /Price was
* greater than 50 :
*
*
* map ?a// to b/<path(1)> process {if '/Price gt 50' continue};
*
*
* And the following shows a more complex statement which would set a field
* according to the value of the input field /Price :
*
*
* map ?a// to b/<path(1)> process {if '/Price lt 50' set(/Tier, 1) elseif '/Price gt 50' set(/Tier, 2)};
*
*
*
* - Process operations
* -
*
* The following operations are supported :
*
*
* Operation
* Description
*
*
*
* set(pointer,
* value)
* Sets the field indicated by the JSON
* pointer to an absolute value. If the field does not exist it
* will be created. The value can be an integer (e.g. 123), a String
* (e.g. "XYZ"), or a boolean (e.g. true or false).
*
* For example :
*
*
* set(/Name, "John")
*
*
* If a hierarchic pointer is specified, the parent object or array must
* exist.
*
*
* set(pointer, calc
* calculation)
* Sets the field indicated by the JSON
* pointer to a value which is the result of the specified
* calculation. If the field does not exist it will be created. The
* calculation can include fields within the input value. The calculation is
* specified as a quoted string. See below for a detailed description of
* calculations.
*
* For example :
*
*
* set(/DoubleValue, calc "/Value * 2")
*
*
* If a hierarchic pointer is specified, the parent object or array must
* exist.
*
*
* remove(pointer)
* Removes the json item at the specified
* pointer. Unlike JSON patch, if the item does not exist the operation
* does not fail.
*
*
* continue
* This is a special operation that indicates that
* the topic view evaluation should continue with the value as it is. This is
* only for use with conditional statements as the default behavior of a
* conditional statement is not to proceed if no condition is satisifed.
*
*
* Operations can be chained by separating them with a ';' as shown in the
* example below:-
*
*
* set(/Amount, calc "/Value * /Number"); remove(/Value); remove (/Number)
*
*
* In this case the operations are all performed on the original value, creating
* a chain of deltas which are only applied to the original value at the end. If
* the 'set' operation fails no reference topic would be generated, however,
* processing will continue if the fields specified in the 'remove's are not
* present.
*
*
*
* - Process calculations
* -
*
* A calculation may be specified as the value of a set operation. A
* calculation is a simple arithmetic calculation upon integer fields. If
* applied to a non integer field the evaluation will not proceed. Floating
* point calculations are not supported.
*
* Arithmetic operators supported are +, - ,
* * and /.
*
* Examples of calculations are:-
*
*
* set(/Value, calc "/Value * 2")
* set(/Result, calc "/Value / 2")
* set(/Bonus, calc "/Salary + 1000")
* set(/Bonus, calc "/Salary + 1000 + /Age * 10")
*
*
* Standard operator precedence is applied, so in the last example above we have
* (/Salary + 1000 + (/Age * 10)) not
* ((/Salary + 1000 + /Age) * 10). Brackets may be used to override
* this.
*
*
* - Process conditional statements
* -
*
* A conditional statement is made up of an if clause, optionally
* followed by one or more elseif clauses and an optional final else
* clause.
*
* The if clause takes the form :
*
*
* if condition operation(s)
*
*
* Where the condition is a quoted string as described in detail below
* and the operation(s) is as described previously. If the condition is
* satisfied, the operations are applied to the value and the process is
* complete. If the condition is not satisfied, processing moves on to any
* elseif or else clauses that follow, but if there are none, the
* topic view evaluation does not proceed and no reference topic is created.
*
* An elseif (else if) clause takes the form :
*
*
* elseif condition operation(s)
*
*
* If the condition is satisfied, the operations are applied to the value and
* the process is complete. If the condition is not satisfied, processing moves
* on to any elseif or else clauses that follow, but if there are
* none, the topic view evaluation does not proceed and no reference topic is
* created. Note that elseif can be abbreviated to elsf.
*
* An else clause takes the form :
*
*
* else operation(s)
*
*
* And will only be reached if no previous if or elseif conditions
* were satisfied. If reached then the operations are applied to the value and
* the topic view evaluation proceeds. The continue operation may be used
* to proceed with an unchanged value.
*
*
* - Process conditions
* -
*
* A condition is of the form:
*
*
* pointer operator constant
*
*
* Where pointer is a JSON pointer, operator is a relational
* operator and constant is a string, integer, or boolean value.
*
* for example:-
*
*
* /Age > 40
* /Name = "Bill"
* /Manager eq true
*
*
* Operators allowed are:
*
*
* Operator
* Variant
* Description
* Supported JSON types
*
*
*
* =
* eq
* Equals
* All
*
*
* >
* gt
* Greater than
* Integer only
*
*
* <
* lt
* Less than
* Integer only
*
*
* !=
* ne
* Not equals
* All
*
*
* >=
* ge
* Greater than or equal
* Integer only
*
*
* <=
* le
* Less than or equal
* Integer only
*
*
*
* Compound conditions are supported by means of boolean operators:
*
* | or or
* & or and
*
* For example:
*
*
* /Age = 50 or /Age > 80
* /Age gt 50 & /Department eq "Accounts"
*
*
* Normal boolean precedence applies but brackets can be used to control
* precedence. For example:
*
*
* (/Age > 50 or /Department eq "Accounts") and /Band > 3
*
*
* Boolean 'not' is also allowed :
*
*
* not (/Age < 65 or /Retired eq false)
*
*
*
*
*
* Patch transformations
*
* Patch transformations indicate that a JSON patch is to be applied to the
* value.
*
* The format of a patch transformation is
*
*
* patch 'patch string'
*
*
* The patch string should be formatted according to the JSON Patch standard
* (see RFC 6902: JavaScript
* Object Notation (JSON) Patch).
*
* Patches are a sequence of JSON Patch operations contained in an array. They
* are applied as an atomic update to the previous value if the resulting update
* is successfully calculated. The following patch will check the value at a
* specific key and update if the expected value is correct:
*
* [{"op":"test", "path":"/price", "value" : 22}, {"op":"add", "path":"/price",
* "value": 23}]
*
* The available operations are:
*
*
* - Add: {@code {"op": "add", "path": "/a/b/c", "value": [
* "foo", "bar" ]}}
*
- Remove: {@code {"op": "remove", "path": "/a/b/c"}}
*
- Replace: {@code {"op": "replace", "path": "/a/b/c",
* "value": 43}}
*
- Move: {@code {"op": "move", "from": "/a/b/c", "path":
* "/a/b/d"}}
*
- Copy: {@code {"op": "copy", "from": "/a/b/c", "path":
* "/a/b/e"}}
*
- Test: {@code {"op": "test", "path": "/a/b/c", "value":
* "foo"}}
*
*
*
* The test operation checks that the CBOR representation of the value of a
* topic is identical to the value provided in the patch after converting it to
* CBOR. If the value is represented differently as CBOR, commonly due to
* different key ordering, then the patch will return the index of the failed
* operation. e.g the values {@code {"foo": "bar", "count": 43}} and
* {@code {"count": 43, "foo": "bar"}} are unequal despite semantic equality due
* to the differences in a byte for byte comparison.
*
* The following patch clause would add the 'price' field and remove the 'name'
* field from an input JSON object.
*
*
* patch '[{"op":"add", "path":"/price", "value" : 22}, {"op":"remove", "path":"/name"}]'
*
*
* Patches can only be applied to JSON arrays or objects and if they fail to
* apply, no resulting reference topic will be created by the view. If an update
* patch fails, any previously created reference topic would be removed.
*
* Insert transformations
*
* Insert transformations are used to insert a value from another topic into the
* current value.
*
* The name of the topic to insert from can be specified in a similar way to the
* path mapping in that constants, path directives, and scalar directives (but
* not expand directives) may be used.
*
* The value from the insertion topic (if found) is inserted into the 'current'
* value at a specified key position. The current value may be the source topic
* value, the value output from expand directives (in which case the insertion
* applies to each value), or the value from a previous transformation.
* Insertion topics may be {@link TopicType#JSON JSON}, {@link TopicType#STRING
* STRING}, {@link TopicType#INT64 INT64}, or {@link TopicType#DOUBLE DOUBLE}.
*
* If, when evaluating a topic view, the insertion topic is not found (or the
* specified key within it is not found), or it is of an unsupported topic type,
* an optional default value may be inserted, but if no default is specified
* then no insertion takes place and the value is passed to the derived
* reference topic unchanged.
*
* The values of the insertion topics are only taken at the point when the
* source topic is evaluated against the topic view (i.e. when the source topic
* is updated). Changes to the value of the insertion topic that occur in the
* interim are not applied to the derived reference topics.
*
* The format of an insert transformation is
*
*
* insert path [key fromKey] at insertionKey
* [default defaultValue]
*
*
* The path is specified in exactly the same way as for the path mapping
* clause, except it may not contain expand directives. path directives
* operate on the path of the source topic, whereas scalar directives
* operate on the 'current' value as defined previously.
*
* key is optional and can specify a fromKey which is a JSON
* pointer indicating the data within the insertion topic that is to be
* inserted. If no key is specified then the whole of the insertion topic value
* is inserted.
*
* at specifies the insertionKey which is a JSON pointer
* indicating where to insert the insertion topic value. If a value already
* exists at the specified key then it is replaced otherwise it is inserted.
* Insertion can only occur if the parent of the key exists in the value and is
* of a compatible type. Array pointers may only be used to replace existing
* entries or append one greater than the last entry. The special pointer value
* /- may be used to append to the end of an existing array.
*
* default is optional and may be used to specify a string
* defaultValue to be inserted if the insertion topic does not exist, it
* is of an unsupported topic type, or the specified key within it could not be
* found.
*
* The following insert transformation would cause the whole value of the topic
* named AnyTopic to be inserted into the current value at key /T, assuming that
* the current value is an object.
*
*
* insert AnyTopic at /T
*
*
* The following insert transformation would cause the whole value of the topic
* named AnyTopic to be inserted into the current value at key /T/MyKey,
* assuming that an object with the key T exists in the current value.
*
*
* insert AnyTopic at /T/MyKey
*
*
* The following insert transformation would cause the whole value of the topic
* named AnyTopic to be appended to the array at the key T in the current value.
*
*
* insert AnyTopic at /T/-
*
*
* The following insert transformation would cause the value at the key name
* within the topic named AnyTopic to be appended to the array at the key T in
* the current value.
*
*
* insert AnyTopic key /name at /T/-
*
*
* In the above examples no insertion would take place if the insertion value
* was not found, but a default value can be specified to insert into the
* current value in this case.
*
*
* insert AnyTopic at /T/- default "unknown"
*
*
* The path of the topic to insert from can be built from parts of the source
* topic path and/or scalar values within the current value. For example:
*
*
* insert AC/<path(1,1)>/<scalar(/myval)> at /T
*
*
* Options
*
* Options are specified after the mapping and any transformations. Any number
* of options may be specified but any particular option may only be specified
* once.
*
Topic property mapping
*
* The {@link TopicSpecification topic specification} of a reference topic is
* derived from the topic specification of the source topics. A reference topic
* has the same topic type as its source topic.
*
*
* The topic properties of a reference topic are derived from the source topic.
* Some topic properties can be tuned using the topic property mapping option.
* The following table describes the behavior for each topic property.
*
*
*
*
*
* Source topic property
* Reference topic specification default
* Can be set by topic property mapping?
* Notes
*
*
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#COMPRESSION
* COMPRESSION}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#CONFLATION
* CONFLATION}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#DONT_RETAIN_VALUE
* DONT_RETAIN_VALUE}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#OWNER
* OWNER}
* Not set
* No
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#PERSISTENT
* PERSISTENT}
* Not set
* No
* Reference topics are not persisted. Topic views are persisted, so a
* reference topic will be recreated on server restart if its source is
* persistent.
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#PRIORITY
* PRIORITY}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#PUBLISH_VALUES_ONLY
* PUBLISH_VALUES_ONLY}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#REMOVAL
* REMOVAL}
* Not set
* No
* Reference topics cannot be removed directly.
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#SCHEMA
* SCHEMA}
* Copied from source topic specification
* No
* A {@link TopicType#RECORD_V2 RECORD_V2} reference topic has the same
* schema as its source topic.
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#TIDY_ON_UNSUBSCRIBE
* TIDY_ON_UNSUBSCRIBE}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#TIME_SERIES_EVENT_VALUE_TYPE
* TIME_SERIES_EVENT_VALUE_TYPE}
* Copied from source topic specification
* No
* A {@link TopicType#TIME_SERIES TIME_SERIES} reference topic has the same
* value type as its source topic.
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#TIME_SERIES_RETAINED_RANGE
* TIME_SERIES_RETAINED_RANGE}
* Copied from source topic specification
* Yes, with restrictions
* A topic property mapping cannot increase the time series retained range
* by overriding the {@code TIME_SERIES_RETAINED_RANGE} property. The retained
* range of a reference time series topic will be constrained to be no greater
* than that of its source topic.
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#TIME_SERIES_SUBSCRIPTION_RANGE
* TIME_SERIES_SUBSCRIPTION_RANGE}
* Copied from source topic specification
* Yes
*
*
* {@link com.pushtechnology.diffusion.client.topics.details.TopicSpecification#VALIDATE_VALUES
* VALIDATE_VALUES}
* Not set
* No
* A reference topic reflects updates to its source topic. It cannot reject
* updates.
*
*
*
*
* A topic property option begins with the keywords {@code with properties} and
* consists of a comma-separated list of topic property keys and values, each
* separated by a colon. For example, the following topic view specification
* maps all topics below the path a to reference topics below the
* path b, and disables both conflation and compression for the
* reference topics.
*
*
* map ?a// to b/<path(1)> with properties CONFLATION:off, COMPRESSION:false
*
*
* Topic value option
*
* By default, a reference topic's value is a copy of the source topic value, or
* part of the source value produced by an expand path mapping directive and/or
* modified by transformations. For {@link TopicType#JSON JSON} source topics or
* {@link TopicType#TIME_SERIES TIME_SERIES} topics with a {@code JSON} event
* type, the value option can be applied to extract part of the resulting value
* (the latest value in the case of TIME_SERIES topics).
*
*
* A topic value option begins with the keyword {@code as} and is followed by a
* value directive. A value directive is delimited by angle brackets ({@code <},
* {@code >}), and consists of the {@code value} keywords and a single JSON
* pointer parameter. The JSON pointer selects the part of the current value to
* copy.
*
* For example, given a current value of
*
*
* {
* "account" : "1234",
* "balance" : { "amount" : 12.57, "currency" : "USD" }
* }
*
*
* and the value option {@code as }, the reference topic value
* will be
*
*
* {
* "amount" : 12.57,
* "currency" : "USD"
* }
*
*
*
* Value mappings that follow expand directives and/or transformations apply to
* the current derived value and not the source topic value.
*
*
* Topic value mappings only alter the reference topic value; only the path
* mapping determines whether a reference topic should exist. If the topic value
* mapping's JSON pointer fails to select anything from the source topic value,
* the reference topic will have the JSON value {@code null}.
*
*
* Topic value mappings are often used with path value mappings to avoid
* repeating information in the path and the value. For example:
*
*
* map ?accounts// to balances/<scalar(/account)> as <value(/balance)>
*
*
* Throttle option
*
* The throttle option can be used to constrain the rate at which a reference
* topic is updated when its source topic is updated. The primary application of
* a throttle option is to restrict the number of updates sent to reference
* topic subscribers, reducing network utilization or the processing each
* subscriber must do. Throttling also restricts the rate at which client
* sessions can observe changes to reference topic values using the fetch API.
*
*
* The throttle option has the form
* throttle to X updates every period, where
* X is a positive integer, and period is a positive integer
* followed by a time unit which is one of seconds,
* minutes, or hours.
*
*
* For example, the following topic view specification maps all topics below the
* path a to reference topics below the path b, but
* updates the value of each reference topic at most twice every five seconds:
*
*
* map ?a// to b/<path(1)> throttle to 2 updates every 5 seconds
*
*
*
* To improve readability, the throttle option allows 1 update as
* an alternative to 1 updates, and every second as an
* alternative to every 1 seconds (and so on, for other time
* units). For example, the following topic view specification maps all topics
* below the path a to reference topics below the path
* b, but updates the value of each reference topic at most once
* every hour:
*
*
* map ?a// to b/<path(1)> throttle to 1 update every minute
*
*
*
* The throttle option is only applied when a source topic is updated more
* frequently than the configured rate. If a source topic is updated less
* frequently, updates are passed on unconstrained. If the rate is exceeded, a
* reference topic will not be updated again until the configured period has
* expired. At this time, the reference topic will be updated based on the
* source topic updates that happened in the interim, and a single value will be
* published. Thus, the throttle option provides topic-scoped
* conflation.
*
*
* The throttle option is ignored for time series topics because time series
* updates do not support efficient conflation. Updates to source time series
* topics are passed on immediately to the corresponding reference topics,
* regardless of any throttle clause.
*
*
Delay option
*
* The delay option causes a change to a view's source topic to be delayed by a
* fixed time before it is reflected in reference topics. Topic additions,
* updates, and removals are all delayed. Delays can range from one second to
* many days.
*
*
* Such a publication delay is a useful way to devalue topic data so it can be
* given away to non-paying users.
*
*
* The delay option has the form delay by duration, where
* duration is a positive integer followed by a time unit which is one
* of seconds, minutes, or hours.
*
*
* For example, the following topic view specification maps all topics below the
* path a to reference topics below the path b, but
* changes to a source topic are delayed by five minutes before they are
* reflected in the corresponding reference topic.
*
*
* map ?a// to b/<path(1)> delay by 5 minutes
*
*
*
* Views with the delay option specified initially create reference topics in an
* unpublished state. The topics are published once the delay time has expired.
* A topic in the unpublished state prevents a lower priority topic view from
* creating a reference topic with the same path. Sessions with the rights to
* read the source topic can browse unpublished topics using the
* {@link Topics.FetchRequest#withUnpublishedDelayedTopics
* withUnpublishedDelayedTopics} fetch request option.
*
*
*
Separator option
*
* Views can use the scalar and expand directives in path mappings to extract
* text from the source value. By default, any {@code /} characters in the text
* are interpreted as path separators and will introduce extra levels in
* reference topic paths. If this is undesirable, the separator option can be
* used to replace any {@code /} characters produced by the scalar and expand
* directives with a constant string. The replacement can contain path
* separators but must have no empty path segments ({@code //}).
*
*
* Here is an example replacing all path separators with an alternate character:
*
*
* map ?a/path/ to b/<scalar(/x/y)> separator '%'
*
*
*
* If the value at '/x/y' in the source topic was foo/bar, this would cause the
* reference topic to be created at /b/foo%bar instead of /b/foo/bar.
*
*
Preserve topics option
*
* The default behavior for a topic view is that only the reference topics that
* can be derived from the current value of the source topic are maintained.
* This applies to views using directives that derive the path of the reference
* topic(s) from a value within the source topic (e.g. scalar or expand). For
* example if a view uses a scalar directive to determine the path of the
* reference topic and the source topic is updated such that the selected scalar
* value changes then the previously created reference topic will be removed and
* a new one created. Similarly an expand directive may create many reference
* topics from the source value but if the source is updated then only those
* reference topics that still have a corresponding value within the source
* value will be updated (or new ones added if not in the previous value),
* whilst others will be removed.
*
* There are situations where an application may not want reference topics to be
* removed because of source value changes. In such cases the 'preserve topics'
* option may be used to indicate that all reference topics created by a view
* should be retained (and updated by later updates that apply to it) until
* either the source topic or the creating view are removed.
*
* For example:
*
*
* map ?a/path/ to b/<expand()> preserve topics
*
*
* Topic type option
*
* The 'type' option can be used to specify the {@link TopicType} of the target
* reference topic. If the current source value of type indicated by the source
* topic's type can be converted to the target type, a reference topic of the
* specified type will be created.
*
* For example:
*
*
* map ?a/ to b/<path(1)> type STRING
*
*
* The specified type must be one of the supported target types
* ({@link TopicType#STRING STRING}, {@link TopicType#INT64 INT64},
* {@link TopicType#DOUBLE DOUBLE}, {@link TopicType#JSON JSON},
* {@link TopicType#TIME_SERIES TIME_SERIES}, or {@link TopicType#BINARY
* BINARY}), case insensitive.
*
* The following table describes the supported conversions from the source topic
* type (the left column) to the supported target types. The number in brackets
* indicates a note at the foot of the table describing exactly how the
* conversion is processed. Where there is no number, no conversion is necessary
* and the derived source value is simply mapped to the target reference topic
* as if the 'type' option was not specified. Where there is no entry for the
* source topic type in the left column, no conversion is supported and topics
* of unsupported types will not be processed by the topic view. An 'x'
* indicates that the conversion is not supported.
*
* In all cases the value being processed will be the 'current' value as derived
* from other mappings within the topic view (e.g. expand) which is not
* necessarily the value of the source topic.
*
*
* Source topic
* {@link TopicType#STRING STRING}
* {@link TopicType#INT64 INT64}
* {@link TopicType#DOUBLE DOUBLE}
* {@link TopicType#JSON JSON}
* {@link TopicType#TIME_SERIES
* TIME_SERIES}
* {@link TopicType#BINARY BINARY}
*
*
*
* {@link TopicType#STRING STRING}
* (1)
* (1)
* (2)
* (3)
* x
*
*
* {@link TopicType#INT64 INT64}
* (4)
* (5)
* (2)
* (3)
* x
*
*
* {@link TopicType#DOUBLE DOUBLE}
* (4)
* (6)
* (2)
* (3)
* x
*
*
* {@link TopicType#JSON JSON}
* (7)
* (7)
* (7)
* (3)
* x
*
*
* {@link TopicType#TIME_SERIES TIME_SERIES}
* (8)
* (8)
* (8)
* (8)
* (9)
*
*
* {@link TopicType#BINARY BINARY}
* x
* x
* x
* x
* (9)
*
*
* Conversion notes:-
*
* - {@link TopicType#STRING STRING} to number conversions will only occur if
* the value of the string can be converted to the target number type. If the
* string cannot be converted then no reference topic will be created.
* - Primitive types to {@link TopicType#JSON JSON} will result in a JSON
* topic containing just the scalar value.
* - Conversions to {@link TopicType#TIME_SERIES TIME_SERIES} will result in a
* time series topic with an event type matching the source topic. Every update
* to the source topic will result in a new value being appended to the
* reference time series topic. It is not possible to convert to a time series
* topic with a different event type from the source topic.
* - Conversions from number types to {@link TopicType#STRING STRING} will
* result in a simple string representation of the number in the reference topic
* value.
* - {@link TopicType#INT64 INT64} to {@link TopicType#DOUBLE DOUBLE}
* conversions perform a simple conversion. For example, 123 becomes 123.0.
* - {@link TopicType#DOUBLE DOUBLE} to {@link TopicType#INT64 INT64}
* conversions perform rounding to the nearest integer value. For example 12.51
* becomes 13.
* - {@link TopicType#JSON JSON} to primitive type conversions only occur if
* the JSON value is a scalar which can be read as a string and converted to the
* target type. Currently, only string and integer scalar values can be read as
* a string (i.e. not doubles). The string representation of the value will be
* converted in the same way as specified for {@link TopicType#STRING STRING} to
* other primitive types. If the JSON value is a structure or cannot be
* converted then no conversion takes place and no reference topic will be
* created.
* - The conversion of {@link TopicType#TIME_SERIES TIME_SERIES} to other
* types follows the same rules as for conversion from the source topic type
* that matches the source time series topic's event value type. So if the time
* series event type is 'double' then the conversion rules from source topic
* type {@link TopicType#DOUBLE DOUBLE} to the target type will apply. Each
* value appended to the source time series topic will result in an update to
* the reference topic. If a failure to convert occurs at any point then the
* reference topic would be removed and only recreated if a value is appended
* that can be converted.
* - {@link TopicType#BINARY BINARY} to {@link TopicType#TIME_SERIES
* TIME_SERIES}, and vice-versa is supported in the same way as for other time
* series conversions.
*
*
* Escaping and quoting special characters
*
* Each part of a topic view expression has characters with special
* significance. Source topic clauses and path mapping clauses are delimited by
* white space. Directives in path and topic property mapping clauses are
* delimited by the {@code <} and {@code >} characters, and each directive
* parameter is terminated by {@code ,} or {@code )}. Topic property mapping
* clauses are delimited by white space, and the {@code :} and {@code ,}
* characters.
*
*
* Sometimes a topic view must refer to or generate topics with paths that
* containing special characters, or use a JSON pointer containing special
* characters. The escape sequence {@code \x} can be used to literally insert
* any character {@code x}, with a one exception: {@code \/} cannot be used in
* path fragments since the path delimiter {@code /} is always significant.
*
*
* Here is an example topic view expression containing escape sequences. It maps
* the topic path {@code a topic} a reference topic with the path
* {@code another topic}.
*
*
* map a\ topic to another\ topic
*
*
*
* Here is an example with a source value directive that uses the JSON pointer
* {@code /x()/y} to extract the target path from the source value. The
* {@code )} character in the JSON pointer must be escaped so it is not treated
* as the end of the parameter list.
*
*
* map ?a// to <scalar(/x(\)/y)>
*
*
*
* To insert {@code \}, the escape sequence {@code \\} must be used.
*
*
* There is no need to escape white space in JSON pointers directive parameters.
* However, white space is significant. For example, the following expressions
* have different topic value mapping clauses since the JSON pointer in the
* second expression is {@code /x }; that is, it has a trailing space:
*
*
* map a to b as <value(/x)>
* map a to b as <value(/x )>
*
*
*
* Instead of using escape sequences, white space characters can be included in
* source topic clauses and path mapping clauses using quotes. A clause is
* quoted by wrapping it in single quote ({@code '}) or double quote ({@code "})
* characters. For example:
*
*
* map "a topic" to "another topic"
*
*
*
* Within a quoted clause, quotes of the same type must be escaped:
*
*
* map 'alice\'s topic' to 'bob\'s topic'
*
*
*
* For consistency, the values in topic property mapping clauses can be escaped
* or quoted. However, there is no need to do so because none of the valid
* values for the mappable properties contain special characters.
*
*
*
Dealing with topic path conflicts
*
*
* Reference topics have a lower priority than normal topics created through the
* API, including replicas of normal topics created by topic replication or
* fan-out. A reference topic will only be created if no topic or reference
* topic is already bound to its derived topic path.
*
*
* Topic views have a precedence based on order of creation. If two topic views
* define mappings the same topic path, the earliest-created topic view will
* create a reference topic. If a topic view is updated, it retains its original
* precedence.
*
*
Remote topic views
*
*
* A remote topic view is one that specifies another server as the location of
* the source topics using the from clause as shown in the example
* below:
*
*
* map ?a// from server1 to b/<path(1)>
*
*
* The server name ({@code server1} in this example) refers to the name of a
* remote server created using the {@link RemoteServers} feature.
*
*
* In this case, upon establishing a successful connection with the remote
* server indicated the topic view will create reference topics locally based
* upon the topics selected by the topic view's selector at the remote server.
* It is important to note that the selector only refers to topics that
* match it at the remote server and not on the local server and there is no
* reason why there could not be a source topic at the remote server that has
* the same path as an entirely different topic on the local server.
*
*
* More than one topic view can specify the same remote server.
*
*
* A remote server only makes a physical connection when it is in use, therefore
* the first topic view that specifies a remote server will cause it to
* establish a connection. Similarly, if the last topic view that uses a remote
* server is removed then the connection will be closed.
*
*
* It is not necessary for the named remote server definition to exist before
* creating the topic view, as if it does not then the topic view will simply
* remain dormant until the remote server is created and a successful connection
* to the server specified in its URL is established. Similarly, if a remote
* server that is in use by remote topic views is removed then all of the
* reference topics created by the topic views will be removed and the topic
* views will become dormant until the named remote server is created again or
* the views are changed to name a different remote server.
*
*
* If a remote topic view selects a {@link TopicType#ROUTING ROUTING} topic at
* the remote server then local mappings will only be performed if the routing
* topic mapping at the remote server is able to establish a mapping for the
* remote server connection. The mapping will be done as if from the resolved
* routing topic.
*
*
* The rules of precedence for remote topic views are the same as for other
* topic views. If the remote server for a remote topic view does not exist or
* does not have an established connection then the remote topic view is not
* evaluated (i.e. it is as if the source topics for the view did not exist),
* but if the remote server later connects then the view will be evaluated and
* rules of precedence will determine whether reference topic will replace those
* created by earlier views.
*
*
Topic view persistence and replication
*
* Reference topics are neither replicated nor persisted. They are created and
* removed based on their source topics. However, topic views are replicated and
* persisted. A server that restarts will restore topic views during recovery.
* Each topic view will then create reference topics based on the source topics
* that have been recovered.
*
*
* The server records all changes to topic views in a persistent store. Topic
* views are restored if the server is started.
*
*
* If a server belongs to a cluster, topic views (and remote servers) will be
* replicated to each server in the cluster. Topic views are evaluated locally
* within a server. Replicated topic views that select non-replicated source
* topics can create different reference topics on each server in the cluster.
* When remote topic views are in use, each server in the cluster will make a
* connection to the specified remote server and will separately manage their
* remote topic views.
*
*
* A view with a delay clause uses temporary storage to record delayed events.
* If there is a high volume of updates, temporary per-server disk files will be
* used to save server memory. The storage is per-server, and does not survive
* server restart. When a server is started, no data will be published by a view
* with a delay clause until the delay time has expired.
*
*
Access control
*
* The following access control restrictions are applied:
*
*
* - To {@link #listTopicViews() list the topic views}, a session needs the
* {@link GlobalPermission#READ_TOPIC_VIEWS READ_TOPIC_VIEWS} global permission.
*
*
- To {@link #createTopicView(String, String) create, replace}, or
* {@link #removeTopicView(String) remove} a topic view, a session needs the
* {@link GlobalPermission#MODIFY_TOPIC_VIEWS MODIFY_TOPIC_VIEWS} global
* permission and {@link PathPermission#SELECT_TOPIC SELECT_TOPIC} permission
* for the path prefix of the source topic selector.
*
*
- Each topic view records the principal and security roles of the session
* that created it as the topic view security context. When a topic
* view is evaluated, this security context is used to constrain the creation of
* reference topics. A reference topic will only be created if the security
* context has {@link PathPermission#READ_TOPIC READ_TOPIC} permission for the
* source topic path, and {@link PathPermission#MODIFY_TOPIC MODIFY_TOPIC}
* permission for the reference topic path. The topic view security context is
* copied from the creating session at the time the topic view is created or
* replaced, and is persisted with the topic view. The topic view security
* context is not updated if the roles associated with the session are changed.
*
*
*
* Accessing the feature
*
* This feature may be obtained from a {@link Session session} as follows:
*
*
* TopicViews topicViews = session.feature(TopicViews.class);
*
*
*
* This feature is also extended by the {@link Topics Topics} feature. This
* means is it possible to use the methods described here through the
* {@link Topics Topics} feature.
*
* @author DiffusionData Limited
* @since 6.3
*/
public interface TopicViews extends Feature {
/**
* Create a new named topic view.
*
* If a view with the same name already exists the new view will update the
* existing view.
*
* @param name the name of the view. If the name is empty, the operation
* will throw an InvalidArgumentException.
* @param specification the
* specification of the topic view
* @return a CompletableFuture that completes when a response is received
* from the server, returning the topic view created by the
* operation.
*
*
* If the task fails, the CompletableFuture will complete
* exceptionally with a {@link CompletionException}. Common reasons
* for failure, listed by the exception reported as the
* {@link CompletionException#getCause() cause}, include:
*
*
* - {@link ScriptException} – if {@code specification} is
* invalid;
*
- {@link ClusterRoutingException} – if the operation
* failed due to a transient cluster error;
*
- {@link PermissionsException} – if the calling
* session does not have MODIFY_TOPIC_VIEW permission or appropriate
* path prefix permissions;
*
- {@link SessionClosedException} – if the session is
* closed.
*
*/
CompletableFuture createTopicView(String name,
String specification);
/**
* List all the topic views that have been created.
*
* @return a CompletableFuture that completes when a response is received
* from the server, returning a list of views sorted by their
* creation order.
*
*
* If the task fails, the CompletableFuture will complete
* exceptionally with a {@link CompletionException}. Common reasons
* for failure, listed by the exception reported as the
* {@link CompletionException#getCause() cause}, include:
*
*
* - {@link ClusterRoutingException} – if the operation
* failed due to a transient cluster error;
*
- {@link PermissionsException} – if the calling
* session does not have READ_TOPIC_VIEW permission or appropriate
* path prefix permissions;
*
- {@link SessionClosedException} – if the session is
* closed.
*
*/
CompletableFuture> listTopicViews();
/**
* Get a named Topic View.
*
* If the named view does not exist the completable future will complete
* successfully with a null result.
*
* @param name the name of the view
* @return a CompletableFuture that completes when a response is received
* from the server, returning a named view if it exists
*
*
* If the task fails, the CompletableFuture will complete
* exceptionally with a {@link CompletionException}. Common reasons
* for failure, listed by the exception reported as the
* {@link CompletionException#getCause() cause}, include:
*
*
* - {@link ClusterRoutingException} – if the operation
* failed due to a transient cluster error;
*
- {@link PermissionsException} – if the calling
* session does not have READ_TOPIC_VIEW permission or appropriate
* path prefix permissions;
*
- {@link SessionClosedException} – if the session is
* closed.
*
*/
CompletableFuture getTopicView(String name);
/**
* Remove a named topic view if it exists.
*
* If the named view does not exist the completable future will complete
* successfully.
*
* @param name the name of the view
* @return a CompletableFuture that completes when a response is received
* from the server.
*
*
* If the task fails, the CompletableFuture will complete
* exceptionally with a {@link CompletionException}. Common reasons
* for failure, listed by the exception reported as the
* {@link CompletionException#getCause() cause}, include:
*
*
* - {@link ClusterRoutingException} – if the operation
* failed due to a transient cluster error;
*
- {@link PermissionsException} – if the calling
* session does not have MODIFY_TOPIC_VIEW permission or appropriate
* path prefix permissions;
*
- {@link SessionClosedException} – if the session is
* closed.
*
*/
CompletableFuture removeTopicView(String name);
}