All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • com.hazelcast.jet.impl.connector.HazelcastWriters Maven / Gradle / Ivy

    The newest version!
    /*
 * Copyright (c) 2008-2024, Hazelcast, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.hazelcast.jet.impl.connector;

import com.hazelcast.cache.ICache;
import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.cluster.Address;
import com.hazelcast.collection.IList;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.HazelcastInstanceNotActiveException;
import com.hazelcast.function.BiConsumerEx;
import com.hazelcast.function.BiFunctionEx;
import com.hazelcast.function.BinaryOperatorEx;
import com.hazelcast.function.ConsumerEx;
import com.hazelcast.function.FunctionEx;
import com.hazelcast.internal.serialization.Data;
import com.hazelcast.internal.serialization.SerializationService;
import com.hazelcast.jet.RestartableException;
import com.hazelcast.jet.core.Processor;
import com.hazelcast.jet.core.ProcessorMetaSupplier;
import com.hazelcast.jet.core.ProcessorSupplier;
import com.hazelcast.jet.core.processor.SinkProcessors;
import com.hazelcast.jet.impl.observer.ObservableImpl;
import com.hazelcast.jet.impl.util.ImdgUtil;
import com.hazelcast.map.EntryProcessor;
import com.hazelcast.security.impl.function.SecuredFunctions;
import com.hazelcast.security.permission.RingBufferPermission;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.Serial;
import java.security.Permission;
import java.util.AbstractMap;
import java.util.AbstractMap.SimpleEntry;
import java.util.AbstractSet;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;

import static com.hazelcast.jet.core.ProcessorMetaSupplier.preferLocalParallelismOne;
import static com.hazelcast.jet.impl.connector.AsyncHazelcastWriterP.MAX_PARALLEL_ASYNC_OPS_DEFAULT;
import static com.hazelcast.jet.impl.util.ImdgUtil.asXmlString;
import static com.hazelcast.jet.impl.util.Util.checkSerializable;
import static com.hazelcast.security.PermissionsUtil.cachePutPermission;
import static com.hazelcast.security.PermissionsUtil.listAddPermission;
import static com.hazelcast.security.PermissionsUtil.mapPutPermission;
import static com.hazelcast.security.PermissionsUtil.mapUpdatePermission;
import static com.hazelcast.security.permission.ActionConstants.ACTION_CREATE;
import static com.hazelcast.security.permission.ActionConstants.ACTION_PUT;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;

/**
 * This is private API. Check out the {@link SinkProcessors} class for
 * public factory methods.
 */
public final class HazelcastWriters {

    private HazelcastWriters() {
    }

    /**
     * Update map with key and value functions
     */
    @Nonnull
    public static  ProcessorMetaSupplier writeMapSupplier(
            @Nonnull String name,
            @Nullable ClientConfig clientConfig,
            @Nonnull FunctionEx toKeyFn,
            @Nonnull FunctionEx toValueFn
    ) {
        MapSinkConfiguration params = new MapSinkConfiguration<>(name);
        params.setClientXml(ImdgUtil.asXmlString(clientConfig));
        params.setToKeyFn(toKeyFn);
        params.setToValueFn(toValueFn);

        String clientXml = asXmlString(clientConfig);
        params.setClientXml(clientXml);

        return writeMapSupplier(params);
    }

    /**
     * Update map with key and value functions
     */
    @Nonnull
    public static  ProcessorMetaSupplier writeMapSupplier(MapSinkConfiguration sinkConfig) {
        // Create ProcessorSupplier
        WriteMapP.Supplier supplier = WriteMapP.Supplier.createNew(sinkConfig);
        if (sinkConfig.isRemote()) {
            // ProcessorMetaSupplier uses default local parallelism and no permission
            return preferLocalParallelismOne(supplier);
        } else {
            // Create permission for ProcessorMetaSupplier
            Permission processorMetaSupplierPermission = mapPutPermission(sinkConfig.getMapName());

            // ProcessorMetaSupplier uses default local parallelism
            return preferLocalParallelismOne(processorMetaSupplierPermission, supplier);
        }
    }

    /**
     * Update map with a merge function
     */
    @Nonnull
    public static  ProcessorMetaSupplier mergeMapSupplier(MapSinkConfiguration sinkConfig) {
        // Get reference to functions because MapSinkMergeParams is not serializable
        FunctionEx toKeyFn = sinkConfig.getToKeyFn();
        FunctionEx toValueFn = sinkConfig.getToValueFn();
        BinaryOperatorEx mergeFn = sinkConfig.getMergeFn();

        checkSerializable(toKeyFn, "toKeyFn");
        checkSerializable(toValueFn, "toValueFn");
        checkSerializable(mergeFn, "mergeFn");

        boolean isRemote = sinkConfig.isRemote();
        FunctionEx processorFunction = SecuredFunctions.updateMapProcessorFn(
                sinkConfig.getMapName(),
                isRemote,
                toKeyFn,
                (V oldValue, T item) -> {
                    V newValue = toValueFn.apply(item);
                    if (oldValue == null) {
                        return newValue;
                    }
                    return mergeFn.apply(oldValue, newValue);
                });

        ProcessorFunctionConnectorSupplier processorSupplier = new ProcessorFunctionConnectorSupplier(
                sinkConfig.getDataConnectionName(),
                sinkConfig.getClientXml(),
                processorFunction
        );

        if (isRemote) {
            // ProcessorMetaSupplier uses default local parallelism and no permission
            return ProcessorMetaSupplier.of(processorSupplier);
        } else {
            // Create permission for ProcessorMetaSupplier
            Permission processorMetaSupplierPermission = mapUpdatePermission(sinkConfig.getMapName());

            // ProcessorMetaSupplier uses default local parallelism
            return ProcessorMetaSupplier.of(processorMetaSupplierPermission, processorSupplier);
        }
    }

    /**
     * Update map with an update function
     */
    @Nonnull
    public static  ProcessorMetaSupplier updateMapSupplier(
            @Nonnull String name,
            @Nullable ClientConfig clientConfig,
            @Nonnull FunctionEx toKeyFn,
            @Nonnull BiFunctionEx updateFn
    ) {
        checkSerializable(toKeyFn, "toKeyFn");
        checkSerializable(updateFn, "updateFn");

        MapSinkConfiguration sinkConfig = new MapSinkConfiguration<>(name);
        sinkConfig.setClientXml(ImdgUtil.asXmlString(clientConfig));
        sinkConfig.setToKeyFn(toKeyFn);
        sinkConfig.setUpdateFn(updateFn);

        return updateMapSupplier(sinkConfig);
    }

    /**
     * Update map with an update function
     */
    @Nonnull
    public static  ProcessorMetaSupplier updateMapSupplier(MapSinkConfiguration sinkConfig) {
        checkSerializable(sinkConfig.getToKeyFn(), "toKeyFn");
        checkSerializable(sinkConfig.getUpdateFn(), "updateFn");

        boolean isRemote = sinkConfig.isRemote();
        FunctionEx processorFunction = SecuredFunctions.updateMapProcessorFn(
                sinkConfig.getMapName(),
                isRemote,
                sinkConfig.getToKeyFn(),
                sinkConfig.getUpdateFn());

        ProcessorFunctionConnectorSupplier processorSupplier = new ProcessorFunctionConnectorSupplier(
                sinkConfig.getDataConnectionName(),
                sinkConfig.getClientXml(),
                processorFunction
        );

        if (isRemote) {
            // ProcessorMetaSupplier uses default local parallelism and no permission
            return ProcessorMetaSupplier.of(processorSupplier);
        } else {
            // Create permission for ProcessorMetaSupplier
            Permission processorMetaSupplierPermission = mapUpdatePermission(sinkConfig.getMapName());

            // ProcessorMetaSupplier uses default local parallelism
            return ProcessorMetaSupplier.of(processorMetaSupplierPermission, processorSupplier);
        }
    }

    /**
     * Update map with an EntryProcessor
     */
    @Nonnull
    public static  ProcessorMetaSupplier updateMapSupplier(
            @Nonnull String name,
            @Nullable ClientConfig clientConfig,
            @Nonnull FunctionEx toKeyFn,
            @Nonnull FunctionEx> toEntryProcessorFn
    ) {
        MapSinkEntryProcessorConfiguration sinkConfig = new MapSinkEntryProcessorConfiguration<>(name);
        sinkConfig.setClientXml(ImdgUtil.asXmlString(clientConfig));
        sinkConfig.setMaxParallelAsyncOps(MAX_PARALLEL_ASYNC_OPS_DEFAULT);
        sinkConfig.setToKeyFn(toKeyFn);
        sinkConfig.setToEntryProcessorFn(toEntryProcessorFn);

        return updateMapSupplier(sinkConfig);

    }

    /**
     * Update map with an EntryProcessor
     */
    @Nonnull
    public static  ProcessorMetaSupplier updateMapSupplier(
            MapSinkEntryProcessorConfiguration sinkConfig
    ) {
        checkSerializable(sinkConfig.getToKeyFn(), "toKeyFn");
        checkSerializable(sinkConfig.getToEntryProcessorFn(), "toEntryProcessorFn");

        boolean isRemote = sinkConfig.isRemote();
        FunctionEx processorFunction = SecuredFunctions.updateWithEntryProcessorFn(
                sinkConfig.getMaxParallelAsyncOps(),
                sinkConfig.getMapName(),
                isRemote,
                sinkConfig.getToKeyFn(),
                sinkConfig.getToEntryProcessorFn());

        ProcessorFunctionConnectorSupplier processorSupplier = new ProcessorFunctionConnectorSupplier(
                sinkConfig.getDataConnectionName(),
                sinkConfig.getClientXml(),
                processorFunction
        );
        if (isRemote) {
            // ProcessorMetaSupplier uses default local parallelism and no permission
            return ProcessorMetaSupplier.of(processorSupplier);
        } else {
            // Create permission for ProcessorMetaSupplier
            Permission processorMetaSupplierPermission = mapUpdatePermission(sinkConfig.getMapName());
            // ProcessorMetaSupplier uses default local parallelism
            return ProcessorMetaSupplier.of(processorMetaSupplierPermission, processorSupplier);
        }
    }

    @Nonnull
    public static ProcessorMetaSupplier writeCacheSupplier(@Nonnull String name, @Nullable ClientConfig clientConfig) {
        String clientXml = asXmlString(clientConfig);
        return preferLocalParallelismOne(cachePutPermission(clientXml, name),
                new WriteCachePSupplier<>(clientXml, name));
    }

    @Nonnull
    public static ProcessorMetaSupplier writeListSupplier(@Nonnull String name, @Nullable ClientConfig clientConfig) {
        String clientXml = asXmlString(clientConfig);
        return preferLocalParallelismOne(listAddPermission(clientXml, name),
                new WriteListPSupplier<>(clientXml, name));
    }

    @SuppressWarnings("AnonInnerLength")
    public static ProcessorMetaSupplier writeObservableSupplier(@Nonnull String name) {
        return new ProcessorMetaSupplier() {
            @Serial
            private static final long serialVersionUID = 1L;

            @Nonnull
            @Override
            public Map getTags() {
                return singletonMap(ObservableImpl.OWNED_OBSERVABLE, name);
            }

            @Override
            public int preferredLocalParallelism() {
                return 1;
            }

            @Nonnull @Override
            public Function get(@Nonnull List
     addresses) {
                return address -> new WriteObservableP.Supplier(name);
            }

            @Override
            public Permission getRequiredPermission() {
                return new RingBufferPermission(name, ACTION_CREATE, ACTION_PUT);
            }

            @Override
            public boolean isReusable() {
                return true;
            }

            @Override
            public boolean initIsCooperative() {
                return true;
            }

            @Override
            public boolean closeIsCooperative() {
                return true;
            }
        };
    }

    static RuntimeException handleInstanceNotActive(HazelcastInstanceNotActiveException e, boolean isLocal) {
        // if we are writing to a local instance, restarting the job should resolve the error
        return isLocal ? new RestartableException(e) : e;
    }

    private static class WriteCachePSupplier extends AbstractHazelcastConnectorSupplier {

        @Serial
        private static final long serialVersionUID = 1L;

        private final String name;

        WriteCachePSupplier(@Nullable String clientXml, @Nonnull String name) {
            super(null, clientXml);
            this.name = name;
        }

        @Override
        protected Processor createProcessor(HazelcastInstance instance, SerializationService serializationService) {
            ICache cache = instance.getCacheManager().getCache(name);

            FunctionEx> bufferCreator = context -> new ArrayMap<>();
            BiConsumerEx, Entry> entryReceiver = (buffer, entry) -> {
                Data key = serializationService.toData(entry.getKey());
                Data value = serializationService.toData(entry.getValue());
                buffer.add(new SimpleEntry<>(key, value));
            };
            ConsumerEx> bufferFlusher = buffer -> {
                try {
                    cache.putAll(buffer);
                } catch (HazelcastInstanceNotActiveException e) {
                    throw handleInstanceNotActive(e, isLocal());
                }
                buffer.clear();
            };

            return new WriteBufferedP<>(bufferCreator, entryReceiver, bufferFlusher, ConsumerEx.noop());
        }

        @Override
        public List permissions() {
            return singletonList(cachePutPermission(clientXml, name));
        }
    }

    private static class WriteListPSupplier extends AbstractHazelcastConnectorSupplier {

        @Serial
        private static final long serialVersionUID = 1L;

        private final String name;

        WriteListPSupplier(@Nullable String clientXml, @Nonnull String name) {
            super(null, clientXml);
            this.name = name;
        }

        @Override
        protected Processor createProcessor(HazelcastInstance instance, SerializationService serializationService) {
            IList list = instance.getList(name);

            FunctionEx> bufferCreator = context -> new ArrayList<>();
            BiConsumerEx, T> itemReceiver = (buffer, item) -> buffer.add(serializationService.toData(item));
            ConsumerEx> bufferFlusher = buffer -> {
                try {
                    list.addAll(buffer);
                } catch (HazelcastInstanceNotActiveException e) {
                    throw handleInstanceNotActive(e, isLocal());
                }
                buffer.clear();
            };

            return new WriteBufferedP<>(bufferCreator, itemReceiver, bufferFlusher, ConsumerEx.noop());
        }

        @Override
        public List permissions() {
            return singletonList(listAddPermission(clientXml, name));
        }
    }

    public static final class ArrayMap extends AbstractMap {

        private final List> entries;
        private final ArraySet set = new ArraySet();

        ArrayMap() {
            entries = new ArrayList<>();
        }

        public ArrayMap(int size) {
            entries = new ArrayList<>(size);
        }

        @Override @Nonnull
        public Set> entrySet() {
            return set;
        }

        public void add(Map.Entry entry) {
            entries.add(entry);
        }

        @Override
        public V get(Object key) {
            throw new UnsupportedOperationException();
        }

        private class ArraySet extends AbstractSet> {

            @Override @Nonnull
            public Iterator> iterator() {
                return entries.iterator();
            }

            @Override
            public int size() {
                return entries.size();
            }

            @Override
            public void clear() {
                entries.clear();
            }
        }

        @Override
        public String toString() {
            return entries.toString();
        }
    }
}
    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy