All Downloads are FREE. Search and download functionalities are using the official Maven repository.

io.castled.warehouses.WarehouseService Maven / Gradle / Ivy

There is a newer version: 1.0.0
Show newest version
package io.castled.warehouses;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import io.castled.ObjectRegistry;
import io.castled.caches.WarehouseCache;
import io.castled.constants.CommonConstants;
import io.castled.daos.PipelineDAO;
import io.castled.encryption.EncryptionManager;
import io.castled.events.CastledEventsClient;
import io.castled.events.warehousevents.WarehouseCreatedEvent;
import io.castled.exceptions.CastledException;
import io.castled.exceptions.CastledRuntimeException;
import io.castled.exceptions.connect.ConnectException;
import io.castled.forms.dtos.FieldOptionsDTO;
import io.castled.forms.dtos.FormFieldsDTO;
import io.castled.jarvis.JarvisTaskGroup;
import io.castled.jarvis.JarvisTaskType;
import io.castled.jarvis.taskmanager.JarvisTasksClient;
import io.castled.jarvis.taskmanager.models.RetryCriteria;
import io.castled.jarvis.taskmanager.models.Task;
import io.castled.jarvis.taskmanager.models.TaskStatus;
import io.castled.jarvis.taskmanager.models.requests.TaskCreateRequest;
import io.castled.models.*;
import io.castled.models.users.User;
import io.castled.pubsub.MessagePublisher;
import io.castled.pubsub.registry.WarehouseUpdatedMessage;
import io.castled.resources.validators.ResourceAccessController;
import io.castled.schema.models.RecordSchema;
import io.castled.utils.DocUtils;
import io.castled.utils.JsonUtils;
import io.castled.utils.TimeUtils;
import io.castled.warehouses.dtos.WarehouseAttributes;
import io.castled.warehouses.dtos.WarehouseTypeDTO;
import io.castled.warehouses.optionsfetchers.WarehouseOptionsFetcher;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.jdbi.v3.core.Jdbi;

import javax.ws.rs.BadRequestException;
import java.util.*;
import java.util.stream.Collectors;

@Slf4j
@Singleton
@SuppressWarnings({"rawtypes", "unchecked"})
public class WarehouseService {

    private final Map warehouseConnectors;
    private final EncryptionManager encryptionManager;
    private final WarehouseDAO warehouseDAO;
    private final PipelineDAO pipelineDAO;
    private final WarehouseCache warehouseCache;
    private final MessagePublisher messagePublisher;
    private final ResourceAccessController accessController;
    private final Map warehouseOptionsFetchers;
    private final CastledEventsClient castledEventsClient;

    @Inject
    public WarehouseService(Map warehouseConnectors,
                            EncryptionManager encryptionManager, Jdbi jdbi, WarehouseCache warehouseCache,
                            MessagePublisher messagePublisher, ResourceAccessController accessController,
                            CastledEventsClient castledEventsClient,
                            Map warehouseOptionsFetchers) {
        this.warehouseConnectors = warehouseConnectors;
        this.encryptionManager = encryptionManager;
        this.warehouseDAO = jdbi.onDemand(WarehouseDAO.class);
        this.pipelineDAO = jdbi.onDemand(PipelineDAO.class);
        this.warehouseCache = warehouseCache;
        this.messagePublisher = messagePublisher;
        this.accessController = accessController;
        this.warehouseOptionsFetchers = warehouseOptionsFetchers;
        this.castledEventsClient = castledEventsClient;
    }

    public void testConnection(WarehouseType warehouseType, WarehouseConfig warehouseConfig) throws ConnectException {
        try {
            this.warehouseConnectors.get(warehouseType).testConnectionForDataPoll(warehouseConfig);
        } catch (ConnectException e) {
            throw new BadRequestException(e.getMessage());
        }
    }

    public Long createWarehouse(WarehouseAttributes warehouseAttributes, User user) {

        WarehouseConfig warehouseConfig = warehouseAttributes.getConfig();
        try {
            testConnection(warehouseConfig.getType(), warehouseConfig);
            String config = this.encryptionManager.encryptText(JsonUtils.objectToString(warehouseConfig), user.getTeamId());
            Long warehouseId = warehouseDAO.createWarehouse(warehouseAttributes.getName(), warehouseConfig.getType(),
                    config, user.getTeamId(), false);
            this.castledEventsClient.publishCastledEvent(new WarehouseCreatedEvent(warehouseId));
            return warehouseId;
        } catch (BadRequestException e) {
            throw e;
        } catch (Exception e) {
            log.error("Create warehouse request failed for team {} and type {}", user.getTeamId(), warehouseConfig.getType(), e);
            throw new CastledRuntimeException(e);
        }
    }

    public FormFieldsDTO getFormFields(WarehouseType warehouseType) {
        return this.warehouseConnectors.get(warehouseType).getFormFields();
    }

    public void updateWarehouse(Long warehouseId, WarehouseAttributes warehouseAttributes, User user) throws CastledException {
        try {

            Warehouse warehouse = getWarehouse(warehouseId, false);
            accessController.validateWarehouseAccess(warehouse, user.getTeamId());

            WarehouseConfig warehouseConfig = warehouseAttributes.getConfig();
            testConnection(warehouseAttributes.getConfig().getType(), warehouseConfig);
            String config = this.encryptionManager.encryptText(JsonUtils.objectToString(warehouseConfig), user.getTeamId());
            warehouseDAO.updateWarehouse(warehouseId, warehouseAttributes.getName(), config);
            this.messagePublisher.publishMessage(new WarehouseUpdatedMessage(warehouseId));
        } catch (BadRequestException e) {
            throw e;
        } catch (Exception e) {
            log.error("Update warehouse request failed for id {}", warehouseId, e);
            if (e instanceof ConnectException) {
                throw new BadRequestException(e.getMessage());
            }
            throw new CastledException(e.getMessage());
        }
    }

    public Warehouse getWarehouse(Long warehouseId) {
        return getWarehouse(warehouseId, false);
    }

    public Warehouse getUnrestrictedWarehouseDetails(Long warehouseId) {
        return filterRestrictedConfigDetails(this.warehouseDAO.getWarehouse(warehouseId));
    }

    public Warehouse filterRestrictedConfigDetails(Warehouse warehouse) {
        WarehouseConfig warehouseConfig = this.warehouseConnectors.get(warehouse.getType()).filterRestrictedConfigDetails(warehouse.getConfig());
        warehouse.setConfig(warehouseConfig);
        return warehouse;
    }

    public void deleteWarehouse(Long warehouseId, Long teamId) {
        Warehouse warehouse = getWarehouse(warehouseId, true);
        accessController.validateWarehouseAccess(warehouse, teamId);
        if (this.pipelineDAO.getPipelinesByWhId(warehouseId).size() > 0) {
            throw new BadRequestException("Please delete all pipelines corresponding to the warehouse before deleting it");
        }
        warehouseDAO.deleteWarehouse(warehouseId);
    }

    public List getAllWarehouses(WarehouseType warehouseType, Long teamId) {
        List warehouseList = this.warehouseDAO.listWarehouses(teamId).stream().filter(warehouse -> (warehouseType == null || warehouse.getType() == warehouseType))
                .collect(Collectors.toList());
        warehouseList.forEach(this::filterRestrictedConfigDetails);
        return warehouseList;
    }

    public Warehouse getWarehouse(Long warehouseId, boolean cached) {
        if (cached) {
            return warehouseCache.getValue(warehouseId);
        }
        return this.warehouseDAO.getWarehouse(warehouseId);
    }

    public List getWarehouses(List warehouseIds, boolean cached) {
        if (cached) {
            return warehouseIds.stream().map(warehouseCache::getValue).collect(Collectors.toList());
        }
        return this.warehouseDAO.listWarehouses(warehouseIds);
    }

    public void updateWarehouseConfig(Long warehouseId, WarehouseConfig warehouseConfig, long teamId) {
        try {
            Warehouse warehouse = getWarehouse(warehouseId, false);
            accessController.validateWarehouseAccess(warehouse, teamId);
            String configStr = this.encryptionManager.encryptText(JsonUtils.objectToString(warehouseConfig), warehouse.getTeamId());
            warehouseDAO.updateWarehouseConfig(warehouseId, configStr);
            this.messagePublisher.publishMessage(new WarehouseUpdatedMessage(warehouseId));
        } catch (Exception e) {
            log.error("Update warehouse config failed for warehouse {}", warehouseId);
            throw new CastledRuntimeException(e);
        }
    }

    public RecordSchema fetchSchema(Long warehouseId, String query) throws Exception {
        Warehouse warehouse = getWarehouse(warehouseId);
        return warehouseConnectors.get(warehouse.getType()).getQuerySchema(warehouse.getConfig(), query);
    }

    public QueryId previewQuery(Long warehouseId, String query, long teamId) throws Exception {

        Warehouse warehouse = getWarehouse(warehouseId, true);
        this.accessController.validateWarehouseAccess(warehouse, teamId);
        String queryId = UUID.randomUUID().toString();
        TaskCreateRequest taskCreateRequest = TaskCreateRequest.builder()
                .group(JarvisTaskGroup.OTHERS.name())
                .type(JarvisTaskType.PREVIEW_QUERY.name())
                .expiry(TimeUtils.minutesToMillis(60))
                .params(ImmutableMap.of(CommonConstants.WAREHOUSE_ID, warehouseId,
                        CommonConstants.QUERY, query))
                .searchId(queryId).retryCriteria(new RetryCriteria(3, true))
                .build();
        ObjectRegistry.getInstance(JarvisTasksClient.class).createTaskSync(taskCreateRequest);
        return new QueryId(queryId);
    }

    public QueryStatusAndResults getQueryResults(String queryId) throws Exception {
        //there can be only one task with one search id logically
        List tasks = ObjectRegistry.getInstance(JarvisTasksClient.class).getTasksBySearchId(queryId, JarvisTaskType.PREVIEW_QUERY.name());
        if (CollectionUtils.isEmpty(tasks)) {
            throw new BadRequestException(String.format("Query id %s not found", queryId));
        }
        if (tasks.size() > 1) {
            throw new BadRequestException(String.format("Multiple queries with the same query id %s found", queryId));
        }
        Task task = tasks.get(0);
        if (task.getStatus() == TaskStatus.FAILED) {
            return QueryStatusAndResults.builder().status(QueryStatus.FAILED).failureMessage(task.getFailureMessage()).build();
        }
        if (task.getStatus() == TaskStatus.PROCESSED) {
            QueryResults queryResults = JsonUtils.jsonStringToObject(task.getResult(), QueryResults.class);
            return QueryStatusAndResults.builder().status(QueryStatus.SUCCEEDED).queryResults(queryResults).build();
        }
        return QueryStatusAndResults.builder().status(QueryStatus.PENDING).build();
    }

    public List listWarehouseTypes(User user) {
        List warehouses = this.warehouseDAO.listWarehouses(user.getTeamId());
        return Arrays.stream(WarehouseType.values()).map(warehouseType -> new WarehouseTypeDTO(warehouseType, warehouseType.title(),
                        warehouseType.getAccessType(), warehouseType.getLogoUrl(), DocUtils.constructDocUrl(warehouseType.getDocUrl()),
                        warehouses.stream().filter(warehouse -> warehouse.getType().equals(warehouseType)).count()))
                .collect(Collectors.toList());
    }

    public FieldOptionsDTO getConfigOptions(WarehouseConfig warehouseConfig,
                                            String optionsReference) {
        return Optional.ofNullable(this.warehouseOptionsFetchers.get(optionsReference))
                .map(optionsFetcher -> new FieldOptionsDTO(optionsFetcher.getFieldOptions(warehouseConfig)))
                .orElse(null);
    }

    public List getDemoQueries(Long warehouseId) {
        Warehouse warehouse = getWarehouse(warehouseId, true);
        if (warehouse.isDemo()) {
            return Lists.newArrayList("select * from users", "select * from companies");
        }
        return Lists.newArrayList();
    }

    public int getTotalActiveWarehousesForTeam(Long teamId) {
        return this.warehouseDAO.getTotalActiveWarehousesForTeam(teamId);
    }
}




© 2015 - 2025 Weber Informatics LLC | Privacy Policy