All Downloads are FREE. Search and download functionalities are using the official Maven repository.

edu.internet2.middleware.grouper.app.azure.GrouperAzureTargetDao Maven / Gradle / Ivy

There is a newer version: 5.13.5
Show newest version
package edu.internet2.middleware.grouper.app.azure;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;

import edu.internet2.middleware.grouper.app.loader.GrouperLoaderConfig;
import edu.internet2.middleware.grouper.app.provisioning.ProvisioningEntity;
import edu.internet2.middleware.grouper.app.provisioning.ProvisioningGroup;
import edu.internet2.middleware.grouper.app.provisioning.ProvisioningMembership;
import edu.internet2.middleware.grouper.app.provisioning.ProvisioningObjectChange;
import edu.internet2.middleware.grouper.app.provisioning.ProvisioningObjectChangeAction;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.GrouperProvisionerDaoCapabilities;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.GrouperProvisionerTargetDaoBase;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoDeleteEntitiesRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoDeleteEntitiesResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoDeleteGroupsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoDeleteGroupsResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoDeleteMembershipsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoDeleteMembershipsResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoInsertEntitiesRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoInsertEntitiesResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoInsertGroupsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoInsertGroupsResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoInsertMembershipsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoInsertMembershipsResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveAllEntitiesRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveAllEntitiesResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveAllGroupsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveAllGroupsResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveEntitiesByValuesRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveEntitiesByValuesResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveEntitiesRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveEntitiesResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveGroupsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveGroupsResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveMembershipsByEntityRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveMembershipsByEntityResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveMembershipsByGroupRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoRetrieveMembershipsByGroupResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoTimingInfo;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoUpdateEntitiesRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoUpdateEntitiesResponse;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoUpdateGroupsRequest;
import edu.internet2.middleware.grouper.app.provisioning.targetDao.TargetDaoUpdateGroupsResponse;
import edu.internet2.middleware.grouper.util.GrouperHttpClient;
import edu.internet2.middleware.grouper.util.GrouperHttpClientLog;
import edu.internet2.middleware.grouper.util.GrouperUtil;
import edu.internet2.middleware.grouperClient.collections.MultiKey;
import edu.internet2.middleware.grouperClient.util.ExpirableCache;

public class GrouperAzureTargetDao extends GrouperProvisionerTargetDaoBase {
  
  /**
   * cache of owner identifier to owner url
   */
  private static ExpirableCache ownerIdentifierToOwnerUrl = new ExpirableCache(60);


  @Override
  public boolean loggingStart() {
    return GrouperHttpClient.logStart(new GrouperHttpClientLog());
  }

  @Override
  public String loggingStop() {
    return GrouperHttpClient.logEnd();
  }

  @Override
  public TargetDaoRetrieveAllGroupsResponse retrieveAllGroups(
      TargetDaoRetrieveAllGroupsRequest targetDaoRetrieveAllGroupsRequest) {

    long startNanos = System.nanoTime();

    try {

      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this
          .getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      List results = new ArrayList();
      
      boolean lookupOwners = azureConfiguration.getTargetGroupAttributeNameToConfig().containsKey("groupOwners");

      List grouperAzureGroups = GrouperAzureApiCommands
          .retrieveAzureGroups(azureConfiguration.getAzureExternalSystemConfigId(), lookupOwners);

      for (GrouperAzureGroup grouperAzureGroup : grouperAzureGroups) {
        ProvisioningGroup targetGroup = grouperAzureGroup.toProvisioningGroup();
        results.add(targetGroup);
      }

      return new TargetDaoRetrieveAllGroupsResponse(results);
    } finally {
      this.addTargetDaoTimingInfo(
          new TargetDaoTimingInfo("retrieveAllGroups", startNanos));
    }
  }

  @Override
  public TargetDaoRetrieveAllEntitiesResponse retrieveAllEntities(
      TargetDaoRetrieveAllEntitiesRequest targetDaoRetrieveAllEntitiesRequest) {

    long startNanos = System.nanoTime();

    try {

      //TODO
      //boolean includeAllMembershipsIfApplicable = targetDaoRetrieveAllGroupsRequest == null ? false : targetDaoRetrieveAllGroupsRequest.isIncludeAllMembershipsIfApplicable();

      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this
          .getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      List results = new ArrayList();

      List grouperAzureUsers = GrouperAzureApiCommands
          .retrieveAzureUsers(azureConfiguration.getAzureExternalSystemConfigId());

      Map targetEntityToNativeEntity = new HashMap<>();
      
      for (GrouperAzureUser grouperAzureUser : grouperAzureUsers) {
        ProvisioningEntity targetEntity = grouperAzureUser.toProvisioningEntity();
        results.add(targetEntity);
        targetEntityToNativeEntity.put(targetEntity, grouperAzureUser);
      }
      
      TargetDaoRetrieveAllEntitiesResponse response = new TargetDaoRetrieveAllEntitiesResponse(results);
      
      if (targetDaoRetrieveAllEntitiesRequest.isIncludeNativeEntity()) {
      	response.setTargetEntityToTargetNativeEntity(targetEntityToNativeEntity);
      }

      return response;
    } finally {
      this.addTargetDaoTimingInfo(
          new TargetDaoTimingInfo("retrieveAllEntities", startNanos));
    }
  }

  @Override
  public TargetDaoRetrieveEntitiesResponse retrieveEntities(TargetDaoRetrieveEntitiesRequest targetDaoRetrieveEntitiesRequest) {

    long startNanos = System.nanoTime();
    
    List targetEntities = targetDaoRetrieveEntitiesRequest.getTargetEntities();

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this
          .getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();
      
      List fieldValues = new ArrayList<>();
      for(ProvisioningEntity targetEntity: targetEntities) {
        String attributeValue = targetEntity.retrieveAttributeValueString(targetDaoRetrieveEntitiesRequest.getSearchAttribute());
        if (StringUtils.isNotBlank(attributeValue)) {
          fieldValues.add(attributeValue);
        }
      }
      
      List azureUsers = GrouperAzureApiCommands.retrieveAzureUsers(
          azureConfiguration.getAzureExternalSystemConfigId(), fieldValues, targetDaoRetrieveEntitiesRequest.getSearchAttribute());
      
      List targetEntitiesFromAzure = new ArrayList<>();
      
      Map targetEntityToNativeEntity = new HashMap<>();
      
      for (GrouperAzureUser userFromAuzre: azureUsers) {
        targetEntitiesFromAzure.add(userFromAuzre.toProvisioningEntity());
        targetEntityToNativeEntity.put(userFromAuzre.toProvisioningEntity(), userFromAuzre);
      }
      
      TargetDaoRetrieveEntitiesResponse response = new TargetDaoRetrieveEntitiesResponse();
      response.setTargetEntities(targetEntitiesFromAzure);
      
      if (targetDaoRetrieveEntitiesRequest.isIncludeNativeEntity()) {
    	response.setTargetEntityToTargetNativeEntity(targetEntityToNativeEntity);
      }
      
      return response;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("retrieveEntities", startNanos));
    }
  }

  @Override
  public TargetDaoRetrieveGroupsResponse retrieveGroups(
      TargetDaoRetrieveGroupsRequest targetDaoRetrieveGroupsRequest) {

    long startNanos = System.nanoTime();

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this
          .getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      List grouperTargetGroups = targetDaoRetrieveGroupsRequest.getTargetGroups();
      
      List fieldValues = new ArrayList<>();
      for(ProvisioningGroup targetGroup: grouperTargetGroups) {
        String attributeValue = targetGroup.retrieveAttributeValueString(targetDaoRetrieveGroupsRequest.getSearchAttribute());
        if (StringUtils.isNotBlank(attributeValue)) {
          fieldValues.add(attributeValue);
        }
      }
      
      boolean lookupOwners = azureConfiguration.getTargetGroupAttributeNameToConfig().containsKey("groupOwners");
      
      // we can retrieve by id or displayName
      List azureGroups = GrouperAzureApiCommands.retrieveAzureGroups(
          azureConfiguration.getAzureExternalSystemConfigId(), fieldValues, targetDaoRetrieveGroupsRequest.getSearchAttribute(), lookupOwners);
      
      
      List targetGroupsFromAzure = new ArrayList<>();
      
      for (GrouperAzureGroup groupFromAuzre: azureGroups) {
        targetGroupsFromAzure.add(groupFromAuzre.toProvisioningGroup());
      }
      
      TargetDaoRetrieveGroupsResponse response = new TargetDaoRetrieveGroupsResponse();
      response.setTargetGroups(targetGroupsFromAzure);
      
      return response;

    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("retrieveGroups", startNanos));
    }
  }

  @Override
  public TargetDaoInsertGroupsResponse insertGroups(TargetDaoInsertGroupsRequest targetDaoInsertGroupsRequest) {
    
    long startNanos = System.nanoTime();
    List targetGroups = targetDaoInsertGroupsRequest.getTargetGroups();

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();
      
      List grouperAzureGroups = new ArrayList<>();
      
      Map azureGroupToTargetGroup = new HashMap<>();
      
      Map> groupToFieldNamesToInsert = new HashMap<>();
      
      for (ProvisioningGroup targetGroup: targetGroups) {
        GrouperAzureGroup grouperAzureGroup = GrouperAzureGroup.fromProvisioningGroup(targetGroup, null);
        grouperAzureGroups.add(grouperAzureGroup);
        azureGroupToTargetGroup.put(grouperAzureGroup, targetGroup);
        
        
        // lets make sure we are doing the right thing
        Set fieldNamesToInsert = new HashSet();

        // Initialize with fields that are required but have defaults and may not be configured explicitly
        fieldNamesToInsert.add("mailEnabled");
        fieldNamesToInsert.add("securityEnabled");
        
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
          String fieldName = provisioningObjectChange.getAttributeName();
          if (provisioningObjectChange.getProvisioningObjectChangeAction() == ProvisioningObjectChangeAction.insert) {
            fieldNamesToInsert.add(fieldName);
          }
        }
        
        if (grouperAzureGroup.isGroupTypeUnified()) {
          fieldNamesToInsert.add("groupTypeUnified");
        }
        
        if (GrouperUtil.length(grouperAzureGroup.getOwners()) > 0) {
          // transform it into something that can be ingested by azure
          
          Set ownersToBeSaved = new HashSet<>();
          Set usersToPrefixUrl = new HashSet<>();
          
          for (String owner: grouperAzureGroup.getOwners()) {
            if (owner.startsWith("http://") || owner.startsWith("https://")) { // if it's already proper url format
              ownersToBeSaved.add(owner);
            } else if (ownerIdentifierToOwnerUrl.get(owner) != null) {
              ownersToBeSaved.add(ownerIdentifierToOwnerUrl.get(owner));
            } else {
              // try to go fetch it and populate the cache
              usersToPrefixUrl.add(owner);
            }
          }
          
          
          if (usersToPrefixUrl.size() > 0) {
            
            TargetDaoRetrieveEntitiesByValuesRequest request = new TargetDaoRetrieveEntitiesByValuesRequest();
            request.setSearchValues(usersToPrefixUrl);
            
            TargetDaoRetrieveEntitiesByValuesResponse valuesResponse = this.getGrouperProvisioner()
                .retrieveGrouperProvisioningTargetDaoAdapter().retrieveEntitiesBySearchValues(request);
            Map searchValueToTargetEntity = valuesResponse.getSearchValueToTargetEntity();
            
            for (Object searchValue: searchValueToTargetEntity.keySet()) {
              ProvisioningEntity targetEntity = searchValueToTargetEntity.get(searchValue);
              String id = targetEntity.getId();
              String resourceEndpoint = GrouperLoaderConfig.retrieveConfig().propertyValueString(
                  "grouper.azureConnector."+azureConfiguration.getAzureExternalSystemConfigId()+".resourceEndpoint");
              String url = GrouperUtil.stripLastSlashIfExists(resourceEndpoint) + "/users/"+id;
              ownersToBeSaved.add(url);
              ownerIdentifierToOwnerUrl.put(searchValue, url);
            }
          }
          
          grouperAzureGroup.setOwners(ownersToBeSaved);
          
        }
        
        groupToFieldNamesToInsert.put(grouperAzureGroup, fieldNamesToInsert);
        
      }
      
      
      Map groupToMaybeException = GrouperAzureApiCommands.createAzureGroups(azureConfiguration.getAzureExternalSystemConfigId(), groupToFieldNamesToInsert);
      
      for (GrouperAzureGroup grouperAzureGroup: groupToMaybeException.keySet()) {
        
        Exception exception = groupToMaybeException.get(grouperAzureGroup);
        ProvisioningGroup targetGroup = azureGroupToTargetGroup.get(grouperAzureGroup);
        
        if (exception == null) {
          
          targetGroup.setId(grouperAzureGroup.getId());
          targetGroup.setProvisioned(true);

          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetGroup.setProvisioned(false);
          targetGroup.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }

      return new TargetDaoInsertGroupsResponse();
    } catch (Exception e) {
      for (ProvisioningGroup targetGroup: targetGroups) {
        targetGroup.setProvisioned(false);
        targetGroup.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      throw e;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("insertGroups", startNanos));
    }
  }

  @Override
  public TargetDaoInsertMembershipsResponse insertMemberships(TargetDaoInsertMembershipsRequest targetDaoInsertMembershipsRequest) {
    long startNanos = System.nanoTime();
    List targetMemberships = targetDaoInsertMembershipsRequest.getTargetMemberships();

    try {
      
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      // lets collate by group
      Map> groupIdToUserIds = new LinkedHashMap>();

      // keep track to mark as complete
      Map groupIdUserIdToProvisioningMembership = new HashMap();
      
      for (ProvisioningMembership targetMembership : targetMemberships) {

        groupIdUserIdToProvisioningMembership.put(new MultiKey(targetMembership.getProvisioningGroupId(), targetMembership.getProvisioningEntityId()), targetMembership);
        
        List userIds = groupIdToUserIds.get(targetMembership.getProvisioningGroupId());
        if (userIds == null) {
          userIds = new ArrayList();
          groupIdToUserIds.put(targetMembership.getProvisioningGroupId(), userIds);
        }
        userIds.add(targetMembership.getProvisioningEntityId());
      }

      // send batches by group
      for (String groupId : groupIdToUserIds.keySet()) {

        List userIds = groupIdToUserIds.get(groupId);
        
        RuntimeException runtimeException = null;
        try {
          Map groupIdUserIdToException = GrouperAzureApiCommands.createAzureMemberships(azureConfiguration.getAzureExternalSystemConfigId(), groupId, userIds);
          
          for (MultiKey groupIduserId : groupIdUserIdToException.keySet()) {
            ProvisioningMembership targetMembership = groupIdUserIdToProvisioningMembership.get(groupIduserId);
            
            targetMembership.setProvisioned(false);
            targetMembership.setException(groupIdUserIdToException.get(groupIduserId));
            for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetMembership.getInternal_objectChanges())) {
              provisioningObjectChange.setProvisioned(false);
            }
          }
          
        } catch (RuntimeException e) {
          runtimeException = e;
        }
        boolean success = runtimeException == null;
        for (String userId : userIds) {
          MultiKey groupIdUserId = new MultiKey(groupId, userId);
          
          ProvisioningMembership targetMembership = groupIdUserIdToProvisioningMembership.get(groupIdUserId);
          if (targetMembership.getProvisioned() == null) {
            targetMembership.setProvisioned(success);
            targetMembership.setException(runtimeException);
            for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetMembership.getInternal_objectChanges())) {
              provisioningObjectChange.setProvisioned(success);
              
            }
          }
        }
      }
      
      return new TargetDaoInsertMembershipsResponse();
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("insertMemberships", startNanos));
    }
  }
  
  @Override
  public TargetDaoDeleteMembershipsResponse deleteMemberships(TargetDaoDeleteMembershipsRequest targetDaoDeleteMembershipsRequest) {
    long startNanos = System.nanoTime();
    
    List targetMemberships = targetDaoDeleteMembershipsRequest.getTargetMemberships();

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      Map targetMembershipsWithMaybeException = GrouperAzureApiCommands.deleteAzureMemberships(azureConfiguration.getAzureExternalSystemConfigId(), targetMemberships);

      for (ProvisioningMembership targetMembership: targetMembershipsWithMaybeException.keySet()) {
        
        Exception exception = targetMembershipsWithMaybeException.get(targetMembership);

        if (exception == null) {
          targetMembership.setProvisioned(true);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetMembership.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetMembership.setProvisioned(false);
          targetMembership.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetMembership.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }

      return new TargetDaoDeleteMembershipsResponse();
    } catch (Exception e) {
      
      for (ProvisioningMembership targetMembership: targetMemberships) {
        targetMembership.setProvisioned(false);
        targetMembership.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetMembership.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      
      throw new RuntimeException("Failed to delete Azure memberships", e);
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("deleteMemberships", startNanos));
    }
  }

  @Override
  public TargetDaoUpdateGroupsResponse updateGroups(TargetDaoUpdateGroupsRequest targetDaoUpdateGroupsRequest) {
    
    long startNanos = System.nanoTime();
    List targetGroups = targetDaoUpdateGroupsRequest.getTargetGroups();
    
    GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();
    
    Map azureGroupToTargetGroup = new HashMap<>();
    Map> azureGroupToFieldNamesToUpdate = new HashMap<>();
    
    Map> groupIdsToOwnersSave = new HashMap>();
    Map> groupIdsToOwnersDelete = new HashMap>();
    
    for (ProvisioningGroup targetGroup: targetGroups) {
      GrouperAzureGroup grouperAzureGroup = GrouperAzureGroup.fromProvisioningGroup(targetGroup, null);
      if (StringUtils.isBlank(grouperAzureGroup.getId())) {
        continue;
      }
      azureGroupToTargetGroup.put(grouperAzureGroup, targetGroup);
      
      Set fieldNamesToUpdate = new HashSet();
//      for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
//        String fieldName = provisioningObjectChange.getAttributeName();
//        fieldNamesToUpdate.add(fieldName);
//      }
      
      Set ownersToBeSaved = new HashSet<>();
      Set ownersToBeDeleted = new HashSet<>();
      for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
        String fieldName = provisioningObjectChange.getAttributeName();
        if (StringUtils.equals(fieldName, "groupOwners")) {
          boolean manageGroupOwnersInTarget = GrouperUtil.booleanValue(targetGroup.retrieveAttributeValueBoolean("groupOwnersManage"), true);
          if (!manageGroupOwnersInTarget) {
            continue;
          }
          if (provisioningObjectChange.getProvisioningObjectChangeAction() == ProvisioningObjectChangeAction.insert) {
            String groupOwnersToBeInsertedString = (String) provisioningObjectChange.getNewValue();
            if (StringUtils.isBlank(groupOwnersToBeInsertedString)) {
              continue;
            }
            
            Set usersToPrefixUrl = new HashSet<>();
            String[] ownersArray = GrouperUtil.splitTrim(groupOwnersToBeInsertedString, ",");
            for (String owner: ownersArray) {
              if (owner.startsWith("http://") || owner.startsWith("https://")) { // if it's already proper url format
                ownersToBeSaved.add(owner);
              } else {
                usersToPrefixUrl.add(owner);
              }
            }
            for (String userId: usersToPrefixUrl) {
              String resourceEndpoint = GrouperLoaderConfig.retrieveConfig().propertyValueString(
                  "grouper.azureConnector."+azureConfiguration.getAzureExternalSystemConfigId()+".resourceEndpoint");
              String url = GrouperUtil.stripLastSlashIfExists(resourceEndpoint) + "/users/"+userId;
              ownersToBeSaved.add(url);
            }
            
          } else if (provisioningObjectChange.getProvisioningObjectChangeAction() == ProvisioningObjectChangeAction.delete) {
            String groupOwnersToBeDeletedString = (String) provisioningObjectChange.getOldValue();
            if (StringUtils.isBlank(groupOwnersToBeDeletedString)) {
              continue;
            }
            
            Set usersToPrefixUrl = new HashSet<>();
            String[] ownersArray = GrouperUtil.splitTrim(groupOwnersToBeDeletedString, ",");
            for (String owner: ownersArray) {
              ownersToBeDeleted.add(owner);
            }
            
            for (String userId: usersToPrefixUrl) {
              String resourceEndpoint = GrouperLoaderConfig.retrieveConfig().propertyValueString(
                  "grouper.azureConnector."+azureConfiguration.getAzureExternalSystemConfigId()+".resourceEndpoint");
              String url = GrouperUtil.stripLastSlashIfExists(resourceEndpoint) + "/users/"+userId;
              ownersToBeDeleted.add(url);
            }
          }
          
        } else {          
          fieldNamesToUpdate.add(fieldName);
        }
      }
      
      groupIdsToOwnersDelete.put(grouperAzureGroup.getId(), ownersToBeDeleted);
      groupIdsToOwnersSave.put(grouperAzureGroup.getId(), ownersToBeSaved);
      azureGroupToFieldNamesToUpdate.put(grouperAzureGroup, fieldNamesToUpdate);
      
    }
    
    try {
      
      Map groupToMaybeException = GrouperAzureApiCommands.updateAzureGroups(azureConfiguration.getAzureExternalSystemConfigId(), azureGroupToFieldNamesToUpdate);
      
      GrouperAzureApiCommands.addOwnersToGroup(azureConfiguration.getAzureExternalSystemConfigId(), groupIdsToOwnersSave);
      GrouperAzureApiCommands.removeOwnersFromGroup(azureConfiguration.getAzureExternalSystemConfigId(), groupIdsToOwnersDelete);
      
      for (GrouperAzureGroup grouperAzureGroup: groupToMaybeException.keySet()) {
        
        Exception exception = groupToMaybeException.get(grouperAzureGroup);
        ProvisioningGroup targetGroup = azureGroupToTargetGroup.get(grouperAzureGroup);
        
        if (exception == null) {
          
          targetGroup.setProvisioned(true);

          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetGroup.setProvisioned(false);
          targetGroup.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }
      
      return new TargetDaoUpdateGroupsResponse();
    } catch (Exception e) {
      
      for (ProvisioningGroup targetGroup: targetGroups) {
        targetGroup.setProvisioned(false);
        targetGroup.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      
      throw e;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("updateEntities", startNanos));
    }
    
    
  }
  
  @Override
  public TargetDaoUpdateEntitiesResponse updateEntities(TargetDaoUpdateEntitiesRequest targetDaoUpdateEntitiesRequest) {
    
    long startNanos = System.nanoTime();
    List targetEntities = targetDaoUpdateEntitiesRequest.getTargetEntities();
    
    Map azureUserToTargetEntity = new HashMap<>();
    Map> azureUserToFieldNamesToUpdate = new HashMap<>();
    
    for (ProvisioningEntity targetEntity: targetEntities) {
      GrouperAzureUser grouperAzureUser = GrouperAzureUser.fromProvisioningEntity(targetEntity, null);
      if (StringUtils.isBlank(grouperAzureUser.getId())) {
        continue;
      }
      azureUserToTargetEntity.put(grouperAzureUser, targetEntity);
      
      Set fieldNamesToUpdate = new HashSet();
      for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
        String fieldName = provisioningObjectChange.getAttributeName();
        fieldNamesToUpdate.add(fieldName);
      }
      
      azureUserToFieldNamesToUpdate.put(grouperAzureUser, fieldNamesToUpdate);
      
    }
    
    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();
      
      Map userToMaybeException = GrouperAzureApiCommands.updateAzureUsers(azureConfiguration.getAzureExternalSystemConfigId(), azureUserToFieldNamesToUpdate);
      
      for (GrouperAzureUser grouperAzureUser: userToMaybeException.keySet()) {
        
        Exception exception = userToMaybeException.get(grouperAzureUser);
        ProvisioningEntity targetEntity = azureUserToTargetEntity.get(grouperAzureUser);
        
        if (exception == null) {
          
          targetEntity.setProvisioned(true);

          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetEntity.setProvisioned(false);
          targetEntity.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }
      
      return new TargetDaoUpdateEntitiesResponse();
    } catch (Exception e) {
      
      for (ProvisioningEntity targetEntity: targetEntities) {
        targetEntity.setProvisioned(false);
        targetEntity.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      
      throw e;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("updateEntities", startNanos));
    }
    
  }

  
  @Override
  public TargetDaoDeleteGroupsResponse deleteGroups(TargetDaoDeleteGroupsRequest targetDaoDeleteGroupsRequest) {
    
    long startNanos = System.nanoTime();
    List targetGroups = targetDaoDeleteGroupsRequest.getTargetGroups();

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      
      List grouperAzureGroups = new ArrayList<>();
      
      Map azureGroupToTargetGroup = new HashMap<>();
      
      for (ProvisioningGroup targetGroup: targetGroups) {
        GrouperAzureGroup grouperAzureGroup = GrouperAzureGroup.fromProvisioningGroup(targetGroup, null);
        if (StringUtils.isBlank(grouperAzureGroup.getId())) {
          continue;
        }
        grouperAzureGroups.add(grouperAzureGroup);
        azureGroupToTargetGroup.put(grouperAzureGroup, targetGroup);
      }
      
      Map azureGroupsToMaybeException = GrouperAzureApiCommands.deleteAzureGroups(azureConfiguration.getAzureExternalSystemConfigId(), grouperAzureGroups);
      
      for (GrouperAzureGroup azureGroup: azureGroupsToMaybeException.keySet()) {
        
        Exception exception = azureGroupsToMaybeException.get(azureGroup);

        ProvisioningGroup targetGroup = azureGroupToTargetGroup.get(azureGroup);
        if (exception == null) {
          targetGroup.setProvisioned(true);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetGroup.setProvisioned(false);
          targetGroup.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }
    
      return new TargetDaoDeleteGroupsResponse();
    } catch (Exception e) {
     
      for (ProvisioningGroup targetGroup: targetGroups) {
        targetGroup.setProvisioned(false);
        targetGroup.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetGroup.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      
      throw e;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("deleteGroup", startNanos));
    }
  }
    
  @Override
  public TargetDaoRetrieveMembershipsByEntityResponse retrieveMembershipsByEntity(
      TargetDaoRetrieveMembershipsByEntityRequest targetDaoRetrieveMembershipsByEntityRequest) {
    long startNanos = System.nanoTime();
    
    ProvisioningEntity targetEntity = targetDaoRetrieveMembershipsByEntityRequest.getTargetEntity();

    String targetEntityId = resolveTargetEntityId(targetEntity);
    List provisioningMemberships = new ArrayList();
    
    if (StringUtils.isBlank(targetEntityId)) {
      return new TargetDaoRetrieveMembershipsByEntityResponse(provisioningMemberships);
    }

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      Set groupIds = GrouperAzureApiCommands.retrieveAzureUserGroups(azureConfiguration.getAzureExternalSystemConfigId(), targetEntityId);
      
      for (String groupId : groupIds) {

        ProvisioningMembership targetMembership = new ProvisioningMembership(false);
        targetMembership.setProvisioningGroupId(groupId);
        targetMembership.setProvisioningEntityId(targetEntity.getId());
        provisioningMemberships.add(targetMembership);
      }
  
      return new TargetDaoRetrieveMembershipsByEntityResponse(provisioningMemberships);
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("retrieveMembershipsByEntity", startNanos));
    }
  }

  
  public String resolveTargetGroupId(ProvisioningGroup targetGroup) {
    
    if (targetGroup == null) {
      return null;
    }
    
    if (StringUtils.isNotBlank(targetGroup.getId())) {
      return targetGroup.getId();
    }
    
    TargetDaoRetrieveGroupsRequest targetDaoRetrieveGroupsRequest = new TargetDaoRetrieveGroupsRequest();
    targetDaoRetrieveGroupsRequest.setTargetGroups(GrouperUtil.toList(targetGroup));
    targetDaoRetrieveGroupsRequest.setIncludeAllMembershipsIfApplicable(false);
    TargetDaoRetrieveGroupsResponse targetDaoRetrieveGroupsResponse = this.getGrouperProvisioner().retrieveGrouperProvisioningTargetDaoAdapter().retrieveGroups(
        targetDaoRetrieveGroupsRequest);

    if (targetDaoRetrieveGroupsResponse == null || GrouperUtil.length(targetDaoRetrieveGroupsResponse.getTargetGroups()) == 0) {
      return null;
    }
    
    return targetDaoRetrieveGroupsResponse.getTargetGroups().get(0).getId();
    
  }
  
  
  @Override
  public TargetDaoRetrieveMembershipsByGroupResponse retrieveMembershipsByGroup(TargetDaoRetrieveMembershipsByGroupRequest targetDaoRetrieveMembershipsByGroupRequest) {
    long startNanos = System.nanoTime();
    ProvisioningGroup targetGroup = targetDaoRetrieveMembershipsByGroupRequest.getTargetGroup();
    
    String targetGroupId = resolveTargetGroupId(targetGroup);
    List provisioningMemberships = new ArrayList();
    
    if (StringUtils.isBlank(targetGroupId)) {
      return new TargetDaoRetrieveMembershipsByGroupResponse(provisioningMemberships);
    }
    
    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      Set userIds = GrouperAzureApiCommands.retrieveAzureGroupMembers(azureConfiguration.getAzureExternalSystemConfigId(), targetGroupId);
      
      for (String userId : userIds) {

        ProvisioningMembership targetMembership = new ProvisioningMembership(false);
        targetMembership.setProvisioningGroupId(targetGroup.getId());
        targetMembership.setProvisioningEntityId(userId);
        provisioningMemberships.add(targetMembership);
      }
  
      return new TargetDaoRetrieveMembershipsByGroupResponse(provisioningMemberships);
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("retrieveMembershipsByGroup", startNanos));
    }
  }
  
  @Override
  public TargetDaoInsertEntitiesResponse insertEntities(TargetDaoInsertEntitiesRequest targetDaoInsertEntitiesRequest) {
   
    
    long startNanos = System.nanoTime();
    List targetEntities = targetDaoInsertEntitiesRequest.getTargetEntityInserts();

    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();

      List grouperAzureUsers = new ArrayList<>();
      
      Map azureUserToTargetEntity = new HashMap<>();
      for (ProvisioningEntity targetEntity: targetEntities) {
        GrouperAzureUser grouperAzureUser = GrouperAzureUser.fromProvisioningEntity(targetEntity, null);
        grouperAzureUsers.add(grouperAzureUser);
        azureUserToTargetEntity.put(grouperAzureUser, targetEntity);
      }
      
      Map userToMaybeException = GrouperAzureApiCommands.createAzureUsers(azureConfiguration.getAzureExternalSystemConfigId(), grouperAzureUsers, null);
      
      for (GrouperAzureUser grouperAzureUser: userToMaybeException.keySet()) {
        
        Exception exception = userToMaybeException.get(grouperAzureUser);
        ProvisioningEntity targetEntity = azureUserToTargetEntity.get(grouperAzureUser);
        
        if (exception == null) {
          
          targetEntity.setId(grouperAzureUser.getId());
          targetEntity.setProvisioned(true);

          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetEntity.setProvisioned(false);
          targetEntity.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }

      return new TargetDaoInsertEntitiesResponse();
    } catch (Exception e) {
      for (ProvisioningEntity targetEntity: targetEntities) {
        targetEntity.setProvisioned(false);
        targetEntity.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      
      throw e;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("insertEntities", startNanos));
    }
    
    
  }

  @Override
  public void registerGrouperProvisionerDaoCapabilities(
      GrouperProvisionerDaoCapabilities grouperProvisionerDaoCapabilities) {
    
    grouperProvisionerDaoCapabilities.setDefaultBatchSize(20);
    
    grouperProvisionerDaoCapabilities.setCanDeleteGroups(true);
    
    grouperProvisionerDaoCapabilities.setCanDeleteEntities(true);

    grouperProvisionerDaoCapabilities.setCanDeleteMemberships(true);

    grouperProvisionerDaoCapabilities.setCanInsertEntities(true);

    grouperProvisionerDaoCapabilities.setCanInsertGroups(true);

    grouperProvisionerDaoCapabilities.setCanInsertMemberships(true);
    grouperProvisionerDaoCapabilities.setInsertMembershipsBatchSize(400);

    grouperProvisionerDaoCapabilities.setCanRetrieveAllEntities(true);
    grouperProvisionerDaoCapabilities.setCanRetrieveAllGroups(true);
//    grouperProvisionerDaoCapabilities.setCanRetrieveAllMemberships(true);

    grouperProvisionerDaoCapabilities.setCanRetrieveEntities(true);

    grouperProvisionerDaoCapabilities.setCanRetrieveGroups(true);

    grouperProvisionerDaoCapabilities.setCanRetrieveMembershipsAllByEntity(true);
    grouperProvisionerDaoCapabilities.setCanRetrieveMembershipsAllByGroup(true);

    grouperProvisionerDaoCapabilities.setCanUpdateEntities(true);

    grouperProvisionerDaoCapabilities.setCanUpdateGroups(true);
  }

  public String resolveTargetEntityId(ProvisioningEntity targetEntity) {
    
    if (targetEntity == null) {
      return null;
    }
    
    if (StringUtils.isNotBlank(targetEntity.getId())) {
      return targetEntity.getId();
    }
    
    TargetDaoRetrieveEntitiesRequest targetDaoRetrieveEntitiesRequest = new TargetDaoRetrieveEntitiesRequest();
    targetDaoRetrieveEntitiesRequest.setTargetEntities(GrouperUtil.toList(targetEntity));
    targetDaoRetrieveEntitiesRequest.setIncludeAllMembershipsIfApplicable(false);
    TargetDaoRetrieveEntitiesResponse targetDaoRetrieveEntitiesResponse = this.getGrouperProvisioner().retrieveGrouperProvisioningTargetDaoAdapter().retrieveEntities(
        targetDaoRetrieveEntitiesRequest);

    if (targetDaoRetrieveEntitiesResponse == null || GrouperUtil.length(targetDaoRetrieveEntitiesResponse.getTargetEntities()) == 0) {
      return null;
    }
    
    return targetDaoRetrieveEntitiesResponse.getTargetEntities().get(0).getId();
    
  }

  @Override
  public TargetDaoDeleteEntitiesResponse deleteEntities(TargetDaoDeleteEntitiesRequest targetDaoDeleteEntitiesRequest) {
    
    long startNanos = System.nanoTime();
    List targetEntities = targetDaoDeleteEntitiesRequest.getTargetEntities();
  
    try {
      GrouperAzureConfiguration azureConfiguration = (GrouperAzureConfiguration) this.getGrouperProvisioner().retrieveGrouperProvisioningConfiguration();
  
      List grouperAzureUsers = new ArrayList<>();
      
      Map azureUserToTargetEntity = new HashMap<>();
      
      for (ProvisioningEntity targetEntity: targetEntities) {
        GrouperAzureUser grouperAzureUser = GrouperAzureUser.fromProvisioningEntity(targetEntity, null);
        if (StringUtils.isBlank(grouperAzureUser.getId())) {
          continue;
        }
        grouperAzureUsers.add(grouperAzureUser);
        azureUserToTargetEntity.put(grouperAzureUser, targetEntity);
      }
      
      Map azureUsersToMaybeException = GrouperAzureApiCommands.deleteAzureUsers(azureConfiguration.getAzureExternalSystemConfigId(), grouperAzureUsers);
      
      for (GrouperAzureUser azureUser: azureUsersToMaybeException.keySet()) {
        
        Exception exception = azureUsersToMaybeException.get(azureUser);

        ProvisioningEntity targetEntity = azureUserToTargetEntity.get(azureUser);
        if (exception == null) {
          targetEntity.setProvisioned(true);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(true);
          }
        } else {
          targetEntity.setProvisioned(false);
          targetEntity.setException(exception);
          for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
            provisioningObjectChange.setProvisioned(false);
          }
        }
        
      }
      
      return new TargetDaoDeleteEntitiesResponse();
    } catch (Exception e) {
      for (ProvisioningEntity targetEntity: targetEntities) {
        targetEntity.setProvisioned(false);
        targetEntity.setException(e);
        for (ProvisioningObjectChange provisioningObjectChange : GrouperUtil.nonNull(targetEntity.getInternal_objectChanges())) {
          provisioningObjectChange.setProvisioned(false);
        }
      }
      throw e;
    } finally {
      this.addTargetDaoTimingInfo(new TargetDaoTimingInfo("deleteEntities", startNanos));
    }
  }

}