com.aliyun.openservices.paifeaturestore.domain.Model Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of paifeaturestore-sdk Show documentation
Show all versions of paifeaturestore-sdk Show documentation
SDK for PAI-FeatureStore service
package com.aliyun.openservices.paifeaturestore.domain;
import com.aliyun.openservices.paifeaturestore.constants.FSType;
import com.aliyun.openservices.paifeaturestore.model.ModelFeatures;
import com.aliyun.openservices.paifeaturestore.model.SeqConfig;
import com.aliyun.tea.utils.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class Model {
private final com.aliyun.openservices.paifeaturestore.model.Model model;
private final Project project;
private final Map featureViewMap = new HashMap<>();
private final Map featureEntityMap = new HashMap<>();
// featureview : feature names
private final Map> featureNamesMap = new HashMap<>();
// featureview : alias names
private final Map> aliasNamesMap = new HashMap<>();
// feature entity joinid : featureviews
private final Map> featureEntityJoinIdMap = new HashMap<>();
List featureEntityJoinIdList = new ArrayList<>();
static ExecutorService executorService;
static {
executorService = Executors.newCachedThreadPool(r -> {
Thread thread = new Thread(r);
thread.setName("modelfeature-processor");
thread.setDaemon(true);
return thread;
});
}
public Model(com.aliyun.openservices.paifeaturestore.model.Model model, Project project) {
this.model = model;
this.project = project;
for (ModelFeatures feature : this.model.getFeatures()) {
//IFeatureView featureView = project.getFeatureView(feature.getFeatureViewName());
IFeatureView featureView = project.getFeatureViewMap().get(feature.getFeatureViewName());
FeatureEntity featureEntity = project.getFeatureEntity(featureView.getFeatureView().getFeatureEntityName());
this.featureViewMap.put(feature.getFeatureViewName(), featureView);
this.featureEntityMap.put(featureView.getFeatureView().getFeatureEntityName(), featureEntity);
if (this.featureNamesMap.containsKey(feature.getFeatureViewName())) {
if (featureView instanceof SequenceFeatureView) {
SequenceFeatureView sequenceFeatureView = (SequenceFeatureView) featureView;
for (SeqConfig config : sequenceFeatureView.getSeqConfigs()) {
if (config.getOfflineSeqName().equals(feature.getName())) {
this.featureNamesMap.get(feature.getFeatureViewName()).add(config.getOnlineSeqName());
}
}
} else {
this.featureNamesMap.get(feature.getFeatureViewName()).add(feature.getName());
}
} else {
List names = new ArrayList<>();
if (featureView instanceof SequenceFeatureView) {
SequenceFeatureView sequenceFeatureView = (SequenceFeatureView) featureView;
for (SeqConfig config : sequenceFeatureView.getSeqConfigs()) {
if (config.getOfflineSeqName().equals(feature.getName())) {
names.add(config.getOnlineSeqName());
}
}
} else {
names.add(feature.getName());
}
this.featureNamesMap.put(feature.getFeatureViewName(), names);
}
if (!StringUtils.isEmpty(feature.getAliasName())) {
if (this.aliasNamesMap.containsKey(feature.getFeatureViewName())) {
this.aliasNamesMap.get(feature.getFeatureViewName()).put(feature.getName(), feature.getAliasName());
} else {
Map names = new HashMap<>();
names.put(feature.getName(), feature.getAliasName());
this.aliasNamesMap.put(feature.getFeatureViewName(), names);
}
}
if (this.featureEntityJoinIdMap.containsKey(featureEntity.getFeatureEntity().getFeatureEntityJoinid())) {
this.featureEntityJoinIdMap.get(featureEntity.getFeatureEntity().getFeatureEntityJoinid()).put(feature.getFeatureViewName(), featureView);
} else {
Map featureViewMap1 = new HashMap<>();
featureViewMap1.put(feature.getFeatureViewName(), featureView);
this.featureEntityJoinIdMap.put(featureEntity.getFeatureEntity().getFeatureEntityJoinid(), featureViewMap1);
}
for (String joinId : this.featureEntityJoinIdMap.keySet()) {
this.featureEntityJoinIdList.add(joinId);
}
}
}
public com.aliyun.openservices.paifeaturestore.model.Model getModel() {
return model;
}
public FeatureResult getOnlineFeatures(Map> joinIds) throws Exception {
int size = -1;
for (String joinId : this.featureEntityJoinIdList) {
if (!joinIds.containsKey(joinId)) {
throw new RuntimeException(String.format("join id:%s not found", joinId));
}
if (-1 == size) {
size = joinIds.get(joinId).size();
} else {
if (size != joinIds.get(joinId).size()) {
throw new RuntimeException(String.format("join id:%s length not equal", joinId));
}
}
}
CountDownLatch countDownLatch = new CountDownLatch(joinIds.size());
// thread safe map
Map> joinIdFeaturesMap = new ConcurrentHashMap<>();
for (Map.Entry> entry : joinIds.entrySet()) {
executorService.execute(() -> {
try {
Map featureViewMap = this.featureEntityJoinIdMap.get(entry.getKey());
Stream> completableFutureStream = featureViewMap.values().stream().map(featureView -> CompletableFuture.supplyAsync(() -> {
try {
return featureView.getOnlineFeatures(entry.getValue().toArray(new String[0]),
this.featureNamesMap.get(featureView.getFeatureView().getName()).toArray(new String[0]), this.aliasNamesMap.get(featureView.getFeatureView().getName()));
} catch (Exception e) {
throw new RuntimeException(e);
}
}));
List featureResults = completableFutureStream.map(CompletableFuture::join).collect(Collectors.toList());
// add to map
joinIdFeaturesMap.put(entry.getKey(), featureResults);
} finally {
countDownLatch.countDown();
}
});
}
// wait all featureview get features
countDownLatch.await();
FeatureStoreResult featureStoreResult = new FeatureStoreResult();
List featureFields = new ArrayList<>();
Map featureFieldTypeMap = new HashMap<>();
List> featureDataList = new ArrayList<>();
for (String joinId : joinIds.keySet()) {
if (!joinIdFeaturesMap.isEmpty()) {
for (FeatureResult result : joinIdFeaturesMap.get(joinId)) {
featureFields.addAll(Arrays.asList(result.getFeatureFields()));
if (result.getFeatureFieldTypeMap()==null) {
continue;
}
featureFieldTypeMap.putAll(result.getFeatureFieldTypeMap());
}
}
}
for (int i = 0; i < size; i++) {
Map featuresMap = new HashMap<>();
for (String joinId : this.featureEntityJoinIdList) {
String joinIdValue = joinIds.get(joinId).get(i);
for (FeatureResult result : joinIdFeaturesMap.get(joinId)) {
if (result.getFeatureData()!=null) {
for (Map featureData : result.getFeatureData()) {
if (joinIdValue.equals(String.valueOf(featureData.get(joinId)))) {
featuresMap.putAll(featureData);
}
}
}
}
}
featureDataList.add(featuresMap);
}
featureStoreResult.setFeatureFields(featureFields.toArray(new String[0]));
featureStoreResult.setFeatureDataList(featureDataList);
featureStoreResult.setFeatureFieldTypeMap(featureFieldTypeMap);
return featureStoreResult;
}
/**
* Get features with featureEntityName. JoinIds must contain the joinId of the FeatureEntity.
* @param joinIds
* @param featureEntityName
* @return
* @throws Exception
*/
public FeatureResult getOnlineFeaturesWithEntity(Map> joinIds, String featureEntityName) throws Exception {
FeatureEntity featureEntity = this.featureEntityMap.get(featureEntityName);
if (featureEntity == null) {
throw new RuntimeException(String.format("feature entity name:%s not found", featureEntityName));
}
if (!joinIds.containsKey(featureEntity.getFeatureEntity().getFeatureEntityJoinid())) {
throw new RuntimeException(String.format("join id:%s not found", featureEntity.getFeatureEntity().getFeatureEntityJoinid()));
}
Map featureViewMap = this.featureEntityJoinIdMap.get(featureEntity.getFeatureEntity().getFeatureEntityJoinid());
List> futures = featureViewMap.values().stream().map(featureView -> CompletableFuture.supplyAsync(() -> {
try {
return featureView.getOnlineFeatures(joinIds.get(featureEntity.getFeatureEntity().getFeatureEntityJoinid()).toArray(new String[0]),
this.featureNamesMap.get(featureView.getFeatureView().getName()).toArray(new String[0]), this.aliasNamesMap.get(featureView.getFeatureView().getName()));
} catch (Exception e) {
throw new RuntimeException(e);
}
})).collect(Collectors.toList());
CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
CompletableFuture> allFutureResults = allFutures.thenApply(v ->
futures.stream()
.map(CompletableFuture::join)
.collect(Collectors.toList())
);
List featureResults = null;
try {
featureResults = allFutureResults.get();
} catch (Exception e) {
throw new RuntimeException(e);
}
FeatureStoreResult featureStoreResult = new FeatureStoreResult();
List featureFields = new ArrayList<>();
Map featureFieldTypeMap = new HashMap<>();
List> featureDataList = new ArrayList<>();
for (FeatureResult result : featureResults) {
featureFields.addAll(Arrays.asList(result.getFeatureFields()));
if (result.getFeatureFieldTypeMap()==null) {
continue;
}
featureFieldTypeMap.putAll(result.getFeatureFieldTypeMap());
}
for (String joinIdValue : joinIds.get(featureEntity.getFeatureEntity().getFeatureEntityJoinid())) {
Map featuresMap = new HashMap<>();
for (FeatureResult result : featureResults) {
if (result.getFeatureData()!=null) {
for (Map featureData : result.getFeatureData()) {
if (joinIdValue.equals(String.valueOf(featureData.get(featureEntity.getFeatureEntity().getFeatureEntityJoinid())))) {
featuresMap.putAll(featureData);
}
}
}
}
featureDataList.add(featuresMap);
}
featureStoreResult.setFeatureFields(featureFields.toArray(new String[0]));
featureStoreResult.setFeatureDataList(featureDataList);
featureStoreResult.setFeatureFieldTypeMap(featureFieldTypeMap);
return featureStoreResult;
}
}