com.aliyun.openservices.paifeaturestore.domain.Model Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of paifeaturestore-sdk Show documentation
Show all versions of paifeaturestore-sdk Show documentation
SDK for PAI-FeatureStore service
package com.aliyun.openservices.paifeaturestore.domain;
import com.aliyun.openservices.paifeaturestore.constants.FSType;
import com.aliyun.openservices.paifeaturestore.model.ModelFeatures;
import com.aliyun.openservices.paifeaturestore.model.SeqConfig;
import com.aliyun.tea.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class Model {
public static Logger logger = LoggerFactory.getLogger(Model.class);
private final com.aliyun.openservices.paifeaturestore.model.Model model;
private final Project project;
private final Map featureViewMap = new HashMap<>();
private final Map featureEntityMap = new HashMap<>();
// featureview : feature names
private final Map> featureNamesMap = new HashMap<>();
// featureview : alias names
private final Map> aliasNamesMap = new HashMap<>();
// feature entity joinid : featureviews
private final Map> featureEntityJoinIdMap = new HashMap<>();
List featureEntityJoinIdList = new ArrayList<>();
// entity joinid : feature entity
private final Map entityJoinIdToFeatureEntityMap = new HashMap<>();
private ExecutorService executorService;
public Model(com.aliyun.openservices.paifeaturestore.model.Model model, Project project) {
int parallelism = Runtime.getRuntime().availableProcessors();
executorService = new ThreadPoolExecutor( parallelism*2,Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,new SynchronousQueue<>(), r -> {
Thread thread = new Thread(r);
thread.setName("modelfeature-processor");
thread.setDaemon(true);
return thread;
});
this.model = model;
this.project = project;
for (ModelFeatures feature : this.model.getFeatures()) {
//IFeatureView featureView = project.getFeatureView(feature.getFeatureViewName());
IFeatureView featureView = project.getFeatureView(feature.getFeatureViewName());
if (null == featureView) {
featureView = project.getSeqFeatureView(feature.getFeatureViewName());
}
FeatureEntity featureEntity = project.getFeatureEntity(featureView.getFeatureView().getFeatureEntityName());
this.featureViewMap.put(feature.getFeatureViewName(), featureView);
this.featureEntityMap.put(featureView.getFeatureView().getFeatureEntityName(), featureEntity);
this.entityJoinIdToFeatureEntityMap.put(featureEntity.getFeatureEntity().getFeatureEntityJoinid(), featureEntity);
if (this.featureNamesMap.containsKey(feature.getFeatureViewName())) {
if (featureView instanceof SequenceFeatureView) {
SequenceFeatureView sequenceFeatureView = (SequenceFeatureView) featureView;
for (SeqConfig config : sequenceFeatureView.getSeqConfigs()) {
if (config.getOfflineSeqName().equals(feature.getName())) {
this.featureNamesMap.get(feature.getFeatureViewName()).add(config.getOnlineSeqName());
}
}
} else {
this.featureNamesMap.get(feature.getFeatureViewName()).add(feature.getName());
}
} else {
List names = new ArrayList<>();
if (featureView instanceof SequenceFeatureView) {
SequenceFeatureView sequenceFeatureView = (SequenceFeatureView) featureView;
for (SeqConfig config : sequenceFeatureView.getSeqConfigs()) {
if (config.getOfflineSeqName().equals(feature.getName())) {
names.add(config.getOnlineSeqName());
}
}
} else {
names.add(feature.getName());
}
this.featureNamesMap.put(feature.getFeatureViewName(), names);
}
if (!StringUtils.isEmpty(feature.getAliasName())) {
if (this.aliasNamesMap.containsKey(feature.getFeatureViewName())) {
this.aliasNamesMap.get(feature.getFeatureViewName()).put(feature.getName(), feature.getAliasName());
} else {
Map names = new HashMap<>();
names.put(feature.getName(), feature.getAliasName());
this.aliasNamesMap.put(feature.getFeatureViewName(), names);
}
}
if (this.featureEntityJoinIdMap.containsKey(featureEntity.getFeatureEntity().getFeatureEntityJoinid())) {
this.featureEntityJoinIdMap.get(featureEntity.getFeatureEntity().getFeatureEntityJoinid()).put(feature.getFeatureViewName(), featureView);
} else {
Map featureViewMap1 = new HashMap<>();
featureViewMap1.put(feature.getFeatureViewName(), featureView);
this.featureEntityJoinIdMap.put(featureEntity.getFeatureEntity().getFeatureEntityJoinid(), featureViewMap1);
}
for (String joinId : this.featureEntityJoinIdMap.keySet()) {
this.featureEntityJoinIdList.add(joinId);
}
}
}
public com.aliyun.openservices.paifeaturestore.model.Model getModel() {
return model;
}
public FeatureResult getOnlineFeatures(Map> joinIds) throws Exception {
int size = -1;
for (String joinId : this.featureEntityJoinIdList) {
if (!joinIds.containsKey(joinId)) {
throw new RuntimeException(String.format("join id:%s not found", joinId));
}
if (-1 == size) {
size = joinIds.get(joinId).size();
} else {
if (size != joinIds.get(joinId).size()) {
throw new RuntimeException(String.format("join id:%s length not equal", joinId));
}
}
}
// thread safe map
List featureFields = new CopyOnWriteArrayList<>();
Map featureFieldTypeMap = new ConcurrentHashMap<>();
List> futures = new ArrayList<>();
Map> indexFeatrueMap = new ConcurrentHashMap<>();
for (Map.Entry> entry : joinIds.entrySet()) {
int finalSize = size;
CompletableFuture future = CompletableFuture.runAsync(()->{
try {
FeatureResult featureResult = new FeatureStoreResult();
try {
featureResult = getOnlineFeaturesWithEntity(joinIds, this.entityJoinIdToFeatureEntityMap.get(entry.getKey()).featureEntity.getFeatureEntityName());
} catch (Exception e) {
logger.error("featureview get online features error", e);
}
featureFields.addAll(Arrays.asList(featureResult.getFeatureFields()));
if (featureResult.getFeatureFieldTypeMap()!=null) {
featureFieldTypeMap.putAll(featureResult.getFeatureFieldTypeMap());
}
for (int i = 0; i < finalSize; i++) {
String joinIdValue = entry.getValue().get(i);
for (Map featureData : featureResult.getFeatureData()) {
if (featureData != null) {
if ( joinIdValue.equals(String.valueOf(featureData.get(entry.getKey())))) {
indexFeatrueMap.computeIfAbsent(i, k -> new ConcurrentHashMap<>()).putAll(featureData);
break;
}
}
}
}
} catch (Exception e) {
logger.error("featureview get online features error", e);
}
}, executorService);
futures.add(future);
}
FeatureStoreResult featureStoreResult = new FeatureStoreResult();
List> featureDataList = new ArrayList<>(size);
// wait all featureview get features
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
for (int i = 0; i < size; i++) {
if (indexFeatrueMap.get(i)==null) {
featureDataList.add(new HashMap<>());
} else {
featureDataList.add(indexFeatrueMap.get(i));
}
}
featureStoreResult.setFeatureFields(featureFields.toArray(new String[0]));
featureStoreResult.setFeatureDataList(featureDataList);
featureStoreResult.setFeatureFieldTypeMap(featureFieldTypeMap);
return featureStoreResult;
}
public FeatureResult getOnlineFeaturesWithEntity(Map> joinIds, String featureEntityName) throws Exception {
FeatureEntity featureEntity = this.featureEntityMap.get(featureEntityName);
if (featureEntity == null) {
throw new RuntimeException(String.format("feature entity name:%s not found", featureEntityName));
}
String entityJoinId = featureEntity.getFeatureEntity().getFeatureEntityJoinid();
if (!joinIds.containsKey(entityJoinId)) {
throw new RuntimeException(String.format("join id:%s not found", entityJoinId));
}
Map featureViewMap = this.featureEntityJoinIdMap.get(entityJoinId);
String[] joinIdsArray = joinIds.get(entityJoinId).toArray(new String[0]);
FeatureStoreResult featureStoreResult = new FeatureStoreResult();
List featureFields = new CopyOnWriteArrayList<>();
Map featureFieldTypeMap = new ConcurrentHashMap<>();
Map> joinIdFeaturMap = new ConcurrentHashMap<>();
List> futures = new ArrayList<>();
for (IFeatureView featureView : featureViewMap.values()) {
CompletableFuture future = CompletableFuture.runAsync(()->{
try {
FeatureResult featureResult = featureView.getOnlineFeatures(joinIdsArray,
this.featureNamesMap.get(featureView.getFeatureView().getName()).toArray(new String[0]), this.aliasNamesMap.get(featureView.getFeatureView().getName()));
if (featureResult.getFeatureData()!=null) {
featureFields.addAll(Arrays.asList(featureResult.getFeatureFields()));
if (featureResult.getFeatureFieldTypeMap()!=null) {
featureFieldTypeMap.putAll(featureResult.getFeatureFieldTypeMap());
}
for (Map featureData : featureResult.getFeatureData()) {
if (featureData != null) {
String joinIdValue = String.valueOf(featureData.get(entityJoinId));
// 过滤掉值为 null 的条目
Map filteredData = featureData.entrySet().stream()
.filter(entry -> entry.getValue() != null)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
joinIdFeaturMap.computeIfAbsent(joinIdValue, k -> new ConcurrentHashMap<>()).putAll(filteredData);
}
}
}
} catch (Exception e) {
logger.error("get feature view features error", e);
}
}, executorService );
futures.add(future);
}
List> featureDataList = new ArrayList<>();
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
for (String joinIdValue : joinIdsArray) {
if (joinIdFeaturMap.containsKey(joinIdValue)) {
featureDataList.add(joinIdFeaturMap.get(joinIdValue));
} else {
Map featureMap = new HashMap<>();
featureMap.put(entityJoinId, joinIdValue);
featureDataList.add(featureMap);
}
}
featureStoreResult.setFeatureFields(featureFields.toArray(new String[0]));
featureStoreResult.setFeatureDataList(featureDataList);
featureStoreResult.setFeatureFieldTypeMap(featureFieldTypeMap);
return featureStoreResult;
}
public void close() throws Exception {
if (!executorService.isShutdown()) {
executorService.shutdown();
}
try {
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
executorService.shutdownNow();
}
} catch (InterruptedException e) {
executorService.shutdownNow();
}
}
}