io.cdap.cdap.metadata.DatasetFieldLineageSummary Maven / Gradle / Ivy
/*
* Copyright © 2019 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package io.cdap.cdap.metadata;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.annotations.SerializedName;
import io.cdap.cdap.common.conf.Constants;
import io.cdap.cdap.proto.id.DatasetId;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* The summary about all the field level lineage information about all fields in a dataset.
*/
public class DatasetFieldLineageSummary {
private final Constants.FieldLineage.Direction direction;
@SerializedName("start-ts")
private final long startTs;
@SerializedName("end-ts")
private final long endTs;
@SerializedName("entityId")
private final DatasetId datasetId;
private final Set fields;
private final Set incoming;
private final Set outgoing;
public DatasetFieldLineageSummary(Constants.FieldLineage.Direction direction, long startTs, long endTs,
DatasetId datasetId, Set fields,
Map fieldCounts,
Map> incoming,
Map> outgoing) {
this.direction = direction;
this.startTs = startTs;
this.endTs = endTs;
this.datasetId = datasetId;
this.fields = fields;
this.incoming = incoming.entrySet().stream().map(
entry -> new FieldLineageRelations(entry.getKey(), fieldCounts.getOrDefault(entry.getKey(), 0),
entry.getValue())).collect(Collectors.toSet());
this.outgoing = outgoing.entrySet().stream().map(
entry -> new FieldLineageRelations(entry.getKey(), fieldCounts.getOrDefault(entry.getKey(), 0),
entry.getValue())).collect(Collectors.toSet());
}
public Constants.FieldLineage.Direction getDirection() {
return direction;
}
public long getStartTs() {
return startTs;
}
public long getEndTs() {
return endTs;
}
public DatasetId getDatasetId() {
return datasetId;
}
public Set getFields() {
return fields;
}
public Set getIncoming() {
return incoming;
}
public Set getOutgoing() {
return outgoing;
}
/**
* This class represents an aggregation of the field level lineage about an incoming/outgoing dataset. The entityId
* represents the dataset. And the relations contains all the field level lineage.
*/
public static class FieldLineageRelations {
@SerializedName("entityId")
private final DatasetId datasetId;
private final int fieldCount;
private final Set relations;
// this datasetId can be nullable to represent the field is not related to any other dataset but still
// have other operations around itself, i.e, drop or generate
@VisibleForTesting
public FieldLineageRelations(@Nullable DatasetId datasetId, int fieldCount, Set relations) {
this.datasetId = datasetId;
this.fieldCount = fieldCount;
this.relations = relations;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
DatasetFieldLineageSummary.FieldLineageRelations that = (DatasetFieldLineageSummary.FieldLineageRelations) o;
return Objects.equals(datasetId, that.datasetId) &&
Objects.equals(relations, that.relations) &&
Objects.equals(fieldCount, that.fieldCount);
}
@Override
public int hashCode() {
return Objects.hash(datasetId, fieldCount, relations);
}
public DatasetId getDatasetId() {
return datasetId;
}
public int getFieldCount() {
return fieldCount;
}
public Set getRelations() {
return relations;
}
}
}