com.google.gerrit.index.IndexedField Maven / Gradle / Ivy
// Copyright (C) 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.index;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import com.google.auto.value.AutoValue;
import com.google.common.base.CharMatcher;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.reflect.TypeToken;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.converter.ProtoConverter;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.index.SchemaFieldDefs.Getter;
import com.google.gerrit.index.SchemaFieldDefs.SchemaField;
import com.google.gerrit.index.SchemaFieldDefs.Setter;
import com.google.gerrit.proto.Protos;
import com.google.protobuf.MessageLite;
import java.io.IOException;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.sql.Timestamp;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.stream.StreamSupport;
/**
* Definition of a field stored in the secondary index.
*
* Each IndexedField, stored in index, may have multiple {@link SearchSpec} which defines how it
* can be searched and how the index tokens are generated.
*
*
Index implementations may choose to store IndexedField and {@link SearchSpec} (search tokens)
* separately, however {@link com.google.gerrit.index.query.IndexedQuery} always issues the queries
* to {@link SearchSpec}.
*
*
This allows index implementations to store IndexedField once, while enabling multiple
* tokenization strategies on the same IndexedField with {@link SearchSpec}
*
* @param input type from which documents are created and search results are returned.
* @param type that should be extracted from the input object when converting to an index
* document.
*/
// TODO(mariasavtchouk): revisit the class name after migration is done.
@SuppressWarnings("serial")
@AutoValue
public abstract class IndexedField {
public static final TypeToken INTEGER_TYPE = new TypeToken<>() {};
public static final TypeToken> ITERABLE_INTEGER_TYPE = new TypeToken<>() {};
public static final TypeToken LONG_TYPE = new TypeToken<>() {};
public static final TypeToken> ITERABLE_LONG_TYPE = new TypeToken<>() {};
public static final TypeToken STRING_TYPE = new TypeToken<>() {};
public static final TypeToken> ITERABLE_STRING_TYPE = new TypeToken<>() {};
public static final TypeToken BYTE_ARRAY_TYPE = new TypeToken<>() {};
public static final TypeToken> ITERABLE_BYTE_ARRAY_TYPE = new TypeToken<>() {};
public static final TypeToken TIMESTAMP_TYPE = new TypeToken<>() {};
// Should not be used directly, only used to check if the proto is stored
private static final TypeToken MESSAGE_TYPE = new TypeToken<>() {};
public static Builder builder(String name, TypeToken fieldType) {
return new AutoValue_IndexedField.Builder()
.name(name)
.fieldType(fieldType)
.stored(false)
.required(false);
}
public static Builder> iterableStringBuilder(String name) {
return builder(name, IndexedField.ITERABLE_STRING_TYPE);
}
public static Builder stringBuilder(String name) {
return builder(name, IndexedField.STRING_TYPE);
}
public static Builder integerBuilder(String name) {
return builder(name, IndexedField.INTEGER_TYPE);
}
public static Builder longBuilder(String name) {
return builder(name, IndexedField.LONG_TYPE);
}
public static Builder> iterableIntegerBuilder(String name) {
return builder(name, IndexedField.ITERABLE_INTEGER_TYPE);
}
public static Builder timestampBuilder(String name) {
return builder(name, IndexedField.TIMESTAMP_TYPE);
}
public static Builder byteArrayBuilder(String name) {
return builder(name, IndexedField.BYTE_ARRAY_TYPE);
}
public static Builder> iterableByteArrayBuilder(String name) {
return builder(name, IndexedField.ITERABLE_BYTE_ARRAY_TYPE);
}
/**
* Defines how {@link IndexedField} can be searched and how the index tokens are generated.
*
* Multiple {@link SearchSpec} can be defined on a single {@link IndexedField}.
*
*
Depending on the implementation, indexes can choose to store {@link IndexedField} and {@link
* SearchSpec} separately. The searches are issues to {@link SearchSpec}.
*/
public class SearchSpec implements SchemaField {
private final String name;
private final SearchOption searchOption;
public SearchSpec(String name, SearchOption searchOption) {
checkName(name);
this.name = name;
this.searchOption = searchOption;
}
@Override
public boolean isStored() {
return getField().stored();
}
@Override
public boolean isRepeatable() {
return getField().repeatable();
}
@Override
@Nullable
public T get(I obj) {
return getField().get(obj);
}
@Override
public String getName() {
return name;
}
@Override
public FieldType getType() {
SearchOption searchOption = getSearchOption();
TypeToken fieldType = getField().fieldType();
if (searchOption.equals(SearchOption.STORE_ONLY)) {
return FieldType.STORED_ONLY;
} else if ((fieldType.equals(IndexedField.INTEGER_TYPE)
|| fieldType.equals(IndexedField.ITERABLE_INTEGER_TYPE))
&& searchOption.equals(SearchOption.EXACT)) {
return FieldType.INTEGER;
} else if (fieldType.equals(IndexedField.INTEGER_TYPE)
&& searchOption.equals(SearchOption.RANGE)) {
return FieldType.INTEGER_RANGE;
} else if (fieldType.equals(IndexedField.LONG_TYPE)) {
return FieldType.LONG;
} else if (fieldType.equals(IndexedField.TIMESTAMP_TYPE)) {
return FieldType.TIMESTAMP;
} else if (fieldType.equals(IndexedField.STRING_TYPE)
|| fieldType.equals(IndexedField.ITERABLE_STRING_TYPE)) {
if (searchOption.equals(SearchOption.EXACT)) {
return FieldType.EXACT;
} else if (searchOption.equals(SearchOption.FULL_TEXT)) {
return FieldType.FULL_TEXT;
} else if (searchOption.equals(SearchOption.PREFIX)) {
return FieldType.PREFIX;
}
}
throw new IllegalArgumentException(
String.format(
"search spec [%s, %s] is not supported on field [%s, %s]",
getName(), getSearchOption(), getField().name(), getField().fieldType()));
}
@Override
public boolean setIfPossible(I object, StoredValue doc) {
return getField().setIfPossible(object, doc);
}
/**
* Returns {@link SearchOption} enabled on this field.
*
* @return {@link SearchOption}
*/
public SearchOption getSearchOption() {
return searchOption;
}
/**
* Returns {@link IndexedField} on which this spec was created.
*
* @return original {@link IndexedField} of this spec.
*/
public IndexedField getField() {
return IndexedField.this;
}
private String checkName(String name) {
CharMatcher m = CharMatcher.anyOf("abcdefghijklmnopqrstuvwxyz0123456789_");
checkArgument(name != null && m.matchesAllOf(name), "illegal field name: %s", name);
return name;
}
}
/**
* Adds {@link SearchSpec} to this {@link IndexedField}
*
* @param name the name to use for in the search.
* @param searchOption the tokenization option, enabled by the new {@link SearchSpec}
* @return the added {@link SearchSpec}.
*/
public SearchSpec addSearchSpec(String name, SearchOption searchOption) {
SearchSpec searchSpec = new SearchSpec(name, searchOption);
checkArgument(
!searchSpecs.containsKey(searchSpec.getName()),
"Can not add search spec %s, because it is already defined on field %s",
searchSpec.getName(),
name());
searchSpecs.put(searchSpec.getName(), searchSpec);
return searchSpec;
}
public SearchSpec exact(String name) {
return addSearchSpec(name, SearchOption.EXACT);
}
public SearchSpec fullText(String name) {
return addSearchSpec(name, SearchOption.FULL_TEXT);
}
public SearchSpec range(String name) {
return addSearchSpec(name, SearchOption.RANGE);
}
public SearchSpec integerRange(String name) {
checkState(fieldType().equals(INTEGER_TYPE));
// we currently store all integer range fields, this may change in the future
checkState(stored());
return addSearchSpec(name, SearchOption.RANGE);
}
public SearchSpec integer(String name) {
checkState(fieldType().equals(INTEGER_TYPE) || fieldType().equals(ITERABLE_INTEGER_TYPE));
return addSearchSpec(name, SearchOption.EXACT);
}
public SearchSpec longSearch(String name) {
checkState(fieldType().equals(LONG_TYPE) || fieldType().equals(ITERABLE_LONG_TYPE));
return addSearchSpec(name, SearchOption.EXACT);
}
public SearchSpec prefix(String name) {
return addSearchSpec(name, SearchOption.PREFIX);
}
public SearchSpec storedOnly(String name) {
checkState(stored());
return addSearchSpec(name, SearchOption.STORE_ONLY);
}
public SearchSpec timestamp(String name) {
checkState(fieldType().equals(TIMESTAMP_TYPE));
return addSearchSpec(name, SearchOption.RANGE);
}
/** A builder for {@link IndexedField}. */
@AutoValue.Builder
public abstract static class Builder {
public abstract IndexedField.Builder name(String name);
public abstract IndexedField.Builder description(Optional description);
public abstract IndexedField.Builder description(String description);
public abstract Builder required(boolean required);
public Builder required() {
required(true);
return this;
}
/** Allow reading the actual data from the index. */
public abstract Builder stored(boolean stored);
public Builder stored() {
stored(true);
return this;
}
abstract Builder repeatable(boolean repeatable);
public abstract Builder size(Optional value);
public abstract Builder size(Integer value);
public abstract Builder getter(Getter getter);
public abstract Builder fieldSetter(Optional> setter);
abstract TypeToken fieldType();
public abstract Builder fieldType(TypeToken type);
public abstract Builder protoConverter(
Optional> value);
abstract IndexedField autoBuild(); // not public
public final IndexedField build() {
boolean isRepeatable = fieldType().isSubtypeOf(Iterable.class);
repeatable(isRepeatable);
IndexedField field = autoBuild();
checkName(field.name());
checkArgument(!field.size().isPresent() || field.size().get() > 0);
return field;
}
public final IndexedField build(Getter getter, Setter setter) {
return this.getter(getter).fieldSetter(Optional.of(setter)).build();
}
public final IndexedField build(
Getter getter,
Setter setter,
ProtoConverter protoConverter) {
return this.getter(getter)
.fieldSetter(Optional.of(setter))
.protoConverter(Optional.of(protoConverter))
.build();
}
public final IndexedField build(Getter getter) {
return this.getter(getter).fieldSetter(Optional.empty()).build();
}
private static String checkName(String name) {
String allowedCharacters = "abcdefghijklmnopqrstuvwxyz0123456789_";
CharMatcher m =
CharMatcher.anyOf(allowedCharacters + allowedCharacters.toUpperCase(Locale.US));
checkArgument(name != null && m.matchesAllOf(name), "illegal field name: %s", name);
return name;
}
}
private Map searchSpecs = new HashMap<>();
/**
* The name to store this field under.
*
* The name should use the UpperCamelCase format, see {@link Builder#checkName}.
*/
public abstract String name();
/** Optional description of the field data. */
public abstract Optional description();
/**
* True if this field is mandatory. Default is false.
*
* This property is not enforced by the common indexing logic. It is up to the index
* implementations to enforce that the field is required.
*/
public abstract boolean required();
/** Allow reading the actual data from the index. Default is false. */
public abstract boolean stored();
/** True if this field is repeatable. */
public abstract boolean repeatable();
/**
* Optional size constrain on the field. The size is not constrained if this property is {@link
* Optional#empty()}
*
*
This property is not enforced by the common indexing logic. It is up to the index
* implementations to enforce the size.
*
*
If the field is {@link #repeatable()}, the constraint applies to each element separately.
*/
public abstract Optional size();
/** See {@link Getter} */
public abstract Getter getter();
/** See {@link Setter} */
public abstract Optional> fieldSetter();
/**
* The {@link TypeToken} describing the contents of the field. See static constants for the common
* supported types.
*
* @return {@link TypeToken} of this field.
*/
public abstract TypeToken fieldType();
/** If the {@link #fieldType()} is proto, the converter to use on byte/proto conversions. */
public abstract Optional> protoConverter();
/**
* Returns all {@link SearchSpec}, enabled on this field.
*
* Note: weather or not a search is supported by the index depends on {@link Schema} version.
*/
public ImmutableMap getSearchSpecs() {
return ImmutableMap.copyOf(searchSpecs);
}
/**
* Get the field contents from the input object.
*
* @param input input object.
* @return the field value(s) to index.
*/
@Nullable
public T get(I input) {
try {
return getter().get(input);
} catch (IOException e) {
throw new StorageException(e);
}
}
@SuppressWarnings("unchecked")
public boolean setIfPossible(I object, StoredValue doc) {
if (!fieldSetter().isPresent()) {
return false;
}
if (this.fieldType().equals(STRING_TYPE)) {
fieldSetter().get().set(object, (T) doc.asString());
return true;
} else if (this.fieldType().equals(ITERABLE_STRING_TYPE)) {
fieldSetter().get().set(object, (T) doc.asStrings());
return true;
} else if (this.fieldType().equals(INTEGER_TYPE)) {
fieldSetter().get().set(object, (T) doc.asInteger());
return true;
} else if (this.fieldType().equals(ITERABLE_INTEGER_TYPE)) {
fieldSetter().get().set(object, (T) doc.asIntegers());
return true;
} else if (this.fieldType().equals(LONG_TYPE)) {
fieldSetter().get().set(object, (T) doc.asLong());
return true;
} else if (this.fieldType().equals(ITERABLE_LONG_TYPE)) {
fieldSetter().get().set(object, (T) doc.asLongs());
return true;
} else if (this.fieldType().equals(BYTE_ARRAY_TYPE)) {
fieldSetter().get().set(object, (T) doc.asByteArray());
return true;
} else if (this.fieldType().equals(ITERABLE_BYTE_ARRAY_TYPE)) {
fieldSetter().get().set(object, (T) doc.asByteArrays());
return true;
} else if (this.fieldType().equals(TIMESTAMP_TYPE)) {
checkState(!repeatable(), "can't repeat timestamp values");
fieldSetter().get().set(object, (T) doc.asTimestamp());
return true;
} else if (isProtoType()) {
MessageLite proto = doc.asProto();
if (proto != null) {
fieldSetter().get().set(object, (T) proto);
return true;
}
byte[] bytes = doc.asByteArray();
if (bytes != null && protoConverter().isPresent()) {
fieldSetter().get().set(object, (T) parseProtoFrom(bytes));
return true;
}
} else if (isProtoIterableType()) {
Iterable protos = doc.asProtos();
if (protos != null) {
fieldSetter().get().set(object, (T) protos);
return true;
}
Iterable bytes = doc.asByteArrays();
if (bytes != null && protoConverter().isPresent()) {
fieldSetter().get().set(object, (T) decodeProtos(bytes));
return true;
}
}
return false;
}
/** Returns true if the {@link #fieldType} is a proto message. */
public boolean isProtoType() {
if (repeatable()) {
return false;
}
return MESSAGE_TYPE.isSupertypeOf(fieldType());
}
/** Returns true if the {@link #fieldType} is a list of proto messages. */
public boolean isProtoIterableType() {
if (!repeatable()) {
return false;
}
if (!(fieldType().getType() instanceof ParameterizedType)) {
return false;
}
ParameterizedType parameterizedType = (ParameterizedType) fieldType().getType();
if (parameterizedType.getActualTypeArguments().length != 1) {
return false;
}
Type type = parameterizedType.getActualTypeArguments()[0];
return MESSAGE_TYPE.isSupertypeOf(type);
}
private ImmutableList decodeProtos(Iterable raw) {
return StreamSupport.stream(raw.spliterator(), false)
.map(bytes -> parseProtoFrom(bytes))
.collect(toImmutableList());
}
private MessageLite parseProtoFrom(byte[] bytes) {
return Protos.parseUnchecked(protoConverter().get().getParser(), bytes);
}
}