com.yahoo.vespa.config.server.filedistribution.FileDBRegistry Maven / Gradle / Ivy
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.config.server.filedistribution;
import com.google.common.collect.ImmutableMap;
import com.yahoo.config.FileReference;
import com.yahoo.config.application.api.FileRegistry;
import com.yahoo.net.HostName;
import com.yahoo.path.Path;
import com.yahoo.text.Utf8;
import net.jpountz.xxhash.XXHashFactory;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
/**
* File registry for one application package
*
* @author Tony Vaagenes
*/
public class FileDBRegistry implements FileRegistry {
private final boolean silenceNonExistingFiles;
private final AddFileInterface manager;
private final Map fileReferenceCache = new HashMap<>();
private static final String entryDelimiter = "\t";
private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL);
public FileDBRegistry(AddFileInterface manager) {
this(manager, Map.of(), false);
}
private FileDBRegistry(AddFileInterface manager, Map knownReferences, boolean silenceNonExistingFiles) {
this.silenceNonExistingFiles = silenceNonExistingFiles;
this.manager = manager;
fileReferenceCache.putAll(knownReferences);
}
public static FileDBRegistry create(AddFileInterface manager, Reader persistedState) {
try (BufferedReader reader = new BufferedReader(persistedState)) {
String ignoredFileSourceHost = reader.readLine();
if (ignoredFileSourceHost == null)
throw new RuntimeException("No file source host");
return new FileDBRegistry(manager, decode(reader), true);
} catch (IOException e) {
throw new RuntimeException("Error while reading pre-generated file registry", e);
}
}
static Map decode(BufferedReader reader) {
Map refs = new HashMap<>();
try {
String line;
while ((line = reader.readLine()) != null) {
String[] parts = entryDelimiterPattern.split(line);
if (parts.length < 2)
throw new IllegalArgumentException("Cannot split '" + line + "' into two parts");
refs.put(parts[0], new FileReference(parts[1]));
}
} catch (IOException e) {
throw new RuntimeException("Error while reading pre-generated file registry", e);
}
return refs;
}
@Override
public synchronized FileReference addFile(String relativePath) {
if (relativePath.startsWith("/"))
throw new IllegalArgumentException(relativePath + " is not relative");
Optional cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath));
return cachedReference.orElseGet(() -> {
try {
FileReference newRef = manager.addFile(Path.fromString(relativePath));
fileReferenceCache.put(relativePath, newRef);
return newRef;
} catch (FileNotFoundException e) {
if (silenceNonExistingFiles) {
return new FileReference("non-existing-file");
} else {
throw new IllegalArgumentException(e);
}
} catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
);
}
@Override
public synchronized FileReference addUri(String uri) {
String relativePath = uriToRelativeFile(uri);
Optional cachedReference = Optional.ofNullable(fileReferenceCache.get(uri));
return cachedReference.orElseGet(() -> {
FileReference newRef = manager.addUri(uri, Path.fromString(relativePath));
fileReferenceCache.put(uri, newRef);
return newRef;
});
}
@Override
public synchronized FileReference addBlob(String blobName, ByteBuffer blob) {
String relativePath = blobToRelativeFile(blobName);
Optional cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName));
return cachedReference.orElseGet(() -> {
FileReference newRef = manager.addBlob(blob, Path.fromString(relativePath));
fileReferenceCache.put(blobName, newRef);
return newRef;
});
}
@Override
public synchronized List export() {
List entries = new ArrayList<>();
for (Map.Entry entry : fileReferenceCache.entrySet()) {
entries.add(new Entry(entry.getKey(), entry.getValue()));
}
return entries;
}
// Used for testing only
synchronized Map getMap() {
return ImmutableMap.copyOf(fileReferenceCache);
}
public static String exportRegistry(FileRegistry registry) {
List entries = registry.export();
StringBuilder builder = new StringBuilder();
builder.append(HostName.getLocalhost()).append('\n');
for (FileRegistry.Entry entry : entries) {
builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n');
}
return builder.toString();
}
private static String uriToRelativeFile(String uri) {
String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0);
if (uri.endsWith(".json")) {
relative += ".json";
} else if (uri.endsWith(".json.lz4")) {
relative += ".json.lz4";
} else if (uri.endsWith(".lz4")) {
relative += ".lz4";
}
return relative;
}
private static String blobToRelativeFile(String blobName) {
return "blob/" + blobName;
}
}