jvmTest.okhttp3.internal.publicsuffix.PublicSuffixListGenerator Maven / Gradle / Ivy
/*
* Copyright (C) 2017 Square, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package okhttp3.internal.publicsuffix;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.SortedSet;
import java.util.TreeSet;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okio.BufferedSink;
import okio.BufferedSource;
import okio.ByteString;
import okio.GzipSink;
import okio.Okio;
import okio.Sink;
/**
* Downloads the public suffix list from https://publicsuffix.org/list/public_suffix_list.dat and
* transforms the file into an efficient format used by OkHttp.
*
* The intent is to use this class to update the list periodically by manually running the main
* method. This should be run from the top-level okhttp directory.
*
*
The resulting file is used by {@link PublicSuffixDatabase}.
*/
public final class PublicSuffixListGenerator {
private static final String OKHTTP_RESOURCE_DIR = "okhttp" + File.separator + "src"
+ File.separator + "main" + File.separator + "resources" + File.separator
+ "okhttp3" + File.separator + "internal" + File.separator + "publicsuffix";
private static final String OKHTTP_TEST_RESOURCE_DIR = "okhttp" + File.separator + "src"
+ File.separator + "test" + File.separator + "resources" + File.separator
+ "okhttp3" + File.separator + "internal" + File.separator + "publicsuffix";
private static final File publicSuffixListDotDat
= new File(OKHTTP_TEST_RESOURCE_DIR, "public_suffix_list.dat");
private static final ByteString EXCEPTION_RULE_MARKER = ByteString.encodeUtf8("!");
private static final String WILDCARD_CHAR = "*";
public static void main(String... args) throws IOException {
File resources = new File(OKHTTP_RESOURCE_DIR);
File testResources = publicSuffixListDotDat.getParentFile();
for (File file : Arrays.asList(resources, testResources)) {
if (!file.mkdirs() && !file.exists()) {
throw new RuntimeException("Unable to create resource directory " + file);
}
}
OkHttpClient client = new OkHttpClient.Builder().build();
Request request = new Request.Builder()
.url("https://publicsuffix.org/list/public_suffix_list.dat")
.build();
try (Response response = client.newCall(request).execute()) {
BufferedSource source = response.body().source();
try (Sink fileSink = Okio.sink(publicSuffixListDotDat)) {
source.readAll(fileSink);
}
}
SortedSet sortedRules = new TreeSet<>();
SortedSet sortedExceptionRules = new TreeSet<>();
try (BufferedSource source = Okio.buffer(Okio.source(publicSuffixListDotDat))) {
int totalRuleBytes = 0;
int totalExceptionRuleBytes = 0;
while (!source.exhausted()) {
String line = source.readUtf8LineStrict();
if (line.trim().isEmpty() || line.startsWith("//")) continue;
if (line.contains(WILDCARD_CHAR)) {
assertWildcardRule(line);
}
ByteString rule = ByteString.encodeUtf8(line);
if (rule.startsWith(EXCEPTION_RULE_MARKER)) {
rule = rule.substring(1);
// We use '\n' for end of value.
totalExceptionRuleBytes += rule.size() + 1;
sortedExceptionRules.add(rule);
} else {
totalRuleBytes += rule.size() + 1; // We use '\n' for end of value.
sortedRules.add(rule);
}
}
Sink fileSink = Okio.sink(new File(resources,
PublicSuffixDatabase.PUBLIC_SUFFIX_RESOURCE));
try (BufferedSink sink = Okio.buffer(new GzipSink(fileSink))) {
sink.writeInt(totalRuleBytes);
for (ByteString domain : sortedRules) {
sink.write(domain).writeByte('\n');
}
sink.writeInt(totalExceptionRuleBytes);
for (ByteString domain : sortedExceptionRules) {
sink.write(domain).writeByte('\n');
}
}
}
}
/**
* These assertions ensure the {@link PublicSuffixDatabase} remains correct. The specification is
* very flexible regarding wildcard rules, but this flexibility is not something currently used
* in practice. To simplify the implementation, we've avoided implementing the flexible rules in
* favor of supporting what's actually used in practice. That means if these assertions ever fail,
* the implementation will need to be revisited to support a more flexible rule.
*/
private static void assertWildcardRule(String rule) {
if (rule.indexOf(WILDCARD_CHAR) != 0) {
throw new RuntimeException("Wildcard Assertion Failure: " + rule + "\nA wildcard rule was "
+ "added with a wildcard that is not in leftmost position! We'll need to change the "
+ PublicSuffixDatabase.class.getName() + " to handle this.");
}
if (rule.indexOf(WILDCARD_CHAR, 1) != -1) {
throw new RuntimeException("Wildcard Assertion Failure: " + rule + "\nA wildcard rule was "
+ "added with multiple wildcards! We'll need to change "
+ PublicSuffixDatabase.class.getName() + " to handle this.");
}
if (rule.length() == 1) {
throw new RuntimeException("Wildcard Assertion Failure: " + rule + "\nA wildcard rule was "
+ "added that wildcards the first level! We'll need to change the "
+ PublicSuffixDatabase.class.getName() + " to handle this.");
}
}
}