net.i2p.router.Blocklist.orig Maven / Gradle / Ivy
package net.i2p.router;
/*
* free (adj.): unencumbered; not under the control of others
* Use at your own risk.
* zzz 2008-06
*/
import java.io.BufferedReader;
import java.io.IOException;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.Writer;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import net.i2p.app.ClientAppManager;
import net.i2p.data.Base64;
import net.i2p.data.DataHelper;
import net.i2p.data.Hash;
import net.i2p.data.router.RouterAddress;
import net.i2p.data.router.RouterInfo;
import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade;
import net.i2p.router.transport.GeoIP;
import net.i2p.router.transport.TransportUtil;
import net.i2p.router.transport.udp.UDPTransport;
import net.i2p.update.UpdateManager;
import net.i2p.update.UpdateType;
import net.i2p.util.Addresses;
import net.i2p.util.ConcurrentHashSet;
import net.i2p.util.LHMCache;
import net.i2p.util.Log;
import net.i2p.util.SimpleTimer2;
import net.i2p.util.SystemVersion;
import net.i2p.util.Translate;
/**
* Manage blocking by IP address, in a manner similar to the Banlist,
* which blocks by router hash.
*
* We also try to keep the two lists in sync: if a router at a given IP is
* blocked, we will also banlist it "forever" (until the next reboot).
*
* While the reverse case (blocking the IP of a router banlisted forever)
* is not automatic, the transports will call add() below to block the IP,
* which allows the transports to terminate an inbound connection before
* the router ident handshake.
*
* And the on-disk blocklist can also contain router hashes to be banlisted.
*
* So, this class maintains three separate lists:
*
* 1) The list of IP ranges, read in from a file at startup
* 2) The list of hashes, read in from the same file
* 3) A list of single IPs, initially empty, added to as needed
*
*
* Read in the IP blocklist from a file, store it in-memory as efficiently
* as we can, and perform tests against it as requested.
*
* When queried for a peer that is blocklisted but isn't banlisted,
* banlist it forever, then go back to the file to get the original
* entry so we can add the reason to the banlist text.
*
* On-disk blocklist supports IPv4 only.
* In-memory supports both IPv4 and IPv6.
*/
public class Blocklist {
private final Log _log;
private final RouterContext _context;
private long _blocklist[];
private int _blocklistSize;
private long _countryBlocklist[];
private int _countryBlocklistSize;
private final Object _lock = new Object();
private Entry _wrapSave;
private final Set _inProcess = new HashSet(4);
private final File _blocklistFeedFile;
private final boolean _haveIPv6;
private boolean _started;
private long _lastExpired = 0;
// temp
private final Map _peerBlocklist = new HashMap(4);
private static final String PROP_BLOCKLIST_ENABLED = "router.blocklist.enable";
private static final String PROP_BLOCKLIST_DETAIL = "router.blocklist.detail";
private static final String PROP_BLOCKLIST_FILE = "router.blocklist.file";
private static final String PROP_BLOCKLIST_EXPIRE_INTERVAL = "router.blocklist.expireInterval";
public static final String BLOCKLIST_FILE_DEFAULT = "blocklist.txt";
private static final String BLOCKLIST_FEED_FILE = "docs/feed/blocklist/blocklist.txt";
/** @since 0.9.48 */
public static final String BLOCKLIST_COUNTRY_FILE = "blocklist-country.txt";
/**
* Limits of transient (in-memory) blocklists.
* Note that it's impossible to prevent clogging up
* the tables by a determined attacker, esp. on IPv6
*/
private static final int MAX_IPV4_SINGLES = SystemVersion.isSlow() ? 2048 : 8192;
private static final int MAX_IPV6_SINGLES = SystemVersion.isSlow() ? 256 : 4096;
private final Map _singleIPBlocklist = new LHMCache(MAX_IPV4_SINGLES);
private final Map _singleIPv6Blocklist;
private static final Object DUMMY = Integer.valueOf(0);
/**
* For Update Manager
* @since 0.9.48
*/
public static final String ID_FEED = "feed";
private static final String ID_SYSTEM = "system";
private static final String ID_LOCAL = "local";
private static final String ID_COUNTRY = "country";
private static final String ID_USER = "user";
public static final String ID_SYBIL = "sybil";
/**
* Router MUST call startup()
*/
public Blocklist(RouterContext context) {
_context = context;
_log = context.logManager().getLog(Blocklist.class);
_blocklistFeedFile = new File(context.getConfigDir(), BLOCKLIST_FEED_FILE);
_haveIPv6 = TransportUtil.getIPv6Config(_context, "SSU") != TransportUtil.IPv6Config.IPV6_DISABLED &&
Addresses.isConnectedIPv6();
_singleIPv6Blocklist = _haveIPv6 ? new LHMCache(MAX_IPV6_SINGLES) : null;
}
/** only for testing with main() */
private Blocklist() {
_context = null;
_log = new Log(Blocklist.class);
_blocklistFeedFile = new File(BLOCKLIST_FEED_FILE);
_haveIPv6 = TransportUtil.getIPv6Config(_context, "SSU") != TransportUtil.IPv6Config.IPV6_DISABLED &&
Addresses.isConnectedIPv6();
_singleIPv6Blocklist = _haveIPv6 ? new LHMCache(MAX_IPV6_SINGLES) : null;
}
private int expireInterval(){
String expireIntervalValue = _context.getProperty(PROP_BLOCKLIST_EXPIRE_INTERVAL, "0");
try{
Integer expireIntervalInt = 0;
if (expireIntervalValue.endsWith("s")) {
expireIntervalValue = expireIntervalValue.substring(0, expireIntervalValue.length() - 1);
expireIntervalInt = Integer.parseInt(expireIntervalValue) * 1000;
}else if(expireIntervalValue.endsWith("m")){
expireIntervalValue = expireIntervalValue.substring(0, expireIntervalValue.length() - 1);
expireIntervalInt = Integer.parseInt(expireIntervalValue) * 60000;
}else if(expireIntervalValue.endsWith("h")){
expireIntervalValue = expireIntervalValue.substring(0, expireIntervalValue.length() - 1);
expireIntervalInt = Integer.parseInt(expireIntervalValue) * 3600000;
}else if (expireIntervalValue.endsWith("d")) {
expireIntervalValue = expireIntervalValue.substring(0, expireIntervalValue.length() - 1);
expireIntervalInt = Integer.parseInt(expireIntervalValue) * 86400000;
}else{
expireIntervalInt = Integer.parseInt(expireIntervalValue);
}
if (expireIntervalInt < 0)
expireIntervalInt = 0;
return expireIntervalInt;
}catch(NumberFormatException nfe){
if (_log.shouldLog(_log.ERROR))
_log.error("format error in "+PROP_BLOCKLIST_EXPIRE_INTERVAL, nfe);
}
// if we don't have a valid value in this field, return 0 which is the same as disabling it.
return 0;
}
/**
* Loads the following files in-order:
* $I2P/blocklist.txt
* ~/.i2p/blocklist.txt
* ~/.i2p/docs/feed/blocklist/blocklist.txt
* ~/.i2p/blocklist-countries.txt
* File if specified with router.blocklist.file
*/
public synchronized void startup() {
if (_started)
return;
_started = true;
if (! _context.getBooleanPropertyDefaultTrue(PROP_BLOCKLIST_ENABLED))
return;
List files = new ArrayList(5);
// install dir
File blFile = new File(_context.getBaseDir(), BLOCKLIST_FILE_DEFAULT);
files.add(new BLFile(blFile, ID_SYSTEM));
// config dir
if (!_context.getConfigDir().equals(_context.getBaseDir())) {
blFile = new File(_context.getConfigDir(), BLOCKLIST_FILE_DEFAULT);
files.add(new BLFile(blFile, ID_LOCAL));
}
files.add(new BLFile(_blocklistFeedFile, ID_FEED));
if (_context.router().isHidden() ||
_context.getBooleanProperty(GeoIP.PROP_BLOCK_MY_COUNTRY)) {
blFile = new File(_context.getConfigDir(), BLOCKLIST_COUNTRY_FILE);
files.add(new BLFile(blFile, ID_COUNTRY));
}
// user specified
String file = _context.getProperty(PROP_BLOCKLIST_FILE);
if (file != null && !file.equals(BLOCKLIST_FILE_DEFAULT)) {
blFile = new File(file);
if (!blFile.isAbsolute())
blFile = new File(_context.getConfigDir(), file);
files.add(new BLFile(blFile, ID_USER));
}
Job job = new ReadinJob(files);
// Run immediately, so it's initialized before netdb.
// As this is called by Router.runRouter() before job queue parallel operation,
// this will block StartupJob, and will complete before netdb initialization.
// If there is a huge blocklist, it will delay router startup,
// but it's important to have this initialized before we read in the netdb.
//job.getTiming().setStartAfter(_context.clock().now() + 30*1000);
_context.jobQueue().addJob(job);
if (expireInterval() > 0) {
Job cleanupJob = new CleanupJob();
cleanupJob.getTiming().setStartAfter(_context.clock().now() + expireInterval());
_context.jobQueue().addJob(cleanupJob);
}
}
/**
* @since 0.9.48
*/
private static class BLFile {
public final File file;
public final String id;
public long version;
public BLFile(File f, String s) { file = f; id = s; }
}
/**
* Delay telling update manager until it's there
* @since 0.9.48
*/
private class VersionNotifier extends SimpleTimer2.TimedEvent {
public final List blfs;
public VersionNotifier(List bf) {
super(_context.simpleTimer2(), 2*60*1000L);
blfs = bf;
}
public void timeReached() {
ClientAppManager cmgr = _context.clientAppManager();
if (cmgr != null) {
UpdateManager umgr = (UpdateManager) cmgr.getRegisteredApp(UpdateManager.APP_NAME);
if (umgr != null) {
for (BLFile blf : blfs) {
if (blf.version > 0)
umgr.notifyInstalled(UpdateType.BLOCKLIST, blf.id, Long.toString(blf.version));
}
} else {
_log.warn("No update manager");
}
}
}
}
private class CleanupJob extends JobImpl {
public CleanupJob() {
super(_context);
}
public String getName(){
return "Expire blocklist at user-defined interval of " + expireInterval();
}
public void runJob() {
clear();
_lastExpired = System.currentTimeMillis();
if (_log.shouldLog(Log.DEBUG))
_log.debug("Expiring blocklist entrys at" + _lastExpired);
// schedule the next one
super.requeue(expireInterval());
}
}
private void clear(){
synchronized(_singleIPBlocklist) {
_singleIPBlocklist.clear();
}
if (_singleIPv6Blocklist != null) {
synchronized(_singleIPv6Blocklist) {
_singleIPv6Blocklist.clear();
}
}
}
private class ReadinJob extends JobImpl {
private final List _files;
/**
* @param files not necessarily existing, but avoid dups
*/
public ReadinJob (List files) {
super(_context);
_files = files;
}
public String getName() { return "Read Blocklist"; }
public void runJob() {
synchronized (_lock) {
_blocklist = allocate(_files);
if (_blocklist == null)
return;
int ccount = process();
if (_blocklist == null)
return;
if (ccount <= 0) {
disable();
return;
}
_blocklistSize = merge(_blocklist, ccount);
// we're done with _peerBlocklist, but leave it
// in case we need it for a later readin
//_peerBlocklist = null;
}
// schedules itself
new VersionNotifier(_files);
}
private int process() {
int count = 0;
try {
for (BLFile blf : _files) {
count = readBlocklistFile(blf, _blocklist, count);
}
} catch (OutOfMemoryError oom) {
_log.log(Log.CRIT, "OOM processing the blocklist");
disable();
return 0;
}
for (Hash peer : _peerBlocklist.keySet()) {
String reason;
String comment = _peerBlocklist.get(peer);
if (comment != null)
reason = _x("Banned by router hash: {0}");
else
reason = _x("Banned by router hash");
banlistRouter(peer, reason, comment);
}
_peerBlocklist.clear();
return count;
}
}
private void banlistRouter(Hash peer, String reason, String comment) {
if (expireInterval() > 0)
_context.banlist().banlistRouter(peer, reason, comment, null, expireInterval());
else
_context.banlist().banlistRouterForever(peer, reason, comment);
}
/**
* The blocklist-country.txt file was created or updated.
* Read it in. Not required normally, as the country file
* is read by startup().
* @since 0.9.48
*/
public synchronized void addCountryFile() {
File blFile = new File(_context.getConfigDir(), BLOCKLIST_COUNTRY_FILE);
BLFile blf = new BLFile(blFile, ID_COUNTRY);
List c = Collections.singletonList(blf);
long[] cb = allocate(c);
if (cb == null)
return;
int count = readBlocklistFile(blf, cb, 0);
if (count <= 0)
return;
ClientAppManager cmgr = _context.clientAppManager();
if (cmgr != null) {
UpdateManager umgr = (UpdateManager) cmgr.getRegisteredApp(UpdateManager.APP_NAME);
if (umgr != null)
umgr.notifyInstalled(UpdateType.BLOCKLIST, ID_COUNTRY, Long.toString(blFile.lastModified()));
}
count = merge(cb, count);
_countryBlocklistSize = count;
_countryBlocklist = cb;
}
public void disable() {
// hmm better block out any checks in process
synchronized (_lock) {
_blocklistSize = 0;
_blocklist = null;
}
}
/**
* @return array or null on failure
* @since 0.9.18 split out from readBlocklistFile()
*/
private long[] allocate(List files) {
int maxSize = 0;
for (BLFile blf : files) {
maxSize += getSize(blf.file);
}
try {
return new long[maxSize + files.size()]; // extra for wrapsave
} catch (OutOfMemoryError oom) {
_log.log(Log.CRIT, "OOM creating the blocklist");
return null;
}
}
/**
* Read in and parse the blocklist.
* The blocklist need not be sorted, and may contain overlapping entries.
*
* Acceptable formats (IPV4 only):
* #comment (# must be in column 1)
* comment:IP-IP
* comment:morecomments:IP-IP
* IP-IP
* (comments also allowed before any of the following)
* IP/masklength
* IP
* hostname (DNS looked up at list readin time, not dynamically, so may not be much use)
* 44-byte Base64 router hash
*
* Acceptable formats (IPV6 only):
* comment:IPv6 (must replace : with ; e.g. abcd;1234;0;12;;ff)
* IPv6 (must replace : with ; e.g. abcd;1234;0;12;;ff)
*
* No whitespace allowed after the last ':'.
*
* For further information and downloads:
* http://www.bluetack.co.uk/forums/index.php?autocom=faq&CODE=02&qid=17
* http://blocklist.googlepages.com/
* http://www.cymru.com/Documents/bogon-list.html
*
*
* Must call allocate() before and merge() after.
*
* @param blocklist out parameter, entries stored here
* @param count current number of entries
* @return new number of entries
*/
private int readBlocklistFile(BLFile blf, long[] blocklist, int count) {
File blFile = blf.file;
if (blFile == null || (!blFile.exists()) || blFile.length() <= 0) {
if (_log.shouldLog(Log.WARN))
_log.warn("Blocklist file not found: " + blFile);
return count;
}
long start = _context.clock().now();
int oldcount = count;
int badcount = 0;
int peercount = 0;
int feedcount = 0;
long ipcount = 0;
final boolean isFeedFile = blFile.equals(_blocklistFeedFile);
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(
new FileInputStream(blFile), "UTF-8"));
String source = blFile.toString();
String buf = null;
while ((buf = br.readLine()) != null) {
Entry e = parse(buf, true);
if (e == null) {
badcount++;
continue;
}
if (e.peer != null) {
_peerBlocklist.put(e.peer, e.comment);
peercount++;
continue;
}
byte[] ip1 = e.ip1;
if (ip1.length == 4) {
//if (isFeedFile) {
// // temporary
// add(ip1, source);
// feedcount++;
//} else {
byte[] ip2 = e.ip2;
store(ip1, ip2, blocklist, count++);
ipcount += 1 + toInt(ip2) - toInt(ip1); // includes dups, oh well
//}
} else {
// IPv6
add(ip1, source);
}
}
} catch (IOException ioe) {
if (_log.shouldLog(Log.ERROR))
_log.error("Error reading the blocklist file", ioe);
return count;
} catch (OutOfMemoryError oom) {
disable();
_log.log(Log.CRIT, "OOM reading the blocklist");
return 0;
} finally {
if (br != null) try { br.close(); } catch (IOException ioe) {}
}
if (_wrapSave != null) {
// the extra record generated in parse() by a line that
// wrapped around 128.0.0.0
store(_wrapSave.ip1, _wrapSave.ip2, blocklist, count++);
ipcount += 1 + toInt(_wrapSave.ip2) - toInt(_wrapSave.ip1);
_wrapSave = null;
}
int read = isFeedFile ? feedcount : (count - oldcount);
// save to tell the update manager
if (read > 0)
blf.version = blFile.lastModified();
if (_log.shouldLog(Log.INFO)) {
_log.info("Stats for " + blFile);
_log.info("Removed " + badcount + " bad entries and comment lines");
_log.info("Read " + read + " valid entries from the blocklist " + blFile);
//_log.info("Blocking " + (isFeedFile ? feedcount : ipcount) + " IPs and " + peercount + " hashes");
_log.info("Blocking " + ipcount + " IPs and " + peercount + " hashes");
_log.info("Blocklist processing finished, time: " + (_context.clock().now() - start));
}
return count;
}
/**
* @param count valid entries in blocklist before merge
* @return count valid entries in blocklist after merge
* @since 0.9.18 split out from readBlocklistFile()
*/
private int merge(long[] blocklist, int count) {
long start = _context.clock().now();
// This is a standard signed sort, so the entries will be ordered
// 128.0.0.0 ... 255.255.255.255 0.0.0.0 .... 127.255.255.255
// But that's ok.
int removed = 0;
try {
Arrays.sort(blocklist, 0, count);
removed = removeOverlap(blocklist, count);
if (removed > 0) {
// Sort again to remove the dups that were "zeroed" out as 127.255.255.255-255.255.255.255
Arrays.sort(blocklist, 0, count);
// sorry, no realloc to save memory, don't want to blow up now
}
} catch (OutOfMemoryError oom) {
disable();
_log.log(Log.CRIT, "OOM sorting the blocklist");
return 0;
}
int blocklistSize = count - removed;
if (_log.shouldLog(Log.INFO)) {
_log.info("Merged Stats:\n" +
"Read " + count + " total entries from the blocklists\n" +
"Merged " + removed + " overlapping entries\n" +
"Result is " + blocklistSize + " entries\n" +
"Blocklist processing finished, time: " + (_context.clock().now() - start));
}
return blocklistSize;
}
/**
* The result of parsing one line
*/
private static class Entry {
final String comment;
final byte ip1[];
final byte ip2[];
final Hash peer;
public Entry(String c, Hash h, byte[] i1, byte[] i2) {
comment = c;
peer = h;
ip1 = i1;
ip2 = i2;
}
}
/**
* Parse one line, returning a temp data structure with the result
*/
private Entry parse(String buf, boolean shouldLog) {
byte[] ip1;
byte[] ip2;
int start1 = 0;
int end1 = buf.length();
if (end1 <= 0)
return null; // blank
//if (buf.charAt(end1 - 1) == '\r') { // DataHelper.readLine leaves the \r on there
// buf.deleteCharAt(end1 - 1);
// end1--;
//}
//if (end1 <= 0)
// return null; // blank
int start2 = -1;
int mask = -1;
String comment = null;
int index = buf.indexOf('#');
if (index == 0)
return null; // comment
index = buf.lastIndexOf(':');
if (index >= 0) {
comment = buf.substring(0, index);
start1 = index + 1;
}
if (end1 - start1 == 44 && buf.substring(start1).indexOf('.') < 0) {
byte b[] = Base64.decode(buf.substring(start1));
if (b != null)
return new Entry(comment, Hash.create(b), null, null);
}
index = buf.indexOf('-', start1);
if (index >= 0) {
end1 = index;
start2 = index + 1;
} else {
index = buf.indexOf('/', start1);
if (index >= 0) {
end1 = index;
mask = index + 1;
}
}
if (end1 - start1 <= 0)
return null; // blank
try {
String sip = buf.substring(start1, end1);
// IPv6
sip = sip.replace(';', ':');
InetAddress pi = InetAddress.getByName(sip);
if (pi == null) return null;
ip1 = pi.getAddress();
//if (ip1.length != 4)
// throw new UnknownHostException();
if (start2 >= 0) {
pi = InetAddress.getByName(buf.substring(start2));
if (pi == null) return null;
ip2 = pi.getAddress();
if (ip2.length != 4)
throw new UnknownHostException();
if ((ip1[0] & 0xff) < 0x80 && (ip2[0] & 0xff) >= 0x80) {
if (_wrapSave == null) {
// don't cross the boundary 127.255.255.255 - 128.0.0.0
// because we are sorting using signed arithmetic
_wrapSave = new Entry(comment, null, new byte[] {(byte)0x80,0,0,0}, new byte[] {ip2[0], ip2[1], ip2[2], ip2[3]});
ip2 = new byte[] {127, (byte)0xff, (byte)0xff, (byte)0xff};
} else
// We only save one entry crossing the boundary, throw the rest out
throw new NumberFormatException();
}
for (int i = 0; i < 4; i++) {
if ((ip2[i] & 0xff) > (ip1[i] & 0xff))
break;
if ((ip2[i] & 0xff) < (ip1[i] & 0xff))
throw new NumberFormatException(); // backwards
}
} else if (mask >= 0) {
int m = Integer.parseInt(buf.substring(mask));
if (m < 3 || m > 32)
throw new NumberFormatException();
ip2 = new byte[4];
// ick
for (int i = 0; i < 4; i++)
ip2[i] = ip1[i];
for (int i = 0; i < 32-m; i++)
ip2[(31-i)/8] |= (0x01 << (i%8));
} else {
ip2 = ip1;
}
} catch (UnknownHostException uhe) {
if (shouldLog)
_log.logAlways(Log.WARN, "Format error in the blocklist file: " + buf);
return null;
} catch (NumberFormatException nfe) {
if (shouldLog)
_log.logAlways(Log.WARN, "Format error in the blocklist file: " + buf);
return null;
} catch (IndexOutOfBoundsException ioobe) {
if (shouldLog)
_log.logAlways(Log.WARN, "Format error in the blocklist file: " + buf);
return null;
}
return new Entry(comment, null, ip1, ip2);
}
/**
* Read the file once just to see how many entries are in it,
* so we can size our array.
* This is i/o inefficient, but memory-efficient, which is what we want.
*/
private int getSize(File blFile) {
if ( (!blFile.exists()) || (blFile.length() <= 0) ) return 0;
int lines = 0;
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(
new FileInputStream(blFile), "ISO-8859-1"));
String s;
while ((s = br.readLine()) != null) {
if (s.length() > 0 && !s.startsWith("#"))
lines++;
}
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error reading the blocklist file", ioe);
return 0;
} finally {
if (br != null) try { br.close(); } catch (IOException ioe) {}
}
return lines;
}
/**
* Merge and remove overlapping entries from a sorted list.
* Returns number of removed entries.
* Caller must re-sort if return code is > 0.
*/
private int removeOverlap(long blist[], int count) {
if (count <= 0) return 0;
int lines = 0;
for (int i = 0; i < count - 1; ) {
int removed = 0;
int to = getTo(blist[i]);
for (int next = i + 1; next < count; next++) {
if (to < getFrom(blist[next]))
break;
if (_log.shouldInfo())
_log.info("Combining entries " + toStr(blist[i]) + " and " + toStr(blist[next]));
int nextTo = getTo(blist[next]);
if (nextTo > to) // else entry next is totally inside entry i
store(getFrom(blist[i]), nextTo, blist, i);
blist[next] = Long.MAX_VALUE; // to be removed with another sort
lines++;
removed++;
}
i += removed + 1;
}
return lines;
}
/**
* Maintain a simple in-memory single-IP blocklist
* This is used for new additions, NOT for the main list
* of IP ranges read in from the file.
*
* @param ip IPv4 or IPv6
*/
public void add(String ip) {
if (!_haveIPv6 && ip.indexOf(':') >= 0)
return;
byte[] pib = Addresses.getIPOnly(ip);
if (pib == null) return;
add(pib, null);
}
/**
* Maintain a simple in-memory single-IP blocklist
* This is used for new additions, NOT for the main list
* of IP ranges read in from the file.
*
* @param ip IPv4 or IPv6
* @param source for logging only, may be null
* @since 0.9.57
*/
public void add(String ip, String source) {
if (!_haveIPv6 && ip.indexOf(':') >= 0)
return;
byte[] pib = Addresses.getIPOnly(ip);
if (pib == null) return;
add(pib, source);
}
/**
* Maintain a simple in-memory single-IP blocklist
* This is used for new additions, NOT for the main list
* of IP ranges read in from the file.
*
* @param ip IPv4 or IPv6
*/
public void add(byte ip[]) {
add(ip, null);
}
/**
* Maintain a simple in-memory single-IP blocklist
* This is used for new additions, NOT for the main list
* of IP ranges read in from the file.
*
* @param ip IPv4 or IPv6
* @param source for logging only, may be null
* @since 0.9.57
*/
public void add(byte ip[], String source) {
boolean rv;
if (ip.length == 4) {
// don't ever block ourselves
String us = _context.getProperty(UDPTransport.PROP_IP);
if (us != null) {
byte[] usb = Addresses.getIP(us);
if (usb != null && DataHelper.eq(usb, ip)) {
if (_log.shouldWarn())
_log.warn("Not adding our own IP " + us, new Exception());
return;
}
}
rv = add(toInt(ip));
if (rv)
_context.commSystem().removeExemption(Addresses.toString(ip));
} else if (ip.length == 16) {
if (!_haveIPv6)
return;
// don't ever block ourselves
String us = _context.getProperty(UDPTransport.PROP_IPV6);
if (us != null) {
byte[] usb = Addresses.getIP(us);
if (usb != null && DataHelper.eq(usb, ip)) {
if (_log.shouldWarn())
_log.warn("Not adding our own IP " + us, new Exception());
return;
}
}
rv = add(new BigInteger(1, ip));
if (rv)
_context.commSystem().removeExemption(Addresses.toCanonicalString(ip));
} else {
return;
}
if (rv) {
// lower log level at startup when initializing from blocklist files
if (source == null && _log.shouldWarn())
_log.warn("Added: " + Addresses.toString(ip), new Exception("source"));
else if (_log.shouldDebug())
_log.debug("Added: " + Addresses.toString(ip) + " source: " + source);
}
}
/**
* Remove from the in-memory single-IP blocklist.
* This is only works to undo add()s, NOT for the main list
* of IP ranges read in from the file.
*
* @param ip IPv4 or IPv6
* @since 0.9.28
*/
public void remove(byte ip[]) {
if (ip.length == 4) {
remove(toInt(ip));
} else if (ip.length == 16) {
if (!_haveIPv6)
return;
remove(new BigInteger(1, ip));
}
}
/**
* @return true if it was NOT previously on the list
*/
private boolean add(int ip) {
// save space, don't put in both
if (isPermanentlyBlocklisted(ip))
return false;
Integer iip = Integer.valueOf(ip);
synchronized(_singleIPBlocklist) {
return _singleIPBlocklist.put(iip, DUMMY) == null;
}
}
/**
* @since 0.9.28
*/
private void remove(int ip) {
Integer iip = Integer.valueOf(ip);
synchronized(_singleIPBlocklist) {
_singleIPBlocklist.remove(iip);
}
}
private boolean isOnSingleList(int ip) {
Integer iip = Integer.valueOf(ip);
synchronized(_singleIPBlocklist) {
return _singleIPBlocklist.get(iip) != null;
}
}
/**
* @param ip IPv6 non-negative
* @return true if it was NOT previously on the list
* @since IPv6
*/
private boolean add(BigInteger ip) {
if (_singleIPv6Blocklist != null) {
synchronized(_singleIPv6Blocklist) {
return _singleIPv6Blocklist.put(ip, DUMMY) == null;
}
}
return false;
}
/**
* @param ip IPv6 non-negative
* @since 0.9.28
*/
private void remove(BigInteger ip) {
if (_singleIPv6Blocklist != null) {
synchronized(_singleIPv6Blocklist) {
_singleIPv6Blocklist.remove(ip);
}
}
}
/**
* @param ip IPv6 non-negative
* @since IPv6
*/
private boolean isOnSingleList(BigInteger ip) {
if (_singleIPv6Blocklist != null) {
synchronized(_singleIPv6Blocklist) {
return _singleIPv6Blocklist.get(ip) != null;
}
}
return false;
}
/**
* Will not contain duplicates.
*/
private List getAddresses(Hash peer) {
RouterInfo pinfo = _context.netDb().lookupRouterInfoLocally(peer);
if (pinfo == null)
return Collections.emptyList();
return getAddresses(pinfo);
}
/**
* Will not contain duplicates.
* @since 0.9.29
*/
private List getAddresses(RouterInfo pinfo) {
List rv = new ArrayList(4);
// for each peer address
for (RouterAddress pa : pinfo.getAddresses()) {
byte[] pib = pa.getIP();
if (pib == null) continue;
if (!_haveIPv6 && pib.length == 16)
continue;
// O(n**2)
boolean dup = false;
for (int i = 0; i < rv.size(); i++) {
if (DataHelper.eq(rv.get(i), pib)) {
dup = true;
break;
}
}
if (!dup)
rv.add(pib);
}
return rv;
}
/**
* Does the peer's IP address appear in the blocklist?
* If so, and it isn't banlisted, banlist it forever...
* or, if the user configured an override, ban it for the
* override period.
* @since 0.9.29
*/
public boolean isBlocklisted(Hash peer) {
List ips = getAddresses(peer);
if (ips.isEmpty())
return false;
for (byte[] ip : ips) {
if (isBlocklisted(ip)) {
if (! _context.banlist().isBanlisted(peer))
// nice knowing you...
banlist(peer, ip);
return true;
}
}
return false;
}
/**
* Does the peer's IP address appear in the blocklist?
* If so, and it isn't banlisted, banlist it forever...
* or, if the user configured an override, ban it for the
* override period.
* @since 0.9.29
*/
public boolean isBlocklisted(RouterInfo pinfo) {
List ips = getAddresses(pinfo);
if (ips.isEmpty())
return false;
for (byte[] ip : ips) {
if (isBlocklisted(ip)) {
Hash peer = pinfo.getHash();
if (! _context.banlist().isBanlisted(peer))
// nice knowing you...
banlist(peer, ip);
return true;
}
}
return false;
}
/**
* calling this externally won't banlist the peer, this is just an IP check
*
* @param ip IPv4 or IPv6
*/
public boolean isBlocklisted(String ip) {
if (!_haveIPv6 && ip.indexOf(':') >= 0)
return false;
byte[] pib = Addresses.getIPOnly(ip);
if (pib == null) return false;
return isBlocklisted(pib);
}
/**
* calling this externally won't banlist the peer, this is just an IP check
*
* @param ip IPv4 or IPv6
*/
public boolean isBlocklisted(byte ip[]) {
if (ip.length == 4)
return isBlocklisted(toInt(ip));
if (ip.length == 16) {
if (!_haveIPv6)
return false;
return isOnSingleList(new BigInteger(1, ip));
}
return false;
}
/**
* First check the single-IP list.
* Then do a
* binary search through the in-memory range list which
* is a sorted array of longs.
* The array is sorted in signed order, but we don't care.
* Each long is ((from << 32) | to)
*/
private boolean isBlocklisted(int ip) {
if (isOnSingleList(ip))
return true;
if (_countryBlocklist != null) {
if (isPermanentlyBlocklisted(ip, _countryBlocklist, _countryBlocklistSize))
return true;
}
return isPermanentlyBlocklisted(ip);
}
/**
* Do a binary search through the in-memory range list which
* is a sorted array of longs.
* The array is sorted in signed order, but we don't care.
* Each long is ((from << 32) | to)
*
* Public for console only, not a public API
*
* @since 0.9.45 split out from above, public since 0.9.48 for console
*/
public boolean isPermanentlyBlocklisted(int ip) {
return isPermanentlyBlocklisted(ip, _blocklist, _blocklistSize);
}
/**
* Do a binary search through the in-memory range list which
* is a sorted array of longs.
* The array is sorted in signed order, but we don't care.
* Each long is ((from << 32) | to)
*
* @since 0.9.48 split out from above
*/
private static boolean isPermanentlyBlocklisted(int ip, long[] blocklist, int blocklistSize) {
int hi = blocklistSize - 1;
if (hi <= 0)
return false;
int lo = 0;
int cur = hi / 2;
while (!match(ip, blocklist[cur])) {
if (isHigher(ip, blocklist[cur]))
lo = cur;
else
hi = cur;
// make sure we get the last one
if (hi - lo <= 1) {
if (lo == cur)
cur = hi;
else
cur = lo;
break;
} else {
cur = lo + ((hi - lo) / 2);
}
}
return match(ip, blocklist[cur]);
}
/*
// Is the IP included in the entry _blocklist[cur] ?
private boolean match(int ip, int cur) {
return match(ip, _blocklist[cur]);
}
*/
// Is the IP included in the compressed entry?
private static boolean match(int ip, long entry) {
if (getFrom(entry) > ip)
return false;
return (ip <= getTo(entry));
}
// Is the IP higher than the entry _blocklist[cur] ?
private static boolean isHigher(int ip, long entry) {
return ip > getFrom(entry);
}
// methods to get and store the from/to values in the array
/**
* Public for console only, not a public API
* @since public since 0.9.48
*/
public static int getFrom(long entry) {
return (int) ((entry >> 32) & 0xffffffff);
}
/**
* Public for console only, not a public API
* @since public since 0.9.48
*/
public static int getTo(long entry) {
return (int) (entry & 0xffffffff);
}
/**
* The in-memory blocklist is an array of longs, with the format
* ((from IP) << 32) | (to IP)
* The XOR is so the signed sort is in normal (unsigned) order.
*
* So the size is (cough) almost 2MB for the 240,000 line splist.txt.
*
*/
private static long toEntry(byte ip1[], byte ip2[]) {
long entry = 0;
for (int i = 0; i < 4; i++)
entry |= ((long) (ip2[i] & 0xff)) << ((3-i)*8);
for (int i = 0; i < 4; i++)
entry |= ((long) (ip1[i] & 0xff)) << (32 + ((3-i)*8));
return entry;
}
/**
* IPv4 only
*/
private static void store(byte ip1[], byte ip2[], long[] blocklist, int idx) {
blocklist[idx] = toEntry(ip1, ip2);
}
private static void store(int ip1, int ip2, long[] blocklist, int idx) {
long entry = ((long) ip1) << 32;
entry |= ((long)ip2) & 0xffffffff;
blocklist[idx] = entry;
}
private static int toInt(byte ip[]) {
int rv = 0;
for (int i = 0; i < 4; i++)
rv |= (ip[i] & 0xff) << ((3-i)*8);
return rv;
}
private static String toStr(long entry) {
StringBuilder buf = new StringBuilder(32);
for (int i = 7; i >= 0; i--) {
buf.append((entry >> (8*i)) & 0xff);
if (i == 4)
buf.append('-');
else if (i > 0)
buf.append('.');
}
return buf.toString();
}
/**
* Public for console only, not a public API
* @since public since 0.9.48
*/
public static String toStr(int ip) {
StringBuilder buf = new StringBuilder(16);
for (int i = 3; i >= 0; i--) {
buf.append((ip >> (8*i)) & 0xff);
if (i > 0)
buf.append('.');
}
return buf.toString();
}
/**
* We don't keep the comment field in-memory,
* so we have to go back out to the file to find it.
*
* Put this in a job because we're looking for the
* actual line in the blocklist file, this could take a while.
*
*/
private void banlist(Hash peer, byte[] ip) {
// Don't bother unless we have IPv6
if (!_haveIPv6 && ip.length == 16)
return;
// Temporary reason, until the job finishes
String reason = _x("IP banned by blocklist.txt entry {0}");
String sip = Addresses.toString(ip);
if ("127.0.0.1".equals(sip) ||
"0:0:0:0:0:0:0:1".equals(sip) ||
sip.startsWith("192.168.")) {
// i2pd bug, possibly at startup, don't ban forever
_context.banlist().banlistRouter(peer, reason, sip, null,
_context.clock().now() + Banlist.BANLIST_DURATION_LOCALHOST);
return;
}
banlistRouter(peer, reason, sip);
if (! _context.getBooleanPropertyDefaultTrue(PROP_BLOCKLIST_DETAIL))
return;
boolean shouldRunJob;
int number;
synchronized (_inProcess) {
number = _inProcess.size();
shouldRunJob = _inProcess.add(peer);
}
if (!shouldRunJob)
return;
// get the IPs now because it won't be in the netdb by the time the job runs
Job job = new BanlistJob(peer, getAddresses(peer));
if (number > 0)
job.getTiming().setStartAfter(_context.clock().now() + (30*1000l * number));
_context.jobQueue().addJob(job);
}
private class BanlistJob extends JobImpl {
private final Hash _peer;
private final List _ips;
public BanlistJob (Hash p, List ips) {
super(_context);
_peer = p;
_ips = ips;
}
public String getName() { return "Ban Peer by IP"; }
public void runJob() {
banlistRouter(_peer, _ips, expireInterval());
synchronized (_inProcess) {
_inProcess.remove(_peer);
}
}
}
/**
* Look up the original record so we can record the reason in the banlist.
* That's the only reason to do this.
* Only synchronize to cut down on the I/O load.
* Additional jobs can wait.
* Although could this clog up the job queue runners? Yes.
* So we also stagger these jobs.
*
*/
private void banlistRouter( Hash peer, String reason, String reasonCode, long duration) {
if (duration > 0)
_context.banlist().banlistRouter(peer, reason, reasonCode, null, System.currentTimeMillis()+expireInterval());
else
_context.banlist().banlistRouterForever(peer, reason, reasonCode);
}
private synchronized void banlistRouter(Hash peer, List ips, long duration) {
// This only checks one file for now, pick the best one
// user specified
File blFile = null;
String file = _context.getProperty(PROP_BLOCKLIST_FILE);
if (file != null) {
blFile = new File(file);
if (!blFile.isAbsolute())
blFile = new File(_context.getConfigDir(), file);
if (!blFile.exists())
blFile = null;
}
// install dir
if (blFile == null)
blFile = new File(_context.getBaseDir(), BLOCKLIST_FILE_DEFAULT);
if ((!blFile.exists()) || blFile.length() <= 0) {
// just ban it and be done
if (_log.shouldLog(Log.WARN))
_log.warn("Banlisting " + peer);
banlistRouter(peer, "Banned", "Banned", expireInterval());
return;
}
// look through the file for each address to find which one was the cause
for (Iterator iter = ips.iterator(); iter.hasNext(); ) {
byte ip[] = iter.next();
int ipint = toInt(ip);
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(
new FileInputStream(blFile), "UTF-8"));
String buf = null;
// assume the file is unsorted, so go through the whole thing
while ((buf = br.readLine()) != null) {
Entry e = parse(buf, false);
if (e == null || e.peer != null) {
continue;
}
if (match(ipint, toEntry(e.ip1, e.ip2))) {
try { br.close(); } catch (IOException ioe) {}
String reason = _x("IP banned by blocklist.txt entry {0}");
// only one translate parameter for now
//for (int i = 0; i < 4; i++) {
// reason = reason + (ip[i] & 0xff);
// if (i != 3)
// reason = reason + '.';
//}
//reason = reason + " banned by " + BLOCKLIST_FILE_DEFAULT + " entry \"" + buf + "\"";
if (_log.shouldLog(Log.WARN))
_log.warn("Banlisting " + peer + " " + reason);
banlistRouter(peer, reason, buf.toString(), expireInterval());
return;
}
}
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error reading the blocklist file", ioe);
} finally {
if (br != null) try { br.close(); } catch (IOException ioe) {}
}
}
// We already banlisted in banlist(peer), that's good enough
}
/**
* Single IPs blocked until restart. Unsorted.
*
* Public for console only, not a public API
* As of 0.9.57, will not contain IPs permanently banned,
* except for ones banned permanently after being added to the transient list.
*
* @return a copy, unsorted
* @since 0.9.48
*/
public List getTransientIPv4Blocks() {
synchronized(_singleIPBlocklist) {
return new ArrayList(_singleIPBlocklist.keySet());
}
}
/**
* Single IPs blocked until restart. Unsorted.
*
* Public for console only, not a public API
*
* @return a copy, unsorted
* @since 0.9.48
*/
public List getTransientIPv6Blocks() {
if (!_haveIPv6)
return Collections.emptyList();
if (_singleIPv6Blocklist != null) {
synchronized(_singleIPv6Blocklist) {
return new ArrayList(_singleIPv6Blocklist.keySet());
}
}
return Collections.emptyList();
}
/**
* IP ranges blocked until restart. Sorted,
* but as signed longs, so 128-255 are first
*
* Public for console only, not a public API
*
* @param max maximum entries to return
* @return a copy, sorted
* @since 0.9.48
*/
public synchronized long[] getPermanentBlocks(int max) {
long[] rv;
if (_blocklistSize <= max) {
rv = new long[_blocklistSize];
System.arraycopy(_blocklist, 0, rv, 0, _blocklistSize);
} else {
// skip ahead to the positive entries
int i = 0;
for (; i < _blocklistSize; i++) {
int from = Blocklist.getFrom(_blocklist[i]);
if (from >= 0)
break;
}
int sz = Math.min(_blocklistSize - i, max);
rv = new long[sz];
System.arraycopy(_blocklist, i, rv, 0, sz);
}
return rv;
}
/**
* Size of permanent blocklist
*
* Public for console only, not a public API
*
* @since 0.9.48
*/
public synchronized int getBlocklistSize() {
return _blocklistSize;
}
/**
* Does nothing, moved to console ConfigPeerHelper
*
* @deprecated
*/
@Deprecated
public void renderStatusHTML(Writer out) throws IOException {
}
/**
* Mark a string for extraction by xgettext and translation.
* Use this only in static initializers.
* It does not translate!
* @return s
*/
private static final String _x(String s) {
return s;
}
/****
public static void main(String args[]) throws Exception {
Blocklist b = new Blocklist(new Router().getContext());
if (args != null && args.length == 1) {
File f = new File(args[0]);
b.allocate(Collections.singletonList(f));
int count = b.readBlocklistFile(f, 0);
b.merge(count);
Writer w = new java.io.OutputStreamWriter(System.out);
b.renderStatusHTML(w);
}
System.out.println("Saved " + b._blocklistSize + " records");
String tests[] = {"0.0.0.0", "0.0.0.1", "0.0.0.2", "0.0.0.255", "1.0.0.0",
"3.3.3.3", "77.1.2.3", "127.0.0.0", "127.127.127.127", "128.0.0.0",
"129.1.2.3", "255.255.255.254", "255.255.255.255"};
for (int i = 0; i < tests.length; i++) {
System.out.println("Testing " + tests[i] + " returns " + b.isBlocklisted(tests[i]));
}
}
****/
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy