propagate from branch 'i2p.i2p' (head 45c85fec6458cd0d1a6a6fa2d34b10ee2b9f215c)

to branch 'i2p.i2p.zzz.test2' (head 3ee9968e19867bebb063a98da1184ff4426626cd)

router/java/src/net/i2p/data/i2np/BuildRequestRecord.java

@@ -1,6 +1,9 @@
 package net.i2p.data.i2np;
 
+import java.util.Date;
+
 import net.i2p.I2PAppContext;
+import net.i2p.data.Base64;
 import net.i2p.data.ByteArray;
 import net.i2p.data.DataFormatException;
 import net.i2p.data.DataHelper;
@@ -68,7 +71,7 @@ public class BuildRequestRecord {
     private static final int OFF_SEND_IDENT = OFF_SEND_TUNNEL + 4;
     private static final int OFF_LAYER_KEY = OFF_SEND_IDENT + Hash.HASH_LENGTH;
     private static final int OFF_IV_KEY = OFF_LAYER_KEY + SessionKey.KEYSIZE_BYTES;
-    private static final int OFF_REPLY_KEY = OFF_IV_KEY + SessionKey.KEYSIZE_BYTES;
+    public static final int OFF_REPLY_KEY = OFF_IV_KEY + SessionKey.KEYSIZE_BYTES;
     private static final int OFF_REPLY_IV = OFF_REPLY_KEY + SessionKey.KEYSIZE_BYTES;
     private static final int OFF_FLAG = OFF_REPLY_IV + IV_SIZE;
     private static final int OFF_REQ_TIME = OFF_FLAG + 1;
@@ -156,7 +159,8 @@ public class BuildRequestRecord {
     }
 
     /**
-     * Time that the request was sent (ms), truncated to the nearest hour
+     * Time that the request was sent (ms), truncated to the nearest hour.
+     * This ignores leap seconds.
      */
     public long readRequestTime() {
         return DataHelper.fromLong(_data, OFF_REQ_TIME, 4) * (60 * 60 * 1000L);
@@ -263,6 +267,7 @@ public class BuildRequestRecord {
         long truncatedHour = ctx.clock().now();
         // prevent hop identification at top of the hour
         truncatedHour -= ctx.random().nextInt(90*1000);
+        // this ignores leap seconds
         truncatedHour /= (60l*60l*1000l);
         DataHelper.toLong(buf, OFF_REQ_TIME, 4, truncatedHour);
         DataHelper.toLong(buf, OFF_SEND_MSG_ID, 4, nextMsgId);
@@ -272,4 +277,34 @@ public class BuildRequestRecord {
         if (!DataHelper.eq(iv, wroteIV))
             throw new RuntimeException("foo");
     }
+
+    /**
+     *  @since 0.9.24
+     */
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder(256);
+        buf.append("BRR ");
+        boolean isIBGW = readIsInboundGateway();
+        boolean isOBEP = readIsOutboundEndpoint();
+        if (isIBGW) {
+            buf.append("IBGW in: ").append(readReceiveTunnelId())
+               .append(" out ").append(readNextTunnelId());
+        } else if (isOBEP) {
+            buf.append("OBEP in: ").append(readReceiveTunnelId());
+        } else {
+            buf.append("part. in: ").append(readReceiveTunnelId())
+               .append(" out: ").append(readNextTunnelId());
+        }
+        buf.append(" to: ").append(readNextIdentity())
+           .append(" layer key: ").append(readLayerKey())
+           .append(" IV key: ").append(readIVKey())
+           .append(" reply key: ").append(readReplyKey())
+           .append(" reply IV: ").append(Base64.encode(readReplyIV()))
+           .append(" hour: ").append(new Date(readRequestTime()))
+           .append(" reply msg id: ").append(readReplyMessageId());
+        // to chase i2pd bug
+        //buf.append('\n').append(net.i2p.util.HexDump.dump(readReplyKey().getData()));
+        return buf.toString();
+    }
 }   

router/java/src/net/i2p/data/i2np/DeliveryInstructions.java

@@ -430,7 +430,7 @@ public class DeliveryInstructions extends DataStructureImpl {
         //    _log.debug("Write flags: " + flags + " mode: " + getDeliveryMode() 
         //               + " =?= " + flagMode(flags));
         byte additionalInfo[] = getAdditionalInfo();
-        DataHelper.writeLong(out, 1, flags);
+        out.write((byte) flags);
         if (additionalInfo != null) {
             out.write(additionalInfo);
             out.flush();

router/java/src/net/i2p/data/router/RouterAddress.java

@@ -288,7 +288,7 @@ public class RouterAddress extends DataStructureImpl {
     public void writeBytes(OutputStream out) throws DataFormatException, IOException {
         if (_transportStyle == null)
             throw new DataFormatException("uninitialized");
-        DataHelper.writeLong(out, 1, _cost);
+        out.write((byte) _cost);
         DataHelper.writeLong(out, 8, _expiration);
         DataHelper.writeString(out, _transportStyle);
         DataHelper.writeProperties(out, _options);

router/java/src/net/i2p/data/router/RouterInfo.java

@@ -78,8 +78,11 @@ public class RouterInfo extends DatabaseEntry {
     private volatile boolean _hashCodeInitialized;
     /** should we cache the byte and string versions _byteified ? **/
     private boolean _shouldCache;
-    /** maybe we should check if we are floodfill? */
-    private static final boolean CACHE_ALL = SystemVersion.getMaxMemory() > 128*1024*1024l;
+    /**
+     * Maybe we should check if we are floodfill?
+     * If we do bring this back, don't do on ARM or Android
+     */
+    private static final boolean CACHE_ALL = false; // SystemVersion.getMaxMemory() > 128*1024*1024l;
 
     public static final String PROP_NETWORK_ID = "netId";
     public static final String PROP_CAPABILITIES = "caps";
@@ -308,11 +311,30 @@ public class RouterInfo extends DatabaseEntry {
      */
     protected byte[] getBytes() throws DataFormatException {
         if (_byteified != null) return _byteified;
-        if (_identity == null) throw new DataFormatException("Router identity isn't set?!");
-
-        //long before = Clock.getInstance().now();
         ByteArrayOutputStream out = new ByteArrayOutputStream(2*1024);
         try {
+            writeDataBytes(out);
+        } catch (IOException ioe) {
+            throw new DataFormatException("IO Error getting bytes", ioe);
+        }
+        byte data[] = out.toByteArray();
+        if (CACHE_ALL || _shouldCache)
+            _byteified = data;
+        return data;
+    }
+
+    /** 
+     * Write out the raw payload of the routerInfo, excluding the signature.  This
+     * caches the data in memory if possible.
+     *
+     * @throws DataFormatException if the data is somehow b0rked (missing props, etc)
+     * @throws IOException
+     * @since 0.9.24
+     */
+    private void writeDataBytes(OutputStream out) throws DataFormatException, IOException {
+        if (_identity == null) throw new DataFormatException("Missing identity");
+        if (_published < 0) throw new DataFormatException("Invalid published date: " + _published);
+
             _identity.writeBytes(out);
             // avoid thrashing objects
             //DataHelper.writeDate(out, new Date(_published));
@@ -320,9 +342,9 @@ public class RouterInfo extends DatabaseEntry {
             int sz = _addresses.size();
             if (sz <= 0 || isHidden()) {
                 // Do not send IP address to peers in hidden mode
-                DataHelper.writeLong(out, 1, 0);
+                out.write((byte) 0);
             } else {
-                DataHelper.writeLong(out, 1, sz);
+                out.write((byte) sz);
                 for (RouterAddress addr : _addresses) {
                     addr.writeBytes(out);
                 }
@@ -332,7 +354,7 @@ public class RouterInfo extends DatabaseEntry {
             //         method of trusted links, which isn't implemented in the router
             //         at the moment, and may not be later.
             int psz = _peers == null ? 0 : _peers.size();
-            DataHelper.writeLong(out, 1, psz);
+            out.write((byte) psz);
             if (psz > 0) {
                 Collection<Hash> peers = _peers;
                 if (psz > 1)
@@ -345,17 +367,6 @@ public class RouterInfo extends DatabaseEntry {
                 }
             }
             DataHelper.writeProperties(out, _options);
-        } catch (IOException ioe) {
-            throw new DataFormatException("IO Error getting bytes", ioe);
-        }
-        byte data[] = out.toByteArray();
-        //if (_log.shouldLog(Log.DEBUG)) {
-        //    long after = Clock.getInstance().now();
-        //    _log.debug("getBytes()  took " + (after - before) + "ms");
-        //}
-        if (CACHE_ALL || _shouldCache)
-            _byteified = data;
-        return data;
     }
 
     /**
@@ -434,44 +445,6 @@ public class RouterInfo extends DatabaseEntry {
         return (bwTier);
     }
 
-    /**
-     * Warning, must be called AFTER setOptions().
-     *
-     * @throws IllegalStateException if RouterInfo is already signed
-     */
-    public void addCapability(char cap) {
-        if (_signature != null)
-            throw new IllegalStateException();
-
-            String caps = _options.getProperty(PROP_CAPABILITIES);
-            if (caps == null)
-                _options.setProperty(PROP_CAPABILITIES, String.valueOf(cap));
-            else if (caps.indexOf(cap) == -1)
-                _options.setProperty(PROP_CAPABILITIES, caps + cap);
-    }
-
-    /**
-     * @throws IllegalStateException if RouterInfo is already signed
-     * @deprecated unused
-     */
-    public void delCapability(char cap) {
-        if (_signature != null)
-            throw new IllegalStateException();
-
-            String caps = _options.getProperty(PROP_CAPABILITIES);
-            int idx;
-            if (caps == null) {
-                return;
-            } else if ((idx = caps.indexOf(cap)) == -1) {
-                return;
-	    } else {
-                StringBuilder buf = new StringBuilder(caps);
-		while ( (idx = buf.indexOf(""+cap)) != -1)
-                    buf.deleteCharAt(idx);
-                _options.setProperty(PROP_CAPABILITIES, buf.toString());
-            }
-    }
-
     /**
      * Determine whether the router was published recently (within the given age milliseconds).
      * The age should be large enough to take into consideration any clock fudge factor, so
@@ -525,16 +498,11 @@ public class RouterInfo extends DatabaseEntry {
 
         if (!_isValid) {
             Log log = I2PAppContext.getGlobalContext().logManager().getLog(RouterInfo.class);
-            byte data[] = null;
-            try {
-                data = getBytes();
-            } catch (DataFormatException dfe) {
-                log.error("Error validating", dfe);
-                return;
+            if (log.shouldWarn()) {
+                log.warn("Sig verify fail: " + toString(), new Exception("from"));
+            } else {
+                log.error("RI Sig verify fail: " + _identity.getHash());
             }
-            log.error("Invalid [" + SHA256Generator.getInstance().calculateHash(data).toBase64()
-                           + (log.shouldLog(Log.WARN) ? ("]\n" + toString()) : ""),
-                           new Exception("Signature failed"));
         }
     }
     
@@ -631,18 +599,9 @@ public class RouterInfo extends DatabaseEntry {
      *  This does NOT validate the signature
      */
     public void writeBytes(OutputStream out) throws DataFormatException, IOException {
-        if (_identity == null) throw new DataFormatException("Missing identity");
-        if (_published < 0) throw new DataFormatException("Invalid published date: " + _published);
         if (_signature == null) throw new DataFormatException("Signature is null");
-        //if (!isValid())
-        //    throw new DataFormatException("Data is not valid");
-        ByteArrayOutputStream baos = new ByteArrayOutputStream(2048);
-        baos.write(getBytes());
-        _signature.writeBytes(baos);
-
-        byte data[] = baos.toByteArray();
-        //_log.debug("Writing routerInfo [len=" + data.length + "]: " + toString());
-        out.write(data);
+        writeDataBytes(out);
+        _signature.writeBytes(out);
     }
     
     @Override

router/java/src/net/i2p/router/CommSystemFacade.java

@@ -70,6 +70,7 @@ public abstract class CommSystemFacade implements Service {
      *
      * @deprecated use getStatus()
      */
+    @Deprecated
     public short getReachabilityStatus() { return (short) getStatus().getCode(); }
     
     /**
@@ -81,13 +82,22 @@ public abstract class CommSystemFacade implements Service {
     /**
      * @deprecated unused
      */
+    @Deprecated
     public void recheckReachability() {}
 
-    public boolean isBacklogged(Hash dest) { return false; }
-    public boolean wasUnreachable(Hash dest) { return false; }
-    public boolean isEstablished(Hash dest) { return false; }
+    public boolean isBacklogged(Hash peer) { return false; }
+    public boolean wasUnreachable(Hash peer) { return false; }
+    public boolean isEstablished(Hash peer) { return false; }
     public byte[] getIP(Hash dest) { return null; }
     public void queueLookup(byte[] ip) {}
+    
+    /**
+     * Tell the comm system that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    public void mayDisconnect(Hash peer) {}
 
     /** @since 0.8.11 */
     public String getOurCountry() { return null; }

router/java/src/net/i2p/router/InNetMessagePool.java

@@ -94,7 +94,7 @@ public class InNetMessagePool implements Service {
      * @return previous builder for this message type, or null
      * @throws AIOOBE if i2npMessageType is greater than MAX_I2NP_MESSAGE_TYPE
      */
-    public HandlerJobBuilder registerHandlerJobBuilder(int i2npMessageType, HandlerJobBuilder builder) {
+    public synchronized HandlerJobBuilder registerHandlerJobBuilder(int i2npMessageType, HandlerJobBuilder builder) {
         HandlerJobBuilder old = _handlerJobBuilders[i2npMessageType];
         _handlerJobBuilders[i2npMessageType] = builder;
         return old;
@@ -103,8 +103,10 @@ public class InNetMessagePool implements Service {
     /**
      * @return previous builder for this message type, or null
      * @throws AIOOBE if i2npMessageType is greater than MAX_I2NP_MESSAGE_TYPE
+     * @deprecated unused
      */
-    public HandlerJobBuilder unregisterHandlerJobBuilder(int i2npMessageType) {
+    @Deprecated
+    public synchronized HandlerJobBuilder unregisterHandlerJobBuilder(int i2npMessageType) {
         HandlerJobBuilder old = _handlerJobBuilders[i2npMessageType];
         _handlerJobBuilders[i2npMessageType] = null;
         return old;

router/java/src/net/i2p/router/MessageHistory.java

@@ -10,6 +10,7 @@ import java.util.Queue;
 import java.util.TimeZone;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
 import net.i2p.data.TunnelId;
 import net.i2p.data.i2np.I2NPMessage;
@@ -40,7 +41,7 @@ public class MessageHistory {
     //private SubmitMessageHistoryJob _submitMessageHistoryJob;
     private volatile boolean _firstPass;
     
-    private final static byte[] NL = System.getProperty("line.separator").getBytes();
+    private final static byte[] NL = DataHelper.getUTF8(System.getProperty("line.separator"));
     private final static int FLUSH_SIZE = 1000; // write out at least once every 1000 entries
         
     /** config property determining whether we want to debug with the message history - default false */
@@ -636,7 +637,7 @@ public class MessageHistory {
             fos = new SecureFileOutputStream(f, true);
             String entry;
             while ((entry = _unwrittenEntries.poll()) != null) {
-                fos.write(entry.getBytes());
+                fos.write(DataHelper.getUTF8(entry));
                 fos.write(NL);
             }
         } catch (IOException ioe) {

router/java/src/net/i2p/router/Router.java

@@ -10,6 +10,7 @@ package net.i2p.router;
 
 import java.io.File;
 import java.io.IOException;
+import java.security.GeneralSecurityException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -35,6 +36,7 @@ import net.i2p.data.SigningPublicKey;
 import net.i2p.data.i2np.GarlicMessage;
 import net.i2p.data.router.RouterInfo;
 import net.i2p.router.CommSystemFacade.Status;
+import net.i2p.router.crypto.FamilyKeyCrypto;
 import net.i2p.router.message.GarlicMessageHandler;
 import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade;
 import net.i2p.router.startup.CreateRouterInfoJob;
@@ -88,6 +90,9 @@ public class Router implements RouterClock.ClockShiftListener {
     private final EventLog _eventLog;
     private final Object _stateLock = new Object();
     private State _state = State.UNINITIALIZED;
+    private FamilyKeyCrypto _familyKeyCrypto;
+    private boolean _familyKeyCryptoFail;
+    public final Object _familyKeyLock = new Object();
     
     public final static String PROP_CONFIG_FILE = "router.configLocation";
     
@@ -488,6 +493,9 @@ public class Router implements RouterClock.ClockShiftListener {
     /**
      *  Our current router info.
      *  Warning, may be null if called very early.
+     *
+     *  Warning - risk of deadlock - do not call while holding locks
+     *
      */
     public RouterInfo getRouterInfo() {
         synchronized (_routerInfoLock) {
@@ -498,6 +506,9 @@ public class Router implements RouterClock.ClockShiftListener {
     /**
      *  Caller must ensure info is valid - no validation done here.
      *  Not for external use.
+     *
+     *  Warning - risk of deadlock - do not call while holding locks
+     *
      */
     public void setRouterInfo(RouterInfo info) { 
         synchronized (_routerInfoLock) {
@@ -511,13 +522,19 @@ public class Router implements RouterClock.ClockShiftListener {
 
     /**
      *  Used only by routerconsole.. to be deprecated?
+     *  @return System time, NOT context time
      */
     public long getWhenStarted() { return _started; }
 
-    /** wall clock uptime */
+    /**
+     * Wall clock uptime.
+     * This uses System time, NOT context time, so context clock shifts will
+     * not affect it. This is important if NTP fails and the
+     * clock then shifts from a SSU peer source just after startup.
+     */
     public long getUptime() { 
-        if ( (_context == null) || (_context.clock() == null) ) return 1; // racing on startup
-        return Math.max(1, _context.clock().now() - _context.clock().getOffset() - _started);
+        if (_started <= 0) return 1000; // racing on startup
+        return Math.max(1000, System.currentTimeMillis() - _started);
     }
     
     /**
@@ -555,7 +572,7 @@ public class Router implements RouterClock.ClockShiftListener {
         _eventLog.addEvent(EventLog.STARTED, RouterVersion.FULL_VERSION);
         startupStuff();
         changeState(State.STARTING_2);
-        _started = _context.clock().now();
+        _started = System.currentTimeMillis();
         try {
             Runtime.getRuntime().addShutdownHook(_shutdownHook);
         } catch (IllegalStateException ise) {}
@@ -806,6 +823,9 @@ public class Router implements RouterClock.ClockShiftListener {
      * Rebuild and republish our routerInfo since something significant 
      * has changed.
      * Not for external use.
+     *
+     *  Warning - risk of deadlock - do not call while holding locks
+     *
      */
     public void rebuildRouterInfo(boolean blockingRebuild) {
         if (_log.shouldLog(Log.INFO))
@@ -821,7 +841,7 @@ public class Router implements RouterClock.ClockShiftListener {
      * has changed.
      */
     private void locked_rebuildRouterInfo(boolean blockingRebuild) {
-        RouterInfo ri = null;
+        RouterInfo ri;
         if (_routerInfo != null)
             ri = new RouterInfo(_routerInfo);
         else
@@ -830,12 +850,11 @@ public class Router implements RouterClock.ClockShiftListener {
         try {
             ri.setPublished(_context.clock().now());
             Properties stats = _context.statPublisher().publishStatistics();
-            stats.setProperty(RouterInfo.PROP_NETWORK_ID, NETWORK_ID+"");
             
             ri.setOptions(stats);
+            // deadlock thru createAddresses() thru SSU REA... move outside lock?
             ri.setAddresses(_context.commSystem().createAddresses());
 
-            addCapabilities(ri);
             SigningPrivateKey key = _context.keyManager().getSigningPrivateKey();
             if (key == null) {
                 _log.log(Log.CRIT, "Internal error - signing private key not known? Impossible?");
@@ -855,6 +874,32 @@ public class Router implements RouterClock.ClockShiftListener {
         }
     }
 
+    /**
+     *  Family Key Crypto Signer / Verifier.
+     *  Not for external use.
+     *  If family key is set, first call Will take a while to generate keys.
+     *  Warning - risk of deadlock - do not call while holding locks
+     *  (other than routerInfoLock)
+     *
+     *  @return null on initialization failure
+     *  @since 0.9.24
+     */
+    public FamilyKeyCrypto getFamilyKeyCrypto() {
+        synchronized (_familyKeyLock) {
+            if (_familyKeyCrypto == null) {
+                if (!_familyKeyCryptoFail) {
+                    try {
+                        _familyKeyCrypto = new FamilyKeyCrypto(_context);
+                    } catch (GeneralSecurityException gse) {
+                        _log.error("Failed to initialize family key crypto", gse);
+                        _familyKeyCryptoFail = true;
+                    }
+                }
+            }
+        }
+        return _familyKeyCrypto;
+    }
+
     // publicize our ballpark capacity
     public static final char CAPABILITY_BW12 = 'K';
     public static final char CAPABILITY_BW32 = 'L';
@@ -878,14 +923,11 @@ public class Router implements RouterClock.ClockShiftListener {
     
     /**
      *  For building our RI. Not for external use.
-     *  This does not publish the ri.
-     *  This does not use anything in the ri (i.e. it can be freshly constructed)
      *
-     *  TODO just return a string instead of passing in the RI? See PublishLocalRouterInfoJob.
-     *
-     *  @param ri an unpublished ri we are generating.
+     *  @return a capabilities string to be added to the RI
      */
-    public void addCapabilities(RouterInfo ri) {
+    public String getCapabilities() {
+        StringBuilder rv = new StringBuilder(4);
         int bwLim = Math.min(_context.bandwidthLimiter().getInboundKBytesPerSecond(),
                              _context.bandwidthLimiter().getOutboundKBytesPerSecond());
         bwLim = (int)(bwLim * getSharePercentage());
@@ -894,40 +936,40 @@ public class Router implements RouterClock.ClockShiftListener {
         
         String force = _context.getProperty(PROP_FORCE_BWCLASS);
         if (force != null && force.length() > 0) {
-            ri.addCapability(force.charAt(0));
+            rv.append(force.charAt(0));
         } else if (bwLim < 12) {
-            ri.addCapability(CAPABILITY_BW12);
+            rv.append(CAPABILITY_BW12);
         } else if (bwLim <= 48) {
-            ri.addCapability(CAPABILITY_BW32);
+            rv.append(CAPABILITY_BW32);
         } else if (bwLim <= 64) {
-            ri.addCapability(CAPABILITY_BW64);
+            rv.append(CAPABILITY_BW64);
         } else if (bwLim <= 128) {
-            ri.addCapability(CAPABILITY_BW128);
+            rv.append(CAPABILITY_BW128);
         } else if (bwLim <= 256) {
-            ri.addCapability(CAPABILITY_BW256);
+            rv.append(CAPABILITY_BW256);
         } else if (bwLim <= 2000) {    // TODO adjust threshold
             // 512 supported as of 0.9.18;
             // Add 256 as well for compatibility
-            ri.addCapability(CAPABILITY_BW512);
-            ri.addCapability(CAPABILITY_BW256);
+            rv.append(CAPABILITY_BW512);
+            rv.append(CAPABILITY_BW256);
         } else {
             // Unlimited supported as of 0.9.18;
             // Add 256 as well for compatibility
-            ri.addCapability(CAPABILITY_BW_UNLIMITED);
-            ri.addCapability(CAPABILITY_BW256);
+            rv.append(CAPABILITY_BW_UNLIMITED);
+            rv.append(CAPABILITY_BW256);
         }
         
         // if prop set to true, don't tell people we are ff even if we are
         if (_context.netDb().floodfillEnabled() &&
             !_context.getBooleanProperty("router.hideFloodfillParticipant"))
-            ri.addCapability(FloodfillNetworkDatabaseFacade.CAPABILITY_FLOODFILL);
+            rv.append(FloodfillNetworkDatabaseFacade.CAPABILITY_FLOODFILL);
         
         if(_context.getBooleanProperty(PROP_HIDDEN))
-            ri.addCapability(RouterInfo.CAPABILITY_HIDDEN);
+            rv.append(RouterInfo.CAPABILITY_HIDDEN);
         
         if (_context.getBooleanProperty(PROP_FORCE_UNREACHABLE)) {
-            ri.addCapability(CAPABILITY_UNREACHABLE);
-            return;
+            rv.append(CAPABILITY_UNREACHABLE);
+            return rv.toString();
         }
         switch (_context.commSystem().getStatus()) {
             case OK:
@@ -937,13 +979,13 @@ public class Router implements RouterClock.ClockShiftListener {
             case IPV4_DISABLED_IPV6_OK:
             case IPV4_UNKNOWN_IPV6_OK:
             case IPV4_SNAT_IPV6_OK:
-                ri.addCapability(CAPABILITY_REACHABLE);
+                rv.append(CAPABILITY_REACHABLE);
                 break;
 
             case DIFFERENT:
             case REJECT_UNSOLICITED:
             case IPV4_DISABLED_IPV6_FIREWALLED:
-                ri.addCapability(CAPABILITY_UNREACHABLE);
+                rv.append(CAPABILITY_UNREACHABLE);
                 break;
 
             case DISCONNECTED:
@@ -957,14 +999,22 @@ public class Router implements RouterClock.ClockShiftListener {
                 // no explicit capability
                 break;
         }
+        return rv.toString();
     }
     
+    /*
+     *  This checks the config only. We don't check the current RI
+     *  due to deadlocks.
+     *
+     */
     public boolean isHidden() {
-        RouterInfo ri;
-        synchronized (_routerInfoLock) {
-            ri = _routerInfo;
-        }
-        if ( (ri != null) && (ri.isHidden()) )
+        //RouterInfo ri;
+        //synchronized (_routerInfoLock) {
+        //    ri = _routerInfo;
+        //}
+        //if ( (ri != null) && (ri.isHidden()) )
+        //    return true;
+        if (_context.getBooleanProperty(PROP_HIDDEN))
             return true;
         String h = _context.getProperty(PROP_HIDDEN_HIDDEN);
         if (h != null)
@@ -1164,7 +1214,6 @@ public class Router implements RouterClock.ClockShiftListener {
         try { _context.clientManager().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the client manager", t); }
         try { _context.namingService().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the naming service", t); }
         try { _context.jobQueue().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the job queue", t); }
-        try { _context.statPublisher().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the stats publisher", t); }
         try { _context.tunnelManager().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the tunnel manager", t); }
         try { _context.tunnelDispatcher().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the tunnel dispatcher", t); }
         try { _context.netDb().shutdown(); } catch (Throwable t) { _log.error("Error shutting down the networkDb", t); }

router/java/src/net/i2p/router/RouterContext.java

@@ -261,6 +261,9 @@ public class RouterContext extends I2PAppContext {
     /**
      *  Convenience method for getting the router hash.
      *  Equivalent to context.router().getRouterInfo().getIdentity().getHash()
+     *
+     *  Warning - risk of deadlock - do not call while holding locks
+     *
      *  @return may be null if called very early
      */
     public Hash routerHash() {

router/java/src/net/i2p/router/RouterThrottleImpl.java

@@ -14,7 +14,7 @@ import net.i2p.util.SimpleTimer;
  * requests if the jobQueue lag is too large.  
  *
  */
-class RouterThrottleImpl implements RouterThrottle {
+public class RouterThrottleImpl implements RouterThrottle {
     protected final RouterContext _context;
     private final Log _log;
     private volatile String _tunnelStatus;
@@ -28,8 +28,8 @@ class RouterThrottleImpl implements RouterThrottle {
     private static final long JOB_LAG_LIMIT_NETDB = 2*1000;
     // TODO reduce
     private static final long JOB_LAG_LIMIT_TUNNEL = 500;
-    private static final String PROP_MAX_TUNNELS = "router.maxParticipatingTunnels";
-    private static final int DEFAULT_MAX_TUNNELS = 10*1000;
+    public static final String PROP_MAX_TUNNELS = "router.maxParticipatingTunnels";
+    public static final int DEFAULT_MAX_TUNNELS = 10*1000;
     private static final String PROP_MAX_PROCESSINGTIME = "router.defaultProcessingTimeThrottle";
     private static final long DEFAULT_REJECT_STARTUP_TIME = 10*60*1000;
     private static final String PROP_REJECT_STARTUP_TIME = "router.rejectStartupTime";
@@ -276,7 +276,7 @@ class RouterThrottleImpl implements RouterThrottle {
         
         // ok, we're not hosed, but can we handle the bandwidth requirements 
         // of another tunnel?
-        rs = _context.statManager().getRate("tunnel.participatingMessageCount");
+        rs = _context.statManager().getRate("tunnel.participatingMessageCountAvgPerTunnel");
         r = null;
         double messagesPerTunnel = DEFAULT_MESSAGES_PER_TUNNEL_ESTIMATE;
         if (rs != null) {

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 24;
+    public final static long BUILD = 25;
 
     /** for example "-test" */
     public final static String EXTRA = "-rc";

router/java/src/net/i2p/router/StatisticsManager.java

@@ -9,6 +9,7 @@ package net.i2p.router;
  */
 
 import java.io.Writer;
+import java.security.GeneralSecurityException;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.util.Locale;
@@ -16,6 +17,9 @@ import java.util.Properties;
 
 import net.i2p.CoreVersion;
 import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.router.RouterInfo;
+import net.i2p.router.crypto.FamilyKeyCrypto;
 import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade;
 import net.i2p.stat.Rate;
 import net.i2p.stat.RateStat;
@@ -25,11 +29,12 @@ import net.i2p.util.Log;
  * Publishes some statistics about the router in the netDB.
  *
  */
-public class StatisticsManager implements Service {
+public class StatisticsManager {
     private final Log _log;
     private final RouterContext _context;
     
     public final static String PROP_PUBLISH_RANKINGS = "router.publishPeerRankings";
+    private static final String PROP_CONTACT_NAME = "netdb.contact";
     /** enhance anonymity by only including build stats one out of this many times */
     private static final int RANDOM_INCLUDE_STATS = 16;
 
@@ -43,20 +48,32 @@ public class StatisticsManager implements Service {
         _log = context.logManager().getLog(StatisticsManager.class);
     }
         
-    /** noop */
-    public void shutdown() {}
-
-    /** noop */
-    public void restart() {}
-
-    /** noop */
-    public void startup() {}
-    
-    /** Retrieve a snapshot of the statistics that should be published */
+    /**
+     *  Retrieve a snapshot of the statistics that should be published.
+     *
+     *  This includes all standard options (as of 0.9.24, network ID and caps)
+     */
     public Properties publishStatistics() { 
+        // if hash is null, will be caught in fkc.sign()
+        return publishStatistics(_context.routerHash());
+    }
+    
+    /**
+     *  Retrieve a snapshot of the statistics that should be published.
+     *
+     *  This includes all standard options (as of 0.9.24, network ID and caps)
+     *
+     *  @param h current router hash, non-null
+     *  @since 0.9.24
+     */
+    public Properties publishStatistics(Hash h) { 
         Properties stats = new Properties();
         stats.setProperty("router.version", RouterVersion.VERSION);
-        stats.setProperty("coreVersion", CoreVersion.VERSION);
+        // scheduled for removal, never used
+        if (CoreVersion.VERSION.equals("0.9.23"))
+            stats.setProperty("coreVersion", CoreVersion.VERSION);
+        stats.setProperty(RouterInfo.PROP_NETWORK_ID, Integer.toString(Router.NETWORK_ID));
+        stats.setProperty(RouterInfo.PROP_CAPABILITIES, _context.router().getCapabilities());
 
         // No longer expose, to make build tracking more expensive
         // stats.setProperty("router.id", RouterVersion.ID);
@@ -148,8 +165,10 @@ public class StatisticsManager implements Service {
             //includeRate("tunnel.acceptLoad", stats, new long[] { 10*60*1000 });
         }
 
-        // So that we will still get build requests
-        stats.setProperty("stat_uptime", "90m");
+        // So that we will still get build requests - not required since 0.7.9 2010-01-12
+        // scheduled for removal
+        if (CoreVersion.VERSION.equals("0.9.23"))
+            stats.setProperty("stat_uptime", "90m");
         if (FloodfillNetworkDatabaseFacade.isFloodfill(_context.router().getRouterInfo())) {
             int ri = _context.router().getUptime() > 30*60*1000 ?
                      _context.netDb().getKnownRouters() :
@@ -161,6 +180,48 @@ public class StatisticsManager implements Service {
             stats.setProperty("netdb.knownLeaseSets", String.valueOf(ls));
         }
 
+        String contact = _context.getProperty(PROP_CONTACT_NAME);
+        if (contact != null)
+            stats.setProperty("contact", contact);
+
+        String family = _context.getProperty(FamilyKeyCrypto.PROP_FAMILY_NAME);
+        if (family != null) {
+            stats.setProperty(FamilyKeyCrypto.OPT_NAME, family);
+            String sig = null;
+            String key = null;
+            RouterInfo oldRI = _context.router().getRouterInfo();
+            if (oldRI != null) {
+                // don't do it if family changed
+                if (family.equals(oldRI.getOption(FamilyKeyCrypto.OPT_NAME))) {
+                    // copy over the pubkey and signature
+                    key = oldRI.getOption(FamilyKeyCrypto.OPT_KEY);
+                    if (key != null) {
+                        if (key.contains(";")) {
+                            // we changed the separator from ';' to ':'
+                            key = null;
+                        } else {
+                            stats.setProperty(FamilyKeyCrypto.OPT_KEY, key);
+                            sig = oldRI.getOption(FamilyKeyCrypto.OPT_SIG);
+                            if (sig != null)
+                                stats.setProperty(FamilyKeyCrypto.OPT_SIG, sig);
+                        }
+                    }
+                }
+            }
+            if (sig == null || key == null) {
+                FamilyKeyCrypto fkc = _context.router().getFamilyKeyCrypto();
+                if (fkc != null) {
+                    try {
+                        stats.putAll(fkc.sign(family, h));
+                    } catch (GeneralSecurityException gse) {
+                        _log.error("Failed to sign router family", gse);
+                        stats.remove(FamilyKeyCrypto.OPT_KEY);
+                        stats.remove(FamilyKeyCrypto.OPT_SIG);
+                    }
+                }
+            }
+        }
+
         return stats;
     }
     

router/java/src/net/i2p/router/crypto/FamilyKeyCrypto.java

@@ -0,0 +1,395 @@
+package net.i2p.router.crypto;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.KeyStore;
+import java.security.GeneralSecurityException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import net.i2p.crypto.CertUtil;
+import net.i2p.crypto.KeyStoreUtil;
+import net.i2p.crypto.SigType;
+import net.i2p.crypto.SigUtil;
+import net.i2p.data.Base64;
+import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.Signature;
+import net.i2p.data.SigningPrivateKey;
+import net.i2p.data.SigningPublicKey;
+import net.i2p.data.router.RouterInfo;
+import net.i2p.router.RouterContext;
+import net.i2p.router.StatisticsManager;
+import net.i2p.util.ConcurrentHashSet;
+import net.i2p.util.Log;
+import net.i2p.util.SecureDirectory;
+
+/**
+ * Utilities for creating, storing, retrieving the signing keys for
+ * the netdb family feature
+ *
+ * @since 0.9.24
+ */
+public class FamilyKeyCrypto {
+
+    private final RouterContext _context;
+    private final Log _log;
+    private final Map<Hash, String> _verified;
+    private final Set<Hash> _negativeCache;
+    // following for verification only, otherwise null
+    private final String _fname;
+    private final SigningPrivateKey _privkey;
+    private final SigningPublicKey _pubkey;
+
+    private static final String PROP_KEYSTORE_PASSWORD = "netdb.family.keystorePassword";
+    public static final String PROP_FAMILY_NAME = "netdb.family.name";
+    private static final String PROP_KEY_PASSWORD = "netdb.family.keyPassword";
+    private static final String CERT_SUFFIX = ".crt";
+    private static final String KEYSTORE_PREFIX = "family-";
+    private static final String KEYSTORE_SUFFIX = ".ks";
+    private static final int DEFAULT_KEY_VALID_DAYS = 3652;  // 10 years
+    // Note that we can't use RSA here, as the b64 sig would exceed the 255 char limit for a Mapping
+    // Note that we can't use EdDSA here, as keystore doesn't know how, and encoding/decoding is unimplemented
+    private static final String DEFAULT_KEY_ALGORITHM = SigType.ECDSA_SHA256_P256.isAvailable() ? "EC" : "DSA";
+    private static final int DEFAULT_KEY_SIZE = SigType.ECDSA_SHA256_P256.isAvailable() ? 256 : 1024;
+    private static final String KS_DIR = "keystore";
+    private static final String CERT_DIR = "certificates/family";
+    public static final String OPT_NAME = "family";
+    public static final String OPT_SIG = "family.sig";
+    public static final String OPT_KEY = "family.key";
+
+
+    /** 
+     *  For signing and verification.
+     *
+     *  If the context property netdb.family.name is set, this can be used for signing,
+     *  else only for verification.
+     */
+    public FamilyKeyCrypto(RouterContext context) throws GeneralSecurityException {
+        _context = context;
+        _log = _context.logManager().getLog(FamilyKeyCrypto.class);
+        _fname = _context.getProperty(PROP_FAMILY_NAME);
+        if (_fname != null) {
+            if (_fname.contains("/") || _fname.contains("\\") ||
+                _fname.contains("..") || (new File(_fname)).isAbsolute())
+                throw new GeneralSecurityException("Illegal family name");
+        }
+        _privkey = (_fname != null) ? initialize() : null;
+        _pubkey = (_privkey != null) ? _privkey.toPublic() : null;
+        _verified = new ConcurrentHashMap<Hash, String>(4);
+        _negativeCache = new ConcurrentHashSet<Hash>(4);
+    }
+    
+    /** 
+     * Create (if necessary) and load the key store, then run.
+     */
+    private SigningPrivateKey initialize() throws GeneralSecurityException {
+        File dir = new SecureDirectory(_context.getConfigDir(), KS_DIR);
+        File keyStore = new File(dir, KEYSTORE_PREFIX + _fname + KEYSTORE_SUFFIX);
+        verifyKeyStore(keyStore);
+        return getPrivKey(keyStore);
+    }
+    
+    /** 
+     * Clears the caches
+     */
+    public void shutdown() {
+        _verified.clear();
+        _negativeCache.clear();
+    }
+
+    /** 
+     *  Caller must add family to RI also.
+     *  throws on all errors
+     *
+     *  @param family non-null, must match that we were initialized with or will throw GSE
+     *  @param h non-null
+     *  @return non-null options to be added to the RI
+     *  @throws GeneralSecurityException on null hash, null or changed family, or signing error
+     */
+    public Map<String, String> sign(String family, Hash h) throws GeneralSecurityException {
+        if (_privkey == null) {
+            _log.logAlways(Log.WARN, "family name now set, must restart router to generate or load keys");
+            throw new GeneralSecurityException("family name now set, must restart router to generate or load keys");
+        }
+        if (h == null)
+            throw new GeneralSecurityException("null router hash");
+        if (!_fname.equals(family)) {
+            _log.logAlways(Log.WARN, "family name changed, must restart router to generate or load new keys");
+            throw new GeneralSecurityException("family name changed, must restart router to generate or load new keys");
+        }
+        byte[] nb = DataHelper.getUTF8(_fname);
+        int len = nb.length + Hash.HASH_LENGTH;
+        byte[] b = new byte[len];
+        System.arraycopy(nb, 0, b, 0, nb.length);
+        System.arraycopy(h.getData(), 0, b, nb.length, Hash.HASH_LENGTH);
+        Signature sig = _context.dsa().sign(b, _privkey);
+        if (sig == null)
+            throw new GeneralSecurityException("sig failed");
+        Map<String, String> rv = new HashMap<String, String>(3);
+        rv.put(OPT_NAME, family);
+        rv.put(OPT_KEY, _pubkey.getType().getCode() + ":" + _pubkey.toBase64());
+        rv.put(OPT_SIG, sig.toBase64());
+        return rv;
+    }
+
+    /** 
+     *  Verify the family signature in a RouterInfo.
+     *  @return true if good sig or if no family specified at all
+     */
+    public boolean verify(RouterInfo ri) {
+        String name = ri.getOption(OPT_NAME);
+        if (name == null)
+            return true;
+        Hash h = ri.getHash();
+        String ssig = ri.getOption(OPT_SIG);
+        if (ssig == null) {
+            if (_log.shouldInfo())
+                _log.info("No sig for " + h + ' ' + name);
+            return false;
+        }
+        String nameAndSig = _verified.get(h);
+        String riNameAndSig = name + ssig;
+        if (nameAndSig != null) {
+            if (nameAndSig.equals(riNameAndSig))
+                return true;
+            // name or sig changed
+            _verified.remove(h);
+        }
+        SigningPublicKey spk;
+        if (name.equals(_fname)) {
+            // us
+            spk = _pubkey;
+        } else {
+            if (_negativeCache.contains(h))
+                return false;
+            spk = loadCert(name);
+            if (spk == null) {
+                // look for a b64 key in the RI
+                String skey = ri.getOption(OPT_KEY);
+                if (skey != null) {
+                    int colon = skey.indexOf(':');
+                    // switched from ';' to ':' during dev, remove this later
+                    if (colon < 0)
+                        colon = skey.indexOf(';');
+                    if (colon > 0) {
+                        try {
+                            int code = Integer.parseInt(skey.substring(0, colon));
+                            SigType type = SigType.getByCode(code);
+                            if (type != null) {
+                                byte[] bkey = Base64.decode(skey.substring(colon + 1));
+                                if (bkey != null) {
+                                    spk = new SigningPublicKey(type, bkey);
+                                }
+                            }
+                        } catch (NumberFormatException e) {
+                            if (_log.shouldInfo())
+                                _log.info("Bad b64 family key: " + ri, e);
+                        } catch (IllegalArgumentException e) {
+                            if (_log.shouldInfo())
+                                _log.info("Bad b64 family key: " + ri, e);
+                        } catch (ArrayIndexOutOfBoundsException e) {
+                            if (_log.shouldInfo())
+                                _log.info("Bad b64 family key: " + ri, e);
+                        }
+                    }
+                }
+                if (spk == null) {
+                    _negativeCache.add(h);
+                    if (_log.shouldInfo())
+                        _log.info("No cert or valid key for " + h + ' ' + name);
+                    return false;
+                }
+            }
+        }
+        if (!spk.getType().isAvailable()) {
+            _negativeCache.add(h);
+            if (_log.shouldInfo())
+                _log.info("Unsupported crypto for sig for " + h);
+            return false;
+        }
+        byte[] bsig = Base64.decode(ssig);
+        if (bsig == null) {
+            _negativeCache.add(h);
+            if (_log.shouldInfo())
+                _log.info("Bad sig for " + h + ' ' + name + ' ' + ssig);
+            return false;
+        }
+        Signature sig;
+        try {
+            sig = new Signature(spk.getType(), bsig);
+        } catch (IllegalArgumentException iae) {
+            // wrong size (type mismatch)
+            _negativeCache.add(h);
+            if (_log.shouldInfo())
+                _log.info("Bad sig for " + ri, iae);
+            return false;
+        }
+        byte[] nb = DataHelper.getUTF8(name);
+        byte[] b = new byte[nb.length + Hash.HASH_LENGTH];
+        System.arraycopy(nb, 0, b, 0, nb.length);
+        System.arraycopy(ri.getHash().getData(), 0, b, nb.length, Hash.HASH_LENGTH);
+        boolean rv = _context.dsa().verifySignature(sig, b, spk);
+        if (rv)
+            _verified.put(h, riNameAndSig);
+        else
+            _negativeCache.add(h);
+        if (_log.shouldInfo())
+            _log.info("Verified? " + rv + " for " + h + ' ' + name + ' ' + ssig);
+        return rv;
+    }
+
+    /**
+     *  @return success if it exists and we have a password, or it was created successfully.
+     *  @throws GeneralSecurityException on keystore error
+     */
+    private void verifyKeyStore(File ks) throws GeneralSecurityException {
+        if (ks.exists()) {
+            if (_context.getProperty(PROP_KEY_PASSWORD) == null) {
+                String s ="Family key error, must set " + PROP_KEY_PASSWORD + " in " +
+                          (new File(_context.getConfigDir(), "router.config")).getAbsolutePath();
+                _log.error(s);
+                throw new GeneralSecurityException(s);
+            }
+            return;
+        }
+        File dir = ks.getParentFile();
+        if (!dir.exists()) {
+            File sdir = new SecureDirectory(dir.getAbsolutePath());
+            if (!sdir.mkdirs()) {
+                String s ="Family key error, must set " + PROP_KEY_PASSWORD + " in " +
+                          (new File(_context.getConfigDir(), "router.config")).getAbsolutePath();
+                _log.error(s);
+                throw new GeneralSecurityException(s);
+            }
+        }
+        createKeyStore(ks);
+
+        // Now read it back out of the new keystore and save it in ascii form
+        // where the clients can get to it.
+        exportCert(ks);
+    }
+
+
+    /**
+     * Call out to keytool to create a new keystore with a keypair in it.
+     * Trying to do this programatically is a nightmare, requiring either BouncyCastle
+     * libs or using proprietary Sun libs, and it's a huge mess.
+     * If successful, stores the keystore password and key password in router.config.
+     *
+     * @throws GeneralSecurityException on all errors
+     */
+    private void createKeyStore(File ks) throws GeneralSecurityException {
+        // make a random 48 character password (30 * 8 / 5)
+        String keyPassword = KeyStoreUtil.randomString();
+        // and one for the cname
+        String cname = _fname + ".family.i2p.net";
+
+        boolean success = KeyStoreUtil.createKeys(ks, KeyStoreUtil.DEFAULT_KEYSTORE_PASSWORD, _fname, cname, "family",
+                                                  DEFAULT_KEY_VALID_DAYS, DEFAULT_KEY_ALGORITHM,
+                                                  DEFAULT_KEY_SIZE, keyPassword);
+        if (success) {
+            success = ks.exists();
+            if (success) {
+                Map<String, String> changes = new HashMap<String, String>();
+                changes.put(PROP_KEYSTORE_PASSWORD, KeyStoreUtil.DEFAULT_KEYSTORE_PASSWORD);
+                changes.put(PROP_KEY_PASSWORD, keyPassword);
+                changes.put(PROP_FAMILY_NAME, _fname);
+                _context.router().saveConfig(changes, null);
+            }
+        }
+        if (success) {
+            _log.logAlways(Log.INFO, "Created new private key for netdb family \"" + _fname +
+                           "\" in keystore: " + ks.getAbsolutePath() + "\n" +
+                           "Copy the keystore to the other routers in the family,\n" +
+                           "and add the following entries to their router.config file:\n" +
+                           PROP_FAMILY_NAME + '=' + _fname + '\n' +
+                           PROP_KEYSTORE_PASSWORD + '=' + KeyStoreUtil.DEFAULT_KEYSTORE_PASSWORD + '\n' +
+                           PROP_KEY_PASSWORD + '=' + keyPassword);
+
+        } else {
+            String s = "Failed to create NetDb family keystore.\n" +
+                       "This is for the Sun/Oracle keytool, others may be incompatible.\n" +
+                       "If you create the keystore manually, you must add " + PROP_KEYSTORE_PASSWORD + " and " + PROP_KEY_PASSWORD +
+                       " to " + (new File(_context.getConfigDir(), "router.config")).getAbsolutePath();
+            _log.error(s);
+            throw new GeneralSecurityException(s);
+        }
+    }
+
+    /** 
+     * Pull the cert back OUT of the keystore and save it as ascii
+     * so the clients can get to it.
+     */
+    private void exportCert(File ks) {
+        File sdir = new SecureDirectory(_context.getConfigDir(), CERT_DIR);
+        if (sdir.exists() || sdir.mkdirs()) {
+            String ksPass = _context.getProperty(PROP_KEYSTORE_PASSWORD, KeyStoreUtil.DEFAULT_KEYSTORE_PASSWORD);
+            String name = _fname.replace("@", "_at_") + CERT_SUFFIX;
+            File out = new File(sdir, name);
+            boolean success = KeyStoreUtil.exportCert(ks, ksPass, _fname, out);
+            if (success) {
+                _log.logAlways(Log.INFO, "Created new public key certificate for netdb family \"" + _fname +
+                           "\" in file: " + out.getAbsolutePath() + "\n" +
+                           "The certificate will be associated with your router identity.\n" +
+                           "Copy the certificate to the directory $I2P/" + CERT_DIR + " for each of the other routers in the family.\n" +
+                           "Give this certificate to an I2P developer for inclusion in the next I2P release.");
+            } else {
+                _log.error("Error getting SSL cert to save as ASCII");
+            }
+        } else {
+            _log.error("Error saving ASCII SSL keys");
+        }
+    }
+
+    /** 
+     * Load a public key from a cert.
+     *
+     * @return null on all errors
+     */
+    private SigningPublicKey loadCert(String familyName) {
+        if (familyName.contains("/") || familyName.contains("\\") ||
+            familyName.contains("..") || (new File(familyName)).isAbsolute())
+            return null;
+        familyName = familyName.replace("@", "_at_");
+        File dir = new File(_context.getBaseDir(), CERT_DIR);
+        File file = new File(dir, familyName + CERT_SUFFIX);
+        if (!file.exists())
+            return null;
+        try {
+            PublicKey pk = CertUtil.loadKey(file);
+            return SigUtil.fromJavaKey(pk);
+        } catch (GeneralSecurityException gse) {
+            _log.error("Error loading family key " + familyName, gse);
+        } catch (IOException ioe) {
+            _log.error("Error loading family key " + familyName, ioe);
+        }
+        return null;
+    }
+
+    /** 
+     * Get the private key from the keystore
+     * @return non-null, throws on all errors
+     */
+    private SigningPrivateKey getPrivKey(File ks) throws GeneralSecurityException {
+        String ksPass = _context.getProperty(PROP_KEYSTORE_PASSWORD, KeyStoreUtil.DEFAULT_KEYSTORE_PASSWORD);
+        String keyPass = _context.getProperty(PROP_KEY_PASSWORD);
+        if (keyPass == null)
+            throw new GeneralSecurityException("No key password, set " + PROP_KEY_PASSWORD +
+                       " in " + (new File(_context.getConfigDir(), "router.config")).getAbsolutePath());
+        try {
+            PrivateKey pk = KeyStoreUtil.getPrivateKey(ks, ksPass, _fname, keyPass);
+            if (pk == null)
+                throw new GeneralSecurityException("Family key not found: " + _fname);
+            return SigUtil.fromJavaKey(pk);
+        } catch (IOException ioe) {
+            throw new GeneralSecurityException("Error loading family key " + _fname, ioe);
+        }
+    }
+}

router/java/src/net/i2p/router/message/GarlicMessageBuilder.java

@@ -243,7 +243,7 @@ public class GarlicMessageBuilder {
             if (config instanceof PayloadGarlicConfig) {
                 byte clove[] = buildClove(ctx, (PayloadGarlicConfig)config);
                 baos = new ByteArrayOutputStream(clove.length + 16);
-                DataHelper.writeLong(baos, 1, 1);
+                baos.write((byte) 1);
                 baos.write(clove);
             } else {
                 byte cloves[][] = new byte[config.getCloveCount()][];
@@ -263,7 +263,7 @@ public class GarlicMessageBuilder {
                 for (int i = 0; i < cloves.length; i++)
                     len += cloves[i].length;
                 baos = new ByteArrayOutputStream(len + 16);
-                DataHelper.writeLong(baos, 1, cloves.length);
+                baos.write((byte) cloves.length);
                 for (int i = 0; i < cloves.length; i++)
                     baos.write(cloves[i]);
             }

router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java

@@ -38,10 +38,67 @@ import net.i2p.router.TunnelInfo;
 import net.i2p.util.Log;
 
 /**
- * Send a client message out a random outbound tunnel and into a random inbound
+ * Send a client message out an outbound tunnel and into an inbound
  * tunnel on the target leaseSet.  This also (sometimes) bundles the sender's leaseSet and
  * a DeliveryStatusMessage (for ACKing any sessionTags used in the garlic).
  *
+ * <p>
+ * This class is where we make several important decisions about
+ * what to send and what path to send it over. These decisions
+ * will dramatically affect:
+ * <ul>
+ * <li>Local performance and outbound bandwidth usage
+ * <li>Streaming performance and reliability
+ * <li>Overall network performace and connection congestion
+ * </ul>
+ *
+ * <p>
+ * For the outbound message, we build and encrypt a garlic message,
+ * after making the following decisions:
+ * <ul>
+ * <li>Whether to bundle our leaseset
+ * <li>Whether to bundle session tags, and if so, how many
+ * <li>Whether to bundle an encrypted DeliveryStatusMessage to be returned
+ *     to us as an acknowledgement
+ * </ul>
+ *
+ * <p>
+ * Also, we make the following path selection decisions:
+ * <ul>
+ * <li>What outbound client tunnel of ours to use send the message out
+ * <li>What inbound client tunnel of his (i.e. lease, chosen from his leaseset)
+ *     to use to send the message in
+ * <li>If a DeliveryStatusMessage is bundled, What inbound client tunnel of ours
+ *     do we specify to receive it
+ * </ul>
+ *
+ * <p>
+ * Note that the 4th tunnel in the DeliveryStatusMessage's round trip (his outbound tunnel)
+ * is not selected by us, it is chosen by the recipient.
+ *
+ * <p>
+ * If a DeliveryStatusMessage is sent, a listener is registered to wait for its reply.
+ * When a reply is received, or the timeout is reached, this is noted
+ * and will influence subsequent bundling and path selection decisions.
+ *
+ * <p>
+ * Path selection decisions are cached and reused if still valid and if
+ * previous deliveries were apparently successful. This significantly
+ * reduces out-of-order delivery and network connection congestion.
+ * Caching is based on the local/remote destination pair.
+ *
+ * <p>
+ * Bundling decisions, and both messaging and reply expiration times, are generally
+ * set here but may be overridden by the client on a per-message basis.
+ * Within clients, there may be overall settings or per-message settings.
+ * The streaming lib also overrides defaults for some messages.
+ * A datagram-based DHT application may need significantly different
+ * settings than a streaming application. For an application such as
+ * a bittorrent client that sends both types of traffic on the same tunnels,
+ * it is important to tune the settings for efficiency and performance.
+ * The per-session and per-message overrides are set via I2CP.
+ *
+ *
  */
 public class OutboundClientMessageOneShotJob extends JobImpl {
     private final Log _log;

router/java/src/net/i2p/router/networkdb/PublishLocalRouterInfoJob.java

@@ -82,9 +82,6 @@ public class PublishLocalRouterInfoJob extends JobImpl {
             List<RouterAddress> newAddrs = getContext().commSystem().createAddresses();
             int count = _runCount.incrementAndGet();
             RouterInfo ri = new RouterInfo(oldRI);
-            // this will get overwritten by setOptions() below, must restore it below
-            getContext().router().addCapabilities(ri);
-            String caps = ri.getCapabilities();
             if (_notFirstTime && (count % 4) != 0 && oldAddrs.size() == newAddrs.size()) {
                 // 3 times out of 4, we don't republish if everything is the same...
                 // If something changed, including the cost, then publish,
@@ -116,9 +113,6 @@ public class PublishLocalRouterInfoJob extends JobImpl {
             }
             ri.setPublished(getContext().clock().now());
             Properties stats = getContext().statPublisher().publishStatistics();
-            stats.setProperty(RouterInfo.PROP_NETWORK_ID, String.valueOf(Router.NETWORK_ID));
-            // restore caps generated above
-            stats.setProperty(RouterInfo.PROP_CAPABILITIES, caps);
             ri.setOptions(stats);
             ri.setAddresses(newAddrs);
 

router/java/src/net/i2p/router/networkdb/kademlia/FloodOnlySearchJob.java

@@ -220,7 +220,7 @@ class FloodOnlySearchJob extends FloodSearchJob {
                 getContext().profileManager().dbLookupFailed(h);
         }
         _facade.complete(_key);
-        getContext().statManager().addRateData("netDb.failedTime", time, 0);
+        getContext().statManager().addRateData("netDb.failedTime", time);
         for (Job j : _onFailed) {
             getContext().jobQueue().addJob(j);
         }
@@ -251,7 +251,7 @@ class FloodOnlySearchJob extends FloodSearchJob {
             }
         }
         _facade.complete(_key);
-        getContext().statManager().addRateData("netDb.successTime", time, 0);
+        getContext().statManager().addRateData("netDb.successTime", time);
         for (Job j : _onFind) {
             getContext().jobQueue().addJob(j);
         }

router/java/src/net/i2p/router/networkdb/kademlia/FloodfillDatabaseLookupMessageHandler.java

@@ -43,7 +43,7 @@ public class FloodfillDatabaseLookupMessageHandler implements HandlerJobBuilder
     }
 
     public Job createJob(I2NPMessage receivedMessage, RouterIdentity from, Hash fromHash) {
-        _context.statManager().addRateData("netDb.lookupsReceived", 1, 0);
+        _context.statManager().addRateData("netDb.lookupsReceived", 1);
 
         DatabaseLookupMessage dlm = (DatabaseLookupMessage)receivedMessage;
         if (!_facade.shouldThrottleLookup(dlm.getFrom(), dlm.getReplyTunnel())) {

router/java/src/net/i2p/router/networkdb/kademlia/FloodfillVerifyStoreJob.java

@@ -257,7 +257,7 @@ class FloodfillVerifyStoreJob extends JobImpl {
                     getContext().profileManager().dbLookupSuccessful(_target, delay);
                     if (_sentTo != null)
                         getContext().profileManager().dbStoreSuccessful(_sentTo);
-                    getContext().statManager().addRateData("netDb.floodfillVerifyOK", delay, 0);
+                    getContext().statManager().addRateData("netDb.floodfillVerifyOK", delay);
                     if (_log.shouldLog(Log.INFO))
                         _log.info("Verify success for " + _key);
                     if (_isRouterInfo)
@@ -290,7 +290,7 @@ class FloodfillVerifyStoreJob extends JobImpl {
             // though it is the real problem.
             if (_target != null && !_target.equals(_sentTo))
                 getContext().profileManager().dbLookupFailed(_target);
-            getContext().statManager().addRateData("netDb.floodfillVerifyFail", delay, 0);
+            getContext().statManager().addRateData("netDb.floodfillVerifyFail", delay);
             resend();
         }        
         public void setMessage(I2NPMessage message) { _message = message; }
@@ -328,7 +328,7 @@ class FloodfillVerifyStoreJob extends JobImpl {
             getContext().profileManager().dbLookupFailed(_target);
             //if (_sentTo != null)
             //    getContext().profileManager().dbStoreFailed(_sentTo);
-            getContext().statManager().addRateData("netDb.floodfillVerifyTimeout", getContext().clock().now() - _sendTime, 0);
+            getContext().statManager().addRateData("netDb.floodfillVerifyTimeout", getContext().clock().now() - _sendTime);
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Verify timed out for: " + _key);
             if (_ignore.size() < MAX_PEERS_TO_TRY) {

router/java/src/net/i2p/router/networkdb/kademlia/IterativeSearchJob.java

@@ -10,6 +10,7 @@ import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 
+import net.i2p.crypto.SigType;
 import net.i2p.data.Base64;
 import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
@@ -30,6 +31,7 @@ import net.i2p.router.util.RandomIterator;
 import net.i2p.util.Log;
 import net.i2p.util.NativeBigInteger;
 import net.i2p.util.SystemVersion;
+import net.i2p.util.VersionComparator;
 
 /**
  * A traditional Kademlia search that continues to search
@@ -74,7 +76,7 @@ class IterativeSearchJob extends FloodSearchJob {
     
     private static final int MAX_NON_FF = 3;
     /** Max number of peers to query */
-    private static final int TOTAL_SEARCH_LIMIT = 6;
+    private static final int TOTAL_SEARCH_LIMIT = 5;
     /** Max number of peers to query if we are ff */
     private static final int TOTAL_SEARCH_LIMIT_WHEN_FF = 3;
     /** TOTAL_SEARCH_LIMIT * SINGLE_SEARCH_TIME, plus some extra */
@@ -199,10 +201,12 @@ class IterativeSearchJob extends FloodSearchJob {
             }
         }
         final boolean empty;
+        // outside sync to avoid deadlock
+        final Hash us = getContext().routerHash();
         synchronized(this) {
             _toTry.addAll(floodfillPeers);
             // don't ask ourselves or the target
-            _toTry.remove(getContext().routerHash());
+            _toTry.remove(us);
             _toTry.remove(_key);
             empty = _toTry.isEmpty();
         }
@@ -286,6 +290,20 @@ class IterativeSearchJob extends FloodSearchJob {
     private void sendQuery(Hash peer) {
             TunnelManagerFacade tm = getContext().tunnelManager();
             RouterInfo ri = getContext().netDb().lookupRouterInfoLocally(peer);
+            if (ri != null) {
+                // Now that most of the netdb is Ed RIs and EC LSs, don't even bother
+                // querying old floodfills that don't know about those sig types.
+                // This is also more recent than the version that supports encrypted replies,
+                // so we won't request unencrypted replies anymore either.
+                String v = ri.getVersion();
+                String since = SigType.EdDSA_SHA512_Ed25519.getSupportedSince();
+                if (VersionComparator.comp(v, since) < 0) {
+                    failed(peer, false);
+                    if (_log.shouldLog(Log.WARN))
+                        _log.warn(getJobId() + ": not sending query to old version " + v + ": " + peer);
+                    return;
+                }
+            }
             TunnelInfo outTunnel;
             TunnelInfo replyTunnel;
             boolean isClientReplyTunnel;
@@ -379,7 +397,9 @@ class IterativeSearchJob extends FloodSearchJob {
                 // if we have the ff RI, garlic encrypt it
                 if (ri != null) {
                     // request encrypted reply
-                    if (DatabaseLookupMessage.supportsEncryptedReplies(ri)) {
+                    // now covered by version check above, which is more recent
+                    //if (DatabaseLookupMessage.supportsEncryptedReplies(ri)) {
+                    if (true) {
                         MessageWrapper.OneTimeSession sess;
                         if (isClientReplyTunnel)
                             sess = MessageWrapper.generateSession(getContext(), _fromLocalDest);
@@ -537,16 +557,20 @@ class IterativeSearchJob extends FloodSearchJob {
             unheard = new ArrayList<Hash>(_unheardFrom);
         }
         // blame the unheard-from (others already blamed in failed() above)
-        for (Hash h : unheard)
+        for (Hash h : unheard) {
             getContext().profileManager().dbLookupFailed(h);
+        }
         long time = System.currentTimeMillis() - _created;
         if (_log.shouldLog(Log.INFO)) {
             long timeRemaining = _expiration - getContext().clock().now();
             _log.info(getJobId() + ": ISJ for " + _key + " failed with " + timeRemaining + " remaining after " + time +
                       ", peers queried: " + tries);
         }
-        getContext().statManager().addRateData("netDb.failedTime", time);
-        getContext().statManager().addRateData("netDb.failedRetries", Math.max(0, tries - 1));
+        if (tries > 0) {
+            // don't bias the stats with immediate fails
+            getContext().statManager().addRateData("netDb.failedTime", time);
+            getContext().statManager().addRateData("netDb.failedRetries", tries - 1);
+        }
         for (Job j : _onFailed) {
             getContext().jobQueue().addJob(j);
         }

router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java

@@ -40,6 +40,7 @@ import net.i2p.router.Job;
 import net.i2p.router.NetworkDatabaseFacade;
 import net.i2p.router.Router;
 import net.i2p.router.RouterContext;
+import net.i2p.router.crypto.FamilyKeyCrypto;
 import net.i2p.router.networkdb.PublishLocalRouterInfoJob;
 import net.i2p.router.networkdb.reseed.ReseedChecker;
 import net.i2p.router.peermanager.PeerProfile;
@@ -211,7 +212,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
     public void removeFromExploreKeys(Collection<Hash> toRemove) {
         if (!_initialized) return;
         _exploreKeys.removeAll(toRemove);
-        _context.statManager().addRateData("netDb.exploreKeySet", _exploreKeys.size(), 0);
+        _context.statManager().addRateData("netDb.exploreKeySet", _exploreKeys.size());
     }
 
     public void queueForExploration(Collection<Hash> keys) {
@@ -219,7 +220,7 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
         for (Iterator<Hash> iter = keys.iterator(); iter.hasNext() && _exploreKeys.size() < MAX_EXPLORE_QUEUE; ) {
             _exploreKeys.add(iter.next());
         }
-        _context.statManager().addRateData("netDb.exploreKeySet", _exploreKeys.size(), 0);
+        _context.statManager().addRateData("netDb.exploreKeySet", _exploreKeys.size());
     }
     
     public synchronized void shutdown() {
@@ -894,6 +895,15 @@ public class KademliaNetworkDatabaseFacade extends NetworkDatabaseFacade {
                 _log.warn("Bad network: " + routerInfo);
             return "Not in our network";
         }
+        FamilyKeyCrypto fkc = _context.router().getFamilyKeyCrypto();
+        if (fkc != null) {
+            boolean validFamily = fkc.verify(routerInfo);
+            if (!validFamily) {
+                if (_log.shouldWarn())
+                    _log.warn("Bad family sig: " + routerInfo.getHash());
+            }
+            // todo store in RI
+        }
         return validate(routerInfo);
     }
 

router/java/src/net/i2p/router/networkdb/kademlia/LookupThrottler.java

@@ -45,8 +45,8 @@ class LookupThrottler {
 
     /** yes, we could have a two-level lookup, or just do h.tostring() + id.tostring() */
     private static class ReplyTunnel {
-        public Hash h;
-        public TunnelId id;
+        public final Hash h;
+        public final TunnelId id;
 
         ReplyTunnel(Hash h, TunnelId id) {
             this.h = h;

router/java/src/net/i2p/router/networkdb/kademlia/PeerSelector.java

@@ -112,15 +112,14 @@ class PeerSelector {
     
     /** UNUSED */
     private class MatchSelectionCollector implements SelectionCollector<Hash> {
-        private TreeMap<BigInteger, Hash> _sorted;
-        private Hash _key;
-        private Set<Hash> _toIgnore;
+        private final TreeMap<BigInteger, Hash> _sorted;
+        private final Hash _key;
+        private final Set<Hash> _toIgnore;
         private int _matches;
         public MatchSelectionCollector(Hash key, Set<Hash> toIgnore) {
             _key = key;
             _sorted = new TreeMap<BigInteger, Hash>();
             _toIgnore = toIgnore;
-            _matches = 0;
         }
         public void add(Hash entry) {
             // deadlock seen here, and we don't mark profiles failing anymore

router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java

@@ -165,7 +165,7 @@ public class PersistentDataStore extends TransientDataStore {
     }
     
     private class RemoveJob extends JobImpl {
-        private Hash _key;
+        private final Hash _key;
         public RemoveJob(Hash key) {
             super(PersistentDataStore.this._context);
             _key = key;

router/java/src/net/i2p/router/networkdb/kademlia/RepublishLeaseSetJob.java

@@ -53,7 +53,7 @@ public class RepublishLeaseSetJob extends JobImpl {
                     } else {
                         if (_log.shouldLog(Log.INFO))
                             _log.info("Publishing " + ls);
-                        getContext().statManager().addRateData("netDb.republishLeaseSetCount", 1, 0);
+                        getContext().statManager().addRateData("netDb.republishLeaseSetCount", 1);
                         _facade.sendStore(_dest, ls, null, new OnRepublishFailure(getContext(), this), REPUBLISH_LEASESET_TIMEOUT, null);
                         _lastPublished = getContext().clock().now();
                         //getContext().jobQueue().addJob(new StoreJob(getContext(), _facade, _dest, ls, new OnSuccess(getContext()), new OnFailure(getContext()), REPUBLISH_LEASESET_TIMEOUT));
@@ -105,7 +105,7 @@ public class RepublishLeaseSetJob extends JobImpl {
 
     /** requeue */
     private static class OnRepublishFailure extends JobImpl {
-        private RepublishLeaseSetJob _job;
+        private final RepublishLeaseSetJob _job;
         public OnRepublishFailure(RouterContext ctx, RepublishLeaseSetJob job) { 
             super(ctx); 
             _job = job;

router/java/src/net/i2p/router/networkdb/kademlia/SearchJob.java

@@ -781,10 +781,10 @@ class SearchJob extends JobImpl {
     }
     
     private static class Search {
-        private Job _onFind;
-        private Job _onFail;
-        private long _expiration;
-        private boolean _isLease;
+        private final Job _onFind;
+        private final Job _onFail;
+        private final long _expiration;
+        private final boolean _isLease;
         
         public Search(Job onFind, Job onFail, long expiration, boolean isLease) {
             _onFind = onFind;

router/java/src/net/i2p/router/networkdb/kademlia/SearchReplyJob.java

@@ -100,7 +100,7 @@ class SearchReplyJob extends JobImpl {
                 } else {
                     if (_log.shouldLog(Log.INFO))
                         _log.info("Peer " + _peer.toBase64() + " sends us bad replies, so not verifying " + peer.toBase64());
-                    getContext().statManager().addRateData("netDb.searchReplyValidationSkipped", 1, 0);
+                    getContext().statManager().addRateData("netDb.searchReplyValidationSkipped", 1);
                 }
             }
 
@@ -125,14 +125,14 @@ class SearchReplyJob extends JobImpl {
         if (_log.shouldLog(Log.INFO))
             _log.info("Peer reply from " + _peer.toBase64());
         _repliesPendingVerification--;
-        getContext().statManager().addRateData("netDb.searchReplyValidated", 1, 0);
+        getContext().statManager().addRateData("netDb.searchReplyValidated", 1);
     }
     void replyNotVerified() {
         if (_log.shouldLog(Log.INFO))
             _log.info("Peer reply from " + _peer.toBase64());
         _repliesPendingVerification--;
         _invalidPeers++;
-        getContext().statManager().addRateData("netDb.searchReplyNotValidated", 1, 0);
+        getContext().statManager().addRateData("netDb.searchReplyNotValidated", 1);
     }
 }
 

router/java/src/net/i2p/router/networkdb/kademlia/StoreJob.java

@@ -293,7 +293,8 @@ class StoreJob extends JobImpl {
             throw new IllegalArgumentException("Storing an unknown data type! " + _state.getData());
         }
         msg.setEntry(_state.getData());
-        msg.setMessageExpiration(getContext().clock().now() + _timeoutMs);
+        long now = getContext().clock().now();
+        msg.setMessageExpiration(now + _timeoutMs);
 
         if (router.getIdentity().equals(getContext().router().getRouterInfo().getIdentity())) {
             // don't send it to ourselves
@@ -305,7 +306,7 @@ class StoreJob extends JobImpl {
         if (_log.shouldLog(Log.DEBUG))
             _log.debug(getJobId() + ": Send store timeout is " + responseTime);
 
-        sendStore(msg, router, getContext().clock().now() + responseTime);
+        sendStore(msg, router, now + responseTime);
     }
     
     /**
@@ -315,14 +316,14 @@ class StoreJob extends JobImpl {
      */
     private void sendStore(DatabaseStoreMessage msg, RouterInfo peer, long expiration) {
         if (msg.getEntry().getType() == DatabaseEntry.KEY_TYPE_LEASESET) {
-            getContext().statManager().addRateData("netDb.storeLeaseSetSent", 1, 0);
+            getContext().statManager().addRateData("netDb.storeLeaseSetSent", 1);
             // if it is an encrypted leaseset...
             if (getContext().keyRing().get(msg.getKey()) != null)
                 sendStoreThroughGarlic(msg, peer, expiration);
             else
                 sendStoreThroughClient(msg, peer, expiration);
         } else {
-            getContext().statManager().addRateData("netDb.storeRouterInfoSent", 1, 0);
+            getContext().statManager().addRateData("netDb.storeRouterInfoSent", 1);
             sendDirect(msg, peer, expiration);
         }
     }
@@ -557,9 +558,9 @@ class StoreJob extends JobImpl {
      *
      */
     private class SendSuccessJob extends JobImpl implements ReplyJob {
-        private RouterInfo _peer;
-        private TunnelInfo _sendThrough;
-        private int _msgSize;
+        private final RouterInfo _peer;
+        private final TunnelInfo _sendThrough;
+        private final int _msgSize;
         
         public SendSuccessJob(RouterContext enclosingContext, RouterInfo peer) {
             this(enclosingContext, peer, null, 0);
@@ -615,8 +616,8 @@ class StoreJob extends JobImpl {
      *
      */
     private class FailedJob extends JobImpl {
-        private RouterInfo _peer;
-        private long _sendOn;
+        private final RouterInfo _peer;
+        private final long _sendOn;
 
         public FailedJob(RouterContext enclosingContext, RouterInfo peer, long sendOn) {
             super(enclosingContext);
@@ -635,7 +636,7 @@ class StoreJob extends JobImpl {
             _state.replyTimeout(hash);
 
             getContext().profileManager().dbStoreFailed(hash);
-            getContext().statManager().addRateData("netDb.replyTimeout", getContext().clock().now() - _sendOn, 0);
+            getContext().statManager().addRateData("netDb.replyTimeout", getContext().clock().now() - _sendOn);
             
             sendNext();
         }

router/java/src/net/i2p/router/networkdb/reseed/Reseeder.java

@@ -86,8 +86,7 @@ public class Reseeder {
     /** @since 0.8.2 */
     public static final String DEFAULT_SSL_SEED_URL =
               "https://reseed.i2p-projekt.de/" + "," + // Only HTTPS
-              //"https://netdb.rows.io:444/" + "," + // Only HTTPS and SU3 (v3) support
-              "https://i2pseed.zarrenspry.info/" + "," + // Only HTTPS and SU3 (v3) support
+              //"https://i2pseed.zarrenspry.info/" + "," + // Only HTTPS and SU3 (v3) support
               "https://i2p.mooo.com/netDb/" + "," +
               "https://netdb.i2p2.no/" + "," + // Only SU3 (v3) support, SNI required
               "https://us.reseed.i2p2.no:444/" + "," +

router/java/src/net/i2p/router/peermanager/CapacityCalculator.java

@@ -29,7 +29,7 @@ class CapacityCalculator {
     private static final double PENALTY_UNREACHABLE = 2;
     // we make this a bonus for non-ff, not a penalty for ff, so we
     // don't drive the ffs below the default
-    private static final double BONUS_NON_FLOODFILL = 0.5;
+    private static final double BONUS_NON_FLOODFILL = 1.0;
     
     public static double calc(PeerProfile profile) {
         double capacity;

router/java/src/net/i2p/router/peermanager/DBHistory.java

@@ -16,17 +16,17 @@ import net.i2p.util.Log;
 public class DBHistory {
     private final Log _log;
     private final RouterContext _context;
-    private long _successfulLookups;
-    private long _failedLookups;
+    //private long _successfulLookups;
+    //private long _failedLookups;
     private RateStat _failedLookupRate;
     private RateStat _invalidReplyRate;
-    private long _lookupReplyNew;
-    private long _lookupReplyOld;
-    private long _lookupReplyDuplicate;
-    private long _lookupReplyInvalid;
-    private long _lookupsReceived;
-    private long _avgDelayBetweenLookupsReceived;
-    private long _lastLookupReceived;
+    //private long _lookupReplyNew;
+    //private long _lookupReplyOld;
+    //private long _lookupReplyDuplicate;
+    //private long _lookupReplyInvalid;
+    //private long _lookupsReceived;
+    //private long _avgDelayBetweenLookupsReceived;
+    //private long _lastLookupReceived;
     private long _lastLookupSuccessful;
     private long _lastLookupFailed;
     private long _lastStoreSuccessful;
@@ -39,49 +39,68 @@ public class DBHistory {
         _context = context;
         _log = context.logManager().getLog(DBHistory.class);
         _statGroup = statGroup;
-        _lastLookupReceived = -1;
+        //_lastLookupReceived = -1;
         createRates(statGroup);
     }
     
     /** how many times we have sent them a db lookup and received the value back from them
-     *  @deprecated unused
      */
-    public long getSuccessfulLookups() { return _successfulLookups; }
+    //public long getSuccessfulLookups() { return _successfulLookups; }
+
     /** how many times we have sent them a db lookup and not received the value or a lookup reply
-     *  @deprecated unused
      */
-    public long getFailedLookups() { return _failedLookups; }
+    //public long getFailedLookups() { return _failedLookups; }
+
     /** how many peers that we have never seen before did lookups provide us with?
-     *  @deprecated unused
      */
-    public long getLookupReplyNew() { return _lookupReplyNew; }
+    //public long getLookupReplyNew() { return _lookupReplyNew; }
+
     /** how many peers that we have already seen did lookups provide us with?
-     *  @deprecated unused
      */
-    public long getLookupReplyOld() { return _lookupReplyOld; }
+    //public long getLookupReplyOld() { return _lookupReplyOld; }
+
     /** how many peers that we explicitly asked the peer not to send us did they reply with?
-     *  @deprecated unused
      */
-    public long getLookupReplyDuplicate() { return _lookupReplyDuplicate; }
+    //public long getLookupReplyDuplicate() { return _lookupReplyDuplicate; }
+
     /** how many peers that were incorrectly formatted / expired / otherwise illegal did lookups provide us with?
-     *  @deprecated unused
      */
-    public long getLookupReplyInvalid() { return _lookupReplyInvalid; }
+    //public long getLookupReplyInvalid() { return _lookupReplyInvalid; }
+
     /** how many lookups this peer has sent us?
-     *  @deprecated unused
      */
-    public long getLookupsReceived() { return _lookupsReceived; }
+    //public long getLookupsReceived() { return _lookupsReceived; }
+
     /** how frequently do they send us lookup requests?
-     *  @deprecated unused
      */
-    public long getAvgDelayBetweenLookupsReceived() { return _avgDelayBetweenLookupsReceived; }
+    //public long getAvgDelayBetweenLookupsReceived() { return _avgDelayBetweenLookupsReceived; }
+
     /** when did they last send us a request?
-     *  @deprecated unused
      */
-    public long getLastLookupReceived() { return _lastLookupReceived; }
+   // public long getLastLookupReceived() { return _lastLookupReceived; }
+
+    /**
+     *  Not persisted until 0.9.24
+     *  @since 0.7.8
+     */
     public long getLastLookupSuccessful() { return _lastLookupSuccessful; }
+
+    /**
+     *  Not persisted until 0.9.24
+     *  @since 0.7.8
+     */
     public long getLastLookupFailed() { return _lastLookupFailed; }
+
+    /**
+     *  Not persisted until 0.9.24
+     *  @since 0.7.8
+     */
     public long getLastStoreSuccessful() { return _lastStoreSuccessful; }
+
+    /**
+     *  Not persisted until 0.9.24
+     *  @since 0.7.8
+     */
     public long getLastStoreFailed() { return _lastStoreFailed; }
 
     /** how many times have they sent us data we didn't ask for and that we've never seen? */
@@ -103,8 +122,8 @@ public class DBHistory {
      *
      */
     public void lookupSuccessful() {
-        _successfulLookups++;
-        _failedLookupRate.addData(0, 0);
+        //_successfulLookups++;
+        _failedLookupRate.addData(0);
         _context.statManager().addRateData("peer.failedLookupRate", 0);
         _lastLookupSuccessful = _context.clock().now();
     }
@@ -113,8 +132,8 @@ public class DBHistory {
      * Note that the peer failed to respond to the db lookup in any way
      */
     public void lookupFailed() {
-        _failedLookups++;
-        _failedLookupRate.addData(1, 0);
+        //_failedLookups++;
+        _failedLookupRate.addData(1);
         _context.statManager().addRateData("peer.failedLookupRate", 1);
         _lastLookupFailed = _context.clock().now();
     }
@@ -123,22 +142,25 @@ public class DBHistory {
      * Note that we successfully stored to a floodfill peer and verified the result
      * by asking another floodfill peer
      *
+     *  @since 0.7.8
      */
     public void storeSuccessful() {
         // Fixme, redefined this to include both lookup and store fails,
         // need to fix the javadocs
-        _failedLookupRate.addData(0, 0);
+        _failedLookupRate.addData(0);
         _context.statManager().addRateData("peer.failedLookupRate", 0);
         _lastStoreSuccessful = _context.clock().now();
     }
 
     /**
      * Note that floodfill verify failed
+     *
+     *  @since 0.7.8
      */
     public void storeFailed() {
         // Fixme, redefined this to include both lookup and store fails,
         // need to fix the javadocs
-        _failedLookupRate.addData(1, 0);
+        _failedLookupRate.addData(1);
         _lastStoreFailed = _context.clock().now();
     }
 
@@ -152,19 +174,21 @@ public class DBHistory {
      *                  themselves if they don't know anyone else)
      */
     public void lookupReply(int newPeers, int oldPeers, int invalid, int duplicate) {
-        _lookupReplyNew += newPeers;
-        _lookupReplyOld += oldPeers;
-        _lookupReplyInvalid += invalid;
-        _lookupReplyDuplicate += duplicate;
+        //_lookupReplyNew += newPeers;
+        //_lookupReplyOld += oldPeers;
+        //_lookupReplyInvalid += invalid;
+        //_lookupReplyDuplicate += duplicate;
         
         if (invalid > 0) {
-            _invalidReplyRate.addData(invalid, 0);
+            _invalidReplyRate.addData(invalid);
         }
     }
+
     /**
      * Note that the peer sent us a lookup
      *
      */
+/****
     public void lookupReceived() {
         long now = _context.clock().now();
         long delay = now - _lastLookupReceived;
@@ -179,6 +203,8 @@ public class DBHistory {
                 _avgDelayBetweenLookupsReceived = _avgDelayBetweenLookupsReceived - (delay / _lookupsReceived);
         }
     }
+****/
+
     /**
      * Note that the peer sent us a data point without us asking for it
      * @param wasNew whether we already knew about this data point or not
@@ -190,15 +216,15 @@ public class DBHistory {
             _unpromptedDbStoreOld++;
     }
     
-    public void setSuccessfulLookups(long num) { _successfulLookups = num; }
-    public void setFailedLookups(long num) { _failedLookups = num; }
-    public void setLookupReplyNew(long num) { _lookupReplyNew = num; }
-    public void setLookupReplyOld(long num) { _lookupReplyOld = num; }
-    public void setLookupReplyInvalid(long num) { _lookupReplyInvalid = num; }
-    public void setLookupReplyDuplicate(long num) { _lookupReplyDuplicate = num; }
-    public void setLookupsReceived(long num) { _lookupsReceived = num; }
-    public void setAvgDelayBetweenLookupsReceived(long ms) { _avgDelayBetweenLookupsReceived = ms; }
-    public void setLastLookupReceived(long when) { _lastLookupReceived = when; }
+    //public void setSuccessfulLookups(long num) { _successfulLookups = num; }
+    //public void setFailedLookups(long num) { _failedLookups = num; }
+    //public void setLookupReplyNew(long num) { _lookupReplyNew = num; }
+    //public void setLookupReplyOld(long num) { _lookupReplyOld = num; }
+    //public void setLookupReplyInvalid(long num) { _lookupReplyInvalid = num; }
+    //public void setLookupReplyDuplicate(long num) { _lookupReplyDuplicate = num; }
+    //public void setLookupsReceived(long num) { _lookupsReceived = num; }
+    //public void setAvgDelayBetweenLookupsReceived(long ms) { _avgDelayBetweenLookupsReceived = ms; }
+    //public void setLastLookupReceived(long when) { _lastLookupReceived = when; }
     public void setUnpromptedDbStoreNew(long num) { _unpromptedDbStoreNew = num; }
     public void setUnpromptedDbStoreOld(long num) { _unpromptedDbStoreOld = num; }
     
@@ -217,17 +243,22 @@ public class DBHistory {
         buf.append("#################").append(NL);
         buf.append("# DB history").append(NL);
         buf.append("###").append(NL);
-        add(buf, "successfulLookups", _successfulLookups, "How many times have they successfully given us what we wanted when looking for it?");
-        add(buf, "failedLookups", _failedLookups, "How many times have we sent them a db lookup and they didn't reply?");
-        add(buf, "lookupsReceived", _lookupsReceived, "How many lookups have they sent us?");
-        add(buf, "lookupReplyDuplicate", _lookupReplyDuplicate, "How many of their reply values to our lookups were something we asked them not to send us?");
-        add(buf, "lookupReplyInvalid", _lookupReplyInvalid, "How many of their reply values to our lookups were invalid (expired, forged, corrupted)?");
-        add(buf, "lookupReplyNew", _lookupReplyNew, "How many of their reply values to our lookups were brand new to us?");
-        add(buf, "lookupReplyOld", _lookupReplyOld, "How many of their reply values to our lookups were something we had seen before?");
+        //add(buf, "successfulLookups", _successfulLookups, "How many times have they successfully given us what we wanted when looking for it?");
+        //add(buf, "failedLookups", _failedLookups, "How many times have we sent them a db lookup and they didn't reply?");
+        //add(buf, "lookupsReceived", _lookupsReceived, "How many lookups have they sent us?");
+        //add(buf, "lookupReplyDuplicate", _lookupReplyDuplicate, "How many of their reply values to our lookups were something we asked them not to send us?");
+        //add(buf, "lookupReplyInvalid", _lookupReplyInvalid, "How many of their reply values to our lookups were invalid (expired, forged, corrupted)?");
+        //add(buf, "lookupReplyNew", _lookupReplyNew, "How many of their reply values to our lookups were brand new to us?");
+        //add(buf, "lookupReplyOld", _lookupReplyOld, "How many of their reply values to our lookups were something we had seen before?");
         add(buf, "unpromptedDbStoreNew", _unpromptedDbStoreNew, "How times have they sent us something we didn't ask for and hadn't seen before?");
         add(buf, "unpromptedDbStoreOld", _unpromptedDbStoreOld, "How times have they sent us something we didn't ask for but have seen before?");
-        add(buf, "lastLookupReceived", _lastLookupReceived, "When was the last time they send us a lookup?  (milliseconds since the epoch)");
-        add(buf, "avgDelayBetweenLookupsReceived", _avgDelayBetweenLookupsReceived, "How long is it typically between each db lookup they send us?  (in milliseconds)");
+        //add(buf, "lastLookupReceived", _lastLookupReceived, "When was the last time they send us a lookup?  (milliseconds since the epoch)");
+        //add(buf, "avgDelayBetweenLookupsReceived", _avgDelayBetweenLookupsReceived, "How long is it typically between each db lookup they send us?  (in milliseconds)");
+        // following 4 weren't persisted until 0.9.24
+        add(buf, "lastLookupSuccessful", _lastLookupSuccessful, "When was the last time a lookup from them succeeded?  (milliseconds since the epoch)");
+        add(buf, "lastLookupFailed", _lastLookupFailed, "When was the last time a lookup from them failed?  (milliseconds since the epoch)");
+        add(buf, "lastStoreSuccessful", _lastStoreSuccessful, "When was the last time a store to them succeeded?  (milliseconds since the epoch)");
+        add(buf, "lastStoreFailed", _lastStoreFailed, "When was the last time a store to them failed?  (milliseconds since the epoch)");
         out.write(buf.toString().getBytes("UTF-8"));
         _failedLookupRate.store(out, "dbHistory.failedLookupRate");
         _invalidReplyRate.store(out, "dbHistory.invalidReplyRate");
@@ -240,17 +271,22 @@ public class DBHistory {
     
     
     public void load(Properties props) {
-        _successfulLookups = getLong(props, "dbHistory.successfulLookups");
-        _failedLookups = getLong(props, "dbHistory.failedLookups");
-        _lookupsReceived = getLong(props, "dbHistory.lookupsReceived");
-        _lookupReplyDuplicate = getLong(props, "dbHistory.lookupReplyDuplicate");
-        _lookupReplyInvalid = getLong(props, "dbHistory.lookupReplyInvalid");
-        _lookupReplyNew = getLong(props, "dbHistory.lookupReplyNew");
-        _lookupReplyOld = getLong(props, "dbHistory.lookupReplyOld");
+        //_successfulLookups = getLong(props, "dbHistory.successfulLookups");
+        //_failedLookups = getLong(props, "dbHistory.failedLookups");
+        //_lookupsReceived = getLong(props, "dbHistory.lookupsReceived");
+        //_lookupReplyDuplicate = getLong(props, "dbHistory.lookupReplyDuplicate");
+        //_lookupReplyInvalid = getLong(props, "dbHistory.lookupReplyInvalid");
+        //_lookupReplyNew = getLong(props, "dbHistory.lookupReplyNew");
+        //_lookupReplyOld = getLong(props, "dbHistory.lookupReplyOld");
         _unpromptedDbStoreNew = getLong(props, "dbHistory.unpromptedDbStoreNew");
         _unpromptedDbStoreOld = getLong(props, "dbHistory.unpromptedDbStoreOld");
-        _lastLookupReceived = getLong(props, "dbHistory.lastLookupReceived");
-        _avgDelayBetweenLookupsReceived = getLong(props, "dbHistory.avgDelayBetweenLookupsReceived");
+        //_lastLookupReceived = getLong(props, "dbHistory.lastLookupReceived");
+        //_avgDelayBetweenLookupsReceived = getLong(props, "dbHistory.avgDelayBetweenLookupsReceived");
+        // following 4 weren't persisted until 0.9.24
+        _lastLookupSuccessful = getLong(props, "dbHistory.lastLookupSuccessful");
+        _lastLookupFailed = getLong(props, "dbHistory.lastLookupFailed");
+        _lastStoreSuccessful = getLong(props, "dbHistory.lastStoreSuccessful");
+        _lastStoreFailed = getLong(props, "dbHistory.lastStoreFailed");
         try {
             _failedLookupRate.load(props, "dbHistory.failedLookupRate", true);
             _log.debug("Loading dbHistory.failedLookupRate");
@@ -266,7 +302,7 @@ public class DBHistory {
         }
     }
     
-    private void createRates(String statGroup) {
+    private synchronized void createRates(String statGroup) {
         if (_failedLookupRate == null)
             _failedLookupRate = new RateStat("dbHistory.failedLookupRate", "How often does this peer to respond to a lookup?", statGroup, new long[] { 10*60*1000l, 60*60*1000l, 24*60*60*1000l });
         if (_invalidReplyRate == null)
@@ -276,14 +312,6 @@ public class DBHistory {
     }
     
     private final static long getLong(Properties props, String key) {
-        String val = props.getProperty(key);
-        if (val != null) {
-            try {
-                return Long.parseLong(val);
-            } catch (NumberFormatException nfe) {
-                return 0;
-            }
-        }
-        return 0;
+        return ProfilePersistenceHelper.getLong(props, key);
     }
 }

router/java/src/net/i2p/router/peermanager/PeerProfile.java

@@ -24,6 +24,9 @@ import net.i2p.util.Log;
  * Once it becomes necessary, we can simply compact the poorly performing profiles
  * (keeping only the most basic data) and maintain hundreds of thousands of profiles
  * in memory. Beyond that size, we can simply eject the peers (e.g. keeping the best 100,000).
+ *
+ * TODO most of the methods should be synchronized.
+ *
  */
 
 public class PeerProfile {
@@ -37,7 +40,7 @@ public class PeerProfile {
     private long _lastSentToSuccessfully;
     private long _lastFailedSend;
     private long _lastHeardFrom;
-    private double _tunnelTestResponseTimeAvg;
+    private float _tunnelTestResponseTimeAvg;
     // periodic rates
     //private RateStat _sendSuccessSize = null;
     //private RateStat _receiveSize = null;
@@ -46,17 +49,20 @@ public class PeerProfile {
     private RateStat _tunnelTestResponseTime;
     private RateStat _dbIntroduction;
     // calculation bonuses
-    private long _speedBonus;
-    private long _capacityBonus;
-    private long _integrationBonus;
+    // ints to save some space
+    private int _speedBonus;
+    private int _capacityBonus;
+    private int _integrationBonus;
     // calculation values
-    private double _speedValue;
-    private double _capacityValue;
-    private double _integrationValue;
+    // floats to save some space
+    private float _speedValue;
+    private float _capacityValue;
+    private float _integrationValue;
     private boolean _isFailing;
     // new calculation values, to be updated
-    private double _speedValueNew;
-    private double _capacityValueNew;
+    // floats to save some space
+    private float _speedValueNew;
+    private float _capacityValueNew;
     // are we in coalescing state?
     private boolean _coalescing;
     // good vs bad behavior
@@ -67,6 +73,24 @@ public class PeerProfile {
     private boolean _expandedDB;
     //private int _consecutiveBanlists;
     private final int _distance;
+
+    /** keep track of the fastest 3 throughputs */
+    private static final int THROUGHPUT_COUNT = 3;
+    /** 
+     * fastest 1 minute throughput, in bytes per minute, ordered with fastest
+     * first.  this is not synchronized, as we don't *need* perfection, and we only
+     * reorder/insert values on coallesce
+     */
+    private final float _peakThroughput[] = new float[THROUGHPUT_COUNT];
+    private volatile long _peakThroughputCurrentTotal;
+    private final float _peakTunnelThroughput[] = new float[THROUGHPUT_COUNT];
+    /** total number of bytes pushed through a single tunnel in a 1 minute period */
+    private final float _peakTunnel1mThroughput[] = new float[THROUGHPUT_COUNT];
+    /** once a day, on average, cut the measured throughtput values in half */
+    /** let's try once an hour times 3/4 */
+    private static final int DROP_PERIOD_MINUTES = 60;
+    private static final float DEGRADE_FACTOR = 0.75f;
+    private long _lastCoalesceDate = System.currentTimeMillis();
     
     /**
      *  Countries with more than about a 2% share of the netdb.
@@ -81,6 +105,8 @@ public class PeerProfile {
     }
 
     /**
+     *  Caller should call setLastHeardAbout() and setFirstHeardAbout()
+     *
      *  @param peer non-null
      */
     public PeerProfile(RouterContext context, Hash peer) {
@@ -88,15 +114,18 @@ public class PeerProfile {
     }
 
     /**
+     *  Caller should call setLastHeardAbout() and setFirstHeardAbout()
+     *
      *  @param peer non-null
      *  @param expand must be true (see below)
      */
     private PeerProfile(RouterContext context, Hash peer, boolean expand) {
-        _context = context;
-        _log = context.logManager().getLog(PeerProfile.class);
         if (peer == null)
             throw new NullPointerException();
+        _context = context;
+        _log = context.logManager().getLog(PeerProfile.class);
         _peer = peer;
+        _firstHeardAbout = _context.clock().now();
         // this is always true, and there are several places in the router that will NPE
         // if it is false, so all need to be fixed before we can have non-expanded profiles
         if (expand)
@@ -191,14 +220,36 @@ public class PeerProfile {
     
     /**
      *  When did we first hear about this peer?
-     *  Currently unused, candidate for removal.
+     *  @return greater than zero, set to now in consturctor
      */
-    public long getFirstHeardAbout() { return _firstHeardAbout; }
-    public void setFirstHeardAbout(long when) { _firstHeardAbout = when; }
+    public synchronized long getFirstHeardAbout() { return _firstHeardAbout; }
+
+    /**
+     *  Set when did we first heard about this peer, only if older.
+     *  Package private, only set by profile management subsystem.
+     */
+    synchronized void setFirstHeardAbout(long when) {
+        if (when < _firstHeardAbout)
+            _firstHeardAbout = when;
+    }
     
-    /** when did we last hear about this peer? */
-    public long getLastHeardAbout() { return _lastHeardAbout; }
-    public void setLastHeardAbout(long when) { _lastHeardAbout = when; }
+    /**
+     *  when did we last hear about this peer?
+     *  @return 0 if unset
+     */
+    public synchronized long getLastHeardAbout() { return _lastHeardAbout; }
+
+    /**
+     *  Set when did we last hear about this peer, only if unset or newer
+     *  Also sets FirstHeardAbout if earlier
+     */
+    public synchronized void setLastHeardAbout(long when) {
+        if (_lastHeardAbout <= 0 || when > _lastHeardAbout)
+            _lastHeardAbout = when;
+        // this is called by netdb PersistentDataStore, so fixup first heard
+        if (when < _firstHeardAbout)
+            _firstHeardAbout = when;
+    }
     
     /** when did we last send to this peer successfully? */
     public long getLastSendSuccessful() { return _lastSentToSuccessfully; }
@@ -244,24 +295,24 @@ public class PeerProfile {
      * written to disk to affect how the algorithm ranks speed.  Negative values are
      * penalties
      */
-    public long getSpeedBonus() { return _speedBonus; }
-    public void setSpeedBonus(long bonus) { _speedBonus = bonus; }
+    public int getSpeedBonus() { return _speedBonus; }
+    public void setSpeedBonus(int bonus) { _speedBonus = bonus; }
     
     /**
      * extra factor added to the capacity ranking - this can be updated in the profile
      * written to disk to affect how the algorithm ranks capacity.  Negative values are
      * penalties
      */
-    public long getCapacityBonus() { return _capacityBonus; }
-    public void setCapacityBonus(long bonus) { _capacityBonus = bonus; }
+    public int getCapacityBonus() { return _capacityBonus; }
+    public void setCapacityBonus(int bonus) { _capacityBonus = bonus; }
     
     /**
      * extra factor added to the integration ranking - this can be updated in the profile
      * written to disk to affect how the algorithm ranks integration.  Negative values are
      * penalties
      */
-    public long getIntegrationBonus() { return _integrationBonus; }
-    public void setIntegrationBonus(long bonus) { _integrationBonus = bonus; }
+    public int getIntegrationBonus() { return _integrationBonus; }
+    public void setIntegrationBonus(int bonus) { _integrationBonus = bonus; }
     
     /**
      * How fast is the peer, taking into consideration both throughput and latency.
@@ -269,26 +320,26 @@ public class PeerProfile {
      * (or measured) max rates, allowing this speed to reflect the speed /available/.
      *
      */
-    public double getSpeedValue() { return _speedValue; }
+    public float getSpeedValue() { return _speedValue; }
     /**
      * How many tunnels do we think this peer can handle over the next hour? 
      *
      */
-    public double getCapacityValue() { return _capacityValue; }
+    public float getCapacityValue() { return _capacityValue; }
     /**
      * How well integrated into the network is this peer (as measured by how much they've
      * told us that we didn't already know).  Higher numbers means better integrated
      *
      */
-    public double getIntegrationValue() { return _integrationValue; }
+    public float getIntegrationValue() { return _integrationValue; }
     /**
      * is this peer actively failing (aka not worth touching)?
      * deprecated - unused - always false
      */
     public boolean getIsFailing() { return _isFailing; }
 
-    public double getTunnelTestTimeAverage() { return _tunnelTestResponseTimeAvg; }
-    void setTunnelTestTimeAverage(double avg) { _tunnelTestResponseTimeAvg = avg; }
+    public float getTunnelTestTimeAverage() { return _tunnelTestResponseTimeAvg; }
+    void setTunnelTestTimeAverage(float avg) { _tunnelTestResponseTimeAvg = avg; }
     
     void updateTunnelTestTimeAverage(long ms) {
         if (_tunnelTestResponseTimeAvg <= 0) 
@@ -296,42 +347,32 @@ public class PeerProfile {
         
         // weighted since we want to let the average grow quickly and shrink slowly
         if (ms < _tunnelTestResponseTimeAvg)
-            _tunnelTestResponseTimeAvg = 0.95*_tunnelTestResponseTimeAvg + .05*ms;
+            _tunnelTestResponseTimeAvg = 0.95f * _tunnelTestResponseTimeAvg + .05f * ms;
         else
-            _tunnelTestResponseTimeAvg = 0.75*_tunnelTestResponseTimeAvg + .25*ms;
+            _tunnelTestResponseTimeAvg = 0.75f * _tunnelTestResponseTimeAvg + .25f * ms;
         
         if (_log.shouldLog(Log.INFO))
             _log.info("Updating tunnel test time for " + _peer.toBase64().substring(0,6) 
                       + " to " + _tunnelTestResponseTimeAvg + " via " + ms);
     }
 
-    /** keep track of the fastest 3 throughputs */
-    private static final int THROUGHPUT_COUNT = 3;
-    /** 
-     * fastest 1 minute throughput, in bytes per minute, ordered with fastest
-     * first.  this is not synchronized, as we don't *need* perfection, and we only
-     * reorder/insert values on coallesce
-     */
-    private final double _peakThroughput[] = new double[THROUGHPUT_COUNT];
-    private volatile long _peakThroughputCurrentTotal;
-    public double getPeakThroughputKBps() { 
-        double rv = 0;
+    public float getPeakThroughputKBps() { 
+        float rv = 0;
         for (int i = 0; i < THROUGHPUT_COUNT; i++)
             rv += _peakThroughput[i];
-        rv /= (60d*1024d*THROUGHPUT_COUNT);
+        rv /= (60 * 1024 * THROUGHPUT_COUNT);
         return rv;
     }
-    public void setPeakThroughputKBps(double kBps) {
+    public void setPeakThroughputKBps(float kBps) {
         _peakThroughput[0] = kBps*60*1024;
         //for (int i = 0; i < THROUGHPUT_COUNT; i++)
         //    _peakThroughput[i] = kBps*60;
     }
     void dataPushed(int size) { _peakThroughputCurrentTotal += size; }
     
-    private final double _peakTunnelThroughput[] = new double[THROUGHPUT_COUNT];
     /** the tunnel pushed that much data in its lifetime */
     void tunnelDataTransferred(long tunnelByteLifetime) {
-        double lowPeak = _peakTunnelThroughput[THROUGHPUT_COUNT-1];
+        float lowPeak = _peakTunnelThroughput[THROUGHPUT_COUNT-1];
         if (tunnelByteLifetime > lowPeak) {
             synchronized (_peakTunnelThroughput) {
                 for (int i = 0; i < THROUGHPUT_COUNT; i++) {
@@ -345,22 +386,20 @@ public class PeerProfile {
             }
         }
     }
-    public double getPeakTunnelThroughputKBps() { 
-        double rv = 0;
+    public float getPeakTunnelThroughputKBps() { 
+        float rv = 0;
         for (int i = 0; i < THROUGHPUT_COUNT; i++)
             rv += _peakTunnelThroughput[i];
-        rv /= (10d*60d*1024d*THROUGHPUT_COUNT);
+        rv /= (10 * 60 * 1024 * THROUGHPUT_COUNT);
         return rv;
     }
-    public void setPeakTunnelThroughputKBps(double kBps) {
-        _peakTunnelThroughput[0] = kBps*60d*10d*1024d;
+    public void setPeakTunnelThroughputKBps(float kBps) {
+        _peakTunnelThroughput[0] = kBps * (60 * 10 * 1024);
     }
     
-    /** total number of bytes pushed through a single tunnel in a 1 minute period */
-    private final double _peakTunnel1mThroughput[] = new double[THROUGHPUT_COUNT];
     /** the tunnel pushed that much data in a 1 minute period */
     void dataPushed1m(int size) {
-        double lowPeak = _peakTunnel1mThroughput[THROUGHPUT_COUNT-1];
+        float lowPeak = _peakTunnel1mThroughput[THROUGHPUT_COUNT-1];
         if (size > lowPeak) {
             synchronized (_peakTunnel1mThroughput) {
                 for (int i = 0; i < THROUGHPUT_COUNT; i++) {
@@ -388,14 +427,14 @@ public class PeerProfile {
      *         through this peer. Ever. Except that the peak values are cut in half
      *         once a day by coalesceThroughput(). This seems way too seldom.
      */
-    public double getPeakTunnel1mThroughputKBps() { 
-        double rv = 0;
+    public float getPeakTunnel1mThroughputKBps() { 
+        float rv = 0;
         for (int i = 0; i < THROUGHPUT_COUNT; i++)
             rv += _peakTunnel1mThroughput[i];
-        rv /= (60d*1024d*THROUGHPUT_COUNT);
+        rv /= (60 * 1024 * THROUGHPUT_COUNT);
         return rv;
     }
-    public void setPeakTunnel1mThroughputKBps(double kBps) {
+    public void setPeakTunnel1mThroughputKBps(float kBps) {
         _peakTunnel1mThroughput[0] = kBps*60*1024;
     }
     
@@ -429,7 +468,7 @@ public class PeerProfile {
      * repeatedly
      *
      */
-    public void expandProfile() {
+    public synchronized void expandProfile() {
         String group = (null == _peer ? "profileUnknown" : _peer.toBase64().substring(0,6));
         //if (_sendSuccessSize == null)
         //    _sendSuccessSize = new RateStat("sendSuccessSize", "How large successfully sent messages are", group, new long[] { 5*60*1000l, 60*60*1000l });
@@ -468,17 +507,12 @@ public class PeerProfile {
         _expandedDB = true;
     }
 
-    /** once a day, on average, cut the measured throughtput values in half */
-    /** let's try once an hour times 3/4 */
-    private static final int DROP_PERIOD_MINUTES = 60;
-    private static final double DEGRADE_FACTOR = 0.75;
-    private long _lastCoalesceDate = System.currentTimeMillis();
     private void coalesceThroughput() {
         long now = System.currentTimeMillis();
         long measuredPeriod = now - _lastCoalesceDate;
         if (measuredPeriod >= 60*1000) {
             long tot = _peakThroughputCurrentTotal;
-            double lowPeak = _peakThroughput[THROUGHPUT_COUNT-1];
+            float lowPeak = _peakThroughput[THROUGHPUT_COUNT-1];
             if (tot > lowPeak) {
                 for (int i = 0; i < THROUGHPUT_COUNT; i++) {
                     if (tot > _peakThroughput[i]) {
@@ -562,9 +596,9 @@ public class PeerProfile {
     	_capacityValue = _capacityValueNew;
     }
     
-    private double calculateSpeed() { return SpeedCalculator.calc(this); }
-    private double calculateCapacity() { return CapacityCalculator.calc(this); }
-    private double calculateIntegration() { return IntegrationCalculator.calc(this); }
+    private float calculateSpeed() { return (float) SpeedCalculator.calc(this); }
+    private float calculateCapacity() { return (float) CapacityCalculator.calc(this); }
+    private float calculateIntegration() { return (float) IntegrationCalculator.calc(this); }
     /** deprecated - unused - always false */
     private boolean calculateIsFailing() { return false; }
     /** deprecated - unused - always false */

router/java/src/net/i2p/router/peermanager/ProfileManagerImpl.java

@@ -222,8 +222,8 @@ public class ProfileManagerImpl implements ProfileManager {
         data.setLastHeardFrom(_context.clock().now());
         if (!data.getIsExpandedDB())
             return;
-        DBHistory hist = data.getDBHistory();
-        hist.lookupReceived();
+        //DBHistory hist = data.getDBHistory();
+        //hist.lookupReceived();
     }
     
     /**
@@ -313,8 +313,7 @@ public class ProfileManagerImpl implements ProfileManager {
     public void heardAbout(Hash peer, long when) {
         PeerProfile data = getProfile(peer);
         //if (data == null) return;
-        if (when > data.getLastHeardAbout())
-            data.setLastHeardAbout(when);
+        data.setLastHeardAbout(when);
     }
     
     /**
@@ -340,7 +339,6 @@ public class ProfileManagerImpl implements ProfileManager {
         PeerProfile prof = _context.profileOrganizer().getProfile(peer);
         if (prof == null) {
             prof = new PeerProfile(_context, peer);
-            prof.setFirstHeardAbout(_context.clock().now());
             _context.profileOrganizer().addProfile(prof);
         }
         return prof;

router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java

@@ -17,7 +17,7 @@ import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import net.i2p.crypto.SHA256Generator;
+import net.i2p.data.DataHelper;
 import net.i2p.data.Hash;
 import net.i2p.data.router.RouterAddress;
 import net.i2p.data.router.RouterInfo;
@@ -1234,12 +1234,15 @@ public class ProfileOrganizer {
     }
 
     /**
+      *
+      * As of 0.9.24, checks for a netdb family match as well, unless mask == 0.
+      *
      * @param mask 0-4 Number of bytes to match to determine if peers in the same IP range should
      *             not be in the same tunnel. 0 = disable check; 1 = /8; 2 = /16; 3 = /24; 4 = exact IP match
      */
     private void locked_selectPeers(Map<Hash, PeerProfile> peers, int howMany, Set<Hash> toExclude, Set<Hash> matches, int mask) {
         List<Hash> all = new ArrayList<Hash>(peers.keySet());
-        Set<Integer> IPSet = new HashSet<Integer>(8);
+        Set<String> IPSet = new HashSet<String>(8);
         // use RandomIterator to avoid shuffling the whole thing
         for (Iterator<Hash> iter = new RandomIterator<Hash>(all); (matches.size() < howMany) && iter.hasNext(); ) {
             Hash peer = iter.next();
@@ -1265,11 +1268,14 @@ public class ProfileOrganizer {
     /**
      * Does the peer's IP address NOT match the IP address of any peer already in the set,
      * on any transport, within a given mask?
+     *
+     * As of 0.9.24, checks for a netdb family match as well.
+     *
      * @param mask is 1-4 (number of bytes to match)
      * @param IPMatches all IPs so far, modified by this routine
      */
-    private boolean notRestricted(Hash peer, Set<Integer> IPSet, int mask) {
-        Set<Integer> peerIPs = maskedIPSet(peer, mask);
+    private boolean notRestricted(Hash peer, Set<String> IPSet, int mask) {
+        Set<String> peerIPs = maskedIPSet(peer, mask);
         if (containsAny(IPSet, peerIPs))
             return false;
         IPSet.addAll(peerIPs);
@@ -1280,10 +1286,12 @@ public class ProfileOrganizer {
       * The Set of IPs for this peer, with a given mask.
       * Includes the comm system's record of the IP, and all netDb addresses.
       *
+      * As of 0.9.24, returned set will include netdb family as well.
+      *
       * @return an opaque set of masked IPs for this peer
       */
-    private Set<Integer> maskedIPSet(Hash peer, int mask) {
-        Set<Integer> rv = new HashSet<Integer>(4);
+    private Set<String> maskedIPSet(Hash peer, int mask) {
+        Set<String> rv = new HashSet<String>(4);
         byte[] commIP = _context.commSystem().getIP(peer);
         if (commIP != null)
             rv.add(maskedIP(commIP, mask));
@@ -1296,31 +1304,40 @@ public class ProfileOrganizer {
             if (pib == null) continue;
             rv.add(maskedIP(pib, mask));
         }
+        String family = pinfo.getOption("family");
+        if (family != null) {
+            // TODO should KNDF put a family-verified indicator in the RI,
+            // after checking the sig, or does it matter?
+            // What's the threat here of not avoid ding a router
+            // falsely claiming to be in the family?
+            // Prefix with something so an IP can't be spoofed
+            rv.add('x' + family);
+        }
         return rv;
     }
 
     /**
      * generate an arbitrary unique value for this ip/mask (mask = 1-4)
-     * If IPv6, force mask = 8.
+     * If IPv6, force mask = 6.
      */
-    private static Integer maskedIP(byte[] ip, int mask) {
-        int rv = ip[0];
+    private static String maskedIP(byte[] ip, int mask) {
+        final StringBuilder buf = new StringBuilder(1 + (mask*2));
+        final char delim;
         if (ip.length == 16) {
-            for (int i = 1; i < 8; i++) {
-                rv <<= i * 4;
-                rv ^= ip[i];
-            }
+            mask = 6;
+            delim = ':';
         } else {
-            for (int i = 1; i < mask; i++) {
-                rv <<= 8;
-                rv ^= ip[i];
-            }
+            delim = '.';
         }
-        return Integer.valueOf(rv);
+        buf.append(delim);
+        buf.append(Long.toHexString(DataHelper.fromLong(ip, 0, mask)));
+        return buf.toString();
     }
 
     /** does a contain any of the elements in b? */
     private static <T> boolean  containsAny(Set<T> a, Set<T> b) {
+        if (a.isEmpty() || b.isEmpty())
+            return false;
         for (T o : b) {
             if (a.contains(o))
                 return true;

router/java/src/net/i2p/router/peermanager/ProfilePersistenceHelper.java

@@ -48,12 +48,12 @@ class ProfilePersistenceHelper {
     private static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~";
     
     /**
-     * If we haven't been able to get a message through to the peer in 3 days,
+     * If we haven't been able to get a message through to the peer in this much time,
      * drop the profile.  They may reappear, but if they do, their config may
      * have changed (etc).
      *
      */
-    private static final long EXPIRE_AGE = 3*24*60*60*1000;
+    private static final long EXPIRE_AGE = 15*24*60*60*1000;
     
     private final File _profileDir;
     private Hash _us;
@@ -170,7 +170,7 @@ class ProfilePersistenceHelper {
     }
     
     /** @since 0.8.5 */
-    private static void add(StringBuilder buf, String name, double val, String description) {
+    private static void add(StringBuilder buf, String name, float val, String description) {
         buf.append("# ").append(name).append(NL).append("# ").append(description).append(NL);
         buf.append(name).append('=').append(val).append(NL).append(NL);
     }
@@ -263,29 +263,29 @@ class ProfilePersistenceHelper {
                     file.delete();
             }
             
-            profile.setCapacityBonus(getLong(props, "capacityBonus"));
-            profile.setIntegrationBonus(getLong(props, "integrationBonus"));
-            profile.setSpeedBonus(getLong(props, "speedBonus"));
+            profile.setCapacityBonus((int) getLong(props, "capacityBonus"));
+            profile.setIntegrationBonus((int) getLong(props, "integrationBonus"));
+            profile.setSpeedBonus((int) getLong(props, "speedBonus"));
             
             profile.setLastHeardAbout(getLong(props, "lastHeardAbout"));
             profile.setFirstHeardAbout(getLong(props, "firstHeardAbout"));
             profile.setLastSendSuccessful(getLong(props, "lastSentToSuccessfully"));
             profile.setLastSendFailed(getLong(props, "lastFailedSend"));
             profile.setLastHeardFrom(getLong(props, "lastHeardFrom"));
-            profile.setTunnelTestTimeAverage(getDouble(props, "tunnelTestTimeAverage"));
-            profile.setPeakThroughputKBps(getDouble(props, "tunnelPeakThroughput"));
-            profile.setPeakTunnelThroughputKBps(getDouble(props, "tunnelPeakTunnelThroughput"));
-            profile.setPeakTunnel1mThroughputKBps(getDouble(props, "tunnelPeakTunnel1mThroughput"));
+            profile.setTunnelTestTimeAverage(getFloat(props, "tunnelTestTimeAverage"));
+            profile.setPeakThroughputKBps(getFloat(props, "tunnelPeakThroughput"));
+            profile.setPeakTunnelThroughputKBps(getFloat(props, "tunnelPeakTunnelThroughput"));
+            profile.setPeakTunnel1mThroughputKBps(getFloat(props, "tunnelPeakTunnel1mThroughput"));
             
             profile.getTunnelHistory().load(props);
 
             // In the interest of keeping the in-memory profiles small,
             // don't load the DB info at all unless there is something interesting there
             // (i.e. floodfills)
-            // It seems like we do one or two lookups as a part of handshaking?
-            // Not sure, to be researched.
-            if (getLong(props, "dbHistory.successfulLookups") > 1 ||
-                getLong(props, "dbHistory.failedlLokups") > 1) {
+            if (getLong(props, "dbHistory.lastLookupSuccessful") > 0 ||
+                getLong(props, "dbHistory.lastLookupFailed") > 0 ||
+                getLong(props, "dbHistory.lastStoreSuccessful") > 0 ||
+                getLong(props, "dbHistory.lastStoreFailed") > 0) {
                 profile.expandDBProfile();
                 profile.getDBHistory().load(props);
                 profile.getDbIntroduction().load(props, "dbIntroduction", true);
@@ -300,6 +300,7 @@ class ProfilePersistenceHelper {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Loaded the profile for " + peer.toBase64() + " from " + file.getName());
             
+            fixupFirstHeardAbout(profile);
             return profile;
         } catch (IOException e) {
             if (_log.shouldLog(Log.WARN))
@@ -308,29 +309,78 @@ class ProfilePersistenceHelper {
             return null;
         }
     }
+
+    /**
+     *  First heard about wasn't always set correctly before,
+     *  set it to the minimum of all recorded timestamps.
+     *
+     *  @since 0.9.24
+     */
+    private void fixupFirstHeardAbout(PeerProfile p) {
+        long min = Long.MAX_VALUE;
+        long t = p.getLastHeardAbout();
+        if (t > 0 && t < min) min = t;
+        t = p.getLastSendSuccessful();
+        if (t > 0 && t < min) min = t;
+        t = p.getLastSendFailed();
+        if (t > 0 && t < min) min = t;
+        t = p.getLastHeardFrom();
+        if (t > 0 && t < min) min = t;
+        // the first was never used and the last 4 were never persisted
+        //DBHistory dh = p.getDBHistory();
+        //if (dh != null) {
+        //    t = dh.getLastLookupReceived();
+        //    if (t > 0 && t < min) min = t;
+        //    t = dh.getLastLookupSuccessful();
+        //    if (t > 0 && t < min) min = t;
+        //    t = dh.getLastLookupFailed();
+        //    if (t > 0 && t < min) min = t;
+        //    t = dh.getLastStoreSuccessful();
+        //    if (t > 0 && t < min) min = t;
+        //    t = dh.getLastStoreFailed();
+        //    if (t > 0 && t < min) min = t;
+        //}
+        TunnelHistory th = p.getTunnelHistory();
+        if (th != null) {
+            t = th.getLastAgreedTo();
+            if (t > 0 && t < min) min = t;
+            t = th.getLastRejectedCritical();
+            if (t > 0 && t < min) min = t;
+            t = th.getLastRejectedBandwidth();
+            if (t > 0 && t < min) min = t;
+            t = th.getLastRejectedTransient();
+            if (t > 0 && t < min) min = t;
+            t = th.getLastRejectedProbabalistic();
+            if (t > 0 && t < min) min = t;
+            t = th.getLastFailed();
+            if (t > 0 && t < min) min = t;
+        }
+        long fha = p.getFirstHeardAbout();
+        if (min > 0 && min < Long.MAX_VALUE && (fha <= 0 || min < fha)) {
+            p.setFirstHeardAbout(min);
+            if (_log.shouldDebug())
+                _log.debug("Fixed up the FHA time for " + p.getPeer().toBase64() + " to " + (new Date(min)));
+        }
+    }
     
-    private final static long getLong(Properties props, String key) {
+    static long getLong(Properties props, String key) {
         String val = props.getProperty(key);
         if (val != null) {
             try {
                 return Long.parseLong(val);
-            } catch (NumberFormatException nfe) {
-                return 0;
-            }
+            } catch (NumberFormatException nfe) {}
         }
         return 0;
     }
 
-    private final static double getDouble(Properties props, String key) {
+    private final static float getFloat(Properties props, String key) {
         String val = props.getProperty(key);
         if (val != null) {
             try {
-                return Double.parseDouble(val);
-            } catch (NumberFormatException nfe) {
-                return 0.0;
-            }
+                return Float.parseFloat(val);
+            } catch (NumberFormatException nfe) {}
         }
-        return 0.0;
+        return 0.0f;
     }
     
     private void loadProps(Properties props, File file) throws IOException {

router/java/src/net/i2p/router/peermanager/TunnelHistory.java

@@ -89,10 +89,10 @@ public class TunnelHistory {
         _lifetimeRejected.incrementAndGet();
         if (severity >= TUNNEL_REJECT_CRIT) {
             _lastRejectedCritical = _context.clock().now();
-            _rejectRate.addData(1, 1);
+            _rejectRate.addData(1);
         } else if (severity >= TUNNEL_REJECT_BANDWIDTH) {
             _lastRejectedBandwidth = _context.clock().now();
-            _rejectRate.addData(1, 1);
+            _rejectRate.addData(1);
         } else if (severity >= TUNNEL_REJECT_TRANSIENT_OVERLOAD) {
             _lastRejectedTransient = _context.clock().now();
             // dont increment the reject rate in this case
@@ -108,7 +108,7 @@ public class TunnelHistory {
      */
     public void incrementFailed(int pct) {
         _lifetimeFailed.incrementAndGet();
-        _failRate.addData(pct, 1);
+        _failRate.addData(pct);
         _lastFailed = _context.clock().now();
     }
     
@@ -190,14 +190,6 @@ public class TunnelHistory {
     }
     
     private final static long getLong(Properties props, String key) {
-        String val = props.getProperty(key);
-        if (val != null) {
-            try {
-                return Long.parseLong(val);
-            } catch (NumberFormatException nfe) {
-                return 0;
-            }
-        }
-        return 0;
+        return ProfilePersistenceHelper.getLong(props, key);
     }
 }

router/java/src/net/i2p/router/startup/CreateRouterInfoJob.java

@@ -98,10 +98,6 @@ public class CreateRouterInfoJob extends JobImpl {
         OutputStream fos1 = null;
         try {
             info.setAddresses(getContext().commSystem().createAddresses());
-            Properties stats = getContext().statPublisher().publishStatistics();
-            stats.setProperty(RouterInfo.PROP_NETWORK_ID, Router.NETWORK_ID+"");
-            getContext().router().addCapabilities(info);
-            info.setOptions(stats);
             // not necessary, in constructor
             //info.setPeers(new HashSet());
             info.setPublished(getCurrentPublishDate(getContext()));
@@ -126,6 +122,8 @@ public class CreateRouterInfoJob extends JobImpl {
                 padding = null;
             }
             info.setIdentity(ident);
+            Properties stats = getContext().statPublisher().publishStatistics(ident.getHash());
+            info.setOptions(stats);
             
             info.sign(signingPrivKey);
 

router/java/src/net/i2p/router/startup/LoadClientAppsJob.java

@@ -123,7 +123,7 @@ public class LoadClientAppsJob extends JobImpl {
             for (int i = 0; i < data.length; i++) {
                 switch (data[i]) {
                     case '\'':
-                    case '\"':
+                    case '"':
                         if (isQuoted) {
                             String str = buf.toString().trim();
                             if (str.length() > 0)

router/java/src/net/i2p/router/startup/LoadRouterInfoJob.java

@@ -31,6 +31,7 @@ import net.i2p.data.router.RouterPrivateKeyFile;
 import net.i2p.router.JobImpl;
 import net.i2p.router.Router;
 import net.i2p.router.RouterContext;
+import net.i2p.router.crypto.FamilyKeyCrypto;
 import net.i2p.router.networkdb.kademlia.PersistentDataStore;
 import net.i2p.util.Log;
 
@@ -98,7 +99,13 @@ class LoadRouterInfoJob extends JobImpl {
                     throw new DataFormatException("Our RouterInfo has a bad signature");
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("Reading in routerInfo from " + rif.getAbsolutePath() + " and it has " + info.getAddresses().size() + " addresses");
-                _us = info;
+                // don't reuse if family name changed
+                if (DataHelper.eq(info.getOption(FamilyKeyCrypto.OPT_NAME),
+                                  getContext().getProperty(FamilyKeyCrypto.PROP_FAMILY_NAME))) {
+                    _us = info;
+                } else {
+                    _log.logAlways(Log.WARN, "NetDb family name changed");
+                }
             }
             
             if (keys2Exist || keysExist) {
@@ -114,9 +121,9 @@ class LoadRouterInfoJob extends JobImpl {
                 boolean sigTypeChanged = stype != cstype;
                 if (sigTypeChanged && getContext().getProperty(CreateRouterInfoJob.PROP_ROUTER_SIGTYPE) == null) {
                     // Not explicitly configured, and default has changed
-                    // Give a 15% chance of rekeying for each restart
+                    // Give a 25% chance of rekeying for each restart
                     // TODO reduce to ~3 (i.e. increase probability) in future release
-                    if (getContext().random().nextInt(7) > 0) {
+                    if (getContext().random().nextInt(4) > 0) {
                         sigTypeChanged = false;
                         if (_log.shouldWarn())
                             _log.warn("Deferring RI rekey from " + stype + " to " + cstype);

router/java/src/net/i2p/router/startup/RebuildRouterInfoJob.java

@@ -118,10 +118,8 @@ class RebuildRouterInfoJob extends JobImpl {
             
             try {
                 info.setAddresses(getContext().commSystem().createAddresses());
-                Properties stats = getContext().statPublisher().publishStatistics();
-                stats.setProperty(RouterInfo.PROP_NETWORK_ID, ""+Router.NETWORK_ID);
+                Properties stats = getContext().statPublisher().publishStatistics(info.getHash());
                 info.setOptions(stats);
-                getContext().router().addCapabilities(info);
                 // info.setPeers(new HashSet()); // this would have the trusted peers
                 info.setPublished(CreateRouterInfoJob.getCurrentPublishDate(getContext()));
                 

router/java/src/net/i2p/router/startup/StartupJob.java

@@ -36,7 +36,6 @@ public class StartupJob extends JobImpl {
     public void runJob() {
         if (!SystemVersion.isAndroid())
             getContext().jobQueue().addJob(new LoadClientAppsJob(getContext()));
-        getContext().statPublisher().startup();
         getContext().jobQueue().addJob(new LoadRouterInfoJob(getContext()));
     }
 }

router/java/src/net/i2p/router/time/RouterTimestamper.java

@@ -45,6 +45,7 @@ public class RouterTimestamper extends Timestamper {
     private static final int MAX_CONSECUTIVE_FAILS = 10;
     private static final int DEFAULT_TIMEOUT = 10*1000;
     private static final int SHORT_TIMEOUT = 5*1000;
+    private static final long MAX_WAIT_INITIALIZATION = 45*1000;
     
     public static final String PROP_QUERY_FREQUENCY = "time.queryFrequencyMs";
     public static final String PROP_SERVER_LIST = "time.sntpServerList";
@@ -141,7 +142,7 @@ public class RouterTimestamper extends Timestamper {
         try { 
             synchronized (this) {
                 if (!_initialized)
-                    wait();
+                    wait(MAX_WAIT_INITIALIZATION);
             }
         } catch (InterruptedException ie) {}
     }

router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java

@@ -155,23 +155,34 @@ public class CommSystemFacadeImpl extends CommSystemFacade {
     }
     
     @Override
-    public boolean isBacklogged(Hash dest) { 
-        return _manager.isBacklogged(dest); 
+    public boolean isBacklogged(Hash peer) { 
+        return _manager.isBacklogged(peer); 
     }
     
     @Override
-    public boolean isEstablished(Hash dest) { 
-        return _manager.isEstablished(dest); 
+    public boolean isEstablished(Hash peer) { 
+        return _manager.isEstablished(peer); 
     }
     
     @Override
-    public boolean wasUnreachable(Hash dest) { 
-        return _manager.wasUnreachable(dest); 
+    public boolean wasUnreachable(Hash peer) { 
+        return _manager.wasUnreachable(peer); 
     }
     
     @Override
-    public byte[] getIP(Hash dest) { 
-        return _manager.getIP(dest); 
+    public byte[] getIP(Hash peer) { 
+        return _manager.getIP(peer); 
+    }
+    
+    /**
+     * Tell the comm system that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    @Override
+    public void mayDisconnect(Hash peer) {
+        _manager.mayDisconnect(peer); 
     }
     
     @Override
@@ -196,6 +207,7 @@ public class CommSystemFacadeImpl extends CommSystemFacade {
      * @deprecated unused
      */
     @Override
+    @Deprecated
     public void recheckReachability() { _manager.recheckReachability(); }
 
     @Override

router/java/src/net/i2p/router/transport/OutboundMessageRegistry.java

@@ -258,6 +258,7 @@ public class OutboundMessageRegistry {
     public void renderStatusHTML(Writer out) throws IOException {}
     
     private class CleanupTask extends SimpleTimer2.TimedEvent {
+        /** LOCKING: _selectors */
         private long _nextExpire;
 
         public CleanupTask() {
@@ -325,23 +326,28 @@ public class OutboundMessageRegistry {
 
             if (log) {
                 int e = removing.size();
-                int r = _selectors.size();
+                int r;
+                synchronized(_selectors) {
+                    r = _selectors.size();
+                }
                 int a = _activeMessages.size();
                 if (r > 0 || e > 0 || a > 0)
                     _log.debug("Expired: " + e + " remaining: " + r + " active: " + a);
             }
-            synchronized(this) {
+            synchronized(_selectors) {
                 if (_nextExpire <= now)
                     _nextExpire = now + 10*1000;
                 schedule(_nextExpire - now);
             }
         }
 
-        public synchronized void scheduleExpiration(MessageSelector sel) {
+        public void scheduleExpiration(MessageSelector sel) {
             long now = _context.clock().now();
-            if ( (_nextExpire <= now) || (sel.getExpiration() < _nextExpire) ) {
-                _nextExpire = sel.getExpiration();
-                reschedule(_nextExpire - now);
+            synchronized(_selectors) {
+                if ( (_nextExpire <= now) || (sel.getExpiration() < _nextExpire) ) {
+                    _nextExpire = sel.getExpiration();
+                    reschedule(_nextExpire - now);
+                }
             }
         }
     }

router/java/src/net/i2p/router/transport/Transport.java

@@ -171,4 +171,12 @@ public interface Transport {
     
     public boolean isUnreachable(Hash peer);
     public boolean isEstablished(Hash peer);
+
+    /**
+     * Tell the transport that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    public void mayDisconnect(Hash peer);
 }

router/java/src/net/i2p/router/transport/TransportImpl.java

@@ -141,9 +141,10 @@ public abstract class TransportImpl implements Transport {
         else // shouldn't happen
             maxProp = "i2np." + style.toLowerCase(Locale.US) + ".maxConnections";
         int def = MAX_CONNECTION_FACTOR;
-        RouterInfo ri = _context.router().getRouterInfo();
-        if (ri != null) {
-            char bw = ri.getBandwidthTier().charAt(0);
+        // get it from here, not the RI, to avoid deadlock
+        String caps = _context.router().getCapabilities();
+
+            char bw = caps.charAt(0);
             switch (bw) {
                 case Router.CAPABILITY_BW12:
                 case 'u':  // unknown
@@ -168,7 +169,7 @@ public abstract class TransportImpl implements Transport {
                     def *= 12;
                     break;
             }
-        }
+
         if (_context.netDb().floodfillEnabled()) {
             // && !SystemVersion.isWindows()) {
             def *= 17; def /= 10;  // 425 for Class O ff
@@ -747,13 +748,11 @@ public abstract class TransportImpl implements Transport {
      *  This can be called before startListening() to set an initial address,
      *  or after the transport is running.
      *
-     *  This implementation does nothing. Transports should override if they want notification.
-     *
      *  @param source defined in Transport.java
      *  @param ip typ. IPv4 or IPv6 non-local; may be null to indicate IPv4 failure or port info only
      *  @param port 0 for unknown or unchanged
      */
-    public void externalAddressReceived(AddressSource source, byte[] ip, int port) {}
+    public abstract void externalAddressReceived(AddressSource source, byte[] ip, int port);
 
     /**
      *  Notify a transport of an external address change.
@@ -810,6 +809,7 @@ public abstract class TransportImpl implements Transport {
     /**
      * @deprecated unused
      */
+    @Deprecated
     public void recheckReachability() {}
 
     /**
@@ -819,8 +819,16 @@ public abstract class TransportImpl implements Transport {
         return TransportUtil.isIPv4Firewalled(_context, getStyle());
     }
 
-    public boolean isBacklogged(Hash dest) { return false; }
-    public boolean isEstablished(Hash dest) { return false; }
+    public boolean isBacklogged(Hash peer) { return false; }
+    public boolean isEstablished(Hash peer) { return false; }
+
+    /**
+     * Tell the transport that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    public void mayDisconnect(Hash peer) {}
 
     public boolean isUnreachable(Hash peer) {
         long now = _context.clock().now();

router/java/src/net/i2p/router/transport/TransportManager.java

@@ -407,35 +407,48 @@ public class TransportManager implements TransportEventListener {
     /**
      * @deprecated unused
      */
+    @Deprecated
     public void recheckReachability() { 
         for (Transport t : _transports.values())
             t.recheckReachability();
     }
 
-    public boolean isBacklogged(Hash dest) {
+    public boolean isBacklogged(Hash peer) {
         for (Transport t : _transports.values()) {
-            if (t.isBacklogged(dest))
+            if (t.isBacklogged(peer))
                 return true;
         }
         return false;
     }    
     
-    public boolean isEstablished(Hash dest) {
+    public boolean isEstablished(Hash peer) {
         for (Transport t : _transports.values()) {
-            if (t.isEstablished(dest))
+            if (t.isEstablished(peer))
                 return true;
         }
         return false;
     }    
     
+    /**
+     * Tell the transports that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    public void mayDisconnect(Hash peer) {
+        for (Transport t : _transports.values()) {
+             t.mayDisconnect(peer);
+        }
+    }
+    
     /**
      * Was the peer UNreachable (outbound only) on any transport,
      * based on the last time we tried it for each transport?
      * This is NOT reset if the peer contacts us.
      */
-    public boolean wasUnreachable(Hash dest) {
+    public boolean wasUnreachable(Hash peer) {
         for (Transport t : _transports.values()) {
-            if (!t.wasUnreachable(dest))
+            if (!t.wasUnreachable(peer))
                 return false;
         }
         return true;
@@ -452,8 +465,8 @@ public class TransportManager implements TransportEventListener {
      *
      * @return IPv4 or IPv6 or null
      */
-    public byte[] getIP(Hash dest) {
-        return TransportImpl.getIP(dest);
+    public byte[] getIP(Hash peer) {
+        return TransportImpl.getIP(peer);
     }    
     
     /**
@@ -745,8 +758,8 @@ public class TransportManager implements TransportEventListener {
                    //"<b id=\"def.dev\">").append(_t("Dev")).append("</b>: ").append(_t("The standard deviation of the round trip time in milliseconds")).append("<br>\n" +
                    "<b id=\"def.rto\">RTO</b>: ").append(_t("The retransmit timeout in milliseconds")).append("<br>\n" +
                    "<b id=\"def.mtu\">MTU</b>: ").append(_t("Current maximum send packet size / estimated maximum receive packet size (bytes)")).append("<br>\n" +
-                   "<b id=\"def.send\">").append(_t("TX")).append("</b>: ").append(_t("The total number of packets sent to the peer")).append("<br>\n" +
-                   "<b id=\"def.recv\">").append(_t("RX")).append("</b>: ").append(_t("The total number of packets received from the peer")).append("<br>\n" +
+                   "<b id=\"def.send\">").append(_t("TX")).append("</b>: ").append(_t("The total number of messages sent to the peer")).append("<br>\n" +
+                   "<b id=\"def.recv\">").append(_t("RX")).append("</b>: ").append(_t("The total number of messages received from the peer")).append("<br>\n" +
                    "<b id=\"def.resent\">").append(_t("Dup TX")).append("</b>: ").append(_t("The total number of packets retransmitted to the peer")).append("<br>\n" +
                    "<b id=\"def.dupRecv\">").append(_t("Dup RX")).append("</b>: ").append(_t("The total number of duplicate packets received from the peer")).append("</p>" +
                    "</div>\n");

router/java/src/net/i2p/router/transport/TransportUtil.java

@@ -98,7 +98,7 @@ public abstract class TransportUtil {
      */
     public static boolean isIPv6(RouterAddress addr) {
         // do this the fast way, without calling getIP() to parse the host string
-        String host = addr.getOption(RouterAddress.PROP_HOST);
+        String host = addr.getHost();
         return host != null && host.contains(":");
     }
 

router/java/src/net/i2p/router/transport/crypto/DHSessionKeyBuilder.java

@@ -505,9 +505,10 @@ public class DHSessionKeyBuilder {
                             break;
                         long curCalc = System.currentTimeMillis() - curStart;
                         // for some relief...
-                        try {
-                            Thread.sleep(Math.min(200, Math.max(10, _calcDelay + (curCalc * 3))));
-                        } catch (InterruptedException ie) { // nop
+                        if (!interrupted()) {
+                            try {
+                                Thread.sleep(Math.min(200, Math.max(10, _calcDelay + (curCalc * 3))));
+                            } catch (InterruptedException ie) {}
                         }
                     }
                 }
@@ -541,6 +542,8 @@ public class DHSessionKeyBuilder {
                 if (_log.shouldLog(Log.INFO)) _log.info("No more builders, creating one now");
                 _context.statManager().addRateData("crypto.DHEmpty", 1);
                 builder = precalc();
+                // stop sleeping, wake up, make some more
+                this.interrupt();
             }
             return builder;
         }

router/java/src/net/i2p/router/transport/ntcp/EventPumper.java

@@ -85,6 +85,7 @@ class EventPumper implements Runnable {
     /** tunnel test now disabled, but this should be long enough to allow an active tunnel to get started */
     private static final long MIN_EXPIRE_IDLE_TIME = 120*1000l;
     private static final long MAX_EXPIRE_IDLE_TIME = 11*60*1000l;
+    private static final long MAY_DISCON_TIMEOUT = 10*1000;
 
     /**
      *  Do we use direct buffers for reading? Default false.
@@ -221,7 +222,8 @@ class EventPumper implements Runnable {
                         int failsafeInvalid = 0;
 
                         // Increase allowed idle time if we are well under allowed connections, otherwise decrease
-                        if (_transport.haveCapacity(33))
+                        boolean haveCap = _transport.haveCapacity(33);
+                        if (haveCap)
                             _expireIdleWriteTime = Math.min(_expireIdleWriteTime + 1000, MAX_EXPIRE_IDLE_TIME);
                         else
                             _expireIdleWriteTime = Math.max(_expireIdleWriteTime - 3000, MIN_EXPIRE_IDLE_TIME);
@@ -270,8 +272,16 @@ class EventPumper implements Runnable {
                                     failsafeWrites++;
                                 }
                                 
-                                if ( con.getTimeSinceSend() > _expireIdleWriteTime &&
-                                     con.getTimeSinceReceive() > _expireIdleWriteTime) {
+                                final long expire;
+                                if (!haveCap && con.getMayDisconnect() &&
+                                    con.getMessagesReceived() <= 2 && con.getMessagesSent() <= 1) {
+                                    expire = MAY_DISCON_TIMEOUT;
+                                } else {
+                                    expire = _expireIdleWriteTime;
+                                }
+
+                                if ( con.getTimeSinceSend() > expire &&
+                                     con.getTimeSinceReceive() > expire) {
                                     // we haven't sent or received anything in a really long time, so lets just close 'er up
                                     con.close();
                                     failsafeCloses++;

router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java

@@ -13,6 +13,7 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.Adler32;
 
@@ -115,8 +116,8 @@ class NTCPConnection implements Closeable {
     private byte _prevWriteEnd[];
     /** current partially read I2NP message */
     private final ReadState _curReadState;
-    private final AtomicLong _messagesRead = new AtomicLong();
-    private final AtomicLong _messagesWritten = new AtomicLong();
+    private final AtomicInteger _messagesRead = new AtomicInteger();
+    private final AtomicInteger _messagesWritten = new AtomicInteger();
     private long _lastSendTime;
     private long _lastReceiveTime;
     private long _lastRateUpdated;
@@ -134,6 +135,7 @@ class NTCPConnection implements Closeable {
     /** how many consecutive sends were failed due to (estimated) send queue time */
     //private int _consecutiveBacklog;
     private long _nextInfoTime;
+    private boolean _mayDisconnect;
     
     /*
      *  Update frequency for send/recv rates in console peers page
@@ -325,11 +327,11 @@ class NTCPConnection implements Closeable {
             return _context.clock().now() -_establishedOn; 
     }
 
-    public long getMessagesSent() { return _messagesWritten.get(); }
+    public int getMessagesSent() { return _messagesWritten.get(); }
 
-    public long getMessagesReceived() { return _messagesRead.get(); }
+    public int getMessagesReceived() { return _messagesRead.get(); }
 
-    public long getOutboundQueueSize() {
+    public int getOutboundQueueSize() {
             int queued;
             synchronized(_outbound) {
                 queued = _outbound.size();
@@ -360,6 +362,17 @@ class NTCPConnection implements Closeable {
      */
     public long getCreated() { return _created; }
 
+    /**
+     * Sets to true.
+     * @since 0.9.24
+     */
+    public void setMayDisconnect() { _mayDisconnect = true; }
+
+    /**
+     * @since 0.9.24
+     */
+    public boolean getMayDisconnect() { return _mayDisconnect; }
+
     /**
      *  workaround for EventPumper
      *  @since 0.8.12

router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java

@@ -486,6 +486,21 @@ public class NTCPTransport extends TransportImpl {
             return (con != null) && con.isEstablished() && con.tooBacklogged();
     }
 
+    /**
+     * Tell the transport that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    @Override
+    public void mayDisconnect(final Hash peer) {
+        final NTCPConnection con = _conByIdent.get(peer);
+        if (con != null && con.isEstablished() && con.isInbound() &&
+            con.getMessagesReceived() <= 2 && con.getMessagesSent() <= 1) {
+            con.setMayDisconnect();
+        }
+    }
+
     /**
      * @return usually the con passed in, but possibly a second connection with the same peer...
      */

router/java/src/net/i2p/router/transport/udp/EstablishmentManager.java

@@ -770,7 +770,7 @@ class EstablishmentManager {
                                            // so it needs to be caught in InNetMessagePool.
         dsm.setMessageExpiration(_context.clock().now() + DATA_MESSAGE_TIMEOUT);
         dsm.setMessageId(_context.random().nextLong(I2NPMessage.MAX_ID_VALUE));
-        _transport.send(dsm, peer);
+        // sent below
 
         // just do this inline
         //_context.simpleTimer2().addEvent(new PublishToNewInbound(peer), 0);
@@ -780,8 +780,14 @@ class EstablishmentManager {
                 // ok, we are fine with them, send them our latest info
                 //if (_log.shouldLog(Log.INFO))
                 //    _log.info("Publishing to the peer after confirm plus delay (without banlist): " + peer);
-                sendOurInfo(peer, true);
+                // bundle the two messages together for efficiency
+                DatabaseStoreMessage dbsm = getOurInfo();
+                List<I2NPMessage> msgs = new ArrayList<I2NPMessage>(2);
+                msgs.add(dsm);
+                msgs.add(dbsm);
+                _transport.send(msgs, peer);
             } else {
+                _transport.send(dsm, peer);
                 // nuh uh.
                 if (_log.shouldLog(Log.WARN))
                     _log.warn("NOT publishing to the peer after confirm plus delay (WITH banlist): " + (hash != null ? hash.toString() : "unknown"));
@@ -828,12 +834,14 @@ class EstablishmentManager {
         _transport.setIP(remote.calculateHash(), state.getSentIP());
         
         _context.statManager().addRateData("udp.outboundEstablishTime", state.getLifetime(), 0);
+        DatabaseStoreMessage dbsm = null;
         if (!state.isFirstMessageOurDSM()) {
-            sendOurInfo(peer, false);
+            dbsm = getOurInfo();
         } else if (_log.shouldLog(Log.INFO)) {
             _log.info("Skipping publish: " + state);
         }
         
+        List<OutNetMessage> msgs = new ArrayList<OutNetMessage>(8);
         OutNetMessage msg;
         while ((msg = state.getNextQueuedMessage()) != null) {
             if (now - Router.CLOCK_FUDGE_FACTOR > msg.getExpiration()) {
@@ -841,21 +849,33 @@ class EstablishmentManager {
                 _transport.failed(msg, "Took too long to establish, but it was established");
             } else {
                 msg.timestamp("session fully established and sent");
-                _transport.send(msg);
+                msgs.add(msg);
             }
         }
+        _transport.send(dbsm, msgs, peer);
         return peer;
     }
     
+/****
     private void sendOurInfo(PeerState peer, boolean isInbound) {
         if (_log.shouldLog(Log.INFO))
             _log.info("Publishing to the peer after confirm: " + 
                       (isInbound ? " inbound con from " + peer : "outbound con to " + peer));
-        
+        DatabaseStoreMessage m = getOurInfo();
+        _transport.send(m, peer);
+    }
+****/
+    
+    /**
+     *  A database store message with our router info
+     *  @return non-null
+     *  @since 0.9.24 split from sendOurInfo()
+     */
+    private DatabaseStoreMessage getOurInfo() {
         DatabaseStoreMessage m = new DatabaseStoreMessage(_context);
         m.setEntry(_context.router().getRouterInfo());
         m.setMessageExpiration(_context.clock().now() + DATA_MESSAGE_TIMEOUT);
-        _transport.send(m, peer);
+        return m;
     }
     
     /** the relay tag is a 4-byte field in the protocol */

router/java/src/net/i2p/router/transport/udp/IntroductionManager.java

@@ -3,6 +3,7 @@ package net.i2p.router.transport.udp;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -164,6 +165,9 @@ class IntroductionManager {
      * Also, ping all idle peers that were introducers in the last 2 hours,
      * to keep the connection up, since the netDb can have quite stale information,
      * and we want to keep our introducers valid.
+     *
+     * @param ssuOptions out parameter, options are added
+     * @return number of introducers added
      */
     public int pickInbound(Properties ssuOptions, int howMany) {
         int start = _context.random().nextInt(Integer.MAX_VALUE);
@@ -440,12 +444,33 @@ class IntroductionManager {
         // and we don't read it here.
         // FIXME implement for getting Alice's IPv4 in RelayRequest sent over IPv6?
         // or is that just too easy to spoof?
-        if (!isValid(alice.getIP(), alice.getPort()) || ipSize != 0 || port != 0) {
-            if (_log.shouldLog(Log.WARN)) {
-                byte ip[] = new byte[ipSize];
-                rrReader.readIP(ip, 0);
-                _log.warn("Bad relay req from " + alice + " for " + Addresses.toString(ip, port));
+        byte[] aliceIP = alice.getIP();
+        int alicePort = alice.getPort();
+        if (!isValid(alice.getIP(), alice.getPort())) {
+            if (_log.shouldWarn())
+                _log.warn("Bad relay req from " + alice + " for " + Addresses.toString(aliceIP, alicePort));
+            _context.statManager().addRateData("udp.relayBadIP", 1);
+            return;
+        }
+        // prior to 0.9.24 we rejected any non-zero-length ip
+        // here we reject anything different
+        // TODO relay request over IPv6
+        if (ipSize != 0) {
+            byte ip[] = new byte[ipSize];
+            rrReader.readIP(ip, 0);
+            if (!Arrays.equals(aliceIP, ip)) {
+                if (_log.shouldWarn())
+                    _log.warn("Bad relay req from " + alice + " for " + Addresses.toString(ip, port));
+                _context.statManager().addRateData("udp.relayBadIP", 1);
+                return;
             }
+        }
+        // prior to 0.9.24 we rejected any nonzero port
+        // here we reject anything different
+        // TODO relay request over IPv6
+        if (port != 0 && port != alicePort) {
+            if (_log.shouldWarn())
+                _log.warn("Bad relay req from " + alice + " for " + Addresses.toString(aliceIP, port));
             _context.statManager().addRateData("udp.relayBadIP", 1);
             return;
         }

router/java/src/net/i2p/router/transport/udp/MTU.java

@@ -40,6 +40,14 @@ abstract class MTU {
             ifcs = NetworkInterface.getNetworkInterfaces();
         } catch (SocketException se) {
             return 0;
+        } catch (java.lang.Error e) {
+            // Windows, possibly when IPv6 only...
+            // https://bugs.openjdk.java.net/browse/JDK-8046500
+            // java.lang.Error: IP Helper Library GetIfTable function failed
+            //   at java.net.NetworkInterface.getAll(Native Method)
+            //   at java.net.NetworkInterface.getNetworkInterfaces(Unknown Source)
+            //   at net.i2p.util.Addresses.getAddresses ...
+            return 0;
         }
         if (ifcs != null) {
             while (ifcs.hasMoreElements()) {

router/java/src/net/i2p/router/transport/udp/OutboundEstablishState.java

@@ -109,7 +109,7 @@ class OutboundEstablishState {
      *  @param claimedAddress an IP/port based RemoteHostId, or null if unknown
      *  @param remoteHostId non-null, == claimedAddress if direct, or a hash-based one if indirect
      *  @param remotePeer must have supported sig type
-     *  @param allowExtenededOptions are we allowed to send extended options to Bob?
+     *  @param allowExtendedOptions are we allowed to send extended options to Bob?
      *  @param needIntroduction should we ask Bob to be an introducer for us?
                ignored unless allowExtendedOptions is true
      *  @param introKey Bob's introduction key, as published in the netdb

router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java

@@ -172,11 +172,12 @@ class OutboundMessageFragments {
     }
 
     /**
-     * short circuit the OutNetMessage, letting us send the establish
-     * complete message reliably
+     *  Short circuit the OutNetMessage, letting us send the establish
+     *  complete message reliably.
+     *  If you have multiple messages, use the list variant,
+     *  so the messages may be bundled efficiently.
      */
-    public void add(OutboundMessageState state) {
-        PeerState peer = state.getPeer();
+    public void add(OutboundMessageState state, PeerState peer) {
         if (peer == null)
             throw new RuntimeException("null peer for " + state);
         peer.add(state);
@@ -184,6 +185,22 @@ class OutboundMessageFragments {
         //_context.statManager().addRateData("udp.outboundActiveCount", active, 0);
     }
 
+    /**
+     *  Short circuit the OutNetMessage, letting us send multiple messages
+     *  reliably and efficiently.
+     *  @since 0.9.24
+     */
+    public void add(List<OutboundMessageState> states, PeerState peer) {
+        if (peer == null)
+            throw new RuntimeException("null peer");
+        int sz = states.size();
+        for (int i = 0; i < sz; i++) {
+            peer.add(states.get(i));
+        }
+        add(peer);
+        //_context.statManager().addRateData("udp.outboundActiveCount", active, 0);
+    }
+
     /**
      * Add the peer to the list of peers wanting to transmit something.
      * This wakes up the packet pusher if it is sleeping.
@@ -400,8 +417,10 @@ class OutboundMessageFragments {
         int fragmentsToSend = toSend.size();
         // sort by size, biggest first
         // don't bother unless more than one state (fragments are already sorted within a state)
-        if (fragmentsToSend > 1 && states.size() > 1)
-            Collections.sort(toSend, new FragmentComparator());
+        // This puts the DeliveryStatusMessage after the DatabaseStoreMessage, don't do it for now.
+        // It also undoes the ordering of the priority queue in PeerState.
+        //if (fragmentsToSend > 1 && states.size() > 1)
+        //    Collections.sort(toSend, new FragmentComparator());
 
         List<Fragment> sendNext = new ArrayList<Fragment>(Math.min(toSend.size(), 4));
         List<UDPPacket> rv = new ArrayList<UDPPacket>(toSend.size());
@@ -490,6 +509,7 @@ class OutboundMessageFragments {
      *  Biggest first
      *  @since 0.9.16
      */
+/****
     private static class FragmentComparator implements Comparator<Fragment>, Serializable {
 
         public int compare(Fragment l, Fragment r) {
@@ -497,7 +517,9 @@ class OutboundMessageFragments {
             return r.state.fragmentSize(r.num) - l.state.fragmentSize(l.num);
         }
     }
+****/
 
+    /** throttle */
     public interface ActiveThrottle {
         public void choke(Hash peer);
         public void unchoke(Hash peer);

router/java/src/net/i2p/router/transport/udp/OutboundMessageState.java

@@ -177,11 +177,14 @@ class OutboundMessageState implements CDPQEntry {
     /**
      * Note that we have pushed the message fragments.
      * Increments push count (and max sends... why?)
+     * @return true if this is the first push
      */
-    public synchronized void push() { 
+    public synchronized boolean push() { 
+        boolean rv = _pushCount == 0;
         // these will never be different...
         _pushCount++; 
         _maxSends = _pushCount;
+        return rv;
     }
 
     /**
@@ -290,6 +293,7 @@ class OutboundMessageState implements CDPQEntry {
     public String toString() {
         StringBuilder buf = new StringBuilder(256);
         buf.append("OB Message ").append(_i2npMessage.getUniqueId());
+        buf.append(" type ").append(_i2npMessage.getType());
         buf.append(" with ").append(_numFragments).append(" fragments");
         buf.append(" of size ").append(_messageBuf.length);
         buf.append(" volleys: ").append(_maxSends);
@@ -301,6 +305,7 @@ class OutboundMessageState implements CDPQEntry {
                     buf.append(i).append(' ');
             }
         }
+        //buf.append(" to: ").append(_peer.toString());
         return buf.toString();
     }
 }

router/java/src/net/i2p/router/transport/udp/PeerState.java

@@ -17,9 +17,9 @@ import net.i2p.data.Hash;
 import net.i2p.data.SessionKey;
 import net.i2p.router.OutNetMessage;
 import net.i2p.router.RouterContext;
+import net.i2p.router.util.CachedIteratorArrayList;
 import net.i2p.router.util.CoDelPriorityBlockingQueue;
 import net.i2p.router.util.PriBlockingQueue;
-import net.i2p.util.CachedIteratorArrayList;
 import net.i2p.util.Log;
 import net.i2p.util.ConcurrentHashSet;
 
@@ -198,6 +198,7 @@ class PeerState {
     /** how many dup packets were received within the last RETRANSMISSION_PERIOD_WIDTH packets */
     private int _packetsReceivedDuplicate;
     private int _packetsReceived;
+    private boolean _mayDisconnect;
     
     /** list of InboundMessageState for active message */
     private final Map<Long, InboundMessageState> _inboundMessages;
@@ -447,6 +448,7 @@ class PeerState {
      *  @return false always
      *  @deprecated unused, ECNs are never sent, always returns false
      */
+    @Deprecated
     public boolean getCurrentSecondECNReceived() { return _currentSecondECNReceived; }
 
     /** 
@@ -542,6 +544,7 @@ class PeerState {
      * connection, or null if we are not in the process of rekeying.
      * @deprecated unused
      */
+    @Deprecated
     public void setNextMACKey(SessionKey key) { _nextMACKey = key; }
 
     /** 
@@ -550,6 +553,7 @@ class PeerState {
      * of rekeying.
      * @deprecated unused
      */
+    @Deprecated
     public void setNextCipherKey(SessionKey key) { _nextCipherKey = key; }
 
     /**
@@ -569,6 +573,7 @@ class PeerState {
      * when were the current cipher and MAC keys established/rekeyed?
      * @deprecated unused
      */
+    @Deprecated
     public void setKeyEstablishedTime(long when) { _keyEstablishedTime = when; }
 
     /**
@@ -771,14 +776,23 @@ class PeerState {
     public long getIntroducerTime() { return _lastIntroducerTime; }
     public void setIntroducerTime() { _lastIntroducerTime = _context.clock().now(); }
     
-    /** we received the message specified completely */
+    /** 
+     *  We received the message specified completely.
+     *  @param bytes if less than or equal to zero, message is a duplicate.
+     */
     public void messageFullyReceived(Long messageId, int bytes) { messageFullyReceived(messageId, bytes, false); }
 
-    public synchronized void messageFullyReceived(Long messageId, int bytes, boolean isForACK) {
+    /** 
+     *  We received the message specified completely.
+     *  @param isForACK unused
+     *  @param bytes if less than or equal to zero, message is a duplicate.
+     */
+    private synchronized void messageFullyReceived(Long messageId, int bytes, boolean isForACK) {
         if (bytes > 0) {
             _receiveBytes += bytes;
             //if (isForACK)
             //    _receiveACKBytes += bytes;
+            _messagesReceived++;
         } else {
             //if (true || _retransmissionPeriodStart + 1000 < _context.clock().now()) {
                 _packetsReceivedDuplicate++;
@@ -803,7 +817,6 @@ class PeerState {
         if (_wantACKSendSince <= 0)
             _wantACKSendSince = now;
         _currentACKs.add(messageId);
-        _messagesReceived++;
     }
     
     public void messagePartiallyReceived() {
@@ -958,6 +971,7 @@ class PeerState {
      * @return non-null, possibly empty
      * @deprecated unused
      */
+    @Deprecated
     public List<ACKBitfield> retrieveACKBitfields() { return retrieveACKBitfields(true); }
 
     /**
@@ -1027,10 +1041,6 @@ class PeerState {
                 }
             }
 
-
-
-
-
         int partialIncluded = 0;
         if (bytesRemaining > 4) {
             // ok, there's room to *try* to fit in some partial ACKs, so
@@ -1170,7 +1180,6 @@ class PeerState {
                 _sendWindowBytesRemaining = _sendWindowBytes;
         //}
         
-        _messagesSent++;
         if (numSends < 2) {
             // caller synchs
             //synchronized (this) {
@@ -1274,8 +1283,23 @@ class PeerState {
     /** how skewed are the measured RTTs? */
     public synchronized int getRTTDeviation() { return _rttDeviation; }
     
-    public synchronized int getMessagesSent() { return _messagesSent; }
+    /**
+     *  I2NP messages sent.
+     *  Does not include duplicates.
+     *  As of 0.9.24, incremented when bandwidth is allocated just before sending, not when acked.
+     */
+    public int getMessagesSent() {
+        synchronized (_outboundMessages) {
+            return _messagesSent;
+        }
+    }
+    
+    /**
+     *  I2NP messages received.
+     *  As of 0.9.24, does not include duplicates.
+     */
     public synchronized int getMessagesReceived() { return _messagesReceived; }
+
     public synchronized int getPacketsTransmitted() { return _packetsTransmitted; }
     public synchronized int getPacketsRetransmitted() { return _packetsRetransmitted; }
     //public long getPacketsPeriodTransmitted() { return _packetsPeriodTransmitted; }
@@ -1339,6 +1363,7 @@ class PeerState {
     public long getLastACKSend() { return _lastACKSend; }
 
     /** @deprecated unused */
+    @Deprecated
     public void setLastACKSend(long when) { _lastACKSend = when; }
 
     public long getWantedACKSendSince() { return _wantACKSendSince; }
@@ -1498,6 +1523,18 @@ class PeerState {
         if (_dead) return 0;
         return _outboundMessages.size() + _outboundQueue.size();
     }
+
+    /**
+     * Sets to true.
+     * @since 0.9.24
+     */
+    public void setMayDisconnect() { _mayDisconnect = true; }
+
+    /**
+     * @since 0.9.24
+     */
+    public boolean getMayDisconnect() { return _mayDisconnect; }
+
     
     /**
      * Expire / complete any outbound messages
@@ -1771,7 +1808,8 @@ class PeerState {
                 if (state.getPushCount() > 0)
                     _retransmitter = state;
 
-                state.push();
+                if (state.push())
+                    _messagesSent++;
             
                 int rto = getRTO();
                 state.setNextSendTime(now + rto);
@@ -2062,8 +2100,10 @@ class PeerState {
         buf.append(" cwin: ").append(_sendWindowBytes);
         buf.append(" acwin: ").append(_sendWindowBytesRemaining);
         buf.append(" consecFail: ").append(_consecutiveFailedSends);
-        buf.append(" recv OK/Dup: ").append(_packetsReceived).append('/').append(_packetsReceivedDuplicate);
-        buf.append(" send OK/Dup: ").append(_packetsTransmitted).append('/').append(_packetsRetransmitted);
+        buf.append(" msgs rcvd: ").append(_messagesReceived);
+        buf.append(" msgs sent: ").append(_messagesSent);
+        buf.append(" pkts rcvd OK/Dup: ").append(_packetsReceived).append('/').append(_packetsReceivedDuplicate);
+        buf.append(" pkts sent OK/Dup: ").append(_packetsTransmitted).append('/').append(_packetsRetransmitted);
         buf.append(" IBM: ").append(_inboundMessages.size());
         buf.append(" OBQ: ").append(_outboundQueue.size());
         buf.append(" OBL: ").append(_outboundMessages.size());

router/java/src/net/i2p/router/transport/udp/UDPAddress.java

@@ -68,7 +68,7 @@ class UDPAddress {
             _introKey = null;
             return;
         }
-        _host = addr.getOption(PROP_HOST);
+        _host = addr.getHost();
         _port = addr.getPort();
         try { 
             String mtu = addr.getOption(PROP_MTU);

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -331,6 +331,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             String fixedHost = _context.getProperty(PROP_EXTERNAL_HOST);
             if (fixedHost != null && fixedHost.length() > 0) {
                 try {
+                    // TODO getAllByName(), bind to each
                     String testAddr = InetAddress.getByName(fixedHost).getHostAddress();
                     if (Addresses.getAddresses().contains(testAddr))
                         bindTo = testAddr;
@@ -733,8 +734,16 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             _log.warn("Received address: " + Addresses.toString(ip, port) + " from: " + source);
         if (ip == null)
             return;
+        // this is essentially isValid(ip), but we can't use that because
+        // _haveIPv6Address is not set yet
+        if (!(isPubliclyRoutable(ip) || allowLocal())) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Invalid address: " + Addresses.toString(ip, port) + " from: " + source);
+            return;
+        }
         if (source == SOURCE_INTERFACE && ip.length == 16) {
-            // must be set before isValid() call
+            // NOW we can set it, it's a valid v6 address
+            // (we don't want to set this for Teredo, 6to4, ...)
             _haveIPv6Address = true;
         }
         if (explicitAddressSpecified())
@@ -742,11 +751,6 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         String sources = _context.getProperty(PROP_SOURCES, DEFAULT_SOURCES);
         if (!sources.contains(source.toConfigString()))
             return;
-        if (!isValid(ip)) {
-            if (_log.shouldLog(Log.WARN))
-                _log.warn("Invalid address: " + Addresses.toString(ip, port) + " from: " + source);
-            return;
-        }
         if (!isAlive()) {
             if (source == SOURCE_INTERFACE || source == SOURCE_UPNP) {
                 try {
@@ -1824,14 +1828,74 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
     }
 
     /**
-     *  "injected" message from the EstablishmentManager
+     *  "injected" message from the EstablishmentManager.
+     *  If you have multiple messages, use the list variant,
+     *  so the messages may be bundled efficiently.
+     *
+     *  @param peer all messages MUST be going to this peer
      */
     void send(I2NPMessage msg, PeerState peer) {
         try {
             OutboundMessageState state = new OutboundMessageState(_context, msg, peer);
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Injecting a data message to a new peer: " + peer);
-            _fragments.add(state);
+            _fragments.add(state, peer);
+        } catch (IllegalArgumentException iae) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Shouldnt happen", new Exception("I did it"));
+        }
+    }
+
+    /**
+     *  "injected" message from the EstablishmentManager,
+     *  plus pending messages to send,
+     *  so the messages may be bundled efficiently.
+     *  Called at end of outbound establishment.
+     *
+     *  @param msg may be null if nothing to inject
+     *  @param msgs non-null, may be empty
+     *  @param peer all messages MUST be going to this peer
+     *  @since 0.9.24
+     */
+    void send(I2NPMessage msg, List<OutNetMessage> msgs, PeerState peer) {
+        try {
+            int sz = msgs.size();
+            List<OutboundMessageState> states = new ArrayList<OutboundMessageState>(sz + 1);
+            if (msg != null) {
+                OutboundMessageState state = new OutboundMessageState(_context, msg, peer);
+                states.add(state);
+            }
+            for (int i = 0; i < sz; i++) {
+                OutboundMessageState state = new OutboundMessageState(_context, msgs.get(i), peer);
+                states.add(state);
+            }
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Injecting " + states.size() + " data messages to a new peer: " + peer);
+            _fragments.add(states, peer);
+        } catch (IllegalArgumentException iae) {
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Shouldnt happen", new Exception("I did it"));
+        }
+    }
+
+    /**
+     *  "injected" messages from the EstablishmentManager.
+     *  Called at end of inbound establishment.
+     *
+     *  @param peer all messages MUST be going to this peer
+     *  @since 0.9.24
+     */
+    void send(List<I2NPMessage> msgs, PeerState peer) {
+        try {
+            int sz = msgs.size();
+            List<OutboundMessageState> states = new ArrayList<OutboundMessageState>(sz);
+            for (int i = 0; i < sz; i++) {
+                OutboundMessageState state = new OutboundMessageState(_context, msgs.get(i), peer);
+                states.add(state);
+            }
+            if (_log.shouldLog(Log.DEBUG))
+                _log.debug("Injecting " + sz + " data messages to a new peer: " + peer);
+            _fragments.add(states, peer);
         } catch (IllegalArgumentException iae) {
             if (_log.shouldLog(Log.WARN))
                 _log.warn("Shouldnt happen", new Exception("I did it"));
@@ -2428,6 +2492,22 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         return peer != null && peer.isBacklogged();
     }
 
+    /**
+     * Tell the transport that we may disconnect from this peer.
+     * This is advisory only.
+     *
+     * @since 0.9.24
+     */
+    @Override
+    public void mayDisconnect(final Hash peer) {
+        final PeerState ps =  _peersByIdent.get(peer);
+        if (ps != null && ps.isInbound() &&
+            ps.getWeRelayToThemAs() <= 0 &&
+            ps.getMessagesReceived() <= 2 && ps.getMessagesSent() <= 2) {
+            ps.setMayDisconnect();
+        }
+    }
+
     public boolean allowConnection() {
             return _peersByIdent.size() < getMaxConnections();
     }
@@ -2674,8 +2754,8 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             //buf.append(peer.getMTUDecreases());
             buf.append("</td>");
         
-            long sent = peer.getPacketsTransmitted();
-            long recv = peer.getPacketsReceived();
+            long sent = peer.getMessagesSent();
+            long recv = peer.getMessagesReceived();
             
             buf.append("<td class=\"cells\" align=\"right\">");
             buf.append(sent);
@@ -2816,6 +2896,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         private static final long LONG_LOOP_TIME = 25*1000;
         private static final long EXPIRE_INCREMENT = 15*1000;
         private static final long EXPIRE_DECREMENT = 45*1000;
+        private static final long MAY_DISCON_TIMEOUT = 10*1000;
 
         public ExpirePeerEvent() {
             super(_context.simpleTimer2());
@@ -2825,7 +2906,8 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
 
         public void timeReached() {
             // Increase allowed idle time if we are well under allowed connections, otherwise decrease
-            if (haveCapacity(33)) {
+            boolean haveCap = haveCapacity(33);
+            if (haveCap) {
                 long inc;
                 // don't adjust too quickly if we are looping fast
                 if (_lastLoopShort)
@@ -2844,6 +2926,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             long now = _context.clock().now();
             long shortInactivityCutoff = now - _expireTimeout;
             long longInactivityCutoff = now - EXPIRE_TIMEOUT;
+            final long mayDisconCutoff = now - MAY_DISCON_TIMEOUT;
             long pingCutoff = now - (2 * 60*60*1000);
             long pingFirewallCutoff = now - PING_FIREWALL_CUTOFF;
             boolean shouldPingFirewall = _reachabilityStatus != Status.OK;
@@ -2858,10 +2941,14 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
                     PeerState peer = iter.next();
                     long inactivityCutoff;
                     // if we offered to introduce them, or we used them as introducer in last 2 hours
-                    if (peer.getWeRelayToThemAs() > 0 || peer.getIntroducerTime() > pingCutoff)
+                    if (peer.getWeRelayToThemAs() > 0 || peer.getIntroducerTime() > pingCutoff) {
                         inactivityCutoff = longInactivityCutoff;
-                    else
+                    } else if (!haveCap && peer.getMayDisconnect() &&
+                               peer.getMessagesReceived() <= 2 && peer.getMessagesSent() <= 2) {
+                        inactivityCutoff = mayDisconCutoff;
+                    } else {
                         inactivityCutoff = shortInactivityCutoff;
+                    }
                     if ( (peer.getLastReceiveTime() < inactivityCutoff) && (peer.getLastSendTime() < inactivityCutoff) ) {
                         _expireBuffer.add(peer);
                         iter.remove();

router/java/src/net/i2p/router/tunnel/BuildMessageGenerator.java

@@ -54,6 +54,9 @@ public abstract class BuildMessageGenerator {
      * containing the hop's configuration (as well as the reply info, if it is an outbound endpoint)
      *
      * @param msg out parameter
+     * @param peerKey Encrypt using this key.
+     *                If null, replyRouter and replyTunnel are ignored,
+     *                and the entire record is filled with random data
      * @throws IllegalArgumentException if hop bigger than config
      */
     public static void createRecord(int recordNum, int hop, TunnelBuildMessage msg,

router/java/src/net/i2p/router/tunnel/BuildMessageProcessor.java

@@ -10,23 +10,72 @@ import net.i2p.data.SessionKey;
 import net.i2p.data.i2np.BuildRequestRecord;
 import net.i2p.data.i2np.EncryptedBuildRecord;
 import net.i2p.data.i2np.TunnelBuildMessage;
+import net.i2p.router.RouterThrottleImpl;
 import net.i2p.router.util.DecayingBloomFilter;
 import net.i2p.router.util.DecayingHashSet;
 import net.i2p.util.Log;
+import net.i2p.util.SystemVersion;
 
 /**
  * Receive the build message at a certain hop, decrypt its encrypted record,
  * read the enclosed tunnel request, decide how to reply, write the reply,
  * encrypt the reply record, and return a TunnelBuildMessage to forward on to
- * the next hop
+ * the next hop.
+ *
+ * There is only one of these.
+ * Instantiated by BuildHandler.
+ *
  */
 public class BuildMessageProcessor {
+    private final I2PAppContext ctx;
+    private final Log log;
     private final DecayingBloomFilter _filter;
     
     public BuildMessageProcessor(I2PAppContext ctx) {
-        _filter = new DecayingHashSet(ctx, 60*1000, 32, "TunnelBMP");
+        this.ctx = ctx;
+        log = ctx.logManager().getLog(getClass());
+        _filter = selectFilter();
         // all createRateStat in TunnelDispatcher
     }
+
+    /**
+     *  For N typical part tunnels and rejecting 50%, that's 12N requests per hour.
+     *  This is the equivalent of (12N/600) KBps through the IVValidator filter.
+     *
+     *  Target false positive rate is 1E-5 or lower
+     *
+     *  @since 0.9.24
+     */
+    private DecayingBloomFilter selectFilter() {
+        long maxMemory = SystemVersion.getMaxMemory();
+        int m;
+        if (SystemVersion.isAndroid() || SystemVersion.isARM() || maxMemory < 96*1024*1024L) {
+            // 32 KB
+            // appx 500 part. tunnels or 6K req/hr
+            m = 17;
+        } else if (ctx.getProperty(RouterThrottleImpl.PROP_MAX_TUNNELS, RouterThrottleImpl.DEFAULT_MAX_TUNNELS) >
+                   RouterThrottleImpl.DEFAULT_MAX_TUNNELS && maxMemory > 256*1024*1024L) {
+            // 2 MB
+            // appx 20K part. tunnels or 240K req/hr
+            m = 23;
+        } else if (maxMemory > 256*1024*1024L) {
+            // 1 MB
+            // appx 10K part. tunnels or 120K req/hr
+            m = 22;
+        } else if (maxMemory > 128*1024*1024L) {
+            // 512 KB
+            // appx 5K part. tunnels or 60K req/hr
+            m = 21;
+        } else {
+            // 128 KB
+            // appx 2K part. tunnels or 24K req/hr
+            m = 19;
+        }
+        if (log.shouldInfo())
+            log.info("Selected Bloom filter m = " + m);
+        return new DecayingBloomFilter(ctx, 60*60*1000, 32, "TunnelBMP", m);
+    }
+
     /**
      * Decrypt the record targetting us, encrypting all of the other records with the included 
      * reply key and IV.  The original, encrypted record targetting us is removed from the request
@@ -38,65 +87,75 @@ public class BuildMessageProcessor {
      *
      * @return the current hop's decrypted record or null on failure
      */
-    public BuildRequestRecord decrypt(I2PAppContext ctx, TunnelBuildMessage msg, Hash ourHash, PrivateKey privKey) {
-        Log log = ctx.logManager().getLog(getClass());
+    public BuildRequestRecord decrypt(TunnelBuildMessage msg, Hash ourHash, PrivateKey privKey) {
         BuildRequestRecord rv = null;
         int ourHop = -1;
         long beforeActualDecrypt = 0;
         long afterActualDecrypt = 0;
-        long totalEq = 0;
-        long totalDup = 0;
+        byte[] ourHashData = ourHash.getData();
         long beforeLoop = System.currentTimeMillis();
         for (int i = 0; i < msg.getRecordCount(); i++) {
             EncryptedBuildRecord rec = msg.getRecord(i);
             int len = BuildRequestRecord.PEER_SIZE;
-            long beforeEq = System.currentTimeMillis();
-            boolean eq = DataHelper.eq(ourHash.getData(), 0, rec.getData(), 0, len);
-            totalEq += System.currentTimeMillis()-beforeEq;
+            boolean eq = DataHelper.eq(ourHashData, 0, rec.getData(), 0, len);
             if (eq) {
-                long beforeIsDup = System.currentTimeMillis();
-                boolean isDup = _filter.add(rec.getData(), len, 32);
-                totalDup += System.currentTimeMillis()-beforeIsDup;
-                if (isDup) {
-                    if (log.shouldLog(Log.WARN))
-                        log.debug(msg.getUniqueId() + ": A record matching our hash was found, but it seems to be a duplicate");
-                    ctx.statManager().addRateData("tunnel.buildRequestDup", 1);
-                    return null;
-                }
                 beforeActualDecrypt = System.currentTimeMillis();
                 try {
-                    BuildRequestRecord req = new BuildRequestRecord(ctx, privKey, rec);
+                    rv = new BuildRequestRecord(ctx, privKey, rec);
+                    afterActualDecrypt = System.currentTimeMillis();
+
+                    // i2pd bug
+                    boolean isBad = SessionKey.INVALID_KEY.equals(rv.readReplyKey());
+                    if (isBad) {
+                        if (log.shouldLog(Log.WARN))
+                            log.warn(msg.getUniqueId() + ": Bad reply key: " + rv);
+                        ctx.statManager().addRateData("tunnel.buildRequestBadReplyKey", 1);
+                        return null;
+                    }
+
+                    // The spec says to feed the 32-byte AES-256 reply key into the Bloom filter.
+                    // But we were using the first 32 bytes of the encrypted reply.
+                    // Fixed in 0.9.24
+                    boolean isDup = _filter.add(rv.getData(), BuildRequestRecord.OFF_REPLY_KEY, 32);
+                    if (isDup) {
+                        if (log.shouldLog(Log.WARN))
+                            log.warn(msg.getUniqueId() + ": Dup record: " + rv);
+                        ctx.statManager().addRateData("tunnel.buildRequestDup", 1);
+                        return null;
+                    }
+
                     if (log.shouldLog(Log.DEBUG))
-                        log.debug(msg.getUniqueId() + ": A record matching our hash was found and decrypted");
-                    rv = req;
+                        log.debug(msg.getUniqueId() + ": Matching record: " + rv);
+                    ourHop = i;
+                    // TODO should we keep looking for a second match and fail if found?
+                    break;
                 } catch (DataFormatException dfe) {
-                    if (log.shouldLog(Log.DEBUG))
-                        log.debug(msg.getUniqueId() + ": A record matching our hash was found, but could not be decrypted");
-                    return null; // our hop is invalid?  b0rkage
+                    if (log.shouldLog(Log.WARN))
+                        log.warn(msg.getUniqueId() + ": Matching record decrypt failure", dfe);
+                    // on the microscopic chance that there's another router
+                    // out there with the same first 16 bytes, go around again
+                    continue;
                 }
-                afterActualDecrypt = System.currentTimeMillis();
-                ourHop = i;
             }
         }
         if (rv == null) {
             // none of the records matched, b0rk
-            if (log.shouldLog(Log.DEBUG))
-                log.debug(msg.getUniqueId() + ": No records matching our hash was found");
+            if (log.shouldLog(Log.WARN))
+                log.warn(msg.getUniqueId() + ": No matching record");
             return null;
         }
         
         long beforeEncrypt = System.currentTimeMillis();
         SessionKey replyKey = rv.readReplyKey();
         byte iv[] = rv.readReplyIV();
-        int ivOff = 0;
         for (int i = 0; i < msg.getRecordCount(); i++) {
             if (i != ourHop) {
                 EncryptedBuildRecord data = msg.getRecord(i);
-                if (log.shouldLog(Log.DEBUG))
-                    log.debug("Encrypting record " + i + "/? with replyKey " + replyKey.toBase64() + "/" + Base64.encode(iv, ivOff, 16));
-                // corrupts SDS
-                ctx.aes().encrypt(data.getData(), 0, data.getData(), 0, replyKey, 
-                                  iv, ivOff, data.length());
+                //if (log.shouldLog(Log.DEBUG))
+                //    log.debug("Encrypting record " + i + "/? with replyKey " + replyKey.toBase64() + "/" + Base64.encode(iv));
+                // encrypt in-place, corrupts SDS
+                byte[] bytes = data.getData();
+                ctx.aes().encrypt(bytes, 0, bytes, 0, replyKey, iv, 0, EncryptedBuildRecord.LENGTH);
             }
         }
         long afterEncrypt = System.currentTimeMillis();
@@ -106,8 +165,6 @@ public class BuildMessageProcessor {
                 log.warn("Slow decryption, total=" + (afterEncrypt-beforeLoop) 
                          + " looping=" + (beforeEncrypt-beforeLoop)
                          + " decrypt=" + (afterActualDecrypt-beforeActualDecrypt)
-                         + " eq=" + totalEq
-                         + " dup=" + totalDup
                          + " encrypt=" + (afterEncrypt-beforeEncrypt));
         }
         return rv;

router/java/src/net/i2p/router/tunnel/HopConfig.java

@@ -26,6 +26,7 @@ public class HopConfig {
 
     // these 4 were longs, let's save some space
     // 2 billion * 1KB / 10 minutes = 3 GBps in a single tunnel
+    // we use synchronization instead of an AtomicInteger here to save space
     private int _messagesProcessed;
     private int _oldMessagesProcessed;
     //private int _messagesSent;
@@ -127,12 +128,29 @@ public class HopConfig {
     /**
      *  Take note of a message being pumped through this tunnel.
      *  "processed" is for incoming and "sent" is for outgoing (could be dropped in between)
+     *  We use synchronization instead of an AtomicInteger here to save space.
      */
-    public void incrementProcessedMessages() { _messagesProcessed++; }
+    public synchronized void incrementProcessedMessages() { _messagesProcessed++; }
 
-    public int getProcessedMessagesCount() { return _messagesProcessed; }
+    public synchronized int getProcessedMessagesCount() { return _messagesProcessed; }
 
-    public int getRecentMessagesCount() {
+    /**
+     *  This returns the number of processed messages since
+     *  the last time getAndResetRecentMessagesCount() was called.
+     *  As of 0.9.23, does NOT reset the count, see getAndResetRecentMessagesCount().
+     */
+    public synchronized int getRecentMessagesCount() {
+        return _messagesProcessed - _oldMessagesProcessed;
+    }
+
+    /**
+     *  This returns the number of processed messages since the last time this was called,
+     *  and resets the count. It should only be called by code that updates the router stats.
+     *  See TunnelDispatcher.updateParticipatingStats().
+     *
+     *  @since 0.9.23
+     */
+    synchronized int getAndResetRecentMessagesCount() {
         int rv = _messagesProcessed - _oldMessagesProcessed;
         _oldMessagesProcessed = _messagesProcessed;
         return rv;
@@ -171,8 +189,9 @@ public class HopConfig {
         }
         
         buf.append(" exp. ").append(TunnelCreatorConfig.format(_expiration));
-        if (_messagesProcessed > 0)
-            buf.append(" used ").append(_messagesProcessed).append("KB");
+        int messagesProcessed = getProcessedMessagesCount();
+        if (messagesProcessed > 0)
+            buf.append(" used ").append(messagesProcessed).append("KB");
         return buf.toString();
     }
 }

router/java/src/net/i2p/router/tunnel/HopProcessor.java

@@ -31,10 +31,6 @@ class HopProcessor {
     //static final boolean USE_DOUBLE_IV_ENCRYPTION = true;
     static final int IV_LENGTH = 16;
     
-    /** @deprecated unused */
-    public HopProcessor(I2PAppContext ctx, HopConfig config) {
-        this(ctx, config, createValidator());
-    }
 
     public HopProcessor(I2PAppContext ctx, HopConfig config, IVValidator validator) {
         _context = ctx;
@@ -43,12 +39,6 @@ class HopProcessor {
         _validator = validator;
     }
     
-    /** @deprecated unused */
-    protected static IVValidator createValidator() { 
-        // yeah, we'll use an O(1) validator later (e.g. bloom filter)
-        return new HashSetIVValidator();
-    }
-    
     /**
      * Process the data for the current hop, overwriting the original data with
      * what should be sent to the next peer.  This also validates the previous 

router/java/src/net/i2p/router/tunnel/TunnelCreatorConfig.java

@@ -152,7 +152,6 @@ public class TunnelCreatorConfig implements TunnelInfo {
 
     /**
      *  This calls profile manager tunnelDataPushed1m() for each peer
-     *  @return null for exploratory
      */
     public synchronized void incrementVerifiedBytesTransferred(int bytes) { 
         _verifiedBytesTransferred += bytes; 

router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java

@@ -163,9 +163,14 @@ public class TunnelDispatcher implements Service {
         ctx.statManager().createRateStat("tunnel.participatingMessageDropped", 
                                          "Dropped for exceeding share limit", "Tunnels", 
                                          new long[] { 60*1000l, 60*10*1000l });
+        // count for console
         ctx.statManager().createRequiredRateStat("tunnel.participatingMessageCount", 
                                          "Number of 1KB participating messages", "Tunnels", 
                                          new long[] { 60*1000l, 60*10*1000l, 60*60*1000l });
+        // estimate for RouterThrottleImpl
+        ctx.statManager().createRequiredRateStat("tunnel.participatingMessageCountAvgPerTunnel", 
+                                         "Estimate of participating messages per tunnel lifetime", "Tunnels", 
+                                         new long[] { 60*1000l });
         ctx.statManager().createRateStat("tunnel.ownedMessageCount", 
                                          "How many messages are sent through a tunnel we created (period == failures)?", "Tunnels", 
                                          new long[] { 60*1000l, 10*60*1000l, 60*60*1000l });
@@ -199,6 +204,7 @@ public class TunnelDispatcher implements Service {
         ctx.statManager().createRateStat("tunnel.participantLookupSuccess", "Was a deferred lookup successful?", "Tunnels", new long[] { 60*60*1000 });
         // following is for BuildMessageProcessor
         ctx.statManager().createRateStat("tunnel.buildRequestDup", "How frequently we get dup build request messages", "Tunnels", new long[] { 60*60*1000 });
+        ctx.statManager().createRateStat("tunnel.buildRequestBadReplyKey", "Build requests with bad reply keys", "Tunnels", new long[] { 60*60*1000 });
         // following are for FragmentHandler
         ctx.statManager().createRateStat("tunnel.smallFragments", "How many pad bytes are in small fragments?", 
                                               "Tunnels", RATES);
@@ -445,8 +451,8 @@ public class TunnelDispatcher implements Service {
     public void remove(TunnelCreatorConfig cfg) {
         if (cfg.isInbound()) {
             TunnelId recvId = cfg.getConfig(cfg.getLength()-1).getReceiveTunnel();
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("removing our own inbound " + cfg);
+            if (_log.shouldLog(Log.INFO))
+                _log.info("removing our own inbound " + cfg);
             TunnelParticipant participant = _participants.remove(recvId);
             if (participant == null) {
                 _inboundGateways.remove(recvId);
@@ -464,8 +470,8 @@ public class TunnelDispatcher implements Service {
                 }
             }
         } else {
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("removing our own outbound " + cfg);
+            if (_log.shouldLog(Log.INFO))
+                _log.info("removing our own outbound " + cfg);
             TunnelId outId = cfg.getConfig(0).getSendTunnel();
             TunnelGateway gw = _outboundGateways.remove(outId);
             if (gw != null) {
@@ -492,8 +498,8 @@ public class TunnelDispatcher implements Service {
         
         boolean removed = (null != _participatingConfig.remove(recvId));
         if (removed) {
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("removing " + cfg /* , new Exception() */ );
+            if (_log.shouldLog(Log.INFO))
+                _log.info("removing " + cfg /* , new Exception() */ );
         } else {
             // this is normal, this can get called twice
             if (_log.shouldLog(Log.DEBUG))
@@ -598,7 +604,7 @@ public class TunnelDispatcher implements Service {
                            + " messageId " + msg.getUniqueId()
                            + "/" + msg.getMessage().getUniqueId()
                            + " messageType: " + msg.getMessage().getClass().getSimpleName()
-                           + " existing = " + _inboundGateways.size(), new Exception("source"));
+                           + " existing = " + _inboundGateways.size());
         }
         
         //long dispatchTime = _context.clock().now() - before;
@@ -712,7 +718,7 @@ public class TunnelDispatcher implements Service {
         long tooYoung = _context.clock().now() - 60*1000;
         long tooOld = tooYoung - 9*60*1000;
         for (HopConfig cfg : _participatingConfig.values()) {
-            long c = cfg.getRecentMessagesCount();
+            long c = cfg.getAndResetRecentMessagesCount();
             bw += c;
             //bwOut += cfg.getRecentSentMessagesCount();
             long created = cfg.getCreation();
@@ -721,9 +727,15 @@ public class TunnelDispatcher implements Service {
             tcount++;
             count += c;
         }
+        // This is an estimate of the average number of participating messages per tunnel
+        // in a tunnel lifetime, used only by RouterThrottleImpl
+        // 10 minutes / 50 seconds = 12
         if (tcount > 0)
-            count = count * 30 / tcount;
-        _context.statManager().addRateData("tunnel.participatingMessageCount", count, ms);
+            count = count * (10*60*1000 / ms) / tcount;
+        _context.statManager().addRateData("tunnel.participatingMessageCountAvgPerTunnel", count, ms);
+        // This is a straight count of the total participating messages, used in the router console
+        _context.statManager().addRateData("tunnel.participatingMessageCount", bw, ms);
+        // Bandwidth in bits per second
         _context.statManager().addRateData("tunnel.participatingBandwidth", bw*1024/(ms/1000), ms);
         // moved to FIFOBandwidthRefiller
         //_context.statManager().addRateData("tunnel.participatingBandwidthOut", bwOut*1024/(ms/1000), ms);

router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java

@@ -47,6 +47,10 @@ import net.i2p.util.Log;
  * it used to be called from the BuildExecutor thread loop.
  *
  * Note that 10 minute tunnel expiration is hardcoded in here.
+ *
+ * There is only one of these objects but there may be multiple
+ * threads running it. Instantiated and started by TunnelPoolManager.
+ *
  */
 class BuildHandler implements Runnable {
     private final RouterContext _context;
@@ -122,6 +126,7 @@ class BuildHandler implements Runnable {
         _context.statManager().createRequiredRateStat("tunnel.rejectHopThrottle", "Reject per-hop limit", "Tunnels", new long[] { 60*60*1000 });
         _context.statManager().createRequiredRateStat("tunnel.dropReqThrottle", "Drop per-hop limit", "Tunnels", new long[] { 60*60*1000 });
         _context.statManager().createRequiredRateStat("tunnel.dropLookupThrottle", "Drop next hop lookup", "Tunnels", new long[] { 60*60*1000 });
+        _context.statManager().createRateStat("tunnel.dropDecryptFail", "Can't find our slot", "Tunnels", new long[] { 60*60*1000 });
 
         _context.statManager().createRequiredRateStat("tunnel.rejectOverloaded", "Delay to process rejected request (ms)", "Tunnels", new long[] { 60*1000, 10*60*1000 });
         _context.statManager().createRequiredRateStat("tunnel.acceptLoad", "Delay to process accepted request (ms)", "Tunnels", new long[] { 60*1000, 10*60*1000 });
@@ -443,9 +448,13 @@ class BuildHandler implements Runnable {
      */
     private long handleRequest(BuildMessageState state) {
         long timeSinceReceived = _context.clock().now()-state.recvTime;
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug(state.msg.getUniqueId() + ": handling request after " + timeSinceReceived);
+        //if (_log.shouldLog(Log.DEBUG))
+        //    _log.debug(state.msg.getUniqueId() + ": handling request after " + timeSinceReceived);
         
+        Hash from = state.fromHash;
+        if (from == null && state.from != null)
+            from = state.from.calculateHash();
+
         if (timeSinceReceived > (BuildRequestor.REQUEST_TIMEOUT*3)) {
             // don't even bother, since we are so overloaded locally
             _context.throttle().setTunnelStatus(_x("Dropping tunnel requests: Overloaded"));
@@ -453,21 +462,27 @@ class BuildHandler implements Runnable {
                 _log.warn("Not even trying to handle/decrypt the request " + state.msg.getUniqueId() 
                            + ", since we received it a long time ago: " + timeSinceReceived);
             _context.statManager().addRateData("tunnel.dropLoadDelay", timeSinceReceived);
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return -1;
         }
         // ok, this is not our own tunnel, so we need to do some heavy lifting
         // this not only decrypts the current hop's record, but encrypts the other records
         // with the enclosed reply key
         long beforeDecrypt = System.currentTimeMillis();
-        BuildRequestRecord req = _processor.decrypt(_context, state.msg, _context.routerHash(), _context.keyManager().getPrivateKey());
+        BuildRequestRecord req = _processor.decrypt(state.msg, _context.routerHash(), _context.keyManager().getPrivateKey());
         long decryptTime = System.currentTimeMillis() - beforeDecrypt;
         _context.statManager().addRateData("tunnel.decryptRequestTime", decryptTime);
         if (decryptTime > 500 && _log.shouldLog(Log.WARN))
             _log.warn("Took too long to decrypt the request: " + decryptTime + " for message " + state.msg.getUniqueId() + " received " + (timeSinceReceived+decryptTime) + " ago");
         if (req == null) {
             // no records matched, or the decryption failed.  bah
-            if (_log.shouldLog(Log.WARN))
-                _log.warn("The request " + state.msg.getUniqueId() + " could not be decrypted");
+            if (_log.shouldLog(Log.WARN)) {
+                _log.warn("The request " + state.msg.getUniqueId() + " could not be decrypted from: " + from);
+            }
+            _context.statManager().addRateData("tunnel.dropDecryptFail", 1);
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return -1;
         }
 
@@ -477,7 +492,7 @@ class BuildHandler implements Runnable {
         RouterInfo nextPeerInfo = _context.netDb().lookupRouterInfoLocally(nextPeer);
         long lookupTime = System.currentTimeMillis()-beforeLookup;
         if (lookupTime > 500 && _log.shouldLog(Log.WARN))
-            _log.warn("Took too long to lookup the request: " + lookupTime + "/" + readPeerTime + " for message " + state.msg.getUniqueId() + " received " + (timeSinceReceived+decryptTime) + " ago");
+            _log.warn("Took too long to lookup the request: " + lookupTime + "/" + readPeerTime + " for " + req);
         if (nextPeerInfo == null) {
             // limit concurrent next-hop lookups to prevent job queue overload attacks
             int numTunnels = _context.tunnelManager().getParticipatingCount();
@@ -485,7 +500,7 @@ class BuildHandler implements Runnable {
             int current = _currentLookups.incrementAndGet();
             if (current <= limit) {
                 if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Request " + state.msg.getUniqueId() + '/' + req.readReceiveTunnelId() + '/' + req.readNextTunnelId() 
+                    _log.debug("Request " + req
                                + " handled, lookup next peer " + nextPeer
                                + " lookups: " + current + '/' + limit);
                 _context.netDb().lookupRouterInfo(nextPeer, new HandleReq(_context, state, req, nextPeer),
@@ -493,16 +508,18 @@ class BuildHandler implements Runnable {
             } else {
                 _currentLookups.decrementAndGet();
                 if (_log.shouldLog(Log.WARN))
-                    _log.warn("Drop next hop lookup, limit " + limit);
+                    _log.warn("Drop next hop lookup, limit " + limit + ": " + req);
                 _context.statManager().addRateData("tunnel.dropLookupThrottle", 1);
             }
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return -1;
         } else {
             long beforeHandle = System.currentTimeMillis();
             handleReq(nextPeerInfo, state, req, nextPeer);
             long handleTime = System.currentTimeMillis() - beforeHandle;
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Request " + state.msg.getUniqueId() + " handled and we know the next peer " 
+                _log.debug("Request " + req + " handled and we know the next peer " 
                            + nextPeer + " after " + handleTime
                            + "/" + decryptTime + "/" + lookupTime + "/" + timeSinceReceived);
             return handleTime;
@@ -543,7 +560,7 @@ class BuildHandler implements Runnable {
             // decrement in-progress counter
             _currentLookups.decrementAndGet();
             if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Request " + _state.msg.getUniqueId() + " handled with a successful deferred lookup for the next peer " + _nextPeer);
+                _log.debug("Request " + _state.msg.getUniqueId() + " handled with a successful deferred lookup: " + _req);
 
             RouterInfo ri = getContext().netDb().lookupRouterInfoLocally(_nextPeer);
             if (ri != null) {
@@ -551,7 +568,7 @@ class BuildHandler implements Runnable {
                 getContext().statManager().addRateData("tunnel.buildLookupSuccess", 1);
             } else {
                 if (_log.shouldLog(Log.WARN))
-                    _log.warn("Deferred successfully, but we couldnt find " + _nextPeer);
+                    _log.warn("Deferred successfully, but we couldnt find " + _nextPeer + "? " + _req);
                 getContext().statManager().addRateData("tunnel.buildLookupSuccess", 0);
             }
         }
@@ -576,15 +593,15 @@ class BuildHandler implements Runnable {
             _currentLookups.decrementAndGet();
             getContext().statManager().addRateData("tunnel.rejectTimeout", 1);
             getContext().statManager().addRateData("tunnel.buildLookupSuccess", 0);
-            // logging commented out so class can be static
-            //if (_log.shouldLog(Log.WARN))
-            //    _log.warn("Request " + _state.msg.getUniqueId() 
-            //              + " could no be satisfied, as the next peer could not be found: " + _nextPeer.toBase64());
+            if (_log.shouldLog(Log.WARN))
+                _log.warn("Next hop lookup failure: " + _req);
 
             // ???  should we blame the peer here?   getContext().profileManager().tunnelTimedOut(_nextPeer);
             getContext().messageHistory().tunnelRejected(_state.fromHash, new TunnelId(_req.readReceiveTunnelId()), _nextPeer, 
-                                                         "rejected because we couldn't find " + _nextPeer + ": " +
-                                                         _state.msg.getUniqueId() + "/" + _req.readNextTunnelId());
+                                                         // this is all disabled anyway
+                                                         //"rejected because we couldn't find " + _nextPeer + ": " +
+                                                         //_state.msg.getUniqueId() + "/" + _req.readNextTunnelId());
+                                                         "lookup fail");
         }
     }
     
@@ -629,9 +646,17 @@ class BuildHandler implements Runnable {
         boolean isInGW = req.readIsInboundGateway();
         boolean isOutEnd = req.readIsOutboundEndpoint();
 
+        Hash from = state.fromHash;
+        if (from == null && state.from != null)
+            from = state.from.calculateHash();
+        // warning, from could be null, but it should only
+        // happen if we will be a IBGW and it came from us as a OBEP
+
         if (isInGW && isOutEnd) {
             _context.statManager().addRateData("tunnel.rejectHostile", 1);
-            _log.error("Dropping build request, IBGW+OBEP");
+            _log.error("Dropping build request, IBGW+OBEP: " + req);
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return;
         }
 
@@ -642,31 +667,30 @@ class BuildHandler implements Runnable {
             // No way to recognize if we are every other hop, but see below
             // old i2pd
             if (_log.shouldWarn())
-                _log.warn("Dropping build request, we are the next hop");
+                _log.warn("Dropping build request, we are the next hop: " + req);
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return;
         }
-        // previous test should be sufficient to keep it from getting here but maybe not?
         if (!isInGW) {
-            Hash from = state.fromHash;
-            if (from == null)
-                from = state.from.calculateHash();
-            if (_context.routerHash().equals(from)) {
+            // if from is null, it came via OutboundMessageDistributor.distribute(),
+            // i.e. we were the OBEP, which is fine if we're going to be an IBGW
+            // but if not, something is seriously wrong here.
+            if (from == null || _context.routerHash().equals(from)) {
                 _context.statManager().addRateData("tunnel.rejectHostile", 1);
-                _log.error("Dropping build request, we are the previous hop");
+                _log.error("Dropping build request, we are the previous hop: " + req);
                 return;
             }
         }
         if ((!isOutEnd) && (!isInGW)) {
-            Hash from = state.fromHash;
-            if (from == null)
-                from = state.from.calculateHash();
             // Previous and next hop the same? Don't help somebody be evil. Drop it without a reply.
             // A-B-C-A is not preventable
             if (nextPeer.equals(from)) {
                 // i2pd does this
                 _context.statManager().addRateData("tunnel.rejectHostile", 1);
                 if (_log.shouldLog(Log.WARN))
-                    _log.warn("Dropping build request with the same previous and next hop");
+                    _log.warn("Dropping build request with the same previous and next hop: " + req);
+                _context.commSystem().mayDisconnect(from);
                 return;
             }
         }
@@ -680,13 +704,17 @@ class BuildHandler implements Runnable {
         if (timeDiff > MAX_REQUEST_AGE) {
             _context.statManager().addRateData("tunnel.rejectTooOld", 1);
             if (_log.shouldLog(Log.WARN))
-                _log.warn("Dropping build request too old... replay attack? " + DataHelper.formatDuration(timeDiff));
+                _log.warn("Dropping build request too old... replay attack? " + DataHelper.formatDuration(timeDiff) + ": " + req);
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return;
         }
         if (timeDiff < 0 - MAX_REQUEST_FUTURE) {
             _context.statManager().addRateData("tunnel.rejectFuture", 1);
             if (_log.shouldLog(Log.WARN))
-                _log.warn("Dropping build request too far in future " + DataHelper.formatDuration(0 - timeDiff));
+                _log.warn("Dropping build request too far in future " + DataHelper.formatDuration(0 - timeDiff) + ": " + req);
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
             return;
         }
 
@@ -761,12 +789,9 @@ class BuildHandler implements Runnable {
         // This is at the end as it compares to a percentage of created tunnels.
         // We may need another counter above for requests.
         if (response == 0 && !isInGW) {
-            Hash from = state.fromHash;
-            if (from == null)
-                from = state.from.calculateHash();
             if (from != null && _throttler.shouldThrottle(from)) {
                 if (_log.shouldLog(Log.WARN))
-                    _log.warn("Rejecting tunnel (hop throttle), previous hop: " + from);
+                    _log.warn("Rejecting tunnel (hop throttle), previous hop: " + from + ": " + req);
                 // no setTunnelStatus() indication
                 _context.statManager().addRateData("tunnel.rejectHopThrottle", 1);
                 response = TunnelHistory.TUNNEL_REJECT_BANDWIDTH;
@@ -775,18 +800,12 @@ class BuildHandler implements Runnable {
         if (response == 0 && (!isOutEnd) &&
             _throttler.shouldThrottle(nextPeer)) {
             if (_log.shouldLog(Log.WARN))
-                _log.warn("Rejecting tunnel (hop throttle), next hop: " + nextPeer);
+                _log.warn("Rejecting tunnel (hop throttle), next hop: " + req);
             _context.statManager().addRateData("tunnel.rejectHopThrottle", 1);
             // no setTunnelStatus() indication
             response = TunnelHistory.TUNNEL_REJECT_BANDWIDTH;
         }
 
-        if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Responding to " + state.msg.getUniqueId() + "/" + ourId
-                       + " after " + recvDelay + " with " + response 
-                       + " from " + (state.fromHash != null ? state.fromHash : 
-                                     state.from != null ? state.from.calculateHash() : "tunnel"));
-
         HopConfig cfg = null;
         if (response == 0) {
             cfg = new HopConfig();
@@ -798,10 +817,8 @@ class BuildHandler implements Runnable {
                 // default
                 //cfg.setReceiveFrom(null);
             } else {
-                if (state.fromHash != null) {
-                    cfg.setReceiveFrom(state.fromHash);
-                } else if (state.from != null) {
-                    cfg.setReceiveFrom(state.from.calculateHash());
+                if (from != null) {
+                    cfg.setReceiveFrom(from);
                 } else {
                     // b0rk
                     return;
@@ -827,7 +844,7 @@ class BuildHandler implements Runnable {
                 success = _context.tunnelDispatcher().joinParticipant(cfg);
             if (success) {
                 if (_log.shouldLog(Log.DEBUG))
-                    _log.debug("Joining " + state.msg.getUniqueId() + "/" + cfg.getReceiveTunnel() + "/" + recvDelay + " as " + (isOutEnd ? "outbound endpoint" : isInGW ? "inbound gw" : "participant"));
+                    _log.debug("Joining: " + req);
             } else {
                 // Dup Tunnel ID. This can definitely happen (birthday paradox).
                 // Probability in 11 minutes (per hop type):
@@ -835,31 +852,44 @@ class BuildHandler implements Runnable {
                 response = TunnelHistory.TUNNEL_REJECT_BANDWIDTH;
                 _context.statManager().addRateData("tunnel.rejectDupID", 1);
                 if (_log.shouldLog(Log.WARN))
-                    _log.warn("DUP ID failure " + state.msg.getUniqueId() + "/" + cfg.getReceiveTunnel() + " as " + (isOutEnd ? "outbound endpoint" : isInGW ? "inbound gw" : "participant"));
+                    _log.warn("DUP ID failure: " + req);
             }
         }
+
+        // determination of response is now complete
+
         if (response != 0) {
             _context.statManager().addRateData("tunnel.reject." + response, 1);
-            _context.messageHistory().tunnelRejected(state.fromHash, new TunnelId(ourId), nextPeer, 
-                                                     "rejecting for " + response + ": " +
-                                                     state.msg.getUniqueId() + "/" + ourId + "/" + req.readNextTunnelId() + " delay " +
-                                                     recvDelay + " as " +
-                                                     (isOutEnd ? "outbound endpoint" : isInGW ? "inbound gw" : "participant"));
+            _context.messageHistory().tunnelRejected(from, new TunnelId(ourId), nextPeer, 
+                                                     // this is all disabled anyway
+                                                     //"rejecting for " + response + ": " +
+                                                     //state.msg.getUniqueId() + "/" + ourId + "/" + req.readNextTunnelId() + " delay " +
+                                                     //recvDelay + " as " +
+                                                     //(isOutEnd ? "outbound endpoint" : isInGW ? "inbound gw" : "participant"));
+                                                     Integer.toString(response));
+            if (from != null)
+                _context.commSystem().mayDisconnect(from);
+            // Connection congestion control:
+            // If we rejected the request, are near our conn limits, and aren't connected to the next hop,
+            // just drop it.
+            // 81% = between 75% control measures in Transports and 87% rejection above
+            if ((! _context.routerHash().equals(nextPeer)) &&
+                (! _context.commSystem().haveOutboundCapacity(81)) &&
+                (! _context.commSystem().isEstablished(nextPeer))) {
+                _context.statManager().addRateData("tunnel.dropConnLimits", 1);
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Not sending rejection due to conn limits: " + req);
+                return;
+            }
+        } else if (isInGW && from != null) {
+            // we're the start of the tunnel, no use staying connected
+            _context.commSystem().mayDisconnect(from);
         }
 
-        // Connection congestion control:
-        // If we rejected the request, are near our conn limits, and aren't connected to the next hop,
-        // just drop it.
-        // 81% = between 75% control measures in Transports and 87% rejection above
-        if (response != 0 &&
-            (! _context.routerHash().equals(nextPeer)) &&
-            (! _context.commSystem().haveOutboundCapacity(81)) &&
-            (! _context.commSystem().isEstablished(nextPeer))) {
-            _context.statManager().addRateData("tunnel.dropConnLimits", 1);
-            if (_log.shouldLog(Log.WARN))
-                _log.warn("Not sending rejection due to conn limits");
-            return;
-        }
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("Responding to " + state.msg.getUniqueId()
+                       + " after " + recvDelay + " with " + response 
+                       + " from " + (from != null ? from : "tunnel") + ": " + req);
 
         EncryptedBuildRecord reply = BuildResponseRecord.create(_context, response, req.readReplyKey(), req.readReplyIV(), state.msg.getUniqueId());
         int records = state.msg.getRecordCount();
@@ -876,19 +906,16 @@ class BuildHandler implements Runnable {
         }
 
         if (_log.shouldLog(Log.DEBUG))
-            _log.debug("Read slot " + ourSlot + " containing our hop @ " + _context.routerHash()
-                      + " accepted? " + response + " receiving on " + ourId 
-                      + " sending to " + nextId
-                      + " on " + nextPeer
-                      + " inGW? " + isInGW + " outEnd? " + isOutEnd
-                      + " recvDelay " + recvDelay + " replyMessage " + req.readReplyMessageId()
-                      + " replyKey " + req.readReplyKey() + " replyIV " + Base64.encode(req.readReplyIV()));
+            _log.debug("Read slot " + ourSlot + " containing: " + req
+                      + " accepted? " + response
+                      + " recvDelay " + recvDelay + " replyMessage " + req.readReplyMessageId());
 
         // now actually send the response
+        long expires = _context.clock().now() + NEXT_HOP_SEND_TIMEOUT;
         if (!isOutEnd) {
             state.msg.setUniqueId(req.readReplyMessageId());
-            state.msg.setMessageExpiration(_context.clock().now() + NEXT_HOP_SEND_TIMEOUT);
-            OutNetMessage msg = new OutNetMessage(_context, state.msg, state.msg.getMessageExpiration(), PRIORITY, nextPeerInfo);
+            state.msg.setMessageExpiration(expires);
+            OutNetMessage msg = new OutNetMessage(_context, state.msg, expires, PRIORITY, nextPeerInfo);
             if (response == 0)
                 msg.setOnFailedSendJob(new TunnelBuildNextHopFailJob(_context, cfg));
             _context.outNetMessagePool().add(msg);
@@ -904,20 +931,20 @@ class BuildHandler implements Runnable {
             for (int i = 0; i < records; i++)
                 replyMsg.setRecord(i, state.msg.getRecord(i));
             replyMsg.setUniqueId(req.readReplyMessageId());
-            replyMsg.setMessageExpiration(_context.clock().now() + NEXT_HOP_SEND_TIMEOUT);
+            replyMsg.setMessageExpiration(expires);
             TunnelGatewayMessage m = new TunnelGatewayMessage(_context);
             m.setMessage(replyMsg);
-            m.setMessageExpiration(replyMsg.getMessageExpiration());
+            m.setMessageExpiration(expires);
             m.setTunnelId(new TunnelId(nextId));
             if (_context.routerHash().equals(nextPeer)) {
                 // ok, we are the gateway, so inject it
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("We are the reply gateway for " + nextId
-                              + " when replying to replyMessage " + req.readReplyMessageId());
+                              + " when replying to replyMessage " + req);
                 _context.tunnelDispatcher().dispatch(m);
             } else {
                 // ok, the gateway is some other peer, shove 'er across
-                OutNetMessage outMsg = new OutNetMessage(_context, m, m.getMessageExpiration(), PRIORITY, nextPeerInfo);
+                OutNetMessage outMsg = new OutNetMessage(_context, m, expires, PRIORITY, nextPeerInfo);
                 if (response == 0)
                     outMsg.setOnFailedSendJob(new TunnelBuildNextHopFailJob(_context, cfg));
                 _context.outNetMessagePool().add(outMsg);
@@ -934,15 +961,20 @@ class BuildHandler implements Runnable {
      *  but could also be the reply where we are the IBEP.
      */
     private class TunnelBuildMessageHandlerJobBuilder implements HandlerJobBuilder {
+
+        /**
+         *  Either from or fromHash may be null, but both should be null only if
+         *  we're to be a IBGW and it came from us as a OBEP.
+         */
         public Job createJob(I2NPMessage receivedMessage, RouterIdentity from, Hash fromHash) {
             // need to figure out if this is a reply to an inbound tunnel request (where we are the
             // endpoint, receiving the request at the last hop)
             long reqId = receivedMessage.getUniqueId();
             PooledTunnelCreatorConfig cfg = _exec.removeFromBuilding(reqId);
-            if (_log.shouldLog(Log.DEBUG))
-                _log.debug("Receive tunnel build message " + reqId + " from " 
-                           + (from != null ? from.calculateHash() : fromHash != null ? fromHash : "tunnels") 
-                           + ", found matching tunnel? " + (cfg != null));
+            //if (_log.shouldLog(Log.DEBUG))
+            //    _log.debug("Receive tunnel build message " + reqId + " from " 
+            //               + (from != null ? from.calculateHash() : fromHash != null ? fromHash : "tunnels") 
+            //               + ", found matching tunnel? " + (cfg != null));
             if (cfg != null) {
                 if (!cfg.isInbound()) {
                     // shouldnt happen - should we put it back?
@@ -977,7 +1009,7 @@ class BuildHandler implements Runnable {
                             fh = from.calculateHash();
                         if (fh != null && _requestThrottler.shouldThrottle(fh)) {
                             if (_log.shouldLog(Log.WARN))
-                                _log.warn("Dropping tunnel request (from throttle), previous hop: " + from);
+                                _log.warn("Dropping tunnel request (from throttle), previous hop: " + fh);
                             _context.statManager().addRateData("tunnel.dropReqThrottle", 1);
                             accept = false;
                         }
@@ -1060,6 +1092,10 @@ class BuildHandler implements Runnable {
         final Hash fromHash;
         final long recvTime;
 
+        /**
+         *  Either f or h may be null, but both should be null only if
+         *  we're to be a IBGW and it came from us as a OBEP.
+         */
         public BuildMessageState(RouterContext ctx, I2NPMessage m, RouterIdentity f, Hash h) {
             _ctx = ctx;
             msg = (TunnelBuildMessage)m;
@@ -1134,7 +1170,11 @@ class BuildHandler implements Runnable {
         public String getName() { return "Timeout contacting next peer for tunnel join"; }
 
         public void runJob() {
-            getContext().tunnelDispatcher().remove(_cfg);
+            //  TODO
+            //  This doesn't seem to be a reliable indication of actual failure,
+            //  as we sometimes get subsequent tunnel messages.
+            //  Until this is investigated and fixed, don't remove the tunnel.
+            //getContext().tunnelDispatcher().remove(_cfg);
             getContext().statManager().addRateData("tunnel.rejectTimeout2", 1);
             Log log = getContext().logManager().getLog(BuildHandler.class);
             if (log.shouldLog(Log.WARN))

router/java/src/net/i2p/router/util/CachedIteratorArrayList.java

@@ -0,0 +1,109 @@
+package net.i2p.router.util;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * ArrayList that uses a single iterator.  Useful to avoid object churn
+ * while keeping the conveniences of an iterator.
+ * 
+ * @since 0.9.4 moved from net.i2p.util in 0.9.24
+ * 
+ * @author zab
+ */
+public class CachedIteratorArrayList<E> extends ArrayList<E> {
+
+    private static final long serialVersionUID = 4863212596318574111L;
+    
+    private final CachedIterator iterator = new CachedIterator();
+
+    public CachedIteratorArrayList() {
+        super();
+    }
+
+    public CachedIteratorArrayList(Collection<? extends E> c) {
+        super(c);
+    }
+
+    public CachedIteratorArrayList(int initialCapacity) {
+        super(initialCapacity);
+    }
+    
+    @Override
+    public Iterator<E> iterator() {
+        iterator.reset();
+        return iterator;
+    }
+
+    private class CachedIterator implements Iterator<E>, Serializable {
+        /**
+         * Index of element to be returned by subsequent call to next.
+         */
+        int cursor = 0;
+
+        /**
+         * Index of element returned by most recent call to next or
+         * previous.  Reset to -1 if this element is deleted by a call
+         * to remove.
+         */
+        int lastRet = -1;
+
+        /**
+         * The modCount value that the iterator believes that the backing
+         * List should have.  If this expectation is violated, the iterator
+         * has detected concurrent modification.
+         */
+        int expectedModCount = modCount;
+        
+        void reset() {
+            cursor = 0;
+            lastRet = -1;
+            expectedModCount = modCount;
+        }
+
+        public boolean hasNext() {
+            return cursor != size();
+        }
+
+        public E next() {
+            checkForComodification();
+            try {
+                int i = cursor;
+                E next = get(i);
+                lastRet = i;
+                cursor = i + 1;
+                return next;
+            } catch (IndexOutOfBoundsException e) {
+                checkForComodification();
+                throw new NoSuchElementException();
+            }
+        }
+
+        public void remove() {
+            if (lastRet < 0)
+                throw new IllegalStateException();
+            checkForComodification();
+
+            try {
+                CachedIteratorArrayList.this.remove(lastRet);
+                if (lastRet < cursor)
+                    cursor--;
+                lastRet = -1;
+                expectedModCount = modCount;
+            } catch (IndexOutOfBoundsException e) {
+                throw new ConcurrentModificationException();
+            }
+        }
+
+        final void checkForComodification() {
+            if (modCount != expectedModCount)
+                throw new ConcurrentModificationException();
+        }
+
+    }
+    
+}

router/java/src/net/i2p/router/util/DecayingBloomFilter.java

@@ -61,13 +61,19 @@ public class DecayingBloomFilter {
         _longToEntry = null;
         _longToEntryMask = 0;
         context.addShutdownTask(new Shutdown());
-        _decayEvent = new DecayEvent();
         _keepDecaying = true;
-        _decayEvent.schedule(_durationMs);
+        if (_durationMs == 60*60*1000) {
+            // special mode for BuildMessageProcessor
+            _decayEvent = new DecayHourlyEvent();
+        } else {
+            _decayEvent = new DecayEvent();
+            _decayEvent.schedule(_durationMs);
+        }
     }
 
     /**
      * Create a bloom filter that will decay its entries over time.  
+     * Uses default m of 23, memory usage is 2 MB.
      *
      * @param durationMs entries last for at least this long, but no more than twice this long
      * @param entryBytes how large are the entries to be added?  if this is less than 32 bytes,
@@ -78,7 +84,10 @@ public class DecayingBloomFilter {
         this(context, durationMs, entryBytes, "DBF");
     }
 
-    /** @param name just for logging / debugging / stats */
+    /**
+     * Uses default m of 23, memory usage is 2 MB.
+     * @param name just for logging / debugging / stats
+     */
     public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name) {
         // this is instantiated in four different places, they may have different
         // requirements, but for now use this as a gross method of memory reduction.
@@ -87,6 +96,8 @@ public class DecayingBloomFilter {
     }
 
     /**
+     * Memory usage is 2 * (2**m) bits or 2**(m-2) bytes.
+     *
      * @param m filter size exponent, max is 29
      */
     public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name, int m) {
@@ -123,9 +134,14 @@ public class DecayingBloomFilter {
             _longToEntry = null;
             _longToEntryMask = 0;
         }
-        _decayEvent = new DecayEvent();
         _keepDecaying = true;
-        _decayEvent.schedule(_durationMs);
+        if (_durationMs == 60*60*1000) {
+            // special mode for BuildMessageProcessor
+            _decayEvent = new DecayHourlyEvent();
+        } else {
+            _decayEvent = new DecayEvent();
+            _decayEvent.schedule(_durationMs);
+        }
         if (_log.shouldLog(Log.WARN))
            _log.warn("New DBF " + name + " m = " + m + " k = " + k + " entryBytes = " + entryBytes +
                      " numExtenders = " + numExtenders + " cycle (s) = " + (durationMs / 1000));
@@ -318,10 +334,13 @@ public class DecayingBloomFilter {
     }
     
     private class DecayEvent extends SimpleTimer2.TimedEvent {
+        /**
+         *  Caller MUST schedule.
+         */
         DecayEvent() {
             super(_context.simpleTimer2());
         }
-    	
+
         public void timeReached() {
             if (_keepDecaying) {
                 decay();
@@ -330,6 +349,48 @@ public class DecayingBloomFilter {
         }
     }
     
+    /**
+     *  Decays at 5 minutes after the top of the hour.
+     *  This ignores leap seconds.
+     *  @since 0.9.24
+     */
+    private class DecayHourlyEvent extends SimpleTimer2.TimedEvent {
+        private static final long HOUR = 60 * 60 * 1000L;
+        private static final long LAG = 5 * 60 * 1000L;
+        private volatile long _currentHour;
+
+        /**
+         *  Schedules itself. Caller MUST NOT schedule.
+         */
+        DecayHourlyEvent() {
+            super(_context.simpleTimer2());
+            schedule(getTimeTillNextHour());
+        }
+
+        public void timeReached() {
+            if (_keepDecaying) {
+                long now = _context.clock().now();
+                long currentHour = now / HOUR;
+                // handle possible clock adjustments
+                if (_currentHour != currentHour) {
+                    decay();
+                    _currentHour = currentHour;
+                }
+                long next = ((1 + currentHour) * HOUR) + LAG;
+                schedule(Math.max(5000, next - now));
+            }
+        }
+
+        /** side effect: sets _currentHour */
+        private long getTimeTillNextHour() {
+            long now = _context.clock().now();
+            long currentHour = now / HOUR;
+            _currentHour = currentHour;
+            long next = ((1 + currentHour) * HOUR) + LAG;
+            return Math.max(5000, next - now);
+        }
+    }
+    
     /** @since 0.8.11 moved from DecayingHashSet */
     protected void getReadLock() {
         _reorganizeLock.readLock().lock();

router/java/src/net/i2p/router/util/EventLog.java

@@ -15,6 +15,7 @@ import java.util.TreeMap;
 import net.i2p.I2PAppContext;
 import net.i2p.data.DataHelper;
 import net.i2p.util.SecureFileOutputStream;
+import net.i2p.util.SystemVersion;
 
 /**
  *  Simple event logger for occasional events,
@@ -94,6 +95,8 @@ public class EventLog {
             buf.append(_context.clock().now()).append(' ').append(event);
             if (info != null && info.length() > 0)
                 buf.append(' ').append(info);
+            if (SystemVersion.isWindows())
+                buf.append('\r');
             buf.append('\n');
             out.write(buf.toString().getBytes("UTF-8"));
         } catch (IOException ioe) {
@@ -126,7 +129,7 @@ public class EventLog {
             String line = null;
             while ( (line = br.readLine()) != null) {
                 try {
-                    String[] s = DataHelper.split(line, " ", 3);
+                    String[] s = DataHelper.split(line.trim(), " ", 3);
                     if (!s[1].equals(event))
                         continue;
                     long time = Long.parseLong(s[0]);
@@ -168,7 +171,7 @@ public class EventLog {
             String line = null;
             while ( (line = br.readLine()) != null) {
                 try {
-                    String[] s = DataHelper.split(line, " ", 2);
+                    String[] s = DataHelper.split(line.trim(), " ", 2);
                     if (s.length < 2)
                         continue;
                     long time = Long.parseLong(s[0]);

router/java/src/net/i2p/router/util/RouterPasswordManager.java

@@ -158,13 +158,7 @@ public class RouterPasswordManager extends PasswordManager {
         String pfx = realm;
         if (user != null && user.length() > 0)
             pfx += '.' + user;
-        byte[] salt = new byte[SALT_LENGTH];
-        _context.random().nextBytes(salt);
-        byte[] pwHash = _context.keyGenerator().generateSessionKey(salt, DataHelper.getUTF8(pw)).getData();
-        byte[] shashBytes = new byte[SHASH_LENGTH];
-        System.arraycopy(salt, 0, shashBytes, 0, SALT_LENGTH);
-        System.arraycopy(pwHash, 0, shashBytes, SALT_LENGTH, SessionKey.KEYSIZE_BYTES);
-        String shash = Base64.encode(shashBytes);
+        String shash = createHash(pw);
         Map<String, String> toAdd = Collections.singletonMap(pfx + PROP_SHASH, shash);
         List<String> toDel = new ArrayList<String>(4);
         toDel.add(pfx + PROP_PW);