From 182fe900b8709c754d778ac11738b300ba555d93 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 7 Jun 2013 17:26:20 +0000 Subject: [PATCH] * BlockfileNamingService: - Fix bug that kept reverse index from being updated - Bump DB version to 3 to trigger repopulation of the reverse index - Make metaindex final - Add freelist check to the main() test --- .../client/naming/BlockfileNamingService.java | 32 ++++++---- .../net/metanotion/io/block/BlockFile.java | 64 +++++++++++++------ .../metanotion/io/block/FreeListBlock.java | 12 ++++ history.txt | 10 +++ .../src/net/i2p/router/RouterVersion.java | 2 +- 5 files changed, 88 insertions(+), 32 deletions(-) diff --git a/core/java/src/net/i2p/client/naming/BlockfileNamingService.java b/core/java/src/net/i2p/client/naming/BlockfileNamingService.java index 76807be18..ff92edf08 100644 --- a/core/java/src/net/i2p/client/naming/BlockfileNamingService.java +++ b/core/java/src/net/i2p/client/naming/BlockfileNamingService.java @@ -32,6 +32,7 @@ import net.i2p.data.Hash; import net.i2p.util.LHMCache; import net.i2p.util.Log; import net.i2p.util.SecureFileOutputStream; +import net.i2p.util.VersionComparator; import net.metanotion.io.RAIFile; import net.metanotion.io.Serializer; @@ -112,8 +113,7 @@ public class BlockfileNamingService extends DummyNamingService { private static final String PROP_LISTS = "lists"; private static final String PROP_CREATED = "created"; private static final String PROP_UPGRADED = "upgraded"; - private static final String VERSION = "2"; - private static final String OLD_VERSION = "1"; + private static final String VERSION = "3"; private static final String PROP_ADDED = "a"; private static final String PROP_SOURCE = "s"; @@ -331,10 +331,8 @@ public class BlockfileNamingService extends DummyNamingService { * @since 0.8.9 */ private boolean needsUpgrade(BlockFile bf, String version) throws IOException { - if (VERSION.equals(version)) + if (version != null && VersionComparator.comp(version, VERSION) >= 0) return false; - if (!OLD_VERSION.equals(version)) - throw new IOException("Bad db version: " + version); if (!bf.file.canWrite()) { if (_log.shouldLog(Log.WARN)) _log.warn("Not upgrading read-only database version " + version); @@ -345,15 +343,23 @@ public class BlockfileNamingService extends DummyNamingService { /** * Blockfile must be writable of course. + * + * Version 1->2: Add reverse skiplist and populate + * Version 2->3: Re-populate reverse skiplist as version 2 didn't keep it updated + * after the upgrade. No change to format. + * * @return true if upgraded successfully * @since 0.8.9 */ private boolean upgrade() { try { - // shouldn't ever be there... + // wasn't there in version 1, is there in version 2 SkipList rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); - if (rev == null) + if (rev == null) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Created reverse index"); rev = _bf.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer); + } Map entries = getEntries(); long start = System.currentTimeMillis(); int i = 0; @@ -361,8 +367,9 @@ public class BlockfileNamingService extends DummyNamingService { addReverseEntry(entry.getKey(), entry.getValue()); i++; } + // i may be greater than skiplist keys if there are dups if (_log.shouldLog(Log.WARN)) - _log.warn("Created reverse index with " + i + " entries"); + _log.warn("Updated reverse index with " + i + " entries"); SkipList hdr = _bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer); if (hdr == null) throw new IOException("No db header"); @@ -723,8 +730,9 @@ public class BlockfileNamingService extends DummyNamingService { addEntry(sl, key, d, props); if (changed) { removeCache(hostname); - addReverseEntry(key, d); + // removeReverseEntry(key, oldDest) ??? } + addReverseEntry(key, d); for (NamingServiceListener nsl : _listeners) { if (changed) nsl.entryChanged(this, hostname, d, options); @@ -1188,13 +1196,13 @@ public class BlockfileNamingService extends DummyNamingService { * BlockfileNamingService [force] * force = force writable */ -/**** public static void main(String[] args) { Properties ctxProps = new Properties(); if (args.length > 0 && args[0].equals("force")) ctxProps.setProperty(PROP_FORCE, "true"); I2PAppContext ctx = new I2PAppContext(ctxProps); BlockfileNamingService bns = new BlockfileNamingService(ctx); +/**** List names = null; Properties props = new Properties(); try { @@ -1268,7 +1276,9 @@ public class BlockfileNamingService extends DummyNamingService { //bns.dumpDB(); +****/ bns.close(); +/**** if (true) return; HostsTxtNamingService htns = new HostsTxtNamingService(I2PAppContext.getGlobalContext()); @@ -1284,6 +1294,6 @@ public class BlockfileNamingService extends DummyNamingService { } System.out.println("HTNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); System.out.println("found " + found + " notfound " + notfound); - } ****/ + } } diff --git a/core/java/src/net/metanotion/io/block/BlockFile.java b/core/java/src/net/metanotion/io/block/BlockFile.java index 9744d0e1b..6a386852d 100644 --- a/core/java/src/net/metanotion/io/block/BlockFile.java +++ b/core/java/src/net/metanotion/io/block/BlockFile.java @@ -90,7 +90,8 @@ public class BlockFile { /** I2P was the file locked when we opened it? */ private final boolean _wasMounted; - private BSkipList metaIndex; + private final BSkipList metaIndex; + private boolean _isClosed; /** cached list of free pages, only valid if freListStart > 0 */ private FreeListBlock flb; private final HashMap openIndices = new HashMap(); @@ -120,7 +121,12 @@ public class BlockFile { } /** - * Run an integrity check on the blockfile and all the skiplists in it + * Run an integrity check on the blockfile and all the skiplists in it. + * + * WARNING: + * This only works on skiplists using UTF8StringBytes as a key + * serializer, unless the exception has been coded in bfck below. + * Will CORRUPT other skiplists. */ public static void main(String args[]) { if (args.length != 1) { @@ -470,11 +476,10 @@ public class BlockFile { */ public void close() throws IOException { // added I2P - if (metaIndex == null) + if (_isClosed) return; - + _isClosed = true; metaIndex.close(); - metaIndex = null; Set oi = openIndices.keySet(); Iterator i = oi.iterator(); @@ -497,27 +502,34 @@ public class BlockFile { * @return true if the levels were modified. */ public boolean bfck(boolean fix) { - log.info("magic bytes " + magicBytes); - log.info("fileLen " + fileLen); - log.info("freeListStart " + freeListStart); - log.info("mounted " + mounted); - log.info("spanSize " + spanSize); - log.info("Metaindex"); - log.info("Checking meta index in blockfile " + file); + if (log.shouldLog(Log.INFO)) { + log.info("magic bytes " + magicBytes); + log.info("fileLen " + fileLen); + log.info("freeListStart " + freeListStart); + log.info("mounted " + mounted); + log.info("spanSize " + spanSize); + log.info("Metaindex"); + log.info("Checking meta index in blockfile " + file); + } boolean rv = metaIndex.bslck(fix, true); - if (rv) - log.warn("Repaired meta index in blockfile " + file); - else - log.info("No errors in meta index in blockfile " + file); + if (rv) { + if (log.shouldLog(Log.WARN)) + log.warn("Repaired meta index in blockfile " + file); + } else { + if (log.shouldLog(Log.INFO)) + log.info("No errors in meta index in blockfile " + file); + } int items = 0; for (SkipIterator iter = metaIndex.iterator(); iter.hasNext(); ) { String slname = (String) iter.nextKey(); Integer page = (Integer) iter.next(); - log.info("List " + slname + " page " + page); + if (log.shouldLog(Log.INFO)) + log.info("List " + slname + " page " + page); try { - // This uses IdentityBytes, so the value class won't be right - //Serializer ser = slname.equals("%%__REVERSE__%%") ? new IntBytes() : new UTF8StringBytes(); - BSkipList bsl = getIndex(slname, new UTF8StringBytes(), new IdentityBytes()); + // This uses IdentityBytes, so the value class won't be right, but at least + // it won't fail the out-of-order check + Serializer keyser = slname.equals("%%__REVERSE__%%") ? new IntBytes() : new UTF8StringBytes(); + BSkipList bsl = getIndex(slname, keyser, new IdentityBytes()); if (bsl == null) { log.error("Can't find list? " + slname); continue; @@ -530,6 +542,18 @@ public class BlockFile { } } log.info("Checked meta index and " + items + " skiplists"); + if(freeListStart != 0) { + try { + if (flb == null) + flb = new FreeListBlock(file, freeListStart); + flb.flbck(true); + } catch (IOException ioe) { + log.error("Free list error", ioe); + } + } else { + if (log.shouldLog(Log.INFO)) + log.info("No freelist"); + } return rv; } } diff --git a/core/java/src/net/metanotion/io/block/FreeListBlock.java b/core/java/src/net/metanotion/io/block/FreeListBlock.java index 53435b2d4..d2e8c3351 100644 --- a/core/java/src/net/metanotion/io/block/FreeListBlock.java +++ b/core/java/src/net/metanotion/io/block/FreeListBlock.java @@ -197,6 +197,18 @@ class FreeListBlock { file.writeInt(0); } + /** + * Recursive. + * @since 0.9.7 + */ + public boolean flbck(boolean fix) throws IOException { + Log log = I2PAppContext.getGlobalContext().logManager().getLog(BlockFile.class); + log.info(toString()); + if (nextPage > 0) + (new FreeListBlock(file, nextPage)).flbck(fix); + return true; + } + @Override public String toString() { return "FLB with " + len + " / " + MAX_SIZE + " page " + page + " next page " + nextPage; diff --git a/history.txt b/history.txt index a756618c8..ee9f10cf2 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,13 @@ +2013-06-07 zzz + * BlockfileNamingService: + - Fix bug that kept reverse index from being updated + - Bump DB version to 3 to trigger repopulation of the reverse index + - Make metaindex final + - Add freelist check to the main() test + +2013-06-06 zzz + * BuildRequestRecord: Don't waste entropy on bytes we will overwrite + 2013-06-05 zzz * DatabaseLookupMessage: Change format for encrypted reply request to allow multiple bundled reply tags diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 4e28a2c8d..c10128fe6 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 5; + public final static long BUILD = 6; /** for example "-test" */ public final static String EXTRA = "";