diff --git a/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java b/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java index 79734cd7f..fe9fbd803 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PartialPiece.java @@ -140,15 +140,40 @@ class PartialPiece implements Comparable { * @since 0.9.62 */ public synchronized boolean isComplete() { - return bitfield.complete(); + return bitfield.complete(); } /** - * How many consecutive bytes are good - as set by read(). - * There may be more good bytes after "holes". + * Have any chunks been downloaded? + * + * @since 0.9.63 + */ + public synchronized boolean hasData() { + return bitfield.count() > 0; + } + + /** + * Has this chunk been downloaded? + * + * @since 0.9.63 + */ + public synchronized boolean hasChunk(int chunk) { + return bitfield.get(chunk); + } + + /** + * How many bytes are good - as set by read(). + * As of 0.9.63, accurately counts good bytes after "holes". */ public synchronized int getDownloaded() { - return this.off; + if (bitfield.complete()) + return pclen; + int sz = bitfield.count(); + int rv = sz * PeerState.PARTSIZE; + int rem = pclen % PeerState.PARTSIZE; + if (rem != 0 && bitfield.get(sz - 1)) + rv -= PeerState.PARTSIZE - rem; + return rv; } /** @@ -348,8 +373,10 @@ class PartialPiece implements Comparable { public void release() { if (bs == null) { synchronized (this) { - if (raf != null) + if (raf != null) { locked_release(); + raf = null; + } } //if (raf != null) // I2PAppContext.getGlobalContext().logManager().getLog(PartialPiece.class).warn("Released " + tempfile); @@ -378,7 +405,7 @@ class PartialPiece implements Comparable { int d = this.piece.compareTo(opp.piece); if (d != 0) return d; - return opp.off - this.off; // reverse + return opp.getDownloaded() - getDownloaded(); // reverse } @Override @@ -401,7 +428,7 @@ class PartialPiece implements Comparable { @Override public String toString() { - return "Partial(" + piece.getId() + ',' + off + ',' + pclen + ')'; + return "Partial(" + piece.getId() + ',' + off + ',' + getDownloaded() + ',' + pclen + ')'; } /** diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java index 1845465c8..6980dd96f 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerCoordinator.java @@ -1370,14 +1370,14 @@ class PeerCoordinator implements PeerListener, BandwidthListener synchronized(wantedPieces) { for (Request req : partials) { PartialPiece pp = req.getPartialPiece(); - if (req.off > 0) { + if (pp.hasData()) { // PartialPiece.equals() only compares piece number, which is what we want int idx = partialPieces.indexOf(pp); if (idx < 0) { partialPieces.add(pp); if (_log.shouldLog(Log.INFO)) _log.info("Saving orphaned partial piece (new) " + pp); - } else if (idx >= 0 && pp.getDownloaded() > partialPieces.get(idx).getDownloaded()) { + } else if (pp.getDownloaded() > partialPieces.get(idx).getDownloaded()) { // replace what's there now partialPieces.get(idx).release(); partialPieces.set(idx, pp); @@ -1434,7 +1434,9 @@ class PeerCoordinator implements PeerListener, BandwidthListener for(Piece piece : wantedPieces) { if (piece.getId() == savedPiece) { if (peer.isCompleted() && piece.getPeerCount() > 1 && - wantedPieces.size() > 2*END_GAME_THRESHOLD) { + wantedPieces.size() > 2*END_GAME_THRESHOLD && + partialPieces.size() < 4 && + _random.nextInt(4) != 0) { // Try to preserve rarest-first // by not requesting a partial piece that at least two non-seeders also have // from a seeder @@ -1462,7 +1464,7 @@ class PeerCoordinator implements PeerListener, BandwidthListener iter.remove(); piece.setRequested(peer, true); if (_log.shouldLog(Log.INFO)) { - _log.info("Restoring orphaned partial piece " + pp + + _log.info("Restoring orphaned partial piece " + pp + " to " + peer + " Partial list size now: " + partialPieces.size()); } return pp; diff --git a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java index 8ba610fae..682afe46c 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/PeerState.java +++ b/apps/i2psnark/java/src/org/klomp/snark/PeerState.java @@ -67,7 +67,7 @@ class PeerState implements DataLoader // Outstanding request private final List outstandingRequests = new ArrayList(); /** the tail (NOT the head) of the request queue */ - private Request lastRequest = null; + private Request lastRequest; private int currentMaxPipeline; // FIXME if piece size < PARTSIZE, pipeline could be bigger @@ -425,7 +425,8 @@ class PeerState implements DataLoader // Last chunk needed for this piece? PartialPiece pp = req.getPartialPiece(); - if (pp.isComplete()) + boolean complete = pp.isComplete(); + if (complete) { // warning - may block here for a while if (listener.gotPiece(peer, pp)) @@ -449,6 +450,25 @@ class PeerState implements DataLoader synchronized(this) { pendingRequest = null; } + + // getOutstandingRequest() was called by PeerConnectionIn at the start of the chunk; + // if the bandwidth limiter throttled us to zero requests then, try again now + if (outstandingRequests.isEmpty()) { + addRequest(); + if (!complete) { + synchronized(this) { + if (outstandingRequests.isEmpty()) { + // we MUST return the partial piece to PeerCoordinator, + // or else we will lose it and leak the data + if (_log.shouldWarn()) + _log.warn("Throttled, returned to coord. w/ data " + req); + List pcs = Collections.singletonList(req); + listener.savePartialPieces(this.peer, pcs); + lastRequest = null; + } + } + } + } } /** @@ -577,15 +597,8 @@ class PeerState implements DataLoader List rv = new ArrayList(pcs.size()); for (Integer p : pcs) { Request req = getLowestOutstandingRequest(p.intValue()); - if (req != null) { - PartialPiece pp = req.getPartialPiece(); - synchronized(pp) { - int dl = pp.getDownloaded(); - if (req.off != dl) - req = new Request(pp, dl); - } + if (req != null) rv.add(req); - } } outstandingRequests.clear(); pendingRequest = null; @@ -709,12 +722,6 @@ class PeerState implements DataLoader /** * BEP 6 - * If the peer rejects lower chunks but not higher ones, thus creating holes, - * we won't figure it out and the piece will fail, since we don't currently - * keep a chunk bitmap in PartialPiece. - * As long as the peer rejects all the chunks, or rejects only the last chunks, - * no holes are created and we will be fine. The reject messages may be in any order, - * just don't make a hole when it's over. * * @since 0.9.21 */ @@ -738,19 +745,10 @@ class PeerState implements DataLoader } } if (deletedRequest != null && !haveMoreRequests) { - // We must return the piece to the coordinator - // Create a new fake request so we can set the offset correctly - PartialPiece pp = deletedRequest.getPartialPiece(); - int downloaded = pp.getDownloaded(); - Request req; - if (deletedRequest.off == downloaded) - req = deletedRequest; - else - req = new Request(pp, downloaded, 1); - List pcs = Collections.singletonList(req); + List pcs = Collections.singletonList(deletedRequest); listener.savePartialPieces(this.peer, pcs); if (_log.shouldWarn()) - _log.warn("Returned to coord. w/ offset " + pp.getDownloaded() + " due to reject(" + piece + ',' + begin + ',' + length + ") from " + peer); + _log.warn("Returned to coord. w/ data " + deletedRequest.getPartialPiece().getDownloaded() + " due to reject(" + piece + ',' + begin + ',' + length + ") from " + peer); } if (lastRequest != null && lastRequest.getPiece() == piece && lastRequest.off == begin && lastRequest.len == length) @@ -876,7 +874,7 @@ class PeerState implements DataLoader * * This is called from several places: *
-   *   By getOustandingRequest() when the first part of a chunk comes in
+   *   By getOutstandingRequest() when the first part of a chunk comes in
    *   By havePiece() when somebody got a new piece completed
    *   By chokeMessage() when we receive an unchoke
    *   By setInteresting() when we are now interested
@@ -952,22 +950,32 @@ class PeerState implements DataLoader
                 isLastChunk = lastRequest.off + lastRequest.len == pieceLength;
 
                 // Last part of a piece?
-                if (isLastChunk)
-                  more_pieces = requestNextPiece();
-                else
-                  {
-                        PartialPiece nextPiece = lastRequest.getPartialPiece();
-                        int nextBegin = lastRequest.off + PARTSIZE;
-                        int maxLength = pieceLength - nextBegin;
-                        int nextLength = maxLength > PARTSIZE ? PARTSIZE
+                if (isLastChunk) {
+                    more_pieces = requestNextPiece();
+                } else {
+                    PartialPiece nextPiece = lastRequest.getPartialPiece();
+                    int nextBegin = lastRequest.off + PARTSIZE;
+                    while (true) {
+                        // don't rerequest chunks we already have
+                        if (!nextPiece.hasChunk(nextBegin / PARTSIZE)) {
+                            int maxLength = pieceLength - nextBegin;
+                            int nextLength = maxLength > PARTSIZE ? PARTSIZE
                                                               : maxLength;
-                        Request req
-                          = new Request(nextPiece,nextBegin, nextLength);
-                        outstandingRequests.add(req);
-                        if (!choked)
-                          out.sendRequest(req);
-                        lastRequest = req;
-                  }
+                            Request req = new Request(nextPiece,nextBegin, nextLength);
+                            outstandingRequests.add(req);
+                            if (!choked)
+                              out.sendRequest(req);
+                            lastRequest = req;
+                            break;
+                        } else {
+                            nextBegin += PARTSIZE;
+                            if (nextBegin >= pieceLength) {
+                                more_pieces = requestNextPiece();
+                                break;
+                            }
+                        }
+                    }
+                }
               }
           }
       }
diff --git a/apps/i2psnark/java/src/org/klomp/snark/WebPeer.java b/apps/i2psnark/java/src/org/klomp/snark/WebPeer.java
index 718080097..3c8d756c6 100644
--- a/apps/i2psnark/java/src/org/klomp/snark/WebPeer.java
+++ b/apps/i2psnark/java/src/org/klomp/snark/WebPeer.java
@@ -371,7 +371,7 @@ class WebPeer extends Peer implements EepGet.StatusListener {
               PartialPiece pp = last.getPartialPiece();
               synchronized(pp) {
                   // Last chunk needed for this piece?
-                  if (pp.getLength() == pp.getDownloaded()) {
+                  if (pp.isComplete()) {
                       if (listener.gotPiece(this, pp)) {
                           if (_log.shouldDebug())
                               _log.debug("Got " + piece + ": " + this);
@@ -650,15 +650,8 @@ class WebPeer extends Peer implements EepGet.StatusListener {
       List rv = new ArrayList(pcs.size());
       for (Integer p : pcs) {
           Request req = getLowestOutstandingRequest(p.intValue());
-          if (req != null) {
-              PartialPiece pp = req.getPartialPiece();
-              synchronized(pp) {
-                  int dl = pp.getDownloaded();
-                  if (req.off != dl)
-                      req = new Request(pp, dl);
-              }
+          if (req != null)
               rv.add(req);
-          }
       }
       outstandingRequests.clear();
       return rv;