corruption of log writer detected by log reader

[Peter Reilly]

When running the tests I noticed a log of "corruption of 35 bytes: Partial record without end"

This also happened with using the code :-(

I have tracked down the problem to an off-by-one bug in

leveldb/src/main/java/org/iq80/leveldb/impl/FileChannelLogWriter.java

and

leveldb/src/main/java/org/iq80/leveldb/impl/MMapLogWriter.java

The following patch fixes this:

git diff leveldb/src/main/java/org/iq80/leveldb/impl/FileChannelLogWriter.java leveldb/src/main/java/org/iq80/leveldb/impl/MMapLogWriter.java
diff --git a/leveldb/src/main/java/org/iq80/leveldb/impl/FileChannelLogWriter.java  b/leveldb/src/main/java/org/iq80/leveldb/impl/FileChannelLogWriter.java

index bd675bc..fdb85f1 100644
--- a/leveldb/src/main/java/org/iq80/leveldb/impl/FileChannelLogWriter.java
+++ b/leveldb/src/main/java/org/iq80/leveldb/impl/FileChannelLogWriter.java
@@ -142,7 +142,7 @@ public class FileChannelLogWriter implements LogWriter
// fragment the record; otherwise write to the end of the record
boolean end;
int fragmentLength;
- if (sliceInput.available() >= bytesAvailableInBlock) {
+ if (sliceInput.available() > bytesAvailableInBlock) {
end = false;
fragmentLength = bytesAvailableInBlock;
}
diff --git a/leveldb/src/main/java/org/iq80/leveldb/impl/MMapLogWriter.java b/leveldb/src/main/java/org/iq80/leveldb/impl/MMapLogWriter.java
index 42b6ca8..221a302 100755
--- a/leveldb/src/main/java/org/iq80/leveldb/impl/MMapLogWriter.java
+++ b/leveldb/src/main/java/org/iq80/leveldb/impl/MMapLogWriter.java
@@ -148,7 +148,7 @@ public class MMapLogWriter implements LogWriter
// fragment the record; otherwise write to the end of the record
boolean end;
int fragmentLength;
- if (sliceInput.available() >= bytesAvailableInBlock) {
+ if (sliceInput.available() > bytesAvailableInBlock) {
end = false;
fragmentLength = bytesAvailableInBlock;
}

I have a junit/hamcrest based test that shows the problem/fix:

package org.iq80.leveldb.impl;

import org.iq80.leveldb.util.Slice;
import org.junit.*;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.util.*;
import org.junit.rules.*;

import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;


 public class MMapLogWriterTest {
   @Rule public TemporaryFolder folder = new TemporaryFolder();

   @Test public void simple() throws Exception {
      File file = new File(folder.getRoot(), "10.log");
      LogWriter w = new MMapLogWriter(file, 10);

      //int recordNumber = 7032;
    //int recordSize = 40101;

    int recordNumber = 1;
    int recordSize = LogConstants.BLOCK_SIZE - LogConstants.HEADER_SIZE;
    Slice record = new Slice(recordSize);

    for (int i = 0; i < recordNumber; ++i) {
        w.addRecord(record, false);
    }
    w.close();

    LogMonitor mon = new LogMonitor() {

        @Override
        public void corruption(long bytes, String reason) {
            fail("corruption at " + bytes + " reason: " + reason);
        }

        @Override
        public void corruption(long bytes, Throwable reason) {
            fail("corruption at " + bytes + " reason: " + reason.toString());
        }

    };

    FileChannel channel = new FileInputStream(file).getChannel();

    LogReader logReader = new LogReader(channel, mon, true, 0L);
    int count = 0;
    while (true) {
        Slice s = logReader.readRecord();
        if (s == null) {
            break;
        }
        assertThat(s.length(), is(recordSize));
        count++;
    }
    assertThat(count, is(recordNumber));
}

}

There is a similar bug reported (issue#14) but there seems to be other issues reported there.

[Peter Reilly]

opps

[Dain Sundstrom]

Applied the fix. Thanks