Skip to content

Commit

Permalink
Avoid allocation in BigDecimal serializer with new methods for bytes …
Browse files Browse the repository at this point in the history 
…in Input/Output. (#1018)

Co-authored-by: Wojtek Gdela <wgdela@codedose.com>
  • Loading branch information
@gdela
gdela and Wojtek Gdela committed Nov 5, 2023
1 parent 7bcaeb5 commit 3e7f499
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 27 deletions.
24 changes: 24 additions & 0 deletions src/com/esotericsoftware/kryo/io/ByteBufferInput.java
View file
Expand Up @@ -352,6 +352,30 @@ public void readBytes (byte[] bytes, int offset, int count) throws KryoException
}
}

public int readInt (int count) {
if (count < 0 || count > 4) throw new IllegalArgumentException("count must be >= 0 and <= 4: " + count);
require(count);
position += count;
ByteBuffer byteBuffer = this.byteBuffer;
switch (count) {
case 1:
return byteBuffer.get();
case 2:
return byteBuffer.get() << 8
| byteBuffer.get() & 0xFF;
case 3:
return byteBuffer.get() << 16
| (byteBuffer.get() & 0xFF) << 8
| byteBuffer.get() & 0xFF;
case 4:
return byteBuffer.get() << 24
| (byteBuffer.get() & 0xFF) << 16
| (byteBuffer.get() & 0xFF) << 8
| byteBuffer.get() & 0xFF;
}
throw new IllegalStateException(); // impossible
}

// int:

public int readInt () throws KryoException {
Expand Down
27 changes: 27 additions & 0 deletions src/com/esotericsoftware/kryo/io/ByteBufferOutput.java
View file
Expand Up @@ -276,6 +276,33 @@ public void writeBytes (byte[] bytes, int offset, int count) throws KryoExceptio
}
}

public void writeInt (int bytes, int count) {
if (count < 0 || count > 4) throw new IllegalArgumentException("count must be >= 0 and <= 4: " + count);
require(count);
position += count;
ByteBuffer byteBuffer = this.byteBuffer;
switch (count) {
case 1:
byteBuffer.put((byte)bytes);
break;
case 2:
byteBuffer.put((byte)(bytes >> 8));
byteBuffer.put((byte)bytes);
break;
case 3:
byteBuffer.put((byte)(bytes >> 16));
byteBuffer.put((byte)(bytes >> 8));
byteBuffer.put((byte)bytes);
break;
case 4:
byteBuffer.put((byte)(bytes >> 24));
byteBuffer.put((byte)(bytes >> 16));
byteBuffer.put((byte)(bytes >> 8));
byteBuffer.put((byte)bytes);
break;
}
}

// int:

public void writeInt (int value) throws KryoException {
Expand Down
38 changes: 38 additions & 0 deletions src/com/esotericsoftware/kryo/io/Input.java
View file
Expand Up @@ -373,6 +373,44 @@ public void readBytes (byte[] bytes, int offset, int count) throws KryoException
}
}

/** Reads count bytes and returns them as int, the last byte read will be the lowest byte in the int. */
public int readInt (int count) {
if (count < 0 || count > 4) throw new IllegalArgumentException("count must be >= 0 and <= 4: " + count);
require(count);
int p = position;
position = p + count;
switch (count) {
case 1:
return buffer[p];
case 2:
return buffer[p] << 8
| buffer[p+1] & 0xFF;
case 3:
return buffer[p] << 16
| (buffer[p+1] & 0xFF) << 8
| buffer[p+2] & 0xFF;
case 4:
return buffer[p] << 24
| (buffer[p+1] & 0xFF) << 16
| (buffer[p+2] & 0xFF) << 8
| buffer[p+3] & 0xFF;
}
throw new IllegalStateException(); // impossible
}

/** Reads count bytes and returns them as long, the last byte read will be the lowest byte in the long. */
public long readLong (int count) {
if (count < 0 || count > 8) throw new IllegalArgumentException("count must be >= 0 and <= 8: " + count);
if (count <= 4) {
return readInt(count);
} else {
require(count);
long highBytes = ((long) readInt(count - 4)) << 32;
long lowBytes = ((long) readInt(4)) & (1L << 32) - 1;
return highBytes | lowBytes;
}
}

// int:

/** Reads a 4 byte int. */
Expand Down
43 changes: 42 additions & 1 deletion src/com/esotericsoftware/kryo/io/Output.java
View file
Expand Up @@ -20,7 +20,6 @@
package com.esotericsoftware.kryo.io;

import com.esotericsoftware.kryo.KryoException;
import com.esotericsoftware.kryo.io.KryoBufferOverflowException;
import com.esotericsoftware.kryo.util.Pool.Poolable;
import com.esotericsoftware.kryo.util.Util;

Expand Down Expand Up @@ -274,6 +273,48 @@ public void writeBytes (byte[] bytes, int offset, int count) throws KryoExceptio
}
}

/** Writes count bytes from long, the last byte written is the lowest byte from the long.
* Note the number of bytes is not written. */
public void writeInt (int bytes, int count) {
if (count < 0 || count > 4) throw new IllegalArgumentException("count must be >= 0 and <= 4: " + count);
require(count);
int p = position;
position = p + count;
switch (count) {
case 1:
buffer[p] = (byte)bytes;
break;
case 2:
buffer[p] = (byte)(bytes >> 8);
buffer[p+1] = (byte)bytes;
break;
case 3:
buffer[p] = (byte)(bytes >> 16);
buffer[p+1] = (byte)(bytes >> 8);
buffer[p+2] = (byte)bytes;
break;
case 4:
buffer[p] = (byte)(bytes >> 24);
buffer[p+1] = (byte)(bytes >> 16);
buffer[p+2] = (byte)(bytes >> 8);
buffer[p+3] = (byte)bytes;
break;
}
}

/** Writes count bytes from long, the last byte written is the lowest byte from the long.
* Note the number of bytes is not written. */
public void writeLong (long bytes, int count) {
if (count < 0 || count > 8) throw new IllegalArgumentException("count must be >= 0 and <= 8: " + count);
if (count <= 4) {
writeInt((int) bytes, count);
} else {
require(count);
writeInt((int) (bytes >> 32), count - 4);
writeInt((int) bytes, 4);
}
}

// int:

/** Writes a 4 byte int. */
Expand Down
37 changes: 11 additions & 26 deletions src/com/esotericsoftware/kryo/serializers/DefaultSerializers.java
View file
Expand Up @@ -21,6 +21,7 @@

import static com.esotericsoftware.kryo.Kryo.*;
import static com.esotericsoftware.kryo.util.Util.*;
import static java.lang.Long.numberOfLeadingZeros;

import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.KryoException;
Expand Down Expand Up @@ -282,27 +283,15 @@ public void write (Kryo kryo, Output output, BigDecimal object) {
}

// compatible with writing unscaled value represented as BigInteger's bytes
private static void writeUnscaledLong(Output output, long unscaledLong) {
if (unscaledLong >>> 7 == 0) { // optimize for tiny values
output.writeVarInt(2, true);
output.writeByte((byte) unscaledLong);
} else {
byte[] bytes = new byte[8];
int pos = 8;
do {
bytes[--pos] = (byte) (unscaledLong & 0xFF);
unscaledLong >>= 8;
} while (unscaledLong != 0 && unscaledLong != -1); // out of bits

if (((bytes[pos] ^ unscaledLong) & 0x80) != 0) {
// sign bit didn't fit in previous byte, need to add another byte
bytes[--pos] = (byte) unscaledLong;
}
private static void writeUnscaledLong (Output output, long unscaledLong) {
int insignificantBits = unscaledLong >= 0
? numberOfLeadingZeros(unscaledLong)
: numberOfLeadingZeros(~unscaledLong);
int significantBits = (64 - insignificantBits) + 1; // one more bit is for the sign
int length = (significantBits + (8 - 1)) >> 3; // how many bytes are needed (rounded up)

int length = 8 - pos;
output.writeVarInt(length + 1, true);
output.writeBytes(bytes, pos, length);
}
output.writeByte(length + 1);
output.writeLong(unscaledLong, length);
}

public BigDecimal read (Kryo kryo, Input input, Class<? extends BigDecimal> type) {
Expand All @@ -313,15 +302,11 @@ public BigDecimal read (Kryo kryo, Input input, Class<? extends BigDecimal> type
if (length == NULL) return null;
length--;

byte[] bytes = input.readBytes(length);
if (length > 8) {
byte[] bytes = input.readBytes(length);
unscaledBig = new BigInteger(bytes);
} else {
unscaledLong = bytes[0];
for (int i = 1; i < bytes.length; i++) {
unscaledLong <<= 8;
unscaledLong |= (bytes[i] & 0xFF);
}
unscaledLong = input.readLong(length);
}

int scale = input.readInt(false);
Expand Down
84 changes: 84 additions & 0 deletions test/com/esotericsoftware/kryo/io/InputOutputTest.java
View file
Expand Up @@ -348,6 +348,25 @@ private void runIntTest (Output write) throws IOException {
write.writeInt(-268435455);
write.writeInt(-134217728);
write.writeInt(-268435456);
write.writeInt(0, 1);
write.writeInt(63, 1);
write.writeInt(64, 1);
write.writeInt(127, 1);
write.writeInt(128, 2);
write.writeInt(8192, 2);
write.writeInt(16384, 2);
write.writeInt(2097151, 3);
write.writeInt(1048575, 3);
write.writeInt(134217727, 4);
write.writeInt(268435455, 4);
write.writeInt(134217728, 4);
write.writeInt(268435456, 4);
write.writeInt(-2097151, 3);
write.writeInt(-1048575, 3);
write.writeInt(-134217727, 4);
write.writeInt(-268435455, 4);
write.writeInt(-134217728, 4);
write.writeInt(-268435456, 4);
assertEquals(1, write.writeVarInt(0, true));
assertEquals(1, write.writeVarInt(0, false));
assertEquals(1, write.writeVarInt(63, true));
Expand Down Expand Up @@ -417,6 +436,25 @@ private void runIntTest (Output write) throws IOException {
assertEquals(-268435455, read.readInt());
assertEquals(-134217728, read.readInt());
assertEquals(-268435456, read.readInt());
assertEquals(0, read.readInt(1));
assertEquals(63, read.readInt(1));
assertEquals(64, read.readInt(1));
assertEquals(127, read.readInt(1));
assertEquals(128, read.readInt(2));
assertEquals(8192, read.readInt(2));
assertEquals(16384, read.readInt(2));
assertEquals(2097151, read.readInt(3));
assertEquals(1048575, read.readInt(3));
assertEquals(134217727, read.readInt(4));
assertEquals(268435455, read.readInt(4));
assertEquals(134217728, read.readInt(4));
assertEquals(268435456, read.readInt(4));
assertEquals(-2097151, read.readInt(3));
assertEquals(-1048575, read.readInt(3));
assertEquals(-134217727, read.readInt(4));
assertEquals(-268435455, read.readInt(4));
assertEquals(-134217728, read.readInt(4));
assertEquals(-268435456, read.readInt(4));
assertTrue(read.canReadVarInt());
assertTrue(read.canReadVarInt());
assertTrue(read.canReadVarInt());
Expand Down Expand Up @@ -477,11 +515,15 @@ private void runIntTest (Output write) throws IOException {
write.writeInt(value);
write.writeVarInt(value, true);
write.writeVarInt(value, false);
int numOfBytes = (i % 4) + 1;
write.writeInt(value, numOfBytes);

read.setBuffer(write.toBytes());
assertEquals(value, read.readInt());
assertEquals(value, read.readVarInt(true));
assertEquals(value, read.readVarInt(false));
int numOfBytesMask = numOfBytes == 4 ? -1 : (1 << numOfBytes * 8) - 1;
assertEquals(value & numOfBytesMask, read.readInt(numOfBytes) & numOfBytesMask);
}
}

Expand Down Expand Up @@ -511,6 +553,25 @@ private void runLongTest (Output write) throws IOException {
write.writeLong(-268435455);
write.writeLong(-134217728);
write.writeLong(-268435456);
write.writeLong(0, 1);
write.writeLong(63, 1);
write.writeLong(64, 1);
write.writeLong(127, 1);
write.writeLong(128, 2);
write.writeLong(8192, 2);
write.writeLong(16384, 2);
write.writeLong(2097151, 3);
write.writeLong(1048575, 3);
write.writeLong(134217727, 4);
write.writeLong(268435455, 4);
write.writeLong(134217728, 4);
write.writeLong(268435456, 4);
write.writeLong(-2097151, 3);
write.writeLong(-1048575, 3);
write.writeLong(-134217727, 4);
write.writeLong(-268435455, 4);
write.writeLong(-134217728, 4);
write.writeLong(-268435456, 4);
assertEquals(1, write.writeVarLong(0, true));
assertEquals(1, write.writeVarLong(0, false));
assertEquals(1, write.writeVarLong(63, true));
Expand Down Expand Up @@ -574,6 +635,25 @@ private void runLongTest (Output write) throws IOException {
assertEquals(-268435455, read.readLong());
assertEquals(-134217728, read.readLong());
assertEquals(-268435456, read.readLong());
assertEquals(0, read.readLong(1));
assertEquals(63, read.readLong(1));
assertEquals(64, read.readLong(1));
assertEquals(127, read.readLong(1));
assertEquals(128, read.readLong(2));
assertEquals(8192, read.readLong(2));
assertEquals(16384, read.readLong(2));
assertEquals(2097151, read.readLong(3));
assertEquals(1048575, read.readLong(3));
assertEquals(134217727, read.readLong(4));
assertEquals(268435455, read.readLong(4));
assertEquals(134217728, read.readLong(4));
assertEquals(268435456, read.readLong(4));
assertEquals(-2097151, read.readLong(3));
assertEquals(-1048575, read.readLong(3));
assertEquals(-134217727, read.readLong(4));
assertEquals(-268435455, read.readLong(4));
assertEquals(-134217728, read.readLong(4));
assertEquals(-268435456, read.readLong(4));
assertEquals(0, read.readVarLong(true));
assertEquals(0, read.readVarLong(false));
assertEquals(63, read.readVarLong(true));
Expand Down Expand Up @@ -624,11 +704,15 @@ private void runLongTest (Output write) throws IOException {
write.writeLong(value);
write.writeVarLong(value, true);
write.writeVarLong(value, false);
int numOfBytes = (i % 8) + 1;
write.writeLong(value, numOfBytes);

read.setBuffer(write.toBytes());
assertEquals(value, read.readLong());
assertEquals(value, read.readVarLong(true));
assertEquals(value, read.readVarLong(false));
long numOfBytesMask = numOfBytes == 8 ? -1 : (1L << numOfBytes * 8) - 1;
assertEquals(value & numOfBytesMask, read.readLong(numOfBytes) & numOfBytesMask);
}
}

Expand Down

0 comments on commit 3e7f499

Please sign in to comment.