Use of java-nio-buffer ByteBuffer, Netty ByteBuf and Mapped ByteBuffer
Article directory
Environmental Science
idea2018，jdk1.8
Notes on the use of buffer ByteBuffer, Netty ByteBuf and Mapped ByteBuffer are recorded. References:
ByteBuffer:<Java NIO>
Netty's ByteBuf: https://www.cnblogs.com/duanxz/p/3724448.html
MappedByteBuffer:https://www.cnblogs.com/ironPhoenix/p/4204472.html
1.ByteBuffer
1)ByteBuffer
The role of mark:
Mark a position you have read so that you can return to it at some point.
2)ByteBuffer directBuffer=ByteBuffer.allocateDirect(10);
Direct memory reads and writes faster than ordinary Buffers, but it creates and destroys slower than ordinary Buffers. Therefore, direct memory is used in situations requiring large memory space and frequent access, and is not suitable for situations requiring frequent release of memory.
ByteBuffer uses an example:
public static void main(String[] args) throws Exception {
System.out.println("----------Test allocate--------");
//In-heap memory is made up of JVM process memory
ByteBuffer buffer = ByteBuffer.allocate(10240000);
System.out.println("buffer = " + buffer);
// This part of the system memory is used directly, so it has no effect on the memory of the JVM.
ByteBuffer directBuffer = ByteBuffer.allocateDirect(10240000);
System.out.println("directBuffer = " + directBuffer);
System.out.println("----------Test wrap--------");
byte[] bytes = new byte[32];
buffer = ByteBuffer.wrap(bytes);
System.out.println(buffer);
buffer = ByteBuffer.wrap(bytes, 10, 22);
System.out.println(buffer);
}
/**
* @Description: ByteBuffer Operation:
* 1)NIO ByteBuffer There is only one pointer position that identifies the location. When reading and writing, flip() and rewind() methods need to be called manually.
* 2)Direct memory reads and writes faster than ordinary Buffers, but it creates and destroys slower than ordinary Buffers.
* Therefore, direct memory is used in situations requiring large memory space and frequent access. It is not suitable for situations requiring frequent release of memory.
*/
public class TestByteBufferMethod {
public static ByteBuffer getByteBuffer(String str) {
return ByteBuffer.wrap(str.getBytes());
}
public static String getString(ByteBuffer buffer) {
buffer.flip();
Charset charset = null;
CharsetDecoder decoder = null;
CharBuffer charBuffer = null;
try {
charset = Charset.forName("UTF-8");
decoder = charset.newDecoder();
// charBuffer = decoder.decode(buffer); // In this case, only one result can be output, and the second one is empty.
charBuffer = decoder.decode(buffer.asReadOnlyBuffer());
return charBuffer.toString();
} catch (Exception ex) {
ex.printStackTrace();
}
return "";
}
public static void main(String[] args) throws Exception {
System.out.println("--------Test reset----------");
ByteBuffer buffer = ByteBuffer.allocate(32);
//Usually invoked before data is written to Buffer
buffer.clear();
//Set position position
buffer.position(5);
//Calling mark() sets mark to the value of the current position, and later calling reset() sets the position property to the value of mark.
buffer.mark();
buffer.position(10);
System.out.println("before reset:" + buffer);
buffer.reset();
System.out.println("after reset:" + buffer);
System.out.println("--------Test rewind--------");
buffer.clear();
buffer.position(10);
buffer.limit(15);
System.out.println("before rewind:" + buffer);
//Set position to 0, limit unchanged, cancel mark, usually call before rewriting data to Buffer
buffer.rewind();
System.out.println("before rewind:" + buffer);
System.out.println("--------Test compact--------");
buffer.clear();
buffer.put("abcd".getBytes());
System.out.println("before compact:" + buffer);
System.out.println(new String(buffer.array()));
buffer.flip();
System.out.println("after flip:" + buffer);
System.out.println((char) buffer.get());
System.out.println((char) buffer.get());
System.out.println("after 2 gets:" + buffer);
//The compact() method copies all unread data to the Buffer start. Then set position directly after the last unread element
buffer.compact();
System.out.println("after compact:" + buffer);
System.out.println("\t" + new String(buffer.array()));
System.out.println("------Test get-------------");
buffer = ByteBuffer.allocate(32);
String s = "123456";
buffer.put(s.getBytes());
System.out.println("before flip()" + buffer);
// Conversion to read mode allows flip to read limit data from position (set to 0)
buffer.flip();
System.out.println("after flip():" + buffer);
//Relative reading, read a byte from the position position position and place + 1 to prepare for the next reading and writing.
System.out.println((char) buffer.get());
System.out.println("after get():" + buffer);
//Absolute read, read byte with index in bytes at the bottom of byteBuffer, without changing position
System.out.println((char) buffer.get(2));
System.out.println("after get(index):" + buffer);
byte[] dst = new byte[5];
//Relative reading from position, reading length byte s, and writing to the area of dst subscripts from offset to offset+length
buffer.get(dst, 3, 1);
System.out.println("after get(dst, 3, 2):" + buffer);
for (int i = 0; i < dst.length; i++) {
System.out.println("dst[i]:" + dst[i]);
}
System.out.println("buffer now is:" + buffer);
System.out.println("--------Test put-------");
ByteBuffer bb = ByteBuffer.allocate(32);
System.out.println("before put(byte):" + bb);
//In contrast, write a byte to the position and postion+1 to prepare for the next reading and writing.
bb.put((byte) 'z');
System.out.println("after put(byte):" + bb);
//Absolutely write, insert byte b into the bytes at the bottom of byteBuffer with index, without changing position
bb.put(5, (byte) 'c');
System.out.println("after put(5,(byte) 'c'):" + bb);
// Here the buffer is abcdef[pos=3 lim=6 cap=32]
bb.put(buffer);
System.out.println("after put(buffer):" + bb);
byte[] bytes = new byte[4];
ByteBuffer buffer2 = ByteBuffer.wrap(bytes);
bb.put(bytes, 0, bytes.length);
System.out.println("after put(bytes):" + bb);
/////
ByteBuffer directBuffer=ByteBuffer.allocateDirect(10);
}
}
2.MappedByteBuffer
MappedByteBuffer file memory mapping, often used to operate read and write large files.
public class TestMappedByteBuffer {
public static void main(String[] args)throws Exception {
int length = 0x00006;//One Byte occupies 1B
try (FileChannel channel = FileChannel.open(Paths.get("D:\\test\\2\\TestMappedByteBuffer.txt"),
StandardOpenOption.READ, StandardOpenOption.WRITE);) {
MappedByteBuffer mapBuffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, length);
for(int i=0;i<length;i++) {
mapBuffer.put((byte)'a');
}
System.out.println("mapBuffer:"+mapBuffer.toString());
mapBuffer.flip();
for(int i = 0;i<3;i++) {
//Access as an array, but do not change position
System.out.println(mapBuffer.toString()+":"+mapBuffer.get(i));
}
while (mapBuffer.hasRemaining()){
System.out.println(mapBuffer.toString()+":"+mapBuffer.get());
}
}
}
}
3 Netty's ByteBuf
Netty's Zero-coyp is entirely user-friendly (at the Java level), and when manipulating data,
-
No need to copy data buffer s from one memory area to another. One less copy of memory
Netty's Zero-copy is embodied in the following aspects:
*- Netty provides the CompositeByteBuf class, which can merge multiple ByteBufs into a logical ByteBuf, avoiding copying between ByteBufs.
* - Through wrap operation, we can wrap byte [] array, ByteBuf, ByteBuffer and so on into a Netty ByteBuf object, thus avoiding copy operation.
* - ByteBuf supports slice operations, so ByteBuf can be decomposed into multiple ByteBufs sharing the same storage area, avoiding memory copy.
* - FileChannel.tranferTo, packaged by FileRegion, can transmit the data of the file buffer directly to the target Channel, thus avoiding the memory copy problem caused by the traditional cyclic write mode.
- Netty provides the CompositeByteBuf class, which can merge multiple ByteBufs into a logical ByteBuf, avoiding copying between ByteBufs.
public class TestByteBuf {
public static String convertByteBufToString(ByteBuf buf) {
String str;
if (buf.hasArray()) {
// Handling Heap Buffer
str = new String(buf.array(), buf.arrayOffset() + buf.readerIndex(), buf.readableBytes());
} else { // Processing direct buffers and composite buffers
byte[] bytes = new byte[buf.readableBytes()];
buf.getBytes(buf.readerIndex(), bytes);
str = new String(bytes, 0, buf.readableBytes());
}
return str;
}
public static ByteBuf convertStringToByteBuf(String str) {
return Unpooled.wrappedBuffer("abcdef".getBytes());
}
public static void main(String[] args) throws Exception {
//The default is to create heap buffer
System.out.print("\n-------test heapBuf----\n");
ByteBuf byteBuf1 = Unpooled.buffer(5);
//Writing data
byteBuf1.writeBytes("abcdefgh".getBytes("utf-8"));
byteBuf1.setByte(0,(byte)'1');
//Read data
for (int i = 0; i < byteBuf1.capacity(); i++) {
System.out.print("\n-------test heapBuf----buf.getByte:" + (byteBuf1.getByte(i)));
}
System.out.printf("\n-------test heapBuf's capcity:" +byteBuf1.capacity());
System.out.printf("\n-------test heapBuf's ----convertByteBufToString(byteBuf1):" + convertByteBufToString(byteBuf1));
/**
* Realize zero copy through Composite ByteBuf, add ByteBuf to Composite ByteBuf
* CompositeByteBuf Class, which can merge multiple ByteBufs into a logical ByteBuf, avoiding copying between ByteBufs
* CompositeByteBuf Share storage space with the added ByteBuf
*/
System.out.print("\n-------test CompositeByteBuf----\n");
//Combination Buffer
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
//Heap Buffer
ByteBuf heapBuf = Unpooled.buffer(4);
//Direct Buffer
ByteBuf directBuf = Unpooled.directBuffer(4);
heapBuf.writeBytes("a".getBytes());
directBuf.writeBytes("ef".getBytes());
compBuf.addComponents(true, heapBuf, directBuf);
compBuf.writeBytes("cdgh".getBytes());
compBuf.setByte(0,(byte)'g');
System.out.printf("\n-------test CompositeByteBuf:" + heapBuf);
System.out.printf("\n-------test CompositeByteBuf:" + directBuf);
System.out.printf("\n-------test CompositeByteBuf:" + compBuf);
System.out.printf("\n-------test CompositeByteBuf heapBuf:" + convertByteBufToString(heapBuf));
System.out.printf("\n-------test CompositeByteBuf directBuf:" + convertByteBufToString(directBuf));
System.out.printf("\n-------test CompositeByteBuf compBuf:" + convertByteBufToString(compBuf));
//View Memory Address
AddressPrint addressPrint=new AddressPrint();
System.out.printf("\n-------test CompositeByteBuf heapBuf address:%s;directBuf address:%s;compBuf address:%s;",addressPrint.addressOf(heapBuf.getByte(0)),addressPrint.addressOf(directBuf.getByte(0)),addressPrint.addressOf(compBuf.getByte(0)));
/**
* Additional copies copy byte arrays to ByteBuf
*/
/*ByteBuf byteBuf = Unpooled.buffer();
byteBuf.writeBytes(bytes2);*/
/**
* By wrap operation, byte [] array, ByteBuf, ByteBuffer and so on can be wrapped into a Netty ByteBuf object, thus avoiding copy operation.
* ByteBuf Objects share the same storage space with bytes arrays, and changes to bytes are reflected in ByteBuf objects.
*/
ByteBuf byteBuf = Unpooled.wrappedBuffer("abcdef".getBytes());
/**
* By slice operation, a slice operation can slice a ByteBuf into multiple ByteBuf objects sharing a storage area.
*/
ByteBuf byteBuf3 = Unpooled.buffer(16);
byteBuf3.writeBytes("abcdef".getBytes());
ByteBuf header = byteBuf3.slice(0, 5);
ByteBuf body = byteBuf3.slice(5, 10);
System.out.printf("\n-------test wrap,slice----byteBuf3:" + byteBuf3);
System.out.printf("\n-------test wrap,slice----header:" + header);
System.out.printf("\n-------test wrap,slice----body:" + body);
System.out.printf("\n-------test wrap,slice----byteBuf3 address:%s;header address:%s;body address:%s;",addressPrint.addressOf(byteBuf3.getByte(0)),addressPrint.addressOf(header.getByte(0)),addressPrint.addressOf(body.getByte(0)));
/**
*
*/
ByteBuf buf1 = Unpooled.copiedBuffer("Netty in Action test!", CharsetUtil.UTF_8);
//Copy the contents of ByteBuf 0-14
ByteBuf copy = buf1.copy(0, 14);
//Similarly modify the content
buf1.setByte(0, (byte) 'J');
//Copy buffers and original buffers do not share data
System.out.print("\nbuf1.getByte(0)" + (char) buf1.getByte(0));
System.out.print("\ncopy.getByte(0)" + (char) copy.getByte(0));
//////////// netty pooled memory allocation: Pooled ByteBuf Allocator uses jemalloc memory allocation algorithm///////////
ByteBuf poolBuf = PooledByteBufAllocator.DEFAULT.buffer(5);
poolBuf.writeBytes("abc".getBytes());
}
public static class AddressPrint {
private Unsafe unsafe;
{
try
{
Field field = Unsafe.class.getDeclaredField("theUnsafe");
field.setAccessible(true);
unsafe = (Unsafe)field.get(null);
}
catch (Exception e)
{
e.printStackTrace();
}
}
public String addressOf(Object o)
throws Exception
{
Object[] array = new Object[] {o};
long baseOffset = unsafe.arrayBaseOffset(Object[].class);
int addressSize = unsafe.addressSize();
long objectAddress;
switch (addressSize)
{
case 4:
objectAddress = unsafe.getInt(array, baseOffset);
break;
case 8:
objectAddress = unsafe.getLong(array, baseOffset);
break;
default:
throw new Error("unsupported address size: " + addressSize);
}
return(Long.toHexString(objectAddress));
}
public void printBytes(long objectAddress, int num)
{
for (long i = 0; i < num; i++)
{
int cur = unsafe.getByte(objectAddress + i);
System.out.print((char)cur);
}
System.out.println();
}
}
}
4. test
1) The first byte memory address of Composite ByteBuf combined buffer and the first byte memory address relationship of ByteBuf added:
//Combination Buffer
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
//Heap Buffer
ByteBuf heapBuf = Unpooled.buffer(4);
//Direct Buffer
ByteBuf directBuf = Unpooled.directBuffer(4);
heapBuf.writeBytes("a".getBytes());
directBuf.writeBytes("ef".getBytes());
compBuf.addComponents(true, heapBuf, directBuf);
compBuf.writeBytes("cdgh".getBytes());
2) The first byte memory address relationship of each ByteBuf after slice:
ByteBuf byteBuf3 = Unpooled.buffer(16);
byteBuf3.writeBytes("abcdef".getBytes());
ByteBuf header = byteBuf3.slice(0, 5);
ByteBuf body = byteBuf3.slice(5, 10);
