Handwritten RPC core module network protocol module writing -- implementation of codec

The previous foundation has been written. Now let's implement the encoder.
Why do I need an encoder?
netty is only responsible for transmitting data. It doesn't care what the data looks like. As mentioned earlier, the custom protocol is to organize, transmit and decode the data we want to transmit according to our rules. The encoder is to organize the data we want to send.
Netty has made a good package for us. We only need to integrate MessageToByteEncoder to implement its encode method, and then add this encoder to the pipeline of our netty processor.

package com.info.protocol.netty.core.codec;

import com.info.protocol.netty.core.Header;
import com.info.protocol.netty.core.Protocol;
import com.info.protocol.serial.Serializer;
import com.info.protocol.serial.SerializerManager;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.MessageToByteEncoder;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class CustomEncoder extends MessageToByteEncoder<Protocol<Object>> {

        /*
        +--------------------------------------------------------------------------------------------+
        |Magic number 16bit | protocol version 8bit | serialization method 8bit | message length 32bit | message type (request or response) 2bit|messageId 64bit|
        +--------------------------------------------------------------------------------------------+
        */

    @Override
    protected void encode(ChannelHandlerContext ctx, Protocol<Object> msg, ByteBuf out) throws Exception {
        log.info("----------------- begin encode -----------------");
        final Header header = msg.getHeader();
        // Write magic number
        out.writeShort(header.getMagic());
        // Write the protocol version used
        out.writeByte(header.getProtocolVersion());
        // Write serialization mode
        out.writeByte(header.getSerializeType());
        // The message content is serialized because the serialization has been implemented previously, but there is only serialization,
        // We need to get the corresponding serializer through this serialization method, so it is best to have a manager for the serialization method, and provide the corresponding serializer according to the serialization method
        // Gets the function of the specific serializer
        Serializer serializer = SerializerManager.getSerializerByCode(header.getSerializeType());
        // Serialize message
        final Object content = msg.getContent();
        final byte[] data = serializer.serialize(content);
        // Write message length
        out.writeInt(data.length);
        // Write message type
        out.writeByte(header.getMessageType());
        // Write message id
        out.writeLong(header.getMessageId());
        // Write specific message content
        out.writeBytes(data);
    }
}

Serialization Manager

package com.info.serial;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

public class SerializerManager {

    private static Map<Byte, Serializer> serializerMap = new ConcurrentHashMap<>(1 << 1);

    static {
        Serializer javaSerializer = new JavaSerializer();
        Serializer jsonSerializer = new JsonSerializer();
        serializerMap.put(javaSerializer.getType(), javaSerializer);
        serializerMap.put(jsonSerializer.getType(), jsonSerializer);
    }

    public static Serializer getSerializerByCode(byte code) {
        Serializer serializer = serializerMap.get(code);
        if (serializer == null) {
            serializer = new JavaSerializer();
        }
        return serializer;
    }
}

After the encoder is implemented, the decoder decodes the encoded data in reverse, because we need to distinguish the message type (the content returned by the sending request and the receiving request are different), and we have defined the object to be deserialized

package com.info.protocol.netty.core;

import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

import java.io.Serializable;

@Getter
@Setter
@ToString
public class Request implements Serializable {

    private String className;

    private String methodName;

    private Object[] params;

    private Class<?>[] parameterTypes;
}

package com.info.protocol.netty.core;

import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;

@Getter
@Setter
@AllArgsConstructor
public class Response {

    private Object data;

    private String msg;
}

Implementation decoder

package com.info.protocol.netty.core.codec;

import com.info.protocol.constants.CommonConstant;
import com.info.protocol.enums.MessageTypeEnum;
import com.info.protocol.netty.core.Header;
import com.info.protocol.netty.core.Protocol;
import com.info.protocol.netty.core.Request;
import com.info.protocol.netty.core.Response;
import com.info.protocol.serial.Serializer;
import com.info.protocol.serial.SerializerManager;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
import lombok.extern.slf4j.Slf4j;

import java.util.List;

@Slf4j
public class CustomDecoder extends ByteToMessageDecoder {

        /*
        +--------------------------------------------------------------------------------------------+
        |Magic number 16bit | protocol version 8bit | serialization method 8bit | message length 32bit | message type (request or response) 2bit|messageId 64bit|
        +--------------------------------------------------------------------------------------------+
        */

    @Override
    protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
        log.info("------------------ begin decode ------------------");
        if (in.readableBytes() < CommonConstant.HEADER_LENGTH) {
            // The message is not long enough and will not be parsed
            return;
        }
        //Mark an index to read data and reset it if necessary.
        in.markReaderIndex();

        final short magic = in.readShort();
        if (magic != CommonConstant.MAGIC) {
            throw new IllegalArgumentException("illegal request parameter 'magic' " + magic);
        }
        final byte protocolVersionCode = in.readByte();
        final byte serializerTypeCode = in.readByte();
        final int messageLength = in.readInt();
        final byte messageTypeCode = in.readByte();
        final long messageId = in.readLong();
        if (in.readableBytes() < messageLength) {
            // Message insufficient reset read ID
            in.resetReaderIndex();
            return;
        }
        byte[] content = new byte[messageLength];
        in.readBytes(content);

        Header header = new Header();
        header.setMagic(magic)
                .setMessageId(messageId)
                .setMessageLength(messageLength)
                .setMessageType(messageTypeCode)
                .setProtocolVersion(protocolVersionCode)
                .setSerializeType(serializerTypeCode);

        // Get deserializer
        final Serializer serializer = SerializerManager.getSerializerByCode(serializerTypeCode);
        MessageTypeEnum messageType = MessageTypeEnum.getMessageTypeEnumByCode(messageTypeCode);

        switch (messageType) {
            case REQUEST:
                final Request request = serializer.deserialize(content, Request.class);
                Protocol<Request> requestProtocol = new Protocol<>();
                requestProtocol.setHeader(header);
                requestProtocol.setContent(request);
                out.add(requestProtocol);
                break;
            case RESPONSE:
                final Response response = serializer.deserialize(content, Response.class);
                Protocol<Response> responseProtocol = new Protocol<>();
                responseProtocol.setHeader(header);
                responseProtocol.setContent(response);
                out.add(responseProtocol);
                break;
            case HEART_BEAT:
            default:
        }

    }
}

Finally, we need to implement a processor. The purpose is that after the server reads the requested data (has been decoded), it needs to really obtain the result of calling the target method. Reflection is used here. The processor also needs to inherit SimpleChannelInboundHandler and implement its channelRead0 method.

package com.info.protocol.netty.server;

import com.info.protocol.enums.MessageTypeEnum;
import com.info.protocol.netty.core.Header;
import com.info.protocol.netty.core.Protocol;
import com.info.protocol.netty.core.Request;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;

import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

public class CustomServerHandler extends SimpleChannelInboundHandler<Protocol<Request>> {
    @Override
    protected void channelRead0(ChannelHandlerContext ctx, Protocol<Request> msg) throws Exception {
        Protocol protocol = new Protocol<>();
        final Header header = msg.getHeader();
        header.setMessageType(MessageTypeEnum.RESPONSE.getCode());
        // Reflection calls the target method
        Object result = invoke(msg.getContent());
        protocol.setHeader(header);
        protocol.setContent(result);
        ctx.writeAndFlush(protocol);
    }

    // Call target method
    private Object invoke(Request request) {
        final String clzName = request.getClassName();
        try {
            final Class<?> clz = Class.forName(clzName);
            final Constructor<?> constructor = clz.getDeclaredConstructors()[0];
            final Object instance = constructor.newInstance();
            final Method method = clz.getDeclaredMethod(request.getMethodName(), request.getParameterTypes());
            return method.invoke(instance, request.getParams());
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (InstantiationException e) {
            e.printStackTrace();
        } catch (IllegalAccessException e) {
            e.printStackTrace();
        } catch (InvocationTargetException e) {
            e.printStackTrace();
        } catch (NoSuchMethodException e) {
            e.printStackTrace();
        }
        return null;
    }
}

The basic code of the server has been implemented. Now we need to combine our encoder, decoder and processor to make them effective. The method is to add them to the pipeline of netty:

package com.info.protocol.netty.server;

import com.info.protocol.netty.core.codec.CustomDecoder;
import com.info.protocol.netty.core.codec.CustomEncoder;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.socket.SocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.logging.LoggingHandler;

public class CustomServerInitializer extends ChannelInitializer<SocketChannel> {
    @Override
    protected void initChannel(SocketChannel ch) throws Exception {
        ch.pipeline()
                .addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 12, 4, 0, 0))
                // netty provides log processing, which is easy to see the call process
                .addLast(new LoggingHandler())
                .addLast(new CustomEncoder())
                .addLast(new CustomDecoder());
    }
}

Finally, you need to add the components of this initializer (connecting codec) to netty:

ServerBootstrap bootstrap = new ServerBootstrap();
            bootstrap.group(bossGroup, workerGroup)
                    .channel(NioServerSocketChannel.class)
                    // Specify childHandler
                    .childHandler(new CustomServerInitializer());

So far, the server code has been implemented, and the next section starts to implement the client code.

Tags: network rpc Middleware Network Protocol

Posted on Mon, 06 Dec 2021 17:33:32 -0500 by AncientSage