Network Connection Connection
The network connection part is a very important part of cobar, which takes on both forward (user-oriented) and backward (mysql-oriented) connections.By analyzing the network connection code, you can roughly see the data flow direction of cobar.
Let's first look at the entire connection architecture (omit heartbeat related content):
Let's look at the system initialization code again
// startup processors
LOGGER.info("Startup processors ...");
int handler = system.getProcessorHandler();
int executor = system.getProcessorExecutor();
int committer = system.getProcessorCommitter();
processors = new NIOProcessor[system.getProcessors()];
for (int i = 0; i < processors.length; i++) {
processors[i] = new NIOProcessor("Processor" + i, handler, executor, committer);
processors[i].startup();
}
timer.schedule(processorCheck(), 0L, system.getProcessorCheckPeriod());
// startup connector
LOGGER.info("Startup connector ...");
connector = new NIOConnector(NAME + "Connector");
connector.setProcessors(processors);
connector.start();
// init dataNodes
Map<String, MySQLDataNode> dataNodes = config.getDataNodes();
LOGGER.info("Initialize dataNodes ...");
for (MySQLDataNode node : dataNodes.values()) {
node.init(1, 0);
}
timer.schedule(dataNodeIdleCheck(), 0L, system.getDataNodeIdleCheckPeriod());
timer.schedule(dataNodeHeartbeat(), 0L, system.getDataNodeHeartbeatPeriod());
// startup manager
ManagerConnectionFactory mf = new ManagerConnectionFactory();
mf.setCharset(system.getCharset());
mf.setIdleTimeout(system.getIdleTimeout());
manager = new NIOAcceptor(NAME + "Manager", system.getManagerPort(), mf);
manager.setProcessors(processors);
manager.start();
LOGGER.info(manager.getName() + " is started and listening on " + manager.getPort());
// startup server
ServerConnectionFactory sf = new ServerConnectionFactory();
sf.setCharset(system.getCharset());
sf.setIdleTimeout(system.getIdleTimeout());
server = new NIOAcceptor(NAME + "Server", system.getServerPort(), sf);
server.setProcessors(processors);
server.start();
timer.schedule(clusterHeartbeat(), 0L, system.getClusterHeartbeatPeriod());It can be seen from the code that the system first initializes the corresponding number of nio processors according to the number of system cores, then initializes the connector, node initialization, management thread startup, service thread startup.As you can see, the managed and service threads share processors.In addition, there are some timers, which are not explained here for the moment.Manager is used for internal personnel management, so let's shelve it here.NIOProcessor, MySQLDataNode, NIOAcceptor are left.Because NIOProcessor is a shared product, let's first look at its structure.
NIOProcessor
private final String name;
private final NIOReactor reactor;
private final BufferPool bufferPool;
private final NameableExecutor handler;
private final NameableExecutor executor;
private final NameableExecutor committer;
private final ConcurrentMap<Long, FrontendConnection> frontends;
private final ConcurrentMap<Long, BackendConnection> backends;
private final CommandCount commands;
private long netInBytes;
private long netOutBytes;The above is the field information for NIOProcessor, the most important of which is NIOReactor, BufferPool.BufferPool is a buffer pool, not to mention here; let's look at the contents of NIOReactor
private final R reactorR;
private final W reactorW;
public NIOReactor(String name) throws IOException {
this.name = name;
this.reactorR = new R();
this.reactorW = new W();
}
final void startup() {
new Thread(reactorR, name + "-R").start();
new Thread(reactorW, name + "-W").start();
}
final void postRegister(NIOConnection c) {
reactorR.registerQueue.offer(c);
reactorR.selector.wakeup();
}
final void postWrite(NIOConnection c) {
reactorW.writeQueue.offer(c);
}Two threads are maintained in the reactor, and one blockingqueue is assigned to each thread.The R thread is responsible for reading or writing connections from the queue, and the W thread is responsible for writing.
NIOAcceptor
The main function of NIOAcceptor is to receive mysql statements executed by the user based on the specified port, poll for NIOProcessor to process, let's take a look at its accept code.First receive the channel and create a FrontendConnection using the ServerConnectionFactory.
private void accept() {
SocketChannel channel = null;
try {
channel = serverChannel.accept();
channel.configureBlocking(false);
FrontendConnection c = factory.make(channel);
c.setAccepted(true);
c.setId(ID_GENERATOR.getId());
NIOProcessor processor = nextProcessor();
c.setProcessor(processor);
processor.postRegister(c);
} catch (Throwable e) {
closeChannel(channel);
LOGGER.warn(getName(), e);
}
}
protected FrontendConnection getConnection(SocketChannel channel) {
SystemConfig sys = CobarServer.getInstance().getConfig().getSystem();
ServerConnection c = new ServerConnection(channel);
c.setPrivileges(new CobarPrivileges());
c.setQueryHandler(new ServerQueryHandler(c));
// c.setPrepareHandler(new ServerPrepareHandler(c)); TODO prepare
c.setTxIsolation(sys.getTxIsolation());
c.setSession(new BlockingSession(c));
c.setSession2(new NonBlockingSession(c));
return c;
}
The code tells you that for each request, cobar creates a ServerConnection, chooses a NIOProcessor to process, and ServerConnection is a subclass implementation of FrontendConnection.Next, let's analyze ServerConnection in detail.
Forward Connection
The main implementation of a forward connection is ServerConnection (this does not include administrative parts).We see the processor.postRegister(c) statement in the accept() function, so let's see what this statement does.
Submit connection to queue
final void postRegister(NIOConnection c) {
reactorR.registerQueue.offer(c);
reactorR.selector.wakeup();
}
//Register the event type and notice that the key is saved here
public void register(Selector selector) throws IOException {
try {
processKey = channel.register(selector, SelectionKey.OP_READ, this);
isRegistered = true;
} finally {
if (isClosed.get()) {
clearSelectionKey();
}
}
}
//Read data for processing
public void read() throws IOException {
ByteBuffer buffer = this.readBuffer;
int got = channel.read(buffer);
lastReadTime = TimeUtil.currentTimeMillis();
if (got < 0) {
throw new EOFException();
}
netInBytes += got;
processor.addNetInBytes(got);
int offset = readBufferOffset, length = 0, position = buffer.position();
for (;;) {
length = getPacketLength(buffer, offset);
if (length == -1) {
if (!buffer.hasRemaining()) {
checkReadBuffer(buffer, offset, position);
}
break;
}
if (position >= offset + length) {
buffer.position(offset);
byte[] data = new byte[length];
buffer.get(data, 0, length);
handle(data);
offset += length;
if (position == offset) {
if (readBufferOffset != 0) {
readBufferOffset = 0;
}
buffer.clear();
break;
} else {
readBufferOffset = offset;
buffer.position(position);
continue;
}
} else {
if (!buffer.hasRemaining()) {
checkReadBuffer(buffer, offset, position);
}
break;
}
}
}
//Reset Command Processing Type
source.setHandler(new FrontendCommandHandler(source));
ByteBuffer buffer = source.allocate();
source.write(source.writeToBuffer(AUTH_OK, buffer));
//Processing data
public void handle(final byte[] data) {
processor.getHandler().execute(new Runnable() {
@Override
public void run() {
try {
handler.handle(data);
} catch (Throwable t) {
error(ErrorCode.ERR_HANDLE_DATA, t);
}
}
});
}
//Reply to Request
public void write(ByteBuffer buffer) {
if (isClosed.get()) {
processor.getBufferPool().recycle(buffer);
return;
}
if (isRegistered) {
try {
writeQueue.put(buffer);
} catch (InterruptedException e) {
error(ErrorCode.ERR_PUT_WRITE_QUEUE, e);
return;
}
processor.postWrite(this);
} else {
processor.getBufferPool().recycle(buffer);
close();
}
}
private void enableWrite() {
final Lock lock = this.keyLock;
lock.lock();
try {
SelectionKey key = this.processKey;
key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
} finally {
lock.unlock();
}
processKey.selector().wakeup();
}
private void write(NIOConnection c) {
try {
c.writeByQueue();
} catch (Throwable e) {
c.error(ErrorCode.ERR_WRITE_BY_QUEUE, e);
}
}
//Processing data
public void handle(byte[] data) {
switch (data[4]) {
case MySQLPacket.COM_INIT_DB:
commands.doInitDB();
source.initDB(data);
break;
case MySQLPacket.COM_QUERY:
commands.doQuery();
source.query(data);
break;
case MySQLPacket.COM_PING:
commands.doPing();
source.ping();
break;
case MySQLPacket.COM_QUIT:
commands.doQuit();
source.close();
break;
case MySQLPacket.COM_PROCESS_KILL:
commands.doKill();
source.kill(data);
break;
case MySQLPacket.COM_STMT_PREPARE:
commands.doStmtPrepare();
source.stmtPrepare(data);
break;
case MySQLPacket.COM_STMT_EXECUTE:
commands.doStmtExecute();
source.stmtExecute(data);
break;
case MySQLPacket.COM_STMT_CLOSE:
commands.doStmtClose();
source.stmtClose(data);
break;
case MySQLPacket.COM_HEARTBEAT:
commands.doHeartbeat();
source.heartbeat(data);
break;
default:
commands.doOther();
source.writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Unknown command");
}
}
//Take query as an example, and let's see how we go down
public void query(byte[] data) {
if (queryHandler != null) {
MySQLMessage mm = new MySQLMessage(data);
mm.position(5);
String sql = null;
try {
sql = mm.readString(charset);
} catch (UnsupportedEncodingException e) {
writeErrMessage(ErrorCode.ER_UNKNOWN_CHARACTER_SET, "Unknown charset '" + charset + "'");
return;
}
if (sql == null || sql.length() == 0) {
writeErrMessage(ErrorCode.ER_NOT_ALLOWED_COMMAND, "Empty SQL");
return;
}
queryHandler.query(sql);
} else {
writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Query unsupported!");
}
}
@Override
public void query(String sql) {
ServerConnection c = this.source;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(new StringBuilder().append(c).append(sql).toString());
}
int rs = ServerParse.parse(sql);
switch (rs & 0xff) {
case ServerParse.EXPLAIN:
ExplainHandler.handle(sql, c, rs >>> 8);
break;
case ServerParse.SET:
SetHandler.handle(sql, c, rs >>> 8);
break;
case ServerParse.SHOW:
ShowHandler.handle(sql, c, rs >>> 8);
break;
case ServerParse.SELECT:
SelectHandler.handle(sql, c, rs >>> 8);
break;
case ServerParse.START:
StartHandler.handle(sql, c, rs >>> 8);
break;
case ServerParse.BEGIN:
BeginHandler.handle(sql, c);
break;
case ServerParse.SAVEPOINT:
SavepointHandler.handle(sql, c);
break;
case ServerParse.KILL:
KillHandler.handle(sql, rs >>> 8, c);
break;
case ServerParse.KILL_QUERY:
c.writeErrMessage(ErrorCode.ER_UNKNOWN_COM_ERROR, "Unsupported command");
break;
case ServerParse.USE:
UseHandler.handle(sql, c, rs >>> 8);
break;
case ServerParse.COMMIT:
c.commit();
break;
case ServerParse.ROLLBACK:
c.rollback();
break;
default:
c.execute(sql, rs);
}
}
//Let's go back todefaultFor example, see what happens next.You can see that routing is already involved here.
public void execute(String sql, int type) {
if (txInterrupted) {
writeErrMessage(ErrorCode.ER_YES, "Transaction error, need to rollback.");
return;
}
String db = this.schema;
if (db == null) {
writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "No database selected");
return;
}
SchemaConfig schema = CobarServer.getInstance().getConfig().getSchemas().get(db);
if (schema == null) {
writeErrMessage(ErrorCode.ER_BAD_DB_ERROR, "Unknown database '" + db + "'");
return;
}
RouteResultset rrs = null;
try {
rrs = ServerRouter.route(schema, sql, this.charset, this);
} catch (SQLNonTransientException e) {
StringBuilder s = new StringBuilder();
LOGGER.warn(s.append(this).append(sql).toString(), e);
String msg = e.getMessage();
writeErrMessage(ErrorCode.ER_PARSE_ERROR, msg == null ? e.getClass().getSimpleName() : msg);
return;
}
session.execute(rrs, type);
}
//Perform sql operations on each node of the route selection
public void execute(RouteResultset rrs, int type) {
if (LOGGER.isDebugEnabled()) {
StringBuilder s = new StringBuilder();
LOGGER.debug(s.append(source).append(rrs).toString());
}
RouteResultsetNode[] nodes = rrs.getNodes();
if (nodes == null || nodes.length == 0) {
source.writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "No dataNode selected");
return;
}
if (nodes.length == 1) {
singleNodeExecutor.execute(nodes[0], this, rrs.getFlag());
} else {
boolean autocommit = source.isAutocommit();
if (autocommit && isModifySQL(type)) {
autocommit = false;
}
multiNodeExecutor.execute(nodes, autocommit, this, rrs.getFlag());
}
}
Not finished, to be continued.