c/c + + background development learning notes use epoll+reactor to realize millions of concurrent servers

reactor

Encapsulate the fd read / write events returned by epoll, set callback functions for each event, store all concerned fd and corresponding events in a data structure, and form a one-to-one correspondence with the red black tree nodes inside epoll. When epoll returns, use data.ptr to get the corresponding entry in our data structure, and then process it
The encapsulation of fd is as follows

typedef int (*NCALLBACK)(int fd, int events, void *arg);

struct ntyevent {
	int fd;
	int events;                     //Events monitored: EPOLLIN or EPOLLOUT
	void *arg;                      //Additional parameters passed to the callback function (reactor pointer)
	NCALLBACK callback;             //Callback function
	
	int used;                       //The current entry is valid
	char buffer[BUFFER_LENGTH];     //Reading and writing share the same buffer, because the currently monitored events can only be read or write, not both
	int length;                     //Length of valid data in buffer
	long last_active;               //Current fd last active event
	int sticky;                     //If set to 1, fd will not be turned off because it is inactive for a long time (for listenfd)
};

The reactor data structure is defined as follows

struct ntyreactor {
	int epfd;
	int block_num;
	struct ntyevent **events; //array of ntyevent *, length = block_num, each block has 1024 ntyevent
};

events is a pointer array that can be expanded. Each element points to a 1024 long ntyevent array

The whole server has the following functions

int accept_cb(int fd, int events, void *arg);
int recv_cb(int fd, int events, void *arg);
int send_cb(int fd, int events, void *arg);

void nty_event_update(struct ntyevent *ev, int fd, NCALLBACK callback, void *arg);
int nty_event_add(int epfd, int events, struct ntyevent *ev);
int nty_event_del(int epfd, struct ntyevent *ev);

int start_listen(short port);
int ntyreactor_init(struct ntyreactor *reactor);
struct ntyevent *ntyreactor_get_event(struct ntyreactor *reactor, int fd);
int ntyreactor_destory(struct ntyreactor *reactor);
int ntyreactor_addlistener(struct ntyreactor *reactor, int listenfd, NCALLBACK acceptor);
int ntyreactor_run(struct ntyreactor *reactor);

First look at the main function

#define BUFFER_LENGTH		4096
#define EVENTS_BLOCK_SIZE	1024
#define EVENT_BATCH_SIZE    1024
#define SERVER_PORT			8888
#define PORT_COUNT          100
#define CLIENT_TIMEOUT      15

int main(int argc, char *argv[]) {

	unsigned short port = SERVER_PORT;
	if (argc == 2) {
		port = atoi(argv[1]);
	}

	struct ntyreactor *reactor = (struct ntyreactor*)calloc(1, sizeof(struct ntyreactor));
	ntyreactor_init(reactor);
	
	int i;
	for(i = 0; i < PORT_COUNT; i++) {
		int listenfd = start_listen(port + i);
		ntyreactor_addlistener(reactor, listenfd, accept_cb);
	}
	ntyreactor_run(reactor);

	ntyreactor_destory(reactor);
	free(reactor);
	return 0;
}

PORT_ When count is set to 100, the program listens to 100 ports starting from 8888. Each port can accept at least 10000 ports, so the number of connections can reach one million. (because the number of clients tested is relatively small, if you do not listen to more ports, the number of (SRC IP, SRC port, DST IP, DST port) quads will not be enough.)

Where ntyreactor_run is the main loop of the program

int ntyreactor_run(struct ntyreactor *reactor) {
	if (reactor == NULL) return -1;
	if (reactor->epfd < 0) return -1;
	if (reactor->events == NULL) return -1;
	
	struct epoll_event events[EVENT_BATCH_SIZE];
	
	int block_idx = 0, check_interval = 10, i, cur; // check every 10 loops

	while (1) {
#if CHECK_TIMEOUT
		cur++;
		if (cur % check_interval == 9) {
			if (block_idx >= reactor->block_num) {
				block_idx = 0;
			}
			ntyreactor_check_timeout(reactor, block_idx++);
		}
#endif
		int nready = epoll_wait(reactor->epfd, events, EVENT_BATCH_SIZE, 1000);
		if (nready < 0) {
			printf("epoll_wait error\n");
			continue;
		}

		for (i = 0; i < nready; i++) {

			struct ntyevent *ev = (struct ntyevent*)events[i].data.ptr;
			int err = 0;
			if ((events[i].events & EPOLLIN) && (ev->events & EPOLLIN)) {
				err = ev->callback(ev->fd, events[i].events, ev->arg);
			}
			if ((events[i].events & EPOLLOUT) && (ev->events & EPOLLOUT)) {
				err = ev->callback(ev->fd, events[i].events, ev->arg);
			}
			
		}

	}
}

In the main loop, the callback function is called directly for each monitored event
accept_cb adds each new connection to epoll and reactor

int accept_cb(int fd, int events, void *arg) {

	struct ntyreactor *reactor = (struct ntyreactor*)arg;
	if (reactor == NULL) return -1;

	struct sockaddr_in client_addr;
	socklen_t len = sizeof(client_addr);

	int clientfd;

	if ((clientfd = accept(fd, (struct sockaddr*)&client_addr, &len)) == -1) {
		if (errno != EAGAIN && errno != EINTR) {
			
		}
		printf("accept: %s\n", strerror(errno));
		return -1;
	}

	int i = 0;
	struct ntyevent *event = ntyreactor_get_event(reactor, clientfd);
	if (event == NULL) return -1;

	do {

		// Set clientfd non blocking
		if (fcntl(clientfd, F_SETFL, O_NONBLOCK) < 0) {
			printf("%s: fcntl nonblocking failed, %d\n", __func__, EVENTS_BLOCK_SIZE);
			break;
		}

		nty_event_update(event, clientfd, recv_cb, reactor);
		nty_event_add(reactor->epfd, EPOLLIN, event);

	} while (0);

	printf("new connect [%s:%d][time:%ld], pos[%d]\n", 
		inet_ntoa(client_addr.sin_addr), ntohs(client_addr.sin_port), event->last_active, i);

	return 0;

}

recv_cb and send_cb are responsible for reading and writing respectively

Here we will mention the problems of horizontal trigger and edge trigger
The horizontal trigger is always triggered as long as the condition is true (for example, epoll will always trigger epolin as long as the data in the buffer has not been read), while the edge trigger is triggered only once when the condition changes from false to true
Generally, horizontal trigger is used when the amount of data is large (when all data in the buffer cannot be read at one time). Edge trigger can be used when the amount of data is small. Edge trigger must use while loop to read all data each time.

This code uses a horizontal trigger.

int recv_cb(int fd, int events, void *arg) {

	struct ntyreactor *reactor = (struct ntyreactor*)arg;
	struct ntyevent *ev =  ntyreactor_get_event(reactor, fd);
	if(ev == NULL) return -1;

	int len = recv(fd, ev->buffer, BUFFER_LENGTH, 0);
	nty_event_del(reactor->epfd, ev);

	if (len > 0) {
		ev->length = len;
		ev->buffer[len] = '\0';
		printf("C[%d]:%s\n", fd, ev->buffer);

		nty_event_update(ev, fd, send_cb, reactor);
		nty_event_add(reactor->epfd, EPOLLOUT, ev);
	} else if (len == 0) {
		close(ev->fd);
		printf("[fd=%d] closed\n", fd);
	} else {
		close(ev->fd);
		printf("recv[fd=%d] error[%d]:%s\n", fd, errno, strerror(errno));
	}

	return len;
}


int send_cb(int fd, int events, void *arg) {

	struct ntyreactor *reactor = (struct ntyreactor*)arg;
	struct ntyevent *ev =  ntyreactor_get_event(reactor, fd);
	if(ev == NULL) return -1;

	int len = send(fd, ev->buffer, ev->length, 0);
	if (len > 0) {
		printf("send[fd=%d], [%d]%s\n", fd, len, ev->buffer);

		nty_event_del(reactor->epfd, ev);
		nty_event_update(ev, fd, recv_cb, reactor);
		nty_event_add(reactor->epfd, EPOLLIN, ev);
		
	} else {

		close(ev->fd);

		nty_event_del(reactor->epfd, ev);
		printf("send[fd=%d] error %s\n", fd, strerror(errno));

	}

	return len;
}

The automatic expansion of reactor is realized through ntyreactor_get_event

//Enter fd to return the corresponding ntyevent
struct ntyevent *ntyreactor_get_event(struct ntyreactor *reactor, int fd) {
	int b = fd / EVENTS_BLOCK_SIZE;
	int i = fd % EVENTS_BLOCK_SIZE;
	if(b >= reactor->block_num) {
		int new_block_num = reactor->block_num == 0 ? 1 : 2 * reactor->block_num;
		while(new_block_num <= b) new_block_num <<= 1;
		struct ntyevent **new_event_blocks = (struct ntyevent **)realloc(
			reactor->events, new_block_num * (sizeof(struct ntyevent *))
		);
		if(new_event_blocks == NULL) {
			printf("cannot allocate block in %s for fd = %d\n", __func__, fd);
			return NULL;
		}
		memset(
			new_event_blocks + reactor->block_num, 0,
			(new_block_num - reactor->block_num) * sizeof(struct ntyevent*));
		reactor->block_num = new_block_num;
		reactor->events = new_event_blocks;
	}
	
	if(reactor->events[b] == NULL) {
		reactor->events[b] = (struct ntyevent *)calloc(EVENTS_BLOCK_SIZE, sizeof(struct ntyevent));
		if (reactor->events[b] == NULL) {
			printf("cannot allocate block in %s for fd = %d\n", __func__, fd);
			return NULL;
		}
	}

	return &reactor->events[b][i];
}

Use ntyreactor_check_timeout to automatically check for long inactive fd in block_idx and close the connection

int ntyreactor_check_timeout(struct ntyreactor *reactor, int block_idx) {
	if(!reactor || !reactor->events || block_idx >= reactor->block_num) return 0;
	struct ntyevent *block = reactor->events[block_idx];
	if (!block) return 0;

	long now = time(NULL);
	int i;
	for(i = 0; i < EVENTS_BLOCK_SIZE; i++) {
		if(!block[i].used || block[i].sticky) continue;
		long duration = now - block[i].last_active;
		if (duration >= 60) {
			close(block[i].fd);
			printf("[fd=%d] timeout\n", block[i].fd);
			nty_event_del(reactor->epfd, &block[i]);
		}
	}
}

nty_event_ * these functions are responsible for adding fd and monitored events to epoll and reactor, and setting callback functions

void nty_event_update(struct ntyevent *ev, int fd, NCALLBACK callback, void *arg) {
	ev->fd = fd;
	ev->callback = callback;
	ev->events = 0;
	ev->arg = arg;
	ev->last_active = time(NULL);
	ev->sticky = 0;
}

// add/update ev on epfd with events
int nty_event_add(int epfd, int events, struct ntyevent *ev) {

	struct epoll_event ep_ev = {0, {0}};
	ep_ev.data.ptr = ev;
	ep_ev.events = ev->events = events;

	int op;
	if (ev->used == 1) {
		op = EPOLL_CTL_MOD;
	} else {
		op = EPOLL_CTL_ADD;
		ev->used = 1;
	}

	if (epoll_ctl(epfd, op, ev->fd, &ep_ev) < 0) {
		printf("event add failed [fd=%d], events[%d]\n", ev->fd, events);
		return -1;
	}

	return 0;
}

// remove ev from epfd
int nty_event_del(int epfd, struct ntyevent *ev) {

	struct epoll_event ep_ev = {0, {0}};

	if (ev->used != 1) {
		return -1;
	}

	ep_ev.data.ptr = ev;
	ev->used = 0;
	epoll_ctl(epfd, EPOLL_CTL_DEL, ev->fd, NULL);

	return 0;
}

There are other functions written here

int start_listen(short port) {

	int fd = socket(AF_INET, SOCK_STREAM, 0);
	fcntl(fd, F_SETFL, O_NONBLOCK);

	struct sockaddr_in server_addr;
	memset(&server_addr, 0, sizeof(server_addr));
	server_addr.sin_family = AF_INET;
	server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
	server_addr.sin_port = htons(port);

	bind(fd, (struct sockaddr*)&server_addr, sizeof(server_addr));

	if (listen(fd, 20) < 0) {
		printf("listen failed : %s\n", strerror(errno));
	}

	printf("listening to port %hd with fd = %d\n", port, fd);
	return fd;
}


int ntyreactor_init(struct ntyreactor *reactor) {

	if (reactor == NULL) return -1;
	memset(reactor, 0, sizeof(struct ntyreactor));

	reactor->epfd = epoll_create(1);
	if (reactor->epfd <= 0) {
		printf("create epfd in %s err %s\n", __func__, strerror(errno));
		return -2;
	}

	reactor->events = (struct ntyevent **)calloc(1, sizeof(struct ntyevent *));
	if (reactor->events == NULL) {
		printf("cannot allocate events block in %s\n", __func__);
		return -3;
	}
	reactor->block_num = 1;
	
	return 0;
}


int ntyreactor_destory(struct ntyreactor *reactor) {
	close(reactor->epfd);
	int i;
	for(i = 0; i < reactor->block_num; i++) {
		free(reactor->events[i]);
	}
	free(reactor->events);
}



int ntyreactor_addlistener(struct ntyreactor *reactor, int listenfd, NCALLBACK acceptor) {

	if (reactor == NULL) return -1;
	if (reactor->events == NULL) return -1;

	struct ntyevent *event = ntyreactor_get_event(reactor, listenfd);
	if (event == NULL) return -1;

	nty_event_update(event, listenfd, acceptor, reactor);
	event->sticky = 1; //don't close listen fd on timeout
	nty_event_add(reactor->epfd, EPOLLIN, event);

	return 0;
}

Other precautions

In order to reach millions of connections, we also need to increase the maximum open files of the system
use

ulimit -a

View open files
Set with - n. if not, change sudo vim /etc/security/limits.conf
Add these two lines at the end

*                soft    nofile          1048576
*                hard    nofile          1048576

And close the shell and log in again
That should be it

reference material

[1] Zero sound education Linux C/C + + back-end server architecture development 2.1 network development

Tags: C Linux Back-end

Posted on Thu, 25 Nov 2021 21:49:24 -0500 by barry_p