462 lines
10 KiB
C
462 lines
10 KiB
C
|
||
|
||
#define _GNU_SOURCE
|
||
#include <errno.h>
|
||
#include <stdio.h>
|
||
#include <sys/socket.h>
|
||
#include <netinet/in.h>
|
||
#include <string.h>
|
||
#include <pthread.h>
|
||
#include <unistd.h>
|
||
#include <poll.h>
|
||
#include <sys/epoll.h>
|
||
#include <errno.h>
|
||
#include <sys/time.h>
|
||
#include <sys/eventfd.h>
|
||
#include <sys/timerfd.h>
|
||
|
||
#include "server.h"
|
||
|
||
|
||
#define CONNECTION_SIZE 1024 // 1024 * 1024
|
||
|
||
#define MAX_PORTS 20
|
||
|
||
#define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
|
||
|
||
|
||
|
||
#if ENABLE_KVSTORE
|
||
|
||
// typedef int (*msg_handler)(char *msg, int length, char *response);
|
||
// typedef int (*msg_handler)(char *request, int request_length, char *response, int *response_length);
|
||
typedef int (*msg_handler)(struct conn* conn);
|
||
|
||
static msg_handler kvs_handler;
|
||
|
||
extern void cleanup_finished_iouring_tasks();
|
||
|
||
|
||
// 0 need more, -1 error, =1 suc
|
||
int kvs_request(struct conn *c) {
|
||
// int consumed_out = kvs_handler(c->rbuffer, c->rlength, c->wbuffer, &c->wlength);
|
||
int consumed_out = kvs_handler(c);
|
||
return consumed_out;
|
||
}
|
||
|
||
int kvs_response(struct conn *c) {
|
||
|
||
|
||
|
||
}
|
||
|
||
|
||
#endif
|
||
|
||
|
||
|
||
int accept_cb(int fd);
|
||
int recv_cb(int fd);
|
||
int send_cb(int fd);
|
||
|
||
|
||
|
||
int epfd = 0;
|
||
struct timeval begin;
|
||
|
||
int wakeup_fd = -1;
|
||
int timer_fd = -1;
|
||
|
||
struct conn conn_list[CONNECTION_SIZE] = {0};
|
||
// fd
|
||
|
||
|
||
// 1 add, 0 mod
|
||
int set_event(int fd, int event, int flag) {
|
||
|
||
if (flag) { // non-zero add
|
||
|
||
struct epoll_event ev;
|
||
ev.events = event;
|
||
ev.data.fd = fd;
|
||
epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev);
|
||
|
||
} else { // zero mod
|
||
|
||
struct epoll_event ev;
|
||
ev.events = event;
|
||
ev.data.fd = fd;
|
||
epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &ev);
|
||
|
||
}
|
||
|
||
|
||
}
|
||
|
||
|
||
int event_register(int fd, int event) {
|
||
|
||
if (fd < 0) return -1;
|
||
|
||
conn_list[fd].fd = fd;
|
||
conn_list[fd].r_action.recv_callback = recv_cb;
|
||
conn_list[fd].send_callback = send_cb;
|
||
|
||
memset(conn_list[fd].rbuffer, 0, BUFFER_LENGTH);
|
||
conn_list[fd].rlength = 0;
|
||
|
||
memset(conn_list[fd].wbuffer, 0, BUFFER_LENGTH*2);
|
||
conn_list[fd].wlength = 0;
|
||
|
||
conn_list[fd].is_stop = 0;
|
||
|
||
set_event(fd, event, 1);
|
||
}
|
||
|
||
|
||
// listenfd(sockfd) --> EPOLLIN --> accept_cb
|
||
int accept_cb(int fd) {
|
||
|
||
struct sockaddr_in clientaddr;
|
||
socklen_t len = sizeof(clientaddr);
|
||
|
||
int clientfd = accept(fd, (struct sockaddr*)&clientaddr, &len);
|
||
if (clientfd < 0) {
|
||
printf("accept errno: %d --> %s\n", errno, strerror(errno));
|
||
return -1;
|
||
}
|
||
|
||
event_register(clientfd, EPOLLIN); // | EPOLLET
|
||
|
||
if ((clientfd % 1000) == 0) {
|
||
|
||
struct timeval current;
|
||
gettimeofday(¤t, NULL);
|
||
|
||
int time_used = TIME_SUB_MS(current, begin);
|
||
memcpy(&begin, ¤t, sizeof(struct timeval));
|
||
|
||
|
||
//printf("accept finshed: %d, time_used: %d\n", clientfd, time_used);
|
||
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
|
||
int recv_cb(int fd) {
|
||
struct conn *c = &conn_list[fd];
|
||
int avail = BUFFER_LENGTH - c->rlength;
|
||
// printf("avail: %d\n", avail);
|
||
if (avail <= 0) {
|
||
// 缓冲满了还没解析出来：协议异常或包过大
|
||
epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL);
|
||
close(fd);
|
||
return 0;
|
||
}
|
||
|
||
int count = recv(fd, c->rbuffer + c->rlength, avail, 0);
|
||
if (count == 0) { // disconnect
|
||
//printf("client disconnect: %d\n", fd);
|
||
|
||
epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL); // unfinished
|
||
close(fd);
|
||
|
||
return 0;
|
||
} else if (count < 0) { //
|
||
|
||
printf("count: %d, errno: %d, %s\n", count, errno, strerror(errno));
|
||
epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL);
|
||
close(fd);
|
||
|
||
return 0;
|
||
}
|
||
|
||
|
||
c->rlength += count;
|
||
//printf("RECV: %s\n", conn_list[fd].rbuffer);
|
||
|
||
#if 0 // echo
|
||
|
||
conn_list[fd].wlength = conn_list[fd].rlength;
|
||
memcpy(conn_list[fd].wbuffer, conn_list[fd].rbuffer, conn_list[fd].wlength);
|
||
|
||
printf("[%d]RECV: %s\n", conn_list[fd].rlength, conn_list[fd].rbuffer);
|
||
|
||
#elif ENABLE_HTTP
|
||
|
||
http_request(&conn_list[fd]);
|
||
|
||
#elif ENABLE_WEBSOCKET
|
||
|
||
ws_request(&conn_list[fd]);
|
||
|
||
#elif ENABLE_KVSTORE
|
||
int consumed = kvs_request(c);
|
||
if(consumed < 0){
|
||
epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL);
|
||
close(fd);
|
||
return 0;
|
||
}
|
||
|
||
// 清理 buffer
|
||
if (consumed > 0 && consumed < c->rlength) {
|
||
// 有剩余未处理数据，搬移到 buffer 头部
|
||
int left = c->rlength - consumed;
|
||
if (left > 0) memmove(c->rbuffer, c->rbuffer + consumed, left);
|
||
c->rlength = left;
|
||
if (c->wlength > 0) set_event(fd, EPOLLOUT, 0);
|
||
return count;
|
||
|
||
}else{
|
||
c->rlength = 0;
|
||
if(c->wlength > 0) set_event(fd, EPOLLOUT, 0);
|
||
return count;
|
||
}
|
||
|
||
#endif
|
||
|
||
|
||
set_event(fd, EPOLLOUT, 0);
|
||
|
||
return count;
|
||
}
|
||
|
||
|
||
int send_cb(int fd) {
|
||
|
||
#if ENABLE_HTTP
|
||
|
||
http_response(&conn_list[fd]);
|
||
|
||
#elif ENABLE_WEBSOCKET
|
||
|
||
ws_response(&conn_list[fd]);
|
||
|
||
#elif ENABLE_KVSTORE
|
||
|
||
kvs_response(&conn_list[fd]);
|
||
|
||
#endif
|
||
|
||
struct conn *c = &conn_list[fd];
|
||
int sent_total = 0;
|
||
|
||
pthread_mutex_lock(&c->g_sync_lock);
|
||
|
||
while (c->wlength > 0) {
|
||
ssize_t n = send(fd, c->wbuffer, (size_t)c->wlength, MSG_NOSIGNAL);
|
||
if (n > 0) {
|
||
sent_total += (int)n;
|
||
|
||
if (n == c->wlength) {
|
||
/* 全部发完 */
|
||
c->wlength = 0;
|
||
break;
|
||
}
|
||
|
||
/* 只发了一部分：把剩余数据搬到 buffer 头部 */
|
||
int left = c->wlength - (int)n;
|
||
memmove(c->wbuffer, c->wbuffer + n, (size_t)left);
|
||
c->wlength = left;
|
||
|
||
/* 不要在这里死循环占用 CPU，交给下一次 EPOLLOUT */
|
||
break;
|
||
}
|
||
|
||
if (n < 0) {
|
||
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||
/* 暂时发不出去，等下一次可写事件 */
|
||
pthread_mutex_unlock(&c->g_sync_lock);
|
||
set_event(fd, EPOLLOUT, 0);
|
||
return sent_total;
|
||
}
|
||
|
||
/* 对端断开 / 其他错误 */
|
||
int e = errno;
|
||
pthread_mutex_unlock(&c->g_sync_lock);
|
||
|
||
printf("send fd=%d errno=%d %s\n", fd, e, strerror(e));
|
||
epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL);
|
||
close(fd);
|
||
return 0;
|
||
}
|
||
|
||
break;
|
||
}
|
||
pthread_mutex_unlock(&c->g_sync_lock);
|
||
|
||
if (c->wlength > 0) {
|
||
/* 还有没发完，继续监听可写 */
|
||
set_event(fd, EPOLLOUT, 0);
|
||
} else {
|
||
/* 发完了，回到读 */
|
||
set_event(fd, EPOLLIN, 0);
|
||
}
|
||
|
||
return sent_total;
|
||
}
|
||
|
||
// wakup fd
|
||
|
||
int handle_wakeup_fd_cb(int fd) {
|
||
uint64_t v;
|
||
while (1) {
|
||
ssize_t n = read(wakeup_fd, &v, sizeof(v));
|
||
if (n == sizeof(v)) continue;
|
||
if (n < 0 && errno == EAGAIN) break; // 已经读空
|
||
break;
|
||
}
|
||
cleanup_finished_iouring_tasks();
|
||
|
||
return 0;
|
||
}
|
||
|
||
int init_wakeup_fd(void) {
|
||
int wfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
|
||
if (wfd < 0) {
|
||
printf("eventfd failed: errno=%d %s\n", errno, strerror(errno));
|
||
return -1;
|
||
}
|
||
|
||
conn_list[wfd].fd = wfd;
|
||
conn_list[wfd].r_action.recv_callback = handle_wakeup_fd_cb;
|
||
set_event(wfd, EPOLLIN, 1);
|
||
|
||
return wfd;
|
||
}
|
||
|
||
// EPOLLOUT
|
||
void sync_wakeup() {
|
||
if (wakeup_fd < 0) return;
|
||
// set_event(wakeup_fd, EPOLLOUT, 0);
|
||
|
||
uint64_t one = 1;
|
||
ssize_t n = write(wakeup_fd, &one, sizeof(one));
|
||
}
|
||
|
||
// #include "diskuring/diskuring.h"
|
||
// extern iouring_ctx_t global_uring_ctx;
|
||
// extern void iouring_tick(iouring_ctx_t *ctx);
|
||
|
||
// 定时器
|
||
int handle_timer_fd_cb(int fd){
|
||
uint64_t v;
|
||
while (1) {
|
||
ssize_t n = read(fd, &v, sizeof(v));
|
||
if (n == sizeof(v)) {
|
||
continue;
|
||
}
|
||
if (n < 0 && errno == EAGAIN) break;
|
||
break;
|
||
}
|
||
// iouring_tick(&global_uring_ctx);
|
||
}
|
||
|
||
int init_timer_fd(void){
|
||
int tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
|
||
|
||
struct itimerspec its = {
|
||
.it_interval = {0, 100 * 1000 * 1000}, // 100ms = 100,000,000 纳秒
|
||
.it_value = {0, 100 * 1000 * 1000}, // 首次 100ms 后触发
|
||
};
|
||
timerfd_settime(tfd, 0, &its, NULL);
|
||
|
||
conn_list[tfd].fd = tfd;
|
||
conn_list[tfd].r_action.recv_callback = handle_timer_fd_cb;
|
||
set_event(tfd, EPOLLIN, 1);
|
||
|
||
return tfd;
|
||
}
|
||
|
||
int r_init_server(unsigned short port) {
|
||
|
||
int sockfd = socket(AF_INET, SOCK_STREAM, 0);
|
||
|
||
int opt = 1;
|
||
if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) < 0) {
|
||
perror("setsockopt");
|
||
close(sockfd);
|
||
}
|
||
|
||
struct sockaddr_in servaddr;
|
||
servaddr.sin_family = AF_INET;
|
||
servaddr.sin_addr.s_addr = htonl(INADDR_ANY); // 0.0.0.0
|
||
servaddr.sin_port = htons(port); // 0-1023,
|
||
|
||
if (-1 == bind(sockfd, (struct sockaddr*)&servaddr, sizeof(struct sockaddr))) {
|
||
printf("bind failed: %s\n", strerror(errno));
|
||
}
|
||
|
||
listen(sockfd, 10);
|
||
//printf("listen finshed: %d\n", sockfd); // 3
|
||
|
||
return sockfd;
|
||
|
||
}
|
||
|
||
int reactor_start(unsigned short port, msg_handler handler) {
|
||
|
||
//unsigned short port = 2000;
|
||
kvs_handler = handler;
|
||
|
||
epfd = epoll_create(1);
|
||
|
||
wakeup_fd = init_wakeup_fd();
|
||
if(wakeup_fd < 0){
|
||
close(epfd);
|
||
return -1;
|
||
}
|
||
|
||
timer_fd = init_timer_fd();
|
||
if(timer_fd < 0){
|
||
close(epfd);
|
||
close(wakeup_fd);
|
||
return -1;
|
||
}
|
||
|
||
int i = 0;
|
||
|
||
for (i = 0;i < MAX_PORTS;i ++) {
|
||
|
||
int sockfd = r_init_server(port + i);
|
||
|
||
conn_list[sockfd].fd = sockfd;
|
||
conn_list[sockfd].r_action.recv_callback = accept_cb;
|
||
conn_list[sockfd].is_stop = 0;
|
||
|
||
set_event(sockfd, EPOLLIN, 1);
|
||
}
|
||
|
||
gettimeofday(&begin, NULL);
|
||
|
||
while (1) { // mainloop
|
||
struct epoll_event events[1024] = {0};
|
||
int nready = epoll_wait(epfd, events, 1024, -1);
|
||
// cleanup_finished_iouring_tasks();
|
||
|
||
int i = 0;
|
||
for (i = 0;i < nready;i ++) {
|
||
|
||
int connfd = events[i].data.fd;
|
||
|
||
if (events[i].events & EPOLLIN) {
|
||
// printf("connlist:%p, r_action:%p, recv_callaback:%p\n", &conn_list[connfd], &conn_list[connfd].r_action, conn_list[connfd].r_action.recv_callback);
|
||
conn_list[connfd].r_action.recv_callback(connfd);
|
||
}
|
||
|
||
if (events[i].events & EPOLLOUT) {
|
||
conn_list[connfd].send_callback(connfd);
|
||
}
|
||
}
|
||
|
||
}
|
||
|
||
if (wakeup_fd >= 0) close(wakeup_fd);
|
||
if (epfd >= 0) close(epfd);
|
||
return 0;
|
||
}
|
||
|
||
|