resp协议定义， xml定义配置文件

.gitignore

@@ -1,9 +1,9 @@
-NtyCo/
 .vscode/
 *.db
 *.copy
 *.o
+*.a
+
 kvstore
 testcase
 
-proactor copy.c

Makefile

@@ -1,9 +1,9 @@
 
 CC = gcc
 # FLAGS = -g -DJEMALLOC_NO_DEMANGLE -I ./NtyCo/core/ -L ./NtyCo/ -lntyco -lpthread -luring -ldl -ljemalloc
-FLAGS = -g -DJEMALLOC_NO_DEMANGLE -I ./NtyCo/core/ -L ./NtyCo/ -lntyco -lpthread -luring -ldl
+FLAGS = -g -DJEMALLOC_NO_DEMANGLE -I./NtyCo/core/ -I/usr/include/libxml2 -L ./NtyCo/ -lntyco -lpthread -luring -ldl -lxml2
 # SRCS = kvstore.c ntyco.c proactor.c reactor.c kvs_array.c kvs_rbtree.c kvs_hash.c  kvs_rw_tools.c
-SRCS = kvstore.c ntyco.c proactor.c reactor.c kvs_array_bin.c kvs_rbtree_bin.c kvs_hash_bin.c kvs_rw_tools.c kvs_cmd_log.c ./mem_pool/mem_pool.c kvs_slave.c
+SRCS = kvstore.c ntyco.c proactor.c reactor.c kvs_array_bin.c kvs_rbtree_bin.c kvs_hash_bin.c kvs_rw_tools.c kvs_cmd_log.c ./mem_pool/mem_pool.c kvs_slave.c ./common/config.c
 TESTCASE_SRCS = testcase.c
 TARGET = kvstore
 SUBDIR = ./NtyCo/

NtyCo/.gitignore

@@ -0,0 +1,4 @@
+_build/
+bin/
+objs/
+lib/

NtyCo/Makefile

@@ -0,0 +1,94 @@
+
+CC = gcc
+ECHO = echo
+
+SUB_DIR = core/
+SAMPLE_DIR = sample/
+ROOT_DIR = $(shell pwd)
+OBJS_DIR = $(ROOT_DIR)/objs
+BIN_DIR = $(ROOT_DIR)/bin
+
+BIN = nty_server nty_client nty_bench nty_server_mulcore hook_tcpserver nty_http_server nty_mysql_client nty_mysql_oper nty_websocket_server nty_http_server_mulcore ntyco_httpd nty_rediscli
+
+LIB = libntyco.a
+
+FLAG = -lpthread -O3 -ldl -I $(ROOT_DIR)/core 
+
+THIRDFLAG = -lcrypto -lssl -lmysqlclient -lhiredis -I /usr/include/mysql/ -I /usr/local/include/hiredis/
+
+CUR_SOURCE = ${wildcard *.c}
+CUR_OBJS = ${patsubst %.c, %.o, %(CUR_SOURCE)}
+
+export CC BIN_DIR OBJS_DIR ROOT_IDR FLAG BIN ECHO EFLAG
+
+
+
+all : check_objs check_bin $(SUB_DIR) $(LIB)
+.PHONY : all
+
+bin: $(SAMPLE_DIR) $(BIN)
+
+# lib: $(SUB_DIR) $(LIB)
+
+$(SUB_DIR) : ECHO
+	make -C $@
+
+#DEBUG : ECHO
+#	make -C bin
+
+ECHO :
+	@echo $(SUB_DIR)
+
+check_objs:
+	if [ ! -d "objs" ]; then \
+		mkdir -p objs;  \
+	fi
+
+check_bin:
+	if [ ! -d "bin" ]; then \
+		mkdir -p bin;   \
+	fi
+
+
+nty_server : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_server.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+nty_client : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_client.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+nty_bench : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_bench.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+nty_server_mulcore : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_server_mulcore.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+nty_http_server : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_http_server.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+nty_websocket_server : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_websocket_server.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) $(THIRDFLAG)
+
+nty_mysql_client : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_mysql_client.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) $(THIRDFLAG)
+
+nty_mysql_oper : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_mysql_oper.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) $(THIRDFLAG)
+
+nty_rediscli : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_rediscli.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) $(THIRDFLAG)
+
+hook_tcpserver: $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/hook_tcpserver.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG)
+
+nty_http_server_mulcore : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/nty_http_server_mulcore.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+ntyco_httpd : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o $(SAMPLE_DIR)/ntyco_httpd.c
+	$(CC) -o $(BIN_DIR)/$@ $^ $(FLAG) 
+
+libntyco.a : $(OBJS_DIR)/nty_socket.o $(OBJS_DIR)/nty_coroutine.o $(OBJS_DIR)/nty_epoll.o $(OBJS_DIR)/nty_schedule.o
+	ar rcs $@ $^
+
+clean :
+	rm -rf $(BIN_DIR)/* $(OBJS_DIR)/* libntyco.a
+

NtyCo/README.md

@@ -0,0 +1,75 @@
+## NtyCo
+
+#### coroutine
+[实现原理](https://github.com/wangbojing/NtyCo/wiki/NtyCo%E7%9A%84%E5%AE%9E%E7%8E%B0)
+[配套视频讲解](https://it.0voice.com/p/t_pc/goods_pc_detail/goods_detail/course_2QFAeORw45TjJA1y9tq8CmdVJTQ)
+
+## details
+#### coroutine FSM
+![](http://bojing.wang/wp-content/uploads/2018/08/status_machine.png)
+
+#### storage structure (ready, wait, sleep, status)
+![](http://bojing.wang/wp-content/uploads/2018/08/6.1.png)
+
+#### nty_server process
+![](https://cos.0voice.com/nty_server_uml.png)
+
+#### compile
+
+```
+编译ntyco的core文件与编译libntyco.a的静态库
+$ make
+
+// 编译sample
+$ make bin
+```
+
+#### err info
+```
+nty_mysql_oper.c:8:19: fatal error: mysql.h: No such file or directory
+
+解决方案：
+# sudo apt-get install libmysqlclient-dev
+
+nty_rediscli.c:11:21: fatal error: hiredis.h: No such file or directory
+
+解决方案：
+需要编译安装hiredis: https://github.com/redis/hiredis
+
+```
+
+
+#### server 
+```
+$ ./bin/nty_server
+```
+#### client
+```
+./bin/nty_client
+```
+
+#### mul_process, mul_core
+```
+$ ./bin/nty_server_mulcore
+```
+#### websocket
+```
+$ ./bin/nty_websocket_server
+```
+
+#### bench
+```
+$ ./bin/nty_bench
+```
+![](http://bojing.wang/wp-content/uploads/2018/08/nty_bench_ntyco.png)
+![](http://bojing.wang/wp-content/uploads/2018/08/nty_bench_nginx.png)
+
+
+#### http server
+```
+$ ./bin/nty_http_server_mulcore
+```
+
+![](http://bojing.wang/wp-content/uploads/2018/08/ntyco_ab.png)![](http://bojing.wang/wp-content/uploads/2018/08/nginx_ab.png)
+
+##### [对应视频讲解](https://ke.qq.com/course/2705727?tuin=1bf84273)

NtyCo/core/CMakeLists.txt

@@ -0,0 +1,3 @@
+aux_source_directory(. SRC)
+
+add_library(nty_core ${SRC})

NtyCo/core/Makefile

@@ -0,0 +1,16 @@
+
+
+CUR_SOURCE = ${wildcard *.c}
+
+CUR_OBJS = ${patsubst %.c, %.o, $(CUR_SOURCE)}
+
+all : $(SUB_DIR) $(CUR_OBJS)
+
+$(SUB_DIR) : ECHO
+	make -C $@
+
+$(CUR_OBJS) : %.o : %.c
+	$(CC) -c $^ -o $(OBJS_DIR)/$@ $(FLAG) 
+
+ECHO : 
+	@echo $(SUB_DIR)

NtyCo/core/nty_coroutine.c

@@ -0,0 +1,353 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+#include "nty_coroutine.h"
+
+pthread_key_t global_sched_key;
+static pthread_once_t sched_key_once = PTHREAD_ONCE_INIT;
+
+// https://github.com/halayli/lthread/blob/master/src/lthread.c#L58
+
+#ifdef _USE_UCONTEXT
+
+static void
+_save_stack(nty_coroutine *co) {
+	char* top = co->sched->stack + co->sched->stack_size;
+	char dummy = 0;
+	assert(top - &dummy <= NTY_CO_MAX_STACKSIZE);
+	if (co->stack_size < top - &dummy) {
+		co->stack = realloc(co->stack, top - &dummy);
+		assert(co->stack != NULL);
+	}
+	co->stack_size = top - &dummy;
+	memcpy(co->stack, &dummy, co->stack_size);
+}
+
+static void
+_load_stack(nty_coroutine *co) {
+	memcpy(co->sched->stack + co->sched->stack_size - co->stack_size, co->stack, co->stack_size);
+}
+
+static void _exec(void *lt) {
+	nty_coroutine *co = (nty_coroutine*)lt;
+	co->func(co->arg);
+	co->status |= (BIT(NTY_COROUTINE_STATUS_EXITED) | BIT(NTY_COROUTINE_STATUS_FDEOF) | BIT(NTY_COROUTINE_STATUS_DETACH));
+	nty_coroutine_yield(co);
+}
+
+#else
+
+int _switch(nty_cpu_ctx *new_ctx, nty_cpu_ctx *cur_ctx);
+
+#ifdef __i386__
+__asm__ (
+"    .text                                  \n"
+"    .p2align 2,,3                          \n"
+".globl _switch                             \n"
+"_switch:                                   \n"
+"__switch:                                  \n"
+"movl 8(%esp), %edx      # fs->%edx         \n"
+"movl %esp, 0(%edx)      # save esp         \n"
+"movl %ebp, 4(%edx)      # save ebp         \n"
+"movl (%esp), %eax       # save eip         \n"
+"movl %eax, 8(%edx)                         \n"
+"movl %ebx, 12(%edx)     # save ebx,esi,edi \n"
+"movl %esi, 16(%edx)                        \n"
+"movl %edi, 20(%edx)                        \n"
+"movl 4(%esp), %edx      # ts->%edx         \n"
+"movl 20(%edx), %edi     # restore ebx,esi,edi      \n"
+"movl 16(%edx), %esi                                \n"
+"movl 12(%edx), %ebx                                \n"
+"movl 0(%edx), %esp      # restore esp              \n"
+"movl 4(%edx), %ebp      # restore ebp              \n"
+"movl 8(%edx), %eax      # restore eip              \n"
+"movl %eax, (%esp)                                  \n"
+"ret                                                \n"
+);
+#elif defined(__x86_64__)
+
+__asm__ (
+"    .text                                  \n"
+"       .p2align 4,,15                                   \n"
+".globl _switch                                          \n"
+".globl __switch                                         \n"
+"_switch:                                                \n"
+"__switch:                                               \n"
+"       movq %rsp, 0(%rsi)      # save stack_pointer     \n"
+"       movq %rbp, 8(%rsi)      # save frame_pointer     \n"
+"       movq (%rsp), %rax       # save insn_pointer      \n"
+"       movq %rax, 16(%rsi)                              \n"
+"       movq %rbx, 24(%rsi)     # save rbx,r12-r15       \n"
+"       movq %r12, 32(%rsi)                              \n"
+"       movq %r13, 40(%rsi)                              \n"
+"       movq %r14, 48(%rsi)                              \n"
+"       movq %r15, 56(%rsi)                              \n"
+"       movq 56(%rdi), %r15                              \n"
+"       movq 48(%rdi), %r14                              \n"
+"       movq 40(%rdi), %r13     # restore rbx,r12-r15    \n"
+"       movq 32(%rdi), %r12                              \n"
+"       movq 24(%rdi), %rbx                              \n"
+"       movq 8(%rdi), %rbp      # restore frame_pointer  \n"
+"       movq 0(%rdi), %rsp      # restore stack_pointer  \n"
+"       movq 16(%rdi), %rax     # restore insn_pointer   \n"
+"       movq %rax, (%rsp)                                \n"
+"       ret                                              \n"
+);
+#endif
+
+
+static void _exec(void *lt) {
+#if defined(__lvm__) && defined(__x86_64__)
+	__asm__("movq 16(%%rbp), %[lt]" : [lt] "=r" (lt));
+#endif
+
+	nty_coroutine *co = (nty_coroutine*)lt;
+	co->func(co->arg);
+	co->status |= (BIT(NTY_COROUTINE_STATUS_EXITED) | BIT(NTY_COROUTINE_STATUS_FDEOF) | BIT(NTY_COROUTINE_STATUS_DETACH));
+#if 1
+	nty_coroutine_yield(co);
+#else
+	co->ops = 0;
+	_switch(&co->sched->ctx, &co->ctx);
+#endif
+}
+
+static inline void nty_coroutine_madvise(nty_coroutine *co) {
+
+	size_t current_stack = (co->stack + co->stack_size) - co->ctx.esp;
+	assert(current_stack <= co->stack_size);
+
+	if (current_stack < co->last_stack_size &&
+		co->last_stack_size > co->sched->page_size) {
+		size_t tmp = current_stack + (-current_stack & (co->sched->page_size - 1));
+		assert(madvise(co->stack, co->stack_size-tmp, MADV_DONTNEED) == 0);
+	}
+	co->last_stack_size = current_stack;
+}
+
+#endif
+
+extern int nty_schedule_create(int stack_size);
+
+
+
+void nty_coroutine_free(nty_coroutine *co) {
+	if (co == NULL) return ;
+	co->sched->spawned_coroutines --;
+#if 1
+	if (co->stack) {
+		free(co->stack);
+		co->stack = NULL;
+	}
+#endif
+	if (co) {
+		free(co);
+	}
+
+}
+
+static void nty_coroutine_init(nty_coroutine *co) {
+
+#ifdef _USE_UCONTEXT
+	getcontext(&co->ctx);
+	co->ctx.uc_stack.ss_sp = co->sched->stack;
+	co->ctx.uc_stack.ss_size = co->sched->stack_size;
+	co->ctx.uc_link = &co->sched->ctx;
+	// printf("TAG21\n");
+	makecontext(&co->ctx, (void (*)(void)) _exec, 1, (void*)co);
+	// printf("TAG22\n");
+#else
+	void **stack = (void **)(co->stack + co->stack_size);
+
+	stack[-3] = NULL;
+	stack[-2] = (void *)co;
+
+	co->ctx.esp = (void*)stack - (4 * sizeof(void*));
+	co->ctx.ebp = (void*)stack - (3 * sizeof(void*));
+	co->ctx.eip = (void*)_exec;
+#endif
+	co->status = BIT(NTY_COROUTINE_STATUS_READY);
+	
+}
+
+void nty_coroutine_yield(nty_coroutine *co) {
+	co->ops = 0;
+#ifdef _USE_UCONTEXT
+	if ((co->status & BIT(NTY_COROUTINE_STATUS_EXITED)) == 0) {
+		_save_stack(co);
+	}
+	swapcontext(&co->ctx, &co->sched->ctx);
+#else
+	_switch(&co->sched->ctx, &co->ctx);
+#endif
+}
+
+int nty_coroutine_resume(nty_coroutine *co) {
+	
+	if (co->status & BIT(NTY_COROUTINE_STATUS_NEW)) {
+		nty_coroutine_init(co);
+	} 
+#ifdef _USE_UCONTEXT	
+	else {
+		_load_stack(co);
+	}
+#endif
+	nty_schedule *sched = nty_coroutine_get_sched();
+	sched->curr_thread = co;
+#ifdef _USE_UCONTEXT
+	swapcontext(&sched->ctx, &co->ctx);
+#else
+	_switch(&co->ctx, &co->sched->ctx);
+	nty_coroutine_madvise(co);
+#endif
+	sched->curr_thread = NULL;
+
+#if 1
+	if (co->status & BIT(NTY_COROUTINE_STATUS_EXITED)) {
+		if (co->status & BIT(NTY_COROUTINE_STATUS_DETACH)) {
+			nty_coroutine_free(co);
+		}
+		return -1;
+	} 
+#endif
+	return 0;
+}
+
+
+void nty_coroutine_renice(nty_coroutine *co) {
+	co->ops ++;
+#if 1
+	if (co->ops < 5) return ;
+#endif
+	TAILQ_INSERT_TAIL(&nty_coroutine_get_sched()->ready, co, ready_next);
+	nty_coroutine_yield(co);
+}
+
+
+void nty_coroutine_sleep(uint64_t msecs) {
+	nty_coroutine *co = nty_coroutine_get_sched()->curr_thread;
+
+	if (msecs == 0) {
+		TAILQ_INSERT_TAIL(&co->sched->ready, co, ready_next);
+		nty_coroutine_yield(co);
+	} else {
+		nty_schedule_sched_sleepdown(co, msecs);
+	}
+}
+
+void nty_coroutine_detach(void) {
+	nty_coroutine *co = nty_coroutine_get_sched()->curr_thread;
+	co->status |= BIT(NTY_COROUTINE_STATUS_DETACH);
+}
+
+static void nty_coroutine_sched_key_destructor(void *data) {
+	free(data);
+}
+
+static void __attribute__((constructor(1000))) nty_coroutine_sched_key_creator(void) {
+	assert(pthread_key_create(&global_sched_key, nty_coroutine_sched_key_destructor) == 0);
+	assert(pthread_setspecific(global_sched_key, NULL) == 0);
+	
+	return ;
+}
+
+
+// coroutine --> 
+// create 
+//
+int nty_coroutine_create(nty_coroutine **new_co, proc_coroutine func, void *arg) {
+
+	assert(pthread_once(&sched_key_once, nty_coroutine_sched_key_creator) == 0);
+	nty_schedule *sched = nty_coroutine_get_sched();
+
+	if (sched == NULL) {
+		nty_schedule_create(0);
+		
+		sched = nty_coroutine_get_sched();
+		if (sched == NULL) {
+			printf("Failed to create scheduler\n");
+			return -1;
+		}
+	}
+
+	nty_coroutine *co = calloc(1, sizeof(nty_coroutine));
+	if (co == NULL) {
+		printf("Failed to allocate memory for new coroutine\n");
+		return -2;
+	}
+
+#ifdef _USE_UCONTEXT
+	co->stack = NULL;
+	co->stack_size = 0;
+#else
+	int ret = posix_memalign(&co->stack, getpagesize(), sched->stack_size);
+	if (ret) {
+		printf("Failed to allocate stack for new coroutine\n");
+		free(co);
+		return -3;
+	}
+	co->stack_size = sched->stack_size;
+#endif
+	co->sched = sched;
+	co->status = BIT(NTY_COROUTINE_STATUS_NEW); //
+	co->id = sched->spawned_coroutines ++;
+	co->func = func;
+#if CANCEL_FD_WAIT_UINT64
+	co->fd = -1;
+	co->events = 0;
+#else
+	co->fd_wait = -1;
+#endif
+	co->arg = arg;
+	co->birth = nty_coroutine_usec_now();
+	*new_co = co;
+
+	TAILQ_INSERT_TAIL(&co->sched->ready, co, ready_next);
+
+	return 0;
+}
+
+
+
+

NtyCo/core/nty_coroutine.h

@@ -0,0 +1,389 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+#ifndef __NTY_COROUTINE_H__
+#define __NTY_COROUTINE_H__
+
+
+#define _GNU_SOURCE
+#include <dlfcn.h>
+
+#define _USE_UCONTEXT
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <limits.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+#include <netinet/tcp.h>
+
+#ifdef _USE_UCONTEXT
+#include <ucontext.h>
+#endif
+
+#include <sys/epoll.h>
+#include <sys/poll.h>
+
+#include <errno.h>
+
+#include "nty_queue.h"
+#include "nty_tree.h"
+
+#define NTY_CO_MAX_EVENTS		(1024*1024)
+#define NTY_CO_MAX_STACKSIZE	(128*1024) // {http: 16*1024, tcp: 4*1024}
+
+#define BIT(x)	 				(1 << (x))
+#define CLEARBIT(x) 			~(1 << (x))
+
+#define CANCEL_FD_WAIT_UINT64	1
+
+typedef void (*proc_coroutine)(void *);
+
+
+typedef enum {
+	NTY_COROUTINE_STATUS_WAIT_READ,
+	NTY_COROUTINE_STATUS_WAIT_WRITE,
+	NTY_COROUTINE_STATUS_NEW,
+	NTY_COROUTINE_STATUS_READY,
+	NTY_COROUTINE_STATUS_EXITED,
+	NTY_COROUTINE_STATUS_BUSY,
+	NTY_COROUTINE_STATUS_SLEEPING,
+	NTY_COROUTINE_STATUS_EXPIRED,
+	NTY_COROUTINE_STATUS_FDEOF,
+	NTY_COROUTINE_STATUS_DETACH,
+	NTY_COROUTINE_STATUS_CANCELLED,
+	NTY_COROUTINE_STATUS_PENDING_RUNCOMPUTE,
+	NTY_COROUTINE_STATUS_RUNCOMPUTE,
+	NTY_COROUTINE_STATUS_WAIT_IO_READ,
+	NTY_COROUTINE_STATUS_WAIT_IO_WRITE,
+	NTY_COROUTINE_STATUS_WAIT_MULTI
+} nty_coroutine_status;
+
+typedef enum {
+	NTY_COROUTINE_COMPUTE_BUSY,
+	NTY_COROUTINE_COMPUTE_FREE
+} nty_coroutine_compute_status;
+
+typedef enum {
+	NTY_COROUTINE_EV_READ,
+	NTY_COROUTINE_EV_WRITE
+} nty_coroutine_event;
+
+
+LIST_HEAD(_nty_coroutine_link, _nty_coroutine);
+TAILQ_HEAD(_nty_coroutine_queue, _nty_coroutine);
+
+RB_HEAD(_nty_coroutine_rbtree_sleep, _nty_coroutine);
+RB_HEAD(_nty_coroutine_rbtree_wait, _nty_coroutine);
+
+
+
+typedef struct _nty_coroutine_link nty_coroutine_link;
+typedef struct _nty_coroutine_queue nty_coroutine_queue;
+
+typedef struct _nty_coroutine_rbtree_sleep nty_coroutine_rbtree_sleep;
+typedef struct _nty_coroutine_rbtree_wait nty_coroutine_rbtree_wait;
+
+
+#ifndef _USE_UCONTEXT
+typedef struct _nty_cpu_ctx {
+	void *esp; //
+	void *ebp;
+	void *eip;
+	void *edi;
+	void *esi;
+	void *ebx;
+	void *r1;
+	void *r2;
+	void *r3;
+	void *r4;
+	void *r5;
+} nty_cpu_ctx;
+#endif
+
+///
+typedef struct _nty_schedule {
+	uint64_t birth;
+#ifdef _USE_UCONTEXT
+	ucontext_t ctx;
+#else
+	nty_cpu_ctx ctx;
+#endif
+	void *stack;
+	size_t stack_size;
+	int spawned_coroutines;
+	uint64_t default_timeout;
+	struct _nty_coroutine *curr_thread;
+	int page_size;
+
+	int poller_fd;
+	int eventfd;
+	struct epoll_event eventlist[NTY_CO_MAX_EVENTS];
+	int nevents;
+
+	int num_new_events;
+	pthread_mutex_t defer_mutex;
+
+	nty_coroutine_queue ready;
+	nty_coroutine_queue defer;
+
+	nty_coroutine_link busy;
+	
+	nty_coroutine_rbtree_sleep sleeping;
+	nty_coroutine_rbtree_wait waiting;
+
+	//private 
+
+} nty_schedule;
+
+typedef struct _nty_coroutine {
+
+	//private
+	
+#ifdef _USE_UCONTEXT
+	ucontext_t ctx;
+#else
+	nty_cpu_ctx ctx;
+#endif
+	proc_coroutine func;
+	void *arg;
+	void *data;
+	size_t stack_size;
+	size_t last_stack_size;
+	
+	nty_coroutine_status status;
+	nty_schedule *sched;
+
+	uint64_t birth;
+	uint64_t id;
+#if CANCEL_FD_WAIT_UINT64
+	int fd;
+	unsigned short events;  //POLL_EVENT
+#else
+	int64_t fd_wait;
+#endif
+	char funcname[64];
+	struct _nty_coroutine *co_join;
+
+	void **co_exit_ptr;
+	void *stack;
+	void *ebp;
+	uint32_t ops;
+	uint64_t sleep_usecs;
+
+	RB_ENTRY(_nty_coroutine) sleep_node;
+	RB_ENTRY(_nty_coroutine) wait_node;
+
+	LIST_ENTRY(_nty_coroutine) busy_next;
+
+	TAILQ_ENTRY(_nty_coroutine) ready_next;
+	TAILQ_ENTRY(_nty_coroutine) defer_next;
+	TAILQ_ENTRY(_nty_coroutine) cond_next;
+
+	TAILQ_ENTRY(_nty_coroutine) io_next;
+	TAILQ_ENTRY(_nty_coroutine) compute_next;
+
+	struct {
+		void *buf;
+		size_t nbytes;
+		int fd;
+		int ret;
+		int err;
+	} io;
+
+	struct _nty_coroutine_compute_sched *compute_sched;
+	int ready_fds;
+	struct pollfd *pfds;
+	nfds_t nfds;
+} nty_coroutine;
+
+
+typedef struct _nty_coroutine_compute_sched {
+#ifdef _USE_UCONTEXT
+	ucontext_t ctx;
+#else
+	nty_cpu_ctx ctx;
+#endif
+	nty_coroutine_queue coroutines;
+
+	nty_coroutine *curr_coroutine;
+
+	pthread_mutex_t run_mutex;
+	pthread_cond_t run_cond;
+
+	pthread_mutex_t co_mutex;
+	LIST_ENTRY(_nty_coroutine_compute_sched) compute_next;
+	
+	nty_coroutine_compute_status compute_status;
+} nty_coroutine_compute_sched;
+
+extern pthread_key_t global_sched_key;
+static inline nty_schedule *nty_coroutine_get_sched(void) {
+	return pthread_getspecific(global_sched_key);
+}
+
+static inline uint64_t nty_coroutine_diff_usecs(uint64_t t1, uint64_t t2) {
+	return t2-t1;
+}
+
+static inline uint64_t nty_coroutine_usec_now(void) {
+	struct timeval t1 = {0, 0};
+	gettimeofday(&t1, NULL);
+
+	return t1.tv_sec * 1000000 + t1.tv_usec;
+}
+
+
+
+int nty_epoller_create(void);
+
+
+void nty_schedule_cancel_event(nty_coroutine *co);
+void nty_schedule_sched_event(nty_coroutine *co, int fd, nty_coroutine_event e, uint64_t timeout);
+
+void nty_schedule_desched_sleepdown(nty_coroutine *co);
+void nty_schedule_sched_sleepdown(nty_coroutine *co, uint64_t msecs);
+
+nty_coroutine* nty_schedule_desched_wait(int fd);
+void nty_schedule_sched_wait(nty_coroutine *co, int fd, unsigned short events, uint64_t timeout);
+
+void nty_schedule_run(void);
+
+int nty_epoller_ev_register_trigger(void);
+int nty_epoller_wait(struct timespec t);
+int nty_coroutine_resume(nty_coroutine *co);
+void nty_coroutine_free(nty_coroutine *co);
+int nty_coroutine_create(nty_coroutine **new_co, proc_coroutine func, void *arg);
+void nty_coroutine_yield(nty_coroutine *co);
+
+void nty_coroutine_sleep(uint64_t msecs);
+
+
+int nty_socket(int domain, int type, int protocol);
+int nty_accept(int fd, struct sockaddr *addr, socklen_t *len);
+ssize_t nty_recv(int fd, void *buf, size_t len, int flags);
+ssize_t nty_send(int fd, const void *buf, size_t len, int flags);
+int nty_close(int fd);
+int nty_poll(struct pollfd *fds, nfds_t nfds, int timeout);
+int nty_connect(int fd, struct sockaddr *name, socklen_t namelen);
+
+ssize_t nty_sendto(int fd, const void *buf, size_t len, int flags,
+               const struct sockaddr *dest_addr, socklen_t addrlen);
+ssize_t nty_recvfrom(int fd, void *buf, size_t len, int flags,
+                 struct sockaddr *src_addr, socklen_t *addrlen);
+
+
+#define COROUTINE_HOOK 
+
+#ifdef  COROUTINE_HOOK
+
+
+typedef int (*socket_t)(int domain, int type, int protocol);
+extern socket_t socket_f;
+
+typedef int(*connect_t)(int, const struct sockaddr *, socklen_t);
+extern connect_t connect_f;
+
+typedef ssize_t(*read_t)(int, void *, size_t);
+extern read_t read_f;
+
+
+typedef ssize_t(*recv_t)(int sockfd, void *buf, size_t len, int flags);
+extern recv_t recv_f;
+
+typedef ssize_t(*recvfrom_t)(int sockfd, void *buf, size_t len, int flags,
+        struct sockaddr *src_addr, socklen_t *addrlen);
+extern recvfrom_t recvfrom_f;
+
+typedef ssize_t(*write_t)(int, const void *, size_t);
+extern write_t write_f;
+
+typedef ssize_t(*send_t)(int sockfd, const void *buf, size_t len, int flags);
+extern send_t send_f;
+
+typedef ssize_t(*sendto_t)(int sockfd, const void *buf, size_t len, int flags,
+        const struct sockaddr *dest_addr, socklen_t addrlen);
+extern sendto_t sendto_f;
+
+typedef int(*accept_t)(int sockfd, struct sockaddr *addr, socklen_t *addrlen);
+extern accept_t accept_f;
+
+// new-syscall
+typedef int(*close_t)(int);
+extern close_t close_f;
+
+
+int init_hook(void);
+
+
+/*
+
+typedef int(*fcntl_t)(int __fd, int __cmd, ...);
+extern fcntl_t fcntl_f;
+
+typedef int (*getsockopt_t)(int sockfd, int level, int optname,
+        void *optval, socklen_t *optlen);
+extern getsockopt_t getsockopt_f;
+
+typedef int (*setsockopt_t)(int sockfd, int level, int optname,
+        const void *optval, socklen_t optlen);
+extern setsockopt_t setsockopt_f;
+
+*/
+
+#endif
+
+
+
+#endif
+
+

NtyCo/core/nty_epoll.c

@@ -0,0 +1,75 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+#include <sys/eventfd.h>
+
+#include "nty_coroutine.h"
+
+
+int nty_epoller_create(void) {
+	return epoll_create(1024);
+} 
+
+int nty_epoller_wait(struct timespec t) {
+	nty_schedule *sched = nty_coroutine_get_sched();
+	return epoll_wait(sched->poller_fd, sched->eventlist, NTY_CO_MAX_EVENTS, t.tv_sec*1000.0 + t.tv_nsec/1000000.0);
+}
+
+int nty_epoller_ev_register_trigger(void) {
+	nty_schedule *sched = nty_coroutine_get_sched();
+
+	if (!sched->eventfd) {
+		sched->eventfd = eventfd(0, EFD_NONBLOCK);
+		assert(sched->eventfd != -1);
+	}
+
+	struct epoll_event ev;
+	ev.events = EPOLLIN;
+	ev.data.fd = sched->eventfd;
+	int ret = epoll_ctl(sched->poller_fd, EPOLL_CTL_ADD, sched->eventfd, &ev);
+
+	assert(ret != -1);
+}
+
+

NtyCo/core/nty_queue.h

@@ -0,0 +1,589 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+#ifndef __NTY_QUEUE_H__
+#define __NTY_QUEUE_H__
+
+#include <sys/cdefs.h>
+
+#ifdef QUEUE_MACRO_DEBUG
+/* Store the last 2 places the queue element or head was altered */
+struct qm_trace {
+	char * lastfile;
+	int lastline;
+	char * prevfile;
+	int prevline;
+};
+
+#define	TRACEBUF	struct qm_trace trace;
+#define	TRASHIT(x)	do {(x) = (void *)-1;} while (0)
+
+#define	QMD_TRACE_HEAD(head) do {							\
+	(head)->trace.prevline = (head)->trace.lastline;		\
+	(head)->trace.prevfile = (head)->trace.lastfile;		\
+	(head)->trace.lastline = __LINE__;						\
+	(head)->trace.lastfile = __FILE__;						\
+} while (0)
+
+#define	QMD_TRACE_ELEM(elem) do {							\
+	(elem)->trace.prevline = (elem)->trace.lastline;		\
+	(elem)->trace.prevfile = (elem)->trace.lastfile;		\
+	(elem)->trace.lastline = __LINE__;						\
+	(elem)->trace.lastfile = __FILE__;						\
+} while (0)
+
+#else
+#define	QMD_TRACE_ELEM(elem)
+#define	QMD_TRACE_HEAD(head)
+#define	TRACEBUF
+#define	TRASHIT(x)
+#endif	/* QUEUE_MACRO_DEBUG */
+
+
+
+/*
+ * Singly-linked List declarations.
+ */
+#define	SLIST_HEAD(name, type)								\
+struct name {												\
+	struct type *slh_first;	/* first element */			\
+}
+
+#define	SLIST_HEAD_INITIALIZER(head)						\
+	{ NULL }
+
+#define	SLIST_ENTRY(type)									\
+struct {													\
+	struct type *sle_next;	/* next element */			\
+}
+
+/*
+ * Singly-linked List functions.
+ */
+#define	SLIST_EMPTY(head)	((head)->slh_first == NULL)
+
+#define	SLIST_FIRST(head)	((head)->slh_first)
+
+#define	SLIST_FOREACH(var, head, field)						\
+	for ((var) = SLIST_FIRST((head));						\
+	    (var);												\
+	    (var) = SLIST_NEXT((var), field))
+
+#define	SLIST_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = SLIST_FIRST((head));						\
+	    (var) && ((tvar) = SLIST_NEXT((var), field), 1);	\
+	    (var) = (tvar))
+
+#define	SLIST_FOREACH_PREVPTR(var, varp, head, field)		\
+	for ((varp) = &SLIST_FIRST((head));						\
+	    ((var) = *(varp)) != NULL;							\
+	    (varp) = &SLIST_NEXT((var), field))
+
+#define	SLIST_INIT(head) do {								\
+	SLIST_FIRST((head)) = NULL;								\
+} while (0)
+
+#define	SLIST_INSERT_AFTER(slistelm, elm, field) do {			\
+	SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field);	\
+	SLIST_NEXT((slistelm), field) = (elm);						\
+} while (0)
+
+#define	SLIST_INSERT_HEAD(head, elm, field) do {				\
+	SLIST_NEXT((elm), field) = SLIST_FIRST((head));				\
+	SLIST_FIRST((head)) = (elm);								\
+} while (0)
+
+#define	SLIST_NEXT(elm, field)	((elm)->field.sle_next)
+
+#define	SLIST_REMOVE(head, elm, type, field) do {			\
+	if (SLIST_FIRST((head)) == (elm)) {					\
+		SLIST_REMOVE_HEAD((head), field);					\
+	}														\
+	else {													\
+		struct type *curelm = SLIST_FIRST((head));			\
+		while (SLIST_NEXT(curelm, field) != (elm))			\
+			curelm = SLIST_NEXT(curelm, field);				\
+		SLIST_REMOVE_AFTER(curelm, field);					\
+	}														\
+	TRASHIT((elm)->field.sle_next);							\
+} while (0)
+
+#define SLIST_REMOVE_AFTER(elm, field) do {					\
+	SLIST_NEXT(elm, field) =								\
+	    SLIST_NEXT(SLIST_NEXT(elm, field), field);			\
+} while (0)
+
+#define	SLIST_REMOVE_HEAD(head, field) do {					\
+	SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field);	\
+} while (0)
+
+
+
+/*
+ * Singly-linked Tail queue declarations.
+ */
+#define	STAILQ_HEAD(name, type)									\
+struct name {													\
+	struct type *stqh_first;/* first element */				\
+	struct type **stqh_last;/* addr of last next element */	\
+}
+
+#define	STAILQ_HEAD_INITIALIZER(head)							\
+	{ NULL, &(head).stqh_first }
+
+#define	STAILQ_ENTRY(type)										\
+struct {														\
+	struct type *stqe_next;	/* next element */				\
+}
+
+/*
+ * Singly-linked Tail queue functions.
+ */
+#define	STAILQ_CONCAT(head1, head2) do {				\
+	if (!STAILQ_EMPTY((head2))) {						\
+		*(head1)->stqh_last = (head2)->stqh_first;		\
+		(head1)->stqh_last = (head2)->stqh_last;		\
+		STAILQ_INIT((head2));							\
+	}													\
+} while (0)
+
+#define	STAILQ_EMPTY(head)	((head)->stqh_first == NULL)
+
+#define	STAILQ_FIRST(head)	((head)->stqh_first)
+
+#define	STAILQ_FOREACH(var, head, field)				\
+	for((var) = STAILQ_FIRST((head));					\
+	   (var);											\
+	   (var) = STAILQ_NEXT((var), field))
+
+
+#define	STAILQ_FOREACH_SAFE(var, head, field, tvar)				\
+	for ((var) = STAILQ_FIRST((head));						\
+	    (var) && ((tvar) = STAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	STAILQ_INIT(head) do {							\
+	STAILQ_FIRST((head)) = NULL;						\
+	(head)->stqh_last = &STAILQ_FIRST((head));			\
+} while (0)
+
+#define	STAILQ_INSERT_AFTER(head, tqelm, elm, field) do {		\
+	if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+	STAILQ_NEXT((tqelm), field) = (elm);				\
+} while (0)
+
+#define	STAILQ_INSERT_HEAD(head, elm, field) do {			\
+	if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL)	\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+	STAILQ_FIRST((head)) = (elm);					\
+} while (0)
+
+#define	STAILQ_INSERT_TAIL(head, elm, field) do {			\
+	STAILQ_NEXT((elm), field) = NULL;				\
+	*(head)->stqh_last = (elm);					\
+	(head)->stqh_last = &STAILQ_NEXT((elm), field);			\
+} while (0)
+
+#define	STAILQ_LAST(head, type, field)					\
+	(STAILQ_EMPTY((head)) ?						\
+		NULL :							\
+	        ((struct type *)(void *)				\
+		((char *)((head)->stqh_last) - __offsetof(struct type, field))))
+
+#define	STAILQ_NEXT(elm, field)	((elm)->field.stqe_next)
+
+#define	STAILQ_REMOVE(head, elm, type, field) do {			\
+	if (STAILQ_FIRST((head)) == (elm)) {					\
+		STAILQ_REMOVE_HEAD((head), field);					\
+	}														\
+	else {													\
+		struct type *curelm = STAILQ_FIRST((head));			\
+		while (STAILQ_NEXT(curelm, field) != (elm))			\
+			curelm = STAILQ_NEXT(curelm, field);			\
+		STAILQ_REMOVE_AFTER(head, curelm, field);			\
+	}														\
+	TRASHIT((elm)->field.stqe_next);						\
+} while (0)
+
+#define	STAILQ_REMOVE_HEAD(head, field) do {					\
+	if ((STAILQ_FIRST((head)) =								\
+	     STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL)	\
+		(head)->stqh_last = &STAILQ_FIRST((head));				\
+} while (0)
+
+#define STAILQ_REMOVE_AFTER(head, elm, field) do {				\
+	if ((STAILQ_NEXT(elm, field) =								\
+	     STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL)	\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);			\
+} while (0)
+
+#define STAILQ_SWAP(head1, head2, type) do {				\
+	struct type *swap_first = STAILQ_FIRST(head1);			\
+	struct type **swap_last = (head1)->stqh_last;			\
+	STAILQ_FIRST(head1) = STAILQ_FIRST(head2);				\
+	(head1)->stqh_last = (head2)->stqh_last;				\
+	STAILQ_FIRST(head2) = swap_first;						\
+	(head2)->stqh_last = swap_last;							\
+	if (STAILQ_EMPTY(head1))								\
+		(head1)->stqh_last = &STAILQ_FIRST(head1);			\
+	if (STAILQ_EMPTY(head2))								\
+		(head2)->stqh_last = &STAILQ_FIRST(head2);			\
+} while (0)
+
+
+
+/*
+ * List declarations.
+ */
+#define	LIST_HEAD(name, type)							\
+	struct name {										\
+		struct type *lh_first;	/* first element */ 	\
+	}
+	
+#define	LIST_HEAD_INITIALIZER(head)						\
+		{ NULL }
+	
+#define	LIST_ENTRY(type)								\
+	struct {											\
+		struct type *le_next;	/* next element */	\
+		struct type **le_prev;	/* address of previous next element */	\
+	}
+
+
+/*
+ * List functions.
+ */
+
+#if (defined(_KERNEL) && defined(INVARIANTS))
+#define	QMD_LIST_CHECK_HEAD(head, field) do {					\
+	if (LIST_FIRST((head)) != NULL &&							\
+	    LIST_FIRST((head))->field.le_prev !=					\
+	     &LIST_FIRST((head)))									\
+		panic("Bad list head %p first->prev != head", (head));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_NEXT(elm, field) do {					\
+	if (LIST_NEXT((elm), field) != NULL &&						\
+	    LIST_NEXT((elm), field)->field.le_prev !=				\
+	     &((elm)->field.le_next))								\
+	     	panic("Bad link elm %p next->prev != elm", (elm));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_PREV(elm, field) do {					\
+	if (*(elm)->field.le_prev != (elm))							\
+		panic("Bad link elm %p prev->next != elm", (elm));		\
+} while (0)
+#else
+#define	QMD_LIST_CHECK_HEAD(head, field)
+#define	QMD_LIST_CHECK_NEXT(elm, field)
+#define	QMD_LIST_CHECK_PREV(elm, field)
+#endif /* (_KERNEL && INVARIANTS) */
+
+
+
+#define	LIST_EMPTY(head)	((head)->lh_first == NULL)
+
+#define	LIST_FIRST(head)	((head)->lh_first)
+
+#define	LIST_FOREACH(var, head, field)							\
+	for ((var) = LIST_FIRST((head));							\
+	    (var);													\
+	    (var) = LIST_NEXT((var), field))
+
+#define	LIST_FOREACH_SAFE(var, head, field, tvar)				\
+	for ((var) = LIST_FIRST((head));							\
+	    (var) && ((tvar) = LIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	LIST_INIT(head) do {									\
+	LIST_FIRST((head)) = NULL;									\
+} while (0)
+
+#define	LIST_INSERT_AFTER(listelm, elm, field) do {				\
+	QMD_LIST_CHECK_NEXT(listelm, field);						\
+	if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\
+		LIST_NEXT((listelm), field)->field.le_prev =			\
+		    &LIST_NEXT((elm), field);							\
+	LIST_NEXT((listelm), field) = (elm);						\
+	(elm)->field.le_prev = &LIST_NEXT((listelm), field);		\
+} while (0)
+
+#define	LIST_INSERT_BEFORE(listelm, elm, field) do {			\
+	QMD_LIST_CHECK_PREV(listelm, field);						\
+	(elm)->field.le_prev = (listelm)->field.le_prev;			\
+	LIST_NEXT((elm), field) = (listelm);						\
+	*(listelm)->field.le_prev = (elm);							\
+	(listelm)->field.le_prev = &LIST_NEXT((elm), field);		\
+} while (0)
+
+#define	LIST_INSERT_HEAD(head, elm, field) do {					\
+	QMD_LIST_CHECK_HEAD((head), field);							\
+	if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL)	\
+		LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\
+	LIST_FIRST((head)) = (elm);									\
+	(elm)->field.le_prev = &LIST_FIRST((head));					\
+} while (0)
+
+#define	LIST_NEXT(elm, field)	((elm)->field.le_next)
+
+#define	LIST_REMOVE(elm, field) do {					\
+	QMD_LIST_CHECK_NEXT(elm, field);					\
+	QMD_LIST_CHECK_PREV(elm, field);					\
+	if (LIST_NEXT((elm), field) != NULL)				\
+		LIST_NEXT((elm), field)->field.le_prev = 		\
+		    (elm)->field.le_prev;						\
+	*(elm)->field.le_prev = LIST_NEXT((elm), field);	\
+	TRASHIT((elm)->field.le_next);						\
+	TRASHIT((elm)->field.le_prev);						\
+} while (0)
+
+#define LIST_SWAP(head1, head2, type, field) do {		\
+	struct type *swap_tmp = LIST_FIRST((head1));		\
+	LIST_FIRST((head1)) = LIST_FIRST((head2));			\
+	LIST_FIRST((head2)) = swap_tmp;						\
+	if ((swap_tmp = LIST_FIRST((head1))) != NULL)		\
+		swap_tmp->field.le_prev = &LIST_FIRST((head1));	\
+	if ((swap_tmp = LIST_FIRST((head2))) != NULL)		\
+		swap_tmp->field.le_prev = &LIST_FIRST((head2));	\
+} while (0)
+
+
+
+	
+/*
+ * Tail queue declarations.
+ */
+#define	TAILQ_HEAD(name, type)										\
+	struct name {													\
+		struct type *tqh_first; /* first element */ 				\
+		struct type **tqh_last; /* addr of last next element */ 	\
+		TRACEBUF													\
+	}
+	
+#define	TAILQ_HEAD_INITIALIZER(head)								\
+		{ NULL, &(head).tqh_first }
+	
+#define	TAILQ_ENTRY(type)											\
+	struct {														\
+		struct type *tqe_next;	/* next element */				\
+		struct type **tqe_prev; /* address of previous next element */	\
+		TRACEBUF							\
+	}
+
+
+
+/*
+ * Tail queue functions.
+ */
+#if (defined(_KERNEL) && defined(INVARIANTS))
+#define	QMD_TAILQ_CHECK_HEAD(head, field) do {					\
+	if (!TAILQ_EMPTY(head) &&									\
+	    TAILQ_FIRST((head))->field.tqe_prev !=					\
+	     &TAILQ_FIRST((head)))									\
+		panic("Bad tailq head %p first->prev != head", (head));	\
+} while (0)
+
+#define	QMD_TAILQ_CHECK_TAIL(head, field) do {					\
+	if (*(head)->tqh_last != NULL)								\
+	    	panic("Bad tailq NEXT(%p->tqh_last) != NULL", (head)); 	\
+} while (0)
+
+#define	QMD_TAILQ_CHECK_NEXT(elm, field) do {				\
+	if (TAILQ_NEXT((elm), field) != NULL &&					\
+	    TAILQ_NEXT((elm), field)->field.tqe_prev !=			\
+	     &((elm)->field.tqe_next))							\
+		panic("Bad link elm %p next->prev != elm", (elm));	\
+} while (0)
+
+#define	QMD_TAILQ_CHECK_PREV(elm, field) do {				\
+	if (*(elm)->field.tqe_prev != (elm))					\
+		panic("Bad link elm %p prev->next != elm", (elm));	\
+} while (0)
+#else
+#define	QMD_TAILQ_CHECK_HEAD(head, field)
+#define	QMD_TAILQ_CHECK_TAIL(head, headname)
+#define	QMD_TAILQ_CHECK_NEXT(elm, field)
+#define	QMD_TAILQ_CHECK_PREV(elm, field)
+#endif /* (_KERNEL && INVARIANTS) */
+
+#define	TAILQ_CONCAT(head1, head2, field) do {				\
+	if (!TAILQ_EMPTY(head2)) {								\
+		*(head1)->tqh_last = (head2)->tqh_first;			\
+		(head2)->tqh_first->field.tqe_prev = (head1)->tqh_last;	\
+		(head1)->tqh_last = (head2)->tqh_last;			\
+		TAILQ_INIT((head2));					\
+		QMD_TRACE_HEAD(head1);					\
+		QMD_TRACE_HEAD(head2);					\
+	}											\
+} while (0)
+
+#define	TAILQ_EMPTY(head)	((head)->tqh_first == NULL)
+
+#define	TAILQ_FIRST(head)	((head)->tqh_first)
+
+#define	TAILQ_FOREACH(var, head, field)					\
+	for ((var) = TAILQ_FIRST((head));					\
+	    (var);											\
+	    (var) = TAILQ_NEXT((var), field))
+
+#define	TAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = TAILQ_FIRST((head));						\
+	    (var) && ((tvar) = TAILQ_NEXT((var), field), 1);	\
+	    (var) = (tvar))
+
+#define	TAILQ_FOREACH_REVERSE(var, head, headname, field)	\
+	for ((var) = TAILQ_LAST((head), headname);				\
+	    (var);												\
+	    (var) = TAILQ_PREV((var), headname, field))
+
+#define	TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, tvar)	\
+	for ((var) = TAILQ_LAST((head), headname);			\
+	    (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1);	\
+	    (var) = (tvar))
+
+#define	TAILQ_INIT(head) do {						\
+	TAILQ_FIRST((head)) = NULL;						\
+	(head)->tqh_last = &TAILQ_FIRST((head));		\
+	QMD_TRACE_HEAD(head);							\
+} while (0)
+
+#define	TAILQ_INSERT_AFTER(head, listelm, elm, field) do {		\
+	QMD_TAILQ_CHECK_NEXT(listelm, field);						\
+	if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\
+		TAILQ_NEXT((elm), field)->field.tqe_prev = 		\
+		    &TAILQ_NEXT((elm), field);					\
+	else {												\
+		(head)->tqh_last = &TAILQ_NEXT((elm), field);	\
+		QMD_TRACE_HEAD(head);							\
+	}													\
+	TAILQ_NEXT((listelm), field) = (elm);				\
+	(elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field);		\
+	QMD_TRACE_ELEM(&(elm)->field);						\
+	QMD_TRACE_ELEM(&listelm->field);					\
+} while (0)
+
+#define	TAILQ_INSERT_BEFORE(listelm, elm, field) do {		\
+	QMD_TAILQ_CHECK_PREV(listelm, field);					\
+	(elm)->field.tqe_prev = (listelm)->field.tqe_prev;		\
+	TAILQ_NEXT((elm), field) = (listelm);					\
+	*(listelm)->field.tqe_prev = (elm);						\
+	(listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field);	\
+	QMD_TRACE_ELEM(&(elm)->field);							\
+	QMD_TRACE_ELEM(&listelm->field);						\
+} while (0)
+
+#define	TAILQ_INSERT_HEAD(head, elm, field) do {			\
+	QMD_TAILQ_CHECK_HEAD(head, field);						\
+	if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL)	\
+		TAILQ_FIRST((head))->field.tqe_prev =				\
+		    &TAILQ_NEXT((elm), field);						\
+	else													\
+		(head)->tqh_last = &TAILQ_NEXT((elm), field);		\
+	TAILQ_FIRST((head)) = (elm);							\
+	(elm)->field.tqe_prev = &TAILQ_FIRST((head));			\
+	QMD_TRACE_HEAD(head);									\
+	QMD_TRACE_ELEM(&(elm)->field);							\
+} while (0)
+
+#define	TAILQ_INSERT_TAIL(head, elm, field) do {			\
+	QMD_TAILQ_CHECK_TAIL(head, field);						\
+	TAILQ_NEXT((elm), field) = NULL;						\
+	(elm)->field.tqe_prev = (head)->tqh_last;				\
+	*(head)->tqh_last = (elm);								\
+	(head)->tqh_last = &TAILQ_NEXT((elm), field);			\
+	QMD_TRACE_HEAD(head);									\
+	QMD_TRACE_ELEM(&(elm)->field);							\
+} while (0)
+
+#define	TAILQ_LAST(head, headname)							\
+	(*(((struct headname *)((head)->tqh_last))->tqh_last))
+
+#define	TAILQ_NEXT(elm, field) ((elm)->field.tqe_next)
+
+#define	TAILQ_PREV(elm, headname, field)					\
+	(*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
+
+#define	TAILQ_REMOVE(head, elm, field) do {				\
+	QMD_TAILQ_CHECK_NEXT(elm, field);					\
+	QMD_TAILQ_CHECK_PREV(elm, field);					\
+	if ((TAILQ_NEXT((elm), field)) != NULL)			\
+		TAILQ_NEXT((elm), field)->field.tqe_prev = 		\
+		    (elm)->field.tqe_prev;						\
+	else {												\
+		(head)->tqh_last = (elm)->field.tqe_prev;		\
+		QMD_TRACE_HEAD(head);							\
+	}													\
+	*(elm)->field.tqe_prev = TAILQ_NEXT((elm), field);	\
+	TRASHIT((elm)->field.tqe_next);						\
+	TRASHIT((elm)->field.tqe_prev);						\
+	QMD_TRACE_ELEM(&(elm)->field);						\
+} while (0)
+
+#define TAILQ_SWAP(head1, head2, type, field) do {		\
+	struct type *swap_first = (head1)->tqh_first;		\
+	struct type **swap_last = (head1)->tqh_last;		\
+	(head1)->tqh_first = (head2)->tqh_first;			\
+	(head1)->tqh_last = (head2)->tqh_last;				\
+	(head2)->tqh_first = swap_first;					\
+	(head2)->tqh_last = swap_last;						\
+	if ((swap_first = (head1)->tqh_first) != NULL)		\
+		swap_first->field.tqe_prev = &(head1)->tqh_first;	\
+	else								\
+		(head1)->tqh_last = &(head1)->tqh_first;		\
+	if ((swap_first = (head2)->tqh_first) != NULL)		\
+		swap_first->field.tqe_prev = &(head2)->tqh_first;	\
+	else								\
+		(head2)->tqh_last = &(head2)->tqh_first;		\
+} while (0)
+
+
+
+
+#endif
+
+
+
+
+

NtyCo/core/nty_schedule.c

@@ -0,0 +1,369 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+#include "nty_coroutine.h"
+
+
+
+#define FD_KEY(f,e) (((int64_t)(f) << (sizeof(int32_t) * 8)) | e)
+#define FD_EVENT(f) ((int32_t)(f))
+#define FD_ONLY(f) ((f) >> ((sizeof(int32_t) * 8)))
+
+
+static inline int nty_coroutine_sleep_cmp(nty_coroutine *co1, nty_coroutine *co2) {
+	if (co1->sleep_usecs < co2->sleep_usecs) {
+		return -1;
+	}
+	if (co1->sleep_usecs == co2->sleep_usecs) {
+		return 0;
+	}
+	return 1;
+}
+
+static inline int nty_coroutine_wait_cmp(nty_coroutine *co1, nty_coroutine *co2) {
+#if CANCEL_FD_WAIT_UINT64
+	if (co1->fd < co2->fd) return -1;
+	else if (co1->fd == co2->fd) return 0;
+	else return 1;
+#else
+	if (co1->fd_wait < co2->fd_wait) {
+		return -1;
+	}
+	if (co1->fd_wait == co2->fd_wait) {
+		return 0;
+	}
+#endif
+	return 1;
+}
+
+
+RB_GENERATE(_nty_coroutine_rbtree_sleep, _nty_coroutine, sleep_node, nty_coroutine_sleep_cmp);
+RB_GENERATE(_nty_coroutine_rbtree_wait, _nty_coroutine, wait_node, nty_coroutine_wait_cmp);
+
+
+
+void nty_schedule_sched_sleepdown(nty_coroutine *co, uint64_t msecs) {
+	uint64_t usecs = msecs * 1000u;
+
+	nty_coroutine *co_tmp = RB_FIND(_nty_coroutine_rbtree_sleep, &co->sched->sleeping, co);
+	if (co_tmp != NULL) {
+		RB_REMOVE(_nty_coroutine_rbtree_sleep, &co->sched->sleeping, co_tmp);
+	}
+
+	co->sleep_usecs = nty_coroutine_diff_usecs(co->sched->birth, nty_coroutine_usec_now()) + usecs;
+
+	while (msecs) {
+		co_tmp = RB_INSERT(_nty_coroutine_rbtree_sleep, &co->sched->sleeping, co);
+		if (co_tmp) {
+			printf("1111 sleep_usecs %"PRIu64"\n", co->sleep_usecs);
+			co->sleep_usecs ++;
+			continue;
+		}
+		co->status |= BIT(NTY_COROUTINE_STATUS_SLEEPING);
+		break;
+	}
+
+	//yield
+}
+
+void nty_schedule_desched_sleepdown(nty_coroutine *co) {
+	if (co->status & BIT(NTY_COROUTINE_STATUS_SLEEPING)) {
+		RB_REMOVE(_nty_coroutine_rbtree_sleep, &co->sched->sleeping, co);
+
+		co->status &= CLEARBIT(NTY_COROUTINE_STATUS_SLEEPING);
+		co->status |= BIT(NTY_COROUTINE_STATUS_READY);
+		co->status &= CLEARBIT(NTY_COROUTINE_STATUS_EXPIRED);
+	}
+}
+
+nty_coroutine *nty_schedule_search_wait(int fd) {
+	nty_coroutine find_it = {0};
+	find_it.fd = fd;
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	
+	nty_coroutine *co = RB_FIND(_nty_coroutine_rbtree_wait, &sched->waiting, &find_it);
+	co->status = 0;
+
+	return co;
+}
+
+nty_coroutine* nty_schedule_desched_wait(int fd) {
+	
+	nty_coroutine find_it = {0};
+	find_it.fd = fd;
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	
+	nty_coroutine *co = RB_FIND(_nty_coroutine_rbtree_wait, &sched->waiting, &find_it);
+	if (co != NULL) {
+		RB_REMOVE(_nty_coroutine_rbtree_wait, &co->sched->waiting, co);
+	}
+	co->status = 0;
+	nty_schedule_desched_sleepdown(co);
+	
+	return co;
+}
+
+void nty_schedule_sched_wait(nty_coroutine *co, int fd, unsigned short events, uint64_t timeout) {
+	
+	if (co->status & BIT(NTY_COROUTINE_STATUS_WAIT_READ) ||
+		co->status & BIT(NTY_COROUTINE_STATUS_WAIT_WRITE)) {
+		printf("Unexpected event. lt id %"PRIu64" fd %"PRId32" already in %"PRId32" state\n",
+            co->id, co->fd, co->status);
+		assert(0);
+	}
+
+	if (events & POLLIN) {
+		co->status |= NTY_COROUTINE_STATUS_WAIT_READ;
+	} else if (events & POLLOUT) {
+		co->status |= NTY_COROUTINE_STATUS_WAIT_WRITE;
+	} else {
+		printf("events : %d\n", events);
+		assert(0);
+	}
+
+	co->fd = fd;
+	co->events = events;
+	nty_coroutine *co_tmp = RB_INSERT(_nty_coroutine_rbtree_wait, &co->sched->waiting, co);
+
+	assert(co_tmp == NULL);
+
+	//printf("timeout --> %"PRIu64"\n", timeout);
+	if (timeout == 1) return ; //Error
+
+	nty_schedule_sched_sleepdown(co, timeout);
+	
+}
+
+void nty_schedule_cancel_wait(nty_coroutine *co) {
+	RB_REMOVE(_nty_coroutine_rbtree_wait, &co->sched->waiting, co);
+}
+
+void nty_schedule_free(nty_schedule *sched) {
+	if (sched->poller_fd > 0) {
+		close(sched->poller_fd);
+	}
+	if (sched->eventfd > 0) {
+		close(sched->eventfd);
+	}
+	if (sched->stack != NULL) {
+		free(sched->stack);
+	}
+	
+	free(sched);
+
+	assert(pthread_setspecific(global_sched_key, NULL) == 0);
+}
+
+int nty_schedule_create(int stack_size) {
+
+	int sched_stack_size = stack_size ? stack_size : NTY_CO_MAX_STACKSIZE;
+
+	nty_schedule *sched = (nty_schedule*)calloc(1, sizeof(nty_schedule));
+	if (sched == NULL) {
+		printf("Failed to initialize scheduler\n");
+		return -1;
+	}
+
+	assert(pthread_setspecific(global_sched_key, sched) == 0);
+
+	sched->poller_fd = nty_epoller_create();
+	if (sched->poller_fd == -1) {
+		printf("Failed to initialize epoller\n");
+		nty_schedule_free(sched);
+		return -2;
+	}
+
+	nty_epoller_ev_register_trigger();
+
+	sched->stack_size = sched_stack_size;
+	sched->page_size = getpagesize();
+
+#ifdef _USE_UCONTEXT
+	int ret = posix_memalign(&sched->stack, sched->page_size, sched->stack_size);
+	assert(ret == 0);
+#else
+	sched->stack = NULL;
+	bzero(&sched->ctx, sizeof(nty_cpu_ctx));
+#endif
+
+	sched->spawned_coroutines = 0;
+	sched->default_timeout = 3000000u;
+
+	RB_INIT(&sched->sleeping);
+	RB_INIT(&sched->waiting);
+
+	sched->birth = nty_coroutine_usec_now();
+
+	TAILQ_INIT(&sched->ready);
+	TAILQ_INIT(&sched->defer);
+	LIST_INIT(&sched->busy);
+
+}
+
+
+static nty_coroutine *nty_schedule_expired(nty_schedule *sched) {
+	
+	uint64_t t_diff_usecs = nty_coroutine_diff_usecs(sched->birth, nty_coroutine_usec_now());
+	nty_coroutine *co = RB_MIN(_nty_coroutine_rbtree_sleep, &sched->sleeping);
+	if (co == NULL) return NULL;
+	
+	if (co->sleep_usecs <= t_diff_usecs) {
+		RB_REMOVE(_nty_coroutine_rbtree_sleep, &co->sched->sleeping, co);
+		return co;
+	}
+	return NULL;
+}
+
+static inline int nty_schedule_isdone(nty_schedule *sched) {
+	return (RB_EMPTY(&sched->waiting) && 
+		LIST_EMPTY(&sched->busy) &&
+		RB_EMPTY(&sched->sleeping) &&
+		TAILQ_EMPTY(&sched->ready));
+}
+
+static uint64_t nty_schedule_min_timeout(nty_schedule *sched) {
+	uint64_t t_diff_usecs = nty_coroutine_diff_usecs(sched->birth, nty_coroutine_usec_now());
+	uint64_t min = sched->default_timeout;
+
+	nty_coroutine *co = RB_MIN(_nty_coroutine_rbtree_sleep, &sched->sleeping);
+	if (!co) return min;
+
+	min = co->sleep_usecs;
+	if (min > t_diff_usecs) {
+		return min - t_diff_usecs;
+	}
+
+	return 0;
+} 
+
+static int nty_schedule_epoll(nty_schedule *sched) {
+
+	sched->num_new_events = 0;
+
+	struct timespec t = {0, 0};
+	uint64_t usecs = nty_schedule_min_timeout(sched);
+	if (usecs && TAILQ_EMPTY(&sched->ready)) {
+		t.tv_sec = usecs / 1000000u;
+		if (t.tv_sec != 0) {
+			t.tv_nsec = (usecs % 1000u) * 1000u;
+		} else {
+			t.tv_nsec = usecs * 1000u;
+		}
+	} else {
+		return 0;
+	}
+
+	int nready = 0;
+	while (1) {
+		nready = nty_epoller_wait(t);
+		if (nready == -1) {
+			if (errno == EINTR) continue;
+			else assert(0);
+		}
+		break;
+	}
+
+	sched->nevents = 0;
+	sched->num_new_events = nready;
+
+	return 0;
+}
+
+void nty_schedule_run(void) {
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) return ;
+
+	while (!nty_schedule_isdone(sched)) {
+		
+		// 1. expired --> sleep rbtree
+		nty_coroutine *expired = NULL;
+		while ((expired = nty_schedule_expired(sched)) != NULL) {
+			nty_coroutine_resume(expired);
+		}
+		// 2. ready queue
+		nty_coroutine *last_co_ready = TAILQ_LAST(&sched->ready, _nty_coroutine_queue);
+		while (!TAILQ_EMPTY(&sched->ready)) {
+			nty_coroutine *co = TAILQ_FIRST(&sched->ready);
+			TAILQ_REMOVE(&co->sched->ready, co, ready_next);
+
+			if (co->status & BIT(NTY_COROUTINE_STATUS_FDEOF)) {
+				nty_coroutine_free(co);
+				break;
+			}
+
+			nty_coroutine_resume(co);
+			if (co == last_co_ready) break;
+		}
+
+		// 3. wait rbtree
+		nty_schedule_epoll(sched);
+		while (sched->num_new_events) {
+			int idx = --sched->num_new_events;
+			struct epoll_event *ev = sched->eventlist+idx;
+			
+			int fd = ev->data.fd;
+			int is_eof = ev->events & EPOLLHUP;
+			if (is_eof) errno = ECONNRESET;
+
+			nty_coroutine *co = nty_schedule_search_wait(fd);
+			if (co != NULL) {
+				if (is_eof) {
+					co->status |= BIT(NTY_COROUTINE_STATUS_FDEOF);
+				}
+				nty_coroutine_resume(co);
+			}
+
+			is_eof = 0;
+		}
+	}
+
+	nty_schedule_free(sched);
+	
+	return ;
+}
+

NtyCo/core/nty_socket.c

@@ -0,0 +1,672 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+#include "nty_coroutine.h"
+
+
+
+static uint32_t nty_pollevent_2epoll( short events )
+{
+	uint32_t e = 0;	
+	if( events & POLLIN ) 	e |= EPOLLIN;
+	if( events & POLLOUT )  e |= EPOLLOUT;
+	if( events & POLLHUP ) 	e |= EPOLLHUP;
+	if( events & POLLERR )	e |= EPOLLERR;
+	if( events & POLLRDNORM ) e |= EPOLLRDNORM;
+	if( events & POLLWRNORM ) e |= EPOLLWRNORM;
+	return e;
+}
+static short nty_epollevent_2poll( uint32_t events )
+{
+	short e = 0;	
+	if( events & EPOLLIN ) 	e |= POLLIN;
+	if( events & EPOLLOUT ) e |= POLLOUT;
+	if( events & EPOLLHUP ) e |= POLLHUP;
+	if( events & EPOLLERR ) e |= POLLERR;
+	if( events & EPOLLRDNORM ) e |= POLLRDNORM;
+	if( events & EPOLLWRNORM ) e |= POLLWRNORM;
+	return e;
+}
+/*
+ * nty_poll_inner --> 1. sockfd--> epoll, 2 yield, 3. epoll x sockfd
+ * fds : 
+ */
+
+static int nty_poll_inner(struct pollfd *fds, nfds_t nfds, int timeout) {
+
+	if (timeout == 0)
+	{
+		return poll(fds, nfds, timeout);
+	}
+	if (timeout < 0)
+	{
+		timeout = INT_MAX;
+	}
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		printf("scheduler not exit!\n");
+		return -1;
+	}
+	
+	nty_coroutine *co = sched->curr_thread;
+	
+	int i = 0;
+	for (i = 0;i < nfds;i ++) {
+	
+		struct epoll_event ev;
+		ev.events = nty_pollevent_2epoll(fds[i].events);
+		ev.data.fd = fds[i].fd;
+		epoll_ctl(sched->poller_fd, EPOLL_CTL_ADD, fds[i].fd, &ev);
+
+		co->events = fds[i].events;
+		nty_schedule_sched_wait(co, fds[i].fd, fds[i].events, timeout);
+	}
+	nty_coroutine_yield(co); 
+
+	for (i = 0;i < nfds;i ++) {
+	
+		struct epoll_event ev;
+		ev.events = nty_pollevent_2epoll(fds[i].events);
+		ev.data.fd = fds[i].fd;
+		epoll_ctl(sched->poller_fd, EPOLL_CTL_DEL, fds[i].fd, &ev);
+
+		nty_schedule_desched_wait(fds[i].fd);
+	}
+
+	return nfds;
+}
+
+
+int nty_socket(int domain, int type, int protocol) {
+
+	int fd = socket(domain, type, protocol);
+	if (fd == -1) {
+		printf("Failed to create a new socket\n");
+		return -1;
+	}
+	int ret = fcntl(fd, F_SETFL, O_NONBLOCK);
+	if (ret == -1) {
+		close(ret);
+		return -1;
+	}
+	int reuse = 1;
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
+	
+	return fd;
+}
+
+//nty_accept 
+//return failed == -1, success > 0
+
+int nty_accept(int fd, struct sockaddr *addr, socklen_t *len) {
+	int sockfd = -1;
+	int timeout = 1;
+	nty_coroutine *co = nty_coroutine_get_sched()->curr_thread;
+	
+	while (1) {
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLIN | POLLERR | POLLHUP;
+		nty_poll_inner(&fds, 1, timeout);
+
+		sockfd = accept(fd, addr, len);
+		if (sockfd < 0) {
+			if (errno == EAGAIN) {
+				continue;
+			} else if (errno == ECONNABORTED) {
+				printf("accept : ECONNABORTED\n");
+				
+			} else if (errno == EMFILE || errno == ENFILE) {
+				printf("accept : EMFILE || ENFILE\n");
+			}
+			return -1;
+		} else {
+			break;
+		}
+	}
+
+	int ret = fcntl(sockfd, F_SETFL, O_NONBLOCK);
+	if (ret == -1) {
+		close(sockfd);
+		return -1;
+	}
+	int reuse = 1;
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
+	
+	return sockfd;
+}
+
+
+int nty_connect(int fd, struct sockaddr *name, socklen_t namelen) {
+
+	int ret = 0;
+
+	while (1) {
+
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLOUT | POLLERR | POLLHUP;
+		nty_poll_inner(&fds, 1, 1);
+
+		ret = connect(fd, name, namelen);
+		if (ret == 0) break;
+
+		if (ret == -1 && (errno == EAGAIN ||
+			errno == EWOULDBLOCK || 
+			errno == EINPROGRESS)) {
+			continue;
+		} else {
+			break;
+		}
+	}
+
+	return ret;
+}
+
+//recv 
+// add epoll first
+//
+ssize_t nty_recv(int fd, void *buf, size_t len, int flags) {
+	
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN | POLLERR | POLLHUP;
+
+	nty_poll_inner(&fds, 1, 1);
+
+	int ret = recv(fd, buf, len, flags);
+	if (ret < 0) {
+		//if (errno == EAGAIN) return ret;
+		if (errno == ECONNRESET) return -1;
+		//printf("recv error : %d, ret : %d\n", errno, ret);
+		
+	}
+	return ret;
+}
+
+
+ssize_t nty_send(int fd, const void *buf, size_t len, int flags) {
+	
+	int sent = 0;
+
+	int ret = send(fd, ((char*)buf)+sent, len-sent, flags);
+	if (ret == 0) return ret;
+	if (ret > 0) sent += ret;
+
+	while (sent < len) {
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLOUT | POLLERR | POLLHUP;
+
+		nty_poll_inner(&fds, 1, 1);
+		ret = send(fd, ((char*)buf)+sent, len-sent, flags);
+		//printf("send --> len : %d\n", ret);
+		if (ret <= 0) {			
+			break;
+		}
+		sent += ret;
+	}
+
+	if (ret <= 0 && sent == 0) return ret;
+	
+	return sent;
+}
+
+
+ssize_t nty_sendto(int fd, const void *buf, size_t len, int flags,
+               const struct sockaddr *dest_addr, socklen_t addrlen) {
+
+
+	int sent = 0;
+
+	while (sent < len) {
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLOUT | POLLERR | POLLHUP;
+
+		nty_poll_inner(&fds, 1, 1);
+		int ret = sendto(fd, ((char*)buf)+sent, len-sent, flags, dest_addr, addrlen);
+		if (ret <= 0) {
+			if (errno == EAGAIN) continue;
+			else if (errno == ECONNRESET) {
+				return ret;
+			}
+			printf("send errno : %d, ret : %d\n", errno, ret);
+			assert(0);
+		}
+		sent += ret;
+	}
+	return sent;
+	
+}
+
+ssize_t nty_recvfrom(int fd, void *buf, size_t len, int flags,
+                 struct sockaddr *src_addr, socklen_t *addrlen) {
+
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN | POLLERR | POLLHUP;
+
+	nty_poll_inner(&fds, 1, 1);
+
+	int ret = recvfrom(fd, buf, len, flags, src_addr, addrlen);
+	if (ret < 0) {
+		if (errno == EAGAIN) return ret;
+		if (errno == ECONNRESET) return 0;
+		
+		printf("recv error : %d, ret : %d\n", errno, ret);
+		assert(0);
+	}
+	return ret;
+
+}
+
+
+
+
+int nty_close(int fd) {
+#if 0
+	nty_schedule *sched = nty_coroutine_get_sched();
+
+	nty_coroutine *co = sched->curr_thread;
+	if (co) {
+		TAILQ_INSERT_TAIL(&nty_coroutine_get_sched()->ready, co, ready_next);
+		co->status |= BIT(NTY_COROUTINE_STATUS_FDEOF);
+	}
+#endif	
+	return close(fd);
+}
+
+
+#ifdef  COROUTINE_HOOK
+
+socket_t socket_f = NULL;
+
+read_t read_f = NULL;
+recv_t recv_f = NULL;
+recvfrom_t recvfrom_f = NULL;
+
+write_t write_f = NULL;
+send_t send_f = NULL;
+sendto_t sendto_f = NULL;
+
+accept_t accept_f = NULL;
+close_t close_f = NULL;
+connect_t connect_f = NULL;
+
+
+int init_hook(void) {
+
+	socket_f = (socket_t)dlsym(RTLD_NEXT, "socket");
+	
+	read_f = (read_t)dlsym(RTLD_NEXT, "read");
+	recv_f = (recv_t)dlsym(RTLD_NEXT, "recv");
+	recvfrom_f = (recvfrom_t)dlsym(RTLD_NEXT, "recvfrom");
+
+	write_f = (write_t)dlsym(RTLD_NEXT, "write");
+	send_f = (send_t)dlsym(RTLD_NEXT, "send");
+    sendto_f = (sendto_t)dlsym(RTLD_NEXT, "sendto");
+
+	accept_f = (accept_t)dlsym(RTLD_NEXT, "accept");
+	close_f = (close_t)dlsym(RTLD_NEXT, "close");
+	connect_f = (connect_t)dlsym(RTLD_NEXT, "connect");
+
+}
+
+
+
+int socket(int domain, int type, int protocol) {
+
+	if (!socket_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return socket_f(domain, type, protocol);
+	}
+
+	int fd = socket_f(domain, type, protocol);
+	if (fd == -1) {
+		printf("Failed to create a new socket\n");
+		return -1;
+	}
+	int ret = fcntl(fd, F_SETFL, O_NONBLOCK);
+	if (ret == -1) {
+		close(ret);
+		return -1;
+	}
+	int reuse = 1;
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
+	
+	return fd;
+}
+
+ssize_t read(int fd, void *buf, size_t count) {
+
+	if (!read_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return read_f(fd, buf, count);
+	}
+
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN | POLLERR | POLLHUP;
+
+	nty_poll_inner(&fds, 1, 1);
+
+	int ret = read_f(fd, buf, count);
+	if (ret < 0) {
+		//if (errno == EAGAIN) return ret;
+		if (errno == ECONNRESET) return -1;
+		//printf("recv error : %d, ret : %d\n", errno, ret);
+		
+	}
+	return ret;
+}
+
+ssize_t recv(int fd, void *buf, size_t len, int flags) {
+
+	if (!recv_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return recv_f(fd, buf, len, flags);
+	}
+
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN | POLLERR | POLLHUP;
+
+	nty_poll_inner(&fds, 1, 1);
+
+	int ret = recv_f(fd, buf, len, flags);
+	if (ret < 0) {
+		//if (errno == EAGAIN) return ret;
+		if (errno == ECONNRESET) return -1;
+		//printf("recv error : %d, ret : %d\n", errno, ret);
+		
+	}
+	return ret;
+}
+
+
+ssize_t recvfrom(int fd, void *buf, size_t len, int flags,
+                 struct sockaddr *src_addr, socklen_t *addrlen) {
+
+	if (!recvfrom_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return recvfrom_f(fd, buf, len, flags, src_addr, addrlen);
+	}
+
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN | POLLERR | POLLHUP;
+
+	nty_poll_inner(&fds, 1, 1);
+
+	int ret = recvfrom_f(fd, buf, len, flags, src_addr, addrlen);
+	if (ret < 0) {
+		if (errno == EAGAIN) return ret;
+		if (errno == ECONNRESET) return 0;
+		
+		printf("recv error : %d, ret : %d\n", errno, ret);
+		assert(0);
+	}
+	return ret;
+
+}
+
+
+ssize_t write(int fd, const void *buf, size_t count) {
+
+	if (!write_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return write_f(fd, buf, count);
+	}
+
+	int sent = 0;
+
+	int ret = write_f(fd, ((char*)buf)+sent, count-sent);
+	if (ret == 0) return ret;
+	if (ret > 0) sent += ret;
+
+	while (sent < count) {
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLOUT | POLLERR | POLLHUP;
+
+		nty_poll_inner(&fds, 1, 1);
+		ret = write_f(fd, ((char*)buf)+sent, count-sent);
+		//printf("send --> len : %d\n", ret);
+		if (ret <= 0) {			
+			break;
+		}
+		sent += ret;
+	}
+
+	if (ret <= 0 && sent == 0) return ret;
+	
+	return sent;
+}
+
+
+ssize_t send(int fd, const void *buf, size_t len, int flags) {
+
+	if (!send_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return send_f(fd, buf, len, flags);
+	}
+
+	int sent = 0;
+
+	int ret = send_f(fd, ((char*)buf)+sent, len-sent, flags);
+	if (ret == 0) return ret;
+	if (ret > 0) sent += ret;
+
+	while (sent < len) {
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLOUT | POLLERR | POLLHUP;
+
+		nty_poll_inner(&fds, 1, 1);
+		ret = send_f(fd, ((char*)buf)+sent, len-sent, flags);
+		//printf("send --> len : %d\n", ret);
+		if (ret <= 0) {			
+			break;
+		}
+		sent += ret;
+	}
+
+	if (ret <= 0 && sent == 0) return ret;
+	
+	return sent;
+}
+
+ssize_t sendto(int sockfd, const void *buf, size_t len, int flags,
+        const struct sockaddr *dest_addr, socklen_t addrlen) {
+
+	if (!sendto_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return sendto_f(sockfd, buf, len, flags, dest_addr, addrlen);
+	}
+
+	struct pollfd fds;
+	fds.fd = sockfd;
+	fds.events = POLLOUT | POLLERR | POLLHUP;
+
+	nty_poll_inner(&fds, 1, 1);
+
+	int ret = sendto_f(sockfd, buf, len, flags, dest_addr, addrlen);
+	if (ret < 0) {
+		if (errno == EAGAIN) return ret;
+		if (errno == ECONNRESET) return 0;
+		
+		printf("recv error : %d, ret : %d\n", errno, ret);
+		assert(0);
+	}
+	return ret;
+
+}
+
+
+
+int accept(int fd, struct sockaddr *addr, socklen_t *len) {
+
+	if (!accept_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return accept_f(fd, addr, len);
+	}
+
+	int sockfd = -1;
+	int timeout = 1;
+	nty_coroutine *co = nty_coroutine_get_sched()->curr_thread;
+	
+	while (1) {
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLIN | POLLERR | POLLHUP;
+		nty_poll_inner(&fds, 1, timeout);
+
+		sockfd = accept_f(fd, addr, len);
+		if (sockfd < 0) {
+			if (errno == EAGAIN) {
+				continue;
+			} else if (errno == ECONNABORTED) {
+				printf("accept : ECONNABORTED\n");
+				
+			} else if (errno == EMFILE || errno == ENFILE) {
+				printf("accept : EMFILE || ENFILE\n");
+			}
+			return -1;
+		} else {
+			break;
+		}
+	}
+
+	int ret = fcntl(sockfd, F_SETFL, O_NONBLOCK);
+	if (ret == -1) {
+		close(sockfd);
+		return -1;
+	}
+	int reuse = 1;
+	setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
+	
+	return sockfd;
+}
+
+int close(int fd) {
+
+	if (!close_f) init_hook();
+
+	return close_f(fd);
+}
+
+
+
+int connect(int fd, const struct sockaddr *addr, socklen_t addrlen) {
+
+	if (!connect_f) init_hook();
+
+	nty_schedule *sched = nty_coroutine_get_sched();
+	if (sched == NULL) {
+		return connect_f(fd, addr, addrlen);
+	}
+
+	int ret = 0;
+
+	while (1) {
+
+		struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLOUT | POLLERR | POLLHUP;
+		nty_poll_inner(&fds, 1, 1);
+
+		ret = connect_f(fd, addr, addrlen);
+		if (ret == 0) break;
+
+		if (ret == -1 && (errno == EAGAIN ||
+			errno == EWOULDBLOCK || 
+			errno == EINPROGRESS)) {
+			continue;
+		} else {
+			break;
+		}
+	}
+
+	return ret;
+}
+
+
+
+
+
+
+
+
+
+#endif
+
+
+
+
+
+
+
+
+
+
+
+

NtyCo/core/nty_tree.h

@@ -0,0 +1,754 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+#ifndef __NTY_TREE_H__
+#define __NTY_TREE_H__
+
+
+#define SPLAY_HEAD(name, type)						\
+struct name {								\
+	struct type *sph_root; /* root of the tree */			\
+}
+
+#define SPLAY_INITIALIZER(root)						\
+	{ NULL }
+
+#define SPLAY_INIT(root) do {						\
+	(root)->sph_root = NULL;					\
+} while (/*CONSTCOND*/ 0)
+
+#define SPLAY_ENTRY(type)						\
+struct {								\
+	struct type *spe_left; /* left element */			\
+	struct type *spe_right; /* right element */			\
+}
+
+#define SPLAY_LEFT(elm, field)		(elm)->field.spe_left
+#define SPLAY_RIGHT(elm, field)		(elm)->field.spe_right
+#define SPLAY_ROOT(head)		(head)->sph_root
+#define SPLAY_EMPTY(head)		(SPLAY_ROOT(head) == NULL)
+
+/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */
+#define SPLAY_ROTATE_RIGHT(head, tmp, field) do {			\
+	SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field);	\
+	SPLAY_RIGHT(tmp, field) = (head)->sph_root;			\
+	(head)->sph_root = tmp;						\
+} while (/*CONSTCOND*/ 0)
+	
+#define SPLAY_ROTATE_LEFT(head, tmp, field) do {			\
+	SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field);	\
+	SPLAY_LEFT(tmp, field) = (head)->sph_root;			\
+	(head)->sph_root = tmp;						\
+} while (/*CONSTCOND*/ 0)
+
+#define SPLAY_LINKLEFT(head, tmp, field) do {				\
+	SPLAY_LEFT(tmp, field) = (head)->sph_root;			\
+	tmp = (head)->sph_root;						\
+	(head)->sph_root = SPLAY_LEFT((head)->sph_root, field);		\
+} while (/*CONSTCOND*/ 0)
+
+#define SPLAY_LINKRIGHT(head, tmp, field) do {				\
+	SPLAY_RIGHT(tmp, field) = (head)->sph_root;				\
+	tmp = (head)->sph_root;									\
+	(head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\
+} while (/*CONSTCOND*/ 0)
+
+#define SPLAY_ASSEMBLE(head, node, left, right, field) do {			\
+	SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field);	\
+	SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\
+	SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field);	\
+	SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field);	\
+} while (/*CONSTCOND*/ 0)
+
+
+
+#define SPLAY_PROTOTYPE(name, type, field, cmp)				\
+void name##_SPLAY(struct name *, struct type *);			\
+void name##_SPLAY_MINMAX(struct name *, int);				\
+struct type *name##_SPLAY_INSERT(struct name *, struct type *);		\
+struct type *name##_SPLAY_REMOVE(struct name *, struct type *);		\
+		\
+/* Finds the node with the same key as elm */			\
+static __inline struct type *								\
+name##_SPLAY_FIND(struct name *head, struct type *elm)	{ 	\									\
+	if (SPLAY_EMPTY(head))									\
+		return(NULL);										\
+	name##_SPLAY(head, elm);								\
+	if ((cmp)(elm, (head)->sph_root) == 0)					\
+		return (head->sph_root);							\
+	return (NULL);											\
+}															\
+															\
+static __inline struct type *								\
+name##_SPLAY_NEXT(struct name *head, struct type *elm) {	\
+	name##_SPLAY(head, elm);								\
+	if (SPLAY_RIGHT(elm, field) != NULL) {					\
+		elm = SPLAY_RIGHT(elm, field);						\
+		while (SPLAY_LEFT(elm, field) != NULL) {			\
+			elm = SPLAY_LEFT(elm, field);					\
+		}													\
+	} else													\
+		elm = NULL;											\
+	return (elm);											\
+}															\
+															\
+static __inline struct type *								\
+name##_SPLAY_MIN_MAX(struct name *head, int val) {			\
+	name##_SPLAY_MINMAX(head, val);							\
+        return (SPLAY_ROOT(head));							\
+}
+
+/* Main splay operation.
+ * Moves node close to the key of elm to top
+ */
+#define SPLAY_GENERATE(name, type, field, cmp)				\
+struct type *												\
+name##_SPLAY_INSERT(struct name *head, struct type *elm) {	\
+    if (SPLAY_EMPTY(head)) {								\
+	    SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL;	\
+    } else {												\
+	    int __comp;											\
+	    name##_SPLAY(head, elm);							\
+	    __comp = (cmp)(elm, (head)->sph_root);				\
+	    if(__comp < 0) {									\
+		    SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\
+		    SPLAY_RIGHT(elm, field) = (head)->sph_root;		\
+		    SPLAY_LEFT((head)->sph_root, field) = NULL;		\
+	    } else if (__comp > 0) {							\
+		    SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\
+		    SPLAY_LEFT(elm, field) = (head)->sph_root;		\
+		    SPLAY_RIGHT((head)->sph_root, field) = NULL;	\
+	    } else												\
+		    return ((head)->sph_root);						\
+    }														\
+    (head)->sph_root = (elm);								\
+    return (NULL);											\
+}															\
+															\
+struct type *												\
+name##_SPLAY_REMOVE(struct name *head, struct type *elm) {	\
+	struct type *__tmp;										\
+	if (SPLAY_EMPTY(head))									\
+		return (NULL);										\
+	name##_SPLAY(head, elm);								\
+	if ((cmp)(elm, (head)->sph_root) == 0) {				\
+		if (SPLAY_LEFT((head)->sph_root, field) == NULL) {	\
+			(head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\
+		} else {											\
+			__tmp = SPLAY_RIGHT((head)->sph_root, field);	\
+			(head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\
+			name##_SPLAY(head, elm);						\
+			SPLAY_RIGHT((head)->sph_root, field) = __tmp;	\
+		}													\
+		return (elm);										\
+	}														\
+	return (NULL);											\
+}															\
+															\
+void														\
+name##_SPLAY(struct name *head, struct type *elm) {			\
+	struct type __node, *__left, *__right, *__tmp;			\
+	int __comp;												\
+															\
+	SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\
+	__left = __right = &__node;								\
+															\
+	while ((__comp = (cmp)(elm, (head)->sph_root)) != 0) {\
+		if (__comp < 0) {									\
+			__tmp = SPLAY_LEFT((head)->sph_root, field);	\
+			if (__tmp == NULL)								\
+				break;										\
+			if ((cmp)(elm, __tmp) < 0){						\
+				SPLAY_ROTATE_RIGHT(head, __tmp, field);		\
+				if (SPLAY_LEFT((head)->sph_root, field) == NULL)\
+					break;									\
+			}												\
+			SPLAY_LINKLEFT(head, __right, field);			\
+		} else if (__comp > 0) {							\
+			__tmp = SPLAY_RIGHT((head)->sph_root, field);	\
+			if (__tmp == NULL)								\
+				break;										\
+			if ((cmp)(elm, __tmp) > 0){						\
+				SPLAY_ROTATE_LEFT(head, __tmp, field);		\
+				if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\
+					break;									\
+			}												\
+			SPLAY_LINKRIGHT(head, __left, field);			\
+		}													\
+	}														\
+	SPLAY_ASSEMBLE(head, &__node, __left, __right, field);	\
+}															\
+															\
+void name##_SPLAY_MINMAX(struct name *head, int __comp) 	\
+{															\
+	struct type __node, *__left, *__right, *__tmp;			\
+															\
+	SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\
+	__left = __right = &__node;								\
+															\
+	while (1) {												\
+		if (__comp < 0) {									\
+			__tmp = SPLAY_LEFT((head)->sph_root, field);	\
+			if (__tmp == NULL)								\
+				break;										\
+			if (__comp < 0){								\
+				SPLAY_ROTATE_RIGHT(head, __tmp, field);		\
+				if (SPLAY_LEFT((head)->sph_root, field) == NULL)\
+					break;									\
+			}												\
+			SPLAY_LINKLEFT(head, __right, field);			\
+		} else if (__comp > 0) {							\
+			__tmp = SPLAY_RIGHT((head)->sph_root, field);	\
+			if (__tmp == NULL)								\
+				break;										\
+			if (__comp > 0) {								\
+				SPLAY_ROTATE_LEFT(head, __tmp, field);		\
+				if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\
+					break;									\
+			}												\
+			SPLAY_LINKRIGHT(head, __left, field);			\
+		}													\
+	}														\
+	SPLAY_ASSEMBLE(head, &__node, __left, __right, field);	\
+}
+
+#define SPLAY_NEGINF		-1
+#define SPLAY_INF			1
+
+#define SPLAY_INSERT(name, x, y)	name##_SPLAY_INSERT(x, y)
+#define SPLAY_REMOVE(name, x, y)	name##_SPLAY_REMOVE(x, y)
+#define SPLAY_FIND(name, x, y)		name##_SPLAY_FIND(x, y)
+#define SPLAY_NEXT(name, x, y)		name##_SPLAY_NEXT(x, y)
+#define SPLAY_MIN(name, x)		(SPLAY_EMPTY(x) ? NULL	\
+					: name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF))
+#define SPLAY_MAX(name, x)		(SPLAY_EMPTY(x) ? NULL	\
+					: name##_SPLAY_MIN_MAX(x, SPLAY_INF))
+
+#define SPLAY_FOREACH(x, name, head)					\
+	for ((x) = SPLAY_MIN(name, head);					\
+	     (x) != NULL;									\
+	     (x) = SPLAY_NEXT(name, head, x))
+
+
+
+
+/* Macros that define a red-black tree */
+#define RB_HEAD(name, type)						\
+	struct name {								\
+		struct type *rbh_root; /* root of the tree */			\
+	}
+	
+#define RB_INITIALIZER(root)						\
+		{ NULL }
+	
+#define RB_INIT(root) do {							\
+		(root)->rbh_root = NULL;					\
+	} while (0)
+
+
+#define RB_BLACK	0
+#define RB_RED		1
+
+#define RB_ENTRY(type)										\
+struct {													\
+	struct type *rbe_left;		/* left element */		\
+	struct type *rbe_right;		/* right element */		\
+	struct type *rbe_parent;	/* parent element */		\
+	int rbe_color;			/* node color */				\
+}
+
+#define RB_LEFT(elm, field)		(elm)->field.rbe_left
+#define RB_RIGHT(elm, field)		(elm)->field.rbe_right
+#define RB_PARENT(elm, field)		(elm)->field.rbe_parent
+#define RB_COLOR(elm, field)		(elm)->field.rbe_color
+#define RB_ROOT(head)			(head)->rbh_root
+#define RB_EMPTY(head)			(RB_ROOT(head) == NULL)
+
+#define RB_SET(elm, parent, field) do {					\
+	RB_PARENT(elm, field) = parent;						\
+	RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL;	\
+	RB_COLOR(elm, field) = RB_RED;						\
+} while (0)
+
+#define RB_SET_BLACKRED(black, red, field) do {			\
+	RB_COLOR(black, field) = RB_BLACK;					\
+	RB_COLOR(red, field) = RB_RED;						\
+} while (0)
+
+#ifndef RB_AUGMENT
+#define RB_AUGMENT(x)	do {} while (0)
+#endif
+
+
+
+#define RB_ROTATE_LEFT(head, elm, tmp, field) do {				\
+	(tmp) = RB_RIGHT(elm, field);								\
+	if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field)) != NULL) {	\
+		RB_PARENT(RB_LEFT(tmp, field), field) = (elm);			\
+	}															\
+	RB_AUGMENT(elm);											\
+	if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field)) != NULL) {	\
+		if ((elm) == RB_LEFT(RB_PARENT(elm, field), field))	\
+			RB_LEFT(RB_PARENT(elm, field), field) = (tmp);		\
+		else													\
+			RB_RIGHT(RB_PARENT(elm, field), field) = (tmp);		\
+	} else														\
+		(head)->rbh_root = (tmp);								\
+	RB_LEFT(tmp, field) = (elm);								\
+	RB_PARENT(elm, field) = (tmp);								\
+	RB_AUGMENT(tmp);											\
+	if ((RB_PARENT(tmp, field)))								\
+		RB_AUGMENT(RB_PARENT(tmp, field));						\
+} while (0)
+
+	
+#define RB_ROTATE_RIGHT(head, elm, tmp, field) do {				\
+		(tmp) = RB_LEFT(elm, field);							\
+		if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field)) != NULL) { \
+			RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); 	\
+		}														\
+		RB_AUGMENT(elm);										\
+		if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field)) != NULL) {	\
+			if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) 	\
+				RB_LEFT(RB_PARENT(elm, field), field) = (tmp);	\
+			else												\
+				RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \
+		} else													\
+			(head)->rbh_root = (tmp);							\
+		RB_RIGHT(tmp, field) = (elm);							\
+		RB_PARENT(elm, field) = (tmp);							\
+		RB_AUGMENT(tmp);										\
+		if ((RB_PARENT(tmp, field)))							\
+			RB_AUGMENT(RB_PARENT(tmp, field));					\
+	} while (0)
+
+
+#define	RB_PROTOTYPE(name, type, field, cmp)					\
+			RB_PROTOTYPE_INTERNAL(name, type, field, cmp,)
+#define	RB_PROTOTYPE_STATIC(name, type, field, cmp)				\
+			RB_PROTOTYPE_INTERNAL(name, type, field, cmp, __unused static)
+#define RB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr)		\
+		attr void name##_RB_INSERT_COLOR(struct name *, struct type *); 	\
+		attr void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\
+		attr struct type *name##_RB_REMOVE(struct name *, struct type *);	\
+		attr struct type *name##_RB_INSERT(struct name *, struct type *);	\
+		attr struct type *name##_RB_FIND(struct name *, struct type *); 	\
+		attr struct type *name##_RB_NFIND(struct name *, struct type *);	\
+		attr struct type *name##_RB_NEXT(struct type *);					\
+		attr struct type *name##_RB_PREV(struct type *);					\
+		attr struct type *name##_RB_MINMAX(struct name *, int); 			\
+
+/* Main rb operation.
+ * Moves node close to the key of elm to top
+ */
+#define	RB_GENERATE(name, type, field, cmp)									\
+	RB_GENERATE_INTERNAL(name, type, field, cmp,)
+#define	RB_GENERATE_STATIC(name, type, field, cmp)							\
+	RB_GENERATE_INTERNAL(name, type, field, cmp, __unused static)
+#define RB_GENERATE_INTERNAL(name, type, field, cmp, attr)					\
+attr void																	\
+name##_RB_INSERT_COLOR(struct name *head, struct type *elm)	{				\
+	struct type *parent, *gparent, *tmp;									\
+	while ((parent = RB_PARENT(elm, field)) != NULL &&						\
+	    RB_COLOR(parent, field) == RB_RED) {								\
+		gparent = RB_PARENT(parent, field);									\
+		if (parent == RB_LEFT(gparent, field)) {							\
+			tmp = RB_RIGHT(gparent, field);									\
+			if (tmp && RB_COLOR(tmp, field) == RB_RED) {					\
+				RB_COLOR(tmp, field) = RB_BLACK;							\
+				RB_SET_BLACKRED(parent, gparent, field);					\
+				elm = gparent;												\
+				continue;													\
+			}																\
+			if (RB_RIGHT(parent, field) == elm) {							\
+				RB_ROTATE_LEFT(head, parent, tmp, field);					\
+				tmp = parent;												\
+				parent = elm;												\
+				elm = tmp;													\
+			}																\
+			RB_SET_BLACKRED(parent, gparent, field);						\
+			RB_ROTATE_RIGHT(head, gparent, tmp, field);						\
+		} else {															\
+			tmp = RB_LEFT(gparent, field);									\
+			if (tmp && RB_COLOR(tmp, field) == RB_RED) {					\
+				RB_COLOR(tmp, field) = RB_BLACK;							\
+				RB_SET_BLACKRED(parent, gparent, field);					\
+				elm = gparent;												\
+				continue;													\
+			}																\
+			if (RB_LEFT(parent, field) == elm) {							\
+				RB_ROTATE_RIGHT(head, parent, tmp, field);					\
+				tmp = parent;												\
+				parent = elm;												\
+				elm = tmp;													\
+			}																\
+			RB_SET_BLACKRED(parent, gparent, field);						\
+			RB_ROTATE_LEFT(head, gparent, tmp, field);						\
+		}																	\
+	}																		\
+	RB_COLOR(head->rbh_root, field) = RB_BLACK;								\
+}																			\
+																			\
+attr void																	\
+name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \
+{																			\
+	struct type *tmp;														\
+	while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) &&				\
+	    elm != RB_ROOT(head)) {												\
+		if (RB_LEFT(parent, field) == elm) {								\
+			tmp = RB_RIGHT(parent, field);									\
+			if (RB_COLOR(tmp, field) == RB_RED) {							\
+				RB_SET_BLACKRED(tmp, parent, field);						\
+				RB_ROTATE_LEFT(head, parent, tmp, field);					\
+				tmp = RB_RIGHT(parent, field);								\
+			}																\
+			if ((RB_LEFT(tmp, field) == NULL ||							\
+			    RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&		\
+			    (RB_RIGHT(tmp, field) == NULL ||							\
+			    RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {		\
+				RB_COLOR(tmp, field) = RB_RED;								\
+				elm = parent;												\
+				parent = RB_PARENT(elm, field);								\
+			} else {														\
+				if (RB_RIGHT(tmp, field) == NULL ||							\
+				    RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {	\
+					struct type *oleft;										\
+					if ((oleft = RB_LEFT(tmp, field)) 						\
+					    != NULL)											\
+						RB_COLOR(oleft, field) = RB_BLACK;					\
+					RB_COLOR(tmp, field) = RB_RED;							\
+					RB_ROTATE_RIGHT(head, tmp, oleft, field);				\
+					tmp = RB_RIGHT(parent, field);							\
+				}															\
+				RB_COLOR(tmp, field) = RB_COLOR(parent, field);				\
+				RB_COLOR(parent, field) = RB_BLACK;							\
+				if (RB_RIGHT(tmp, field))									\
+					RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;		\
+				RB_ROTATE_LEFT(head, parent, tmp, field);					\
+				elm = RB_ROOT(head);										\
+				break;														\
+			}																\
+		} else {															\
+			tmp = RB_LEFT(parent, field);									\
+			if (RB_COLOR(tmp, field) == RB_RED) {							\
+				RB_SET_BLACKRED(tmp, parent, field);						\
+				RB_ROTATE_RIGHT(head, parent, tmp, field);					\
+				tmp = RB_LEFT(parent, field);								\
+			}																\
+			if ((RB_LEFT(tmp, field) == NULL ||							\
+			    RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&		\
+			    (RB_RIGHT(tmp, field) == NULL ||							\
+			    RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {		\
+				RB_COLOR(tmp, field) = RB_RED;								\
+				elm = parent;												\
+				parent = RB_PARENT(elm, field);								\
+			} else {														\
+				if (RB_LEFT(tmp, field) == NULL ||							\
+				    RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {		\
+					struct type *oright;									\
+					if ((oright = RB_RIGHT(tmp, field)) 					\
+					    != NULL)											\
+						RB_COLOR(oright, field) = RB_BLACK;					\
+					RB_COLOR(tmp, field) = RB_RED;							\
+					RB_ROTATE_LEFT(head, tmp, oright, field);				\
+					tmp = RB_LEFT(parent, field);							\
+				}															\
+				RB_COLOR(tmp, field) = RB_COLOR(parent, field);				\
+				RB_COLOR(parent, field) = RB_BLACK;							\
+				if (RB_LEFT(tmp, field))									\
+					RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;		\
+				RB_ROTATE_RIGHT(head, parent, tmp, field);					\
+				elm = RB_ROOT(head);										\
+				break;														\
+			}																\
+		}																	\
+	}																		\
+	if (elm)																\
+		RB_COLOR(elm, field) = RB_BLACK;									\
+}																			\
+																			\
+attr struct type *															\
+name##_RB_REMOVE(struct name *head, struct type *elm)						\
+{																			\
+	struct type *child, *parent, *old = elm;								\
+	int color;																\
+	if (RB_LEFT(elm, field) == NULL)										\
+		child = RB_RIGHT(elm, field);										\
+	else if (RB_RIGHT(elm, field) == NULL)									\
+		child = RB_LEFT(elm, field);										\
+	else {																	\
+		struct type *left;													\
+		elm = RB_RIGHT(elm, field);											\
+		while ((left = RB_LEFT(elm, field)) != NULL)						\
+			elm = left;														\
+		child = RB_RIGHT(elm, field);										\
+		parent = RB_PARENT(elm, field);										\
+		color = RB_COLOR(elm, field);										\
+		if (child)															\
+			RB_PARENT(child, field) = parent;								\
+		if (parent) {														\
+			if (RB_LEFT(parent, field) == elm)								\
+				RB_LEFT(parent, field) = child;								\
+			else															\
+				RB_RIGHT(parent, field) = child;							\
+			RB_AUGMENT(parent);												\
+		} else																\
+			RB_ROOT(head) = child;											\
+		if (RB_PARENT(elm, field) == old)									\
+			parent = elm;													\
+		(elm)->field = (old)->field;										\
+		if (RB_PARENT(old, field)) {										\
+			if (RB_LEFT(RB_PARENT(old, field), field) == old)				\
+				RB_LEFT(RB_PARENT(old, field), field) = elm;				\
+			else															\
+				RB_RIGHT(RB_PARENT(old, field), field) = elm;				\
+			RB_AUGMENT(RB_PARENT(old, field));								\
+		} else																\
+			RB_ROOT(head) = elm;											\
+		RB_PARENT(RB_LEFT(old, field), field) = elm;						\
+		if (RB_RIGHT(old, field))											\
+			RB_PARENT(RB_RIGHT(old, field), field) = elm;					\
+		if (parent) {														\
+			left = parent;													\
+			do {															\
+				RB_AUGMENT(left);											\
+			} while ((left = RB_PARENT(left, field)) != NULL); 				\
+		}																	\
+		goto color;															\
+	}																		\
+	parent = RB_PARENT(elm, field);											\
+	color = RB_COLOR(elm, field);											\
+	if (child)																\
+		RB_PARENT(child, field) = parent;									\
+	if (parent) {															\
+		if (RB_LEFT(parent, field) == elm)									\
+			RB_LEFT(parent, field) = child;									\
+		else																\
+			RB_RIGHT(parent, field) = child;								\
+		RB_AUGMENT(parent);													\
+	} else																	\
+		RB_ROOT(head) = child;												\
+color:																		\
+	if (color == RB_BLACK)													\
+		name##_RB_REMOVE_COLOR(head, parent, child);						\
+	return (old);															\
+}																			\
+																			\
+/* Inserts a node into the RB tree */									\
+attr struct type *															\
+name##_RB_INSERT(struct name *head, struct type *elm) {						\
+	struct type *tmp;														\
+	struct type *parent = NULL;												\
+	int comp = 0;															\
+	tmp = RB_ROOT(head);													\
+	while (tmp) {															\
+		parent = tmp;														\
+		comp = (cmp)(elm, parent);											\
+		if (comp < 0)														\
+			tmp = RB_LEFT(tmp, field);										\
+		else if (comp > 0)													\
+			tmp = RB_RIGHT(tmp, field);										\
+		else																\
+			return (tmp);													\
+	}																		\
+	RB_SET(elm, parent, field);												\
+	if (parent != NULL) {													\
+		if (comp < 0)														\
+			RB_LEFT(parent, field) = elm;									\
+		else																\
+			RB_RIGHT(parent, field) = elm;									\
+		RB_AUGMENT(parent);													\
+	} else																	\
+		RB_ROOT(head) = elm;												\
+	name##_RB_INSERT_COLOR(head, elm);										\
+	return (NULL);															\
+}																			\
+																			\
+/* Finds the node with the same key as elm */							\
+attr struct type *															\
+name##_RB_FIND(struct name *head, struct type *elm)	{						\
+	struct type *tmp = RB_ROOT(head);										\
+	int comp;																\
+	while (tmp) {															\
+		comp = cmp(elm, tmp);												\
+		if (comp < 0)														\
+			tmp = RB_LEFT(tmp, field);										\
+		else if (comp > 0)													\
+			tmp = RB_RIGHT(tmp, field);										\
+		else																\
+			return (tmp);													\
+	}																		\
+	return (NULL);															\
+}																			\
+																			\
+/* Finds the first node greater than or equal to the search key */	\
+attr struct type *															\
+name##_RB_NFIND(struct name *head, struct type *elm)						\
+{																			\
+	struct type *tmp = RB_ROOT(head);										\
+	struct type *res = NULL;												\
+	int comp;																\
+	while (tmp) {															\
+		comp = cmp(elm, tmp);												\
+		if (comp < 0) {														\
+			res = tmp;														\
+			tmp = RB_LEFT(tmp, field);										\
+		}																	\
+		else if (comp > 0)													\
+			tmp = RB_RIGHT(tmp, field);										\
+		else																\
+			return (tmp);													\
+	}																		\
+	return (res);															\
+}																			\
+																			\
+/* ARGSUSED */																\
+attr struct type *															\
+name##_RB_NEXT(struct type *elm)											\
+{																			\
+	if (RB_RIGHT(elm, field)) {												\
+		elm = RB_RIGHT(elm, field);											\
+		while (RB_LEFT(elm, field))											\
+			elm = RB_LEFT(elm, field);										\
+	} else {																\
+		if (RB_PARENT(elm, field) &&										\
+		    (elm == RB_LEFT(RB_PARENT(elm, field), field)))				\
+			elm = RB_PARENT(elm, field);									\
+		else {																\
+			while (RB_PARENT(elm, field) &&								\
+			    (elm == RB_RIGHT(RB_PARENT(elm, field), field)))			\
+				elm = RB_PARENT(elm, field);								\
+			elm = RB_PARENT(elm, field);									\
+		}																	\
+	}																		\
+	return (elm);															\
+}																			\
+																			\
+/* ARGSUSED */																\
+attr struct type *															\
+name##_RB_PREV(struct type *elm)											\
+{																			\
+	if (RB_LEFT(elm, field)) {												\
+		elm = RB_LEFT(elm, field);											\
+		while (RB_RIGHT(elm, field))										\
+			elm = RB_RIGHT(elm, field);										\
+	} else {																\
+		if (RB_PARENT(elm, field) &&										\
+		    (elm == RB_RIGHT(RB_PARENT(elm, field), field)))				\
+			elm = RB_PARENT(elm, field);									\
+		else {																\
+			while (RB_PARENT(elm, field) &&								\
+			    (elm == RB_LEFT(RB_PARENT(elm, field), field)))			\
+				elm = RB_PARENT(elm, field);								\
+			elm = RB_PARENT(elm, field);									\
+		}																	\
+	}																		\
+	return (elm);															\
+}																			\
+																			\
+attr struct type *															\
+name##_RB_MINMAX(struct name *head, int val)								\
+{																			\
+	struct type *tmp = RB_ROOT(head);										\
+	struct type *parent = NULL;												\
+	while (tmp) {															\
+		parent = tmp;														\
+		if (val < 0)														\
+			tmp = RB_LEFT(tmp, field);										\
+		else																\
+			tmp = RB_RIGHT(tmp, field);										\
+	}																		\
+	return (parent);														\
+}
+
+#define RB_NEGINF	-1
+#define RB_INF	1
+
+#define RB_INSERT(name, x, y)	name##_RB_INSERT(x, y)
+#define RB_REMOVE(name, x, y)	name##_RB_REMOVE(x, y)
+#define RB_FIND(name, x, y)	name##_RB_FIND(x, y)
+#define RB_NFIND(name, x, y)	name##_RB_NFIND(x, y)
+#define RB_NEXT(name, x, y)	name##_RB_NEXT(y)
+#define RB_PREV(name, x, y)	name##_RB_PREV(y)
+#define RB_MIN(name, x)		name##_RB_MINMAX(x, RB_NEGINF)
+#define RB_MAX(name, x)		name##_RB_MINMAX(x, RB_INF)
+
+#define RB_FOREACH(x, name, head)											\
+	for ((x) = RB_MIN(name, head);											\
+	     (x) != NULL;														\
+	     (x) = name##_RB_NEXT(x))
+
+#define RB_FOREACH_FROM(x, name, y)											\
+	for ((x) = (y);															\
+	    ((x) != NULL) && ((y) = name##_RB_NEXT(x), (x) != NULL);			\
+	     (x) = (y))
+
+#define RB_FOREACH_SAFE(x, name, head, y)									\
+	for ((x) = RB_MIN(name, head);											\
+	    ((x) != NULL) && ((y) = name##_RB_NEXT(x), (x) != NULL);			\
+	     (x) = (y))
+
+#define RB_FOREACH_REVERSE(x, name, head)									\
+	for ((x) = RB_MAX(name, head);											\
+	     (x) != NULL;														\
+	     (x) = name##_RB_PREV(x))
+
+#define RB_FOREACH_REVERSE_FROM(x, name, y)									\
+	for ((x) = (y);															\
+	    ((x) != NULL) && ((y) = name##_RB_PREV(x), (x) != NULL);			\
+	     (x) = (y))
+
+#define RB_FOREACH_REVERSE_SAFE(x, name, head, y)							\
+	for ((x) = RB_MAX(name, head);											\
+	    ((x) != NULL) && ((y) = name##_RB_PREV(x), (x) != NULL);			\
+	     (x) = (y))
+
+
+
+#endif
+
+
+
+
+

NtyCo/htdocs/check.cgi

@@ -0,0 +1,20 @@
+#!/usr/bin/perl -Tw
+
+use strict;
+use CGI;
+
+my($cgi) = new CGI;
+
+print $cgi->header('text/html');
+print $cgi->start_html(-title => "Example CGI script",
+                       -BGCOLOR => 'red');
+print $cgi->h1("CGI Example");
+print $cgi->p, "This is an example of CGI\n";
+print $cgi->p, "Parameters given to this script:\n";
+print "<UL>\n";
+foreach my $param ($cgi->param)
+{
+ print "<LI>", "$param ", $cgi->param($param), "\n";
+}
+print "</UL>";
+print $cgi->end_html, "\n";

NtyCo/htdocs/color.cgi

@@ -0,0 +1,15 @@
+#!/usr/bin/perl -Tw
+
+use strict;
+use CGI;
+
+my($cgi) = new CGI;
+
+print $cgi->header;
+my($color) = "blue";
+$color = $cgi->param('color') if defined $cgi->param('color');
+
+print $cgi->start_html(-title => uc($color),
+                       -BGCOLOR => $color);
+print $cgi->h1("This is $color");
+print $cgi->end_html;

NtyCo/htdocs/index.html

@@ -0,0 +1,11 @@
+<HTML>
+<TITLE>Index</TITLE>
+<BODY>
+<P>Welcome to J. David's webserver.
+<H1>CGI demo
+<FORM ACTION="color.cgi" METHOD="POST">
+Enter a color: <INPUT TYPE="text" NAME="color">
+<INPUT TYPE="submit">
+</FORM>
+</BODY>
+</HTML>

NtyCo/sample/CMakeLists.txt

@@ -0,0 +1,15 @@
+
+aux_source_directory(. SRC)
+
+# 编译不过的，先放这里
+set(NOT_COMPILE )
+
+foreach(FILE_PATH ${SRC})
+	string(REGEX REPLACE ".+/(.+)\\..*" "\\1" FILE_NAME ${FILE_PATH})
+	list(FIND NOT_COMPILE ${FILE_NAME} RESULT)
+	if(${RESULT} EQUAL -1)
+		message(${FILE_NAME})
+		add_executable(${FILE_NAME} ${FILE_PATH})
+		target_link_libraries(${FILE_NAME}  nty_core ${LIBS})
+	endif()
+endforeach()

NtyCo/sample/Makefile

@@ -0,0 +1,16 @@
+
+
+CUR_SOURCE = ${wildcard *.c}
+
+CUR_OBJS = ${patsubst %.c, %.o, $(CUR_SOURCE)}
+
+all : $(SUB_DIR) $(CUR_OBJS)
+
+$(SUB_DIR) : ECHO
+	make -C $@
+
+$(CUR_OBJS) : %.o : %.c
+	$(CC) -c $^ -o $(OBJS_DIR)/$@ $(FLAG)
+
+ECHO : 
+	@echo $(SUB_DIR)

NtyCo/sample/hook_tcpserver.c

@@ -0,0 +1,84 @@
+
+
+
+
+#include "nty_coroutine.h"
+
+#include <arpa/inet.h>
+
+
+
+void server_reader(void *arg) {
+	int fd = *(int *)arg;
+	int ret = 0;
+
+ 
+	while (1) {
+		
+		char buf[1024] = {0};
+		ret = recv(fd, buf, 1024, 0);
+		if (ret > 0) {
+			printf("read from server: %.*s\n", ret, buf);
+
+			ret = send(fd, buf, strlen(buf), 0);
+			if (ret == -1) {
+				close(fd);
+				break;
+			}
+		} else if (ret == 0) {	
+			close(fd);
+			break;
+		}
+
+	}
+}
+
+
+
+void server(void *arg) {
+
+	unsigned short port = *(unsigned short *)arg;
+
+	int fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) return ;
+
+	struct sockaddr_in local, remote;
+	local.sin_family = AF_INET;
+	local.sin_port = htons(port);
+	local.sin_addr.s_addr = INADDR_ANY;
+	bind(fd, (struct sockaddr*)&local, sizeof(struct sockaddr_in));
+
+	listen(fd, 20);
+	printf("listen port : %d\n", port);
+
+
+	while (1) {
+		socklen_t len = sizeof(struct sockaddr_in);
+		int cli_fd = accept(fd, (struct sockaddr*)&remote, &len);
+		
+
+		nty_coroutine *read_co;
+		nty_coroutine_create(&read_co, server_reader, &cli_fd);
+
+	}
+	
+}
+
+
+
+
+int main(int argc, char *argv[]) {
+
+	int port = atoi(argv[1]);
+
+	
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, server, &port);
+
+	nty_schedule_run();
+
+}
+
+
+
+

NtyCo/sample/nty_bench.c

@@ -0,0 +1,727 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+/*
+ * (C) Radim Kolar 1997-2004
+ * This is free software, see GNU Public License version 2 for
+ * details.
+ *
+ * Simple forking WWW Server benchmark:
+ *
+ * Usage:
+ *   webbench --help
+ *
+ * Return codes:
+ *    0 - sucess
+ *    1 - benchmark failed (server is not on-line)
+ *    2 - bad param
+ *    3 - internal error, fork failed
+ * 
+ */ 
+
+ 
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/time.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+
+#include "nty_coroutine.h"
+
+#define ENABLE_NTYCO		1
+#define MAX_BUFFER_LENGTH	1024
+
+
+#define write(a, b, c)		nty_send(a, b, c, 0)
+#define send(a, b, c, d)	nty_send(a, b, c, d)
+#define read(a, b, c)		nty_recv(a, b, c, 0)
+#define recv(a, b, c, d)	nty_recv(a, b, c, d)
+#define connect(a, b, c)	nty_connect(a, b, c)
+#define socket(a, b, c)		nty_socket(a, b, c)
+#define close(a)			nty_close(a)
+#define accept(a, b, c)		nty_accept(a, b, c)
+
+
+
+int Socket(const char *host, int clientPort)
+{
+    int sock;
+    unsigned long inaddr;
+    struct sockaddr_in ad;
+    struct hostent *hp;
+    
+    memset(&ad, 0, sizeof(ad));
+    ad.sin_family = AF_INET;
+
+    inaddr = inet_addr(host);
+    if (inaddr != INADDR_NONE)
+        memcpy(&ad.sin_addr, &inaddr, sizeof(inaddr));
+    else
+    {
+        hp = gethostbyname(host);
+        if (hp == NULL)
+            return -1;
+        memcpy(&ad.sin_addr, hp->h_addr, hp->h_length);
+    }
+    ad.sin_port = htons(clientPort);
+    
+    sock = socket(AF_INET, SOCK_STREAM, 0);
+    if (sock < 0)
+        return sock;
+    if (connect(sock, (struct sockaddr *)&ad, sizeof(ad)) < 0)
+        return -1;
+    return sock;
+}
+
+
+#include <unistd.h>
+#include <sys/param.h>
+#include <rpc/types.h>
+#include <getopt.h>
+#include <strings.h>
+#include <time.h>
+#include <signal.h>
+
+/* values */
+volatile int timerexpired=0;
+int speed=0;
+int failed=0;
+int bytes=0;
+/* globals */
+int http10=1; /* 0 - http/0.9, 1 - http/1.0, 2 - http/1.1 */
+/* Allow: GET, HEAD, OPTIONS, TRACE */
+#define METHOD_GET 0
+#define METHOD_HEAD 1
+#define METHOD_OPTIONS 2
+#define METHOD_TRACE 3
+#define PROGRAM_VERSION "1.5"
+int method=METHOD_GET;
+int clients=1;
+
+
+#if ENABLE_NTYCO
+// clients --> fork num
+// nreqs --> http request
+// socket --> io num
+// socket per fork --> sockets / client
+// request per socket --> request / sockets
+
+int nreqs = 0;
+int sockets = 0;
+#endif
+int force=0;
+int force_reload=0;
+int proxyport=80;
+char *proxyhost=NULL;
+int benchtime=30;
+/* internal */
+int mypipe[2];
+char host[MAXHOSTNAMELEN];
+#define REQUEST_SIZE 2048
+char request[REQUEST_SIZE] = {0};
+
+static const struct option long_options[]=
+{
+ {"force",no_argument,&force,1},
+ {"reload",no_argument,&force_reload,1},
+ {"time",required_argument,NULL,'t'},
+ {"help",no_argument,NULL,'?'},
+ {"http09",no_argument,NULL,'9'},
+ {"http10",no_argument,NULL,'1'},
+ {"http11",no_argument,NULL,'2'},
+ {"get",no_argument,&method,METHOD_GET},
+ {"head",no_argument,&method,METHOD_HEAD},
+ {"options",no_argument,&method,METHOD_OPTIONS},
+ {"trace",no_argument,&method,METHOD_TRACE},
+ {"version",no_argument,NULL,'V'},
+ {"proxy",required_argument,NULL,'p'},
+ {"clients",required_argument,NULL,'c'},
+ {NULL,0,NULL,0}
+};
+
+/* prototypes */
+static void benchcore(const char* host,const int port, const char *request);
+static int bench(void);
+static void build_request(const char *url);
+
+static void alarm_handler(int signal)
+{
+   timerexpired=1;
+}	
+
+static void usage(void)
+{
+   fprintf(stderr,
+	"webbench [option]... URL\n"
+	"  -f|--force               Don't wait for reply from server.\n"
+	"  -r|--reload              Send reload request - Pragma: no-cache.\n"
+	"  -t|--time <sec>          Run benchmark for <sec> seconds. Default 30.\n"
+	"  -n|--request <num>       Sum of requestion\n"
+	"  -s|--socket <num>        Sum of socket\n"
+	"  -p|--proxy <server:port> Use proxy server for request.\n"
+	"  -c|--clients <n>         Run <n> HTTP clients at once. Default one.\n"
+	"  -9|--http09              Use HTTP/0.9 style requests.\n"
+	"  -1|--http10              Use HTTP/1.0 protocol.\n"
+	"  -2|--http11              Use HTTP/1.1 protocol.\n"
+	"  --get                    Use GET request method.\n"
+	"  --head                   Use HEAD request method.\n"
+	"  --options                Use OPTIONS request method.\n"
+	"  --trace                  Use TRACE request method.\n"
+	"  -?|-h|--help             This information.\n"
+	"  -V|--version             Display program version.\n"
+	);
+};
+int main(int argc, char *argv[])
+{
+ int opt=0;
+ int options_index=0;
+ char *tmp=NULL;
+
+ if(argc==1)
+ {
+	  usage();
+          return 2;
+ } 
+
+ while((opt=getopt_long(argc,argv,"912Vfrt:p:n:s:c:?h",long_options,&options_index))!=EOF )
+ {
+  switch(opt)
+  {
+   case  0 : break;
+   case 'f': force=1;break;
+   case 'r': force_reload=1;break; 
+   case '9': http10=0;break;
+   case '1': http10=1;break;
+   case '2': http10=2;break;
+   case 'V': printf(PROGRAM_VERSION"\n");exit(0);
+   case 't': benchtime=atoi(optarg);break;	     
+   case 'p': 
+	     /* proxy server parsing server:port */
+	     tmp=strrchr(optarg,':');
+	     proxyhost=optarg;
+	     if(tmp==NULL)
+	     {
+		     break;
+	     }
+	     if(tmp==optarg)
+	     {
+		     fprintf(stderr,"Error in option --proxy %s: Missing hostname.\n",optarg);
+		     return 2;
+	     }
+	     if(tmp==optarg+strlen(optarg)-1)
+	     {
+		     fprintf(stderr,"Error in option --proxy %s Port number is missing.\n",optarg);
+		     return 2;
+	     }
+	     *tmp='\0';
+	     proxyport=atoi(tmp+1);break;
+   case ':':
+   case 'h':
+   case '?': usage();return 2;break;
+   case 'c': clients=atoi(optarg);break;
+#if ENABLE_NTYCO
+   case 'n': nreqs=atoi(optarg);break;
+   case 's': sockets=atoi(optarg);break;
+#endif
+  }
+ }
+ 
+ if(optind==argc) {
+                      fprintf(stderr,"webbench: Missing URL!\n");
+		      usage();
+		      return 2;
+                    }
+
+ if(clients==0) clients=1;
+ if(benchtime==0) benchtime=60;
+ 
+#if ENABLE_NTYCO
+	if (clients > 10) clients = 10;
+	if (nreqs < 1000) nreqs = 1000;
+	//if (sockets < clients) sockets = clients;
+	if (sockets % clients != 0)  sockets = sockets / clients * clients;
+#endif
+
+ /* Copyright */
+ fprintf(stderr,"Webbench - Simple Web Benchmark "PROGRAM_VERSION"\n"
+	 "Copyright (c) Radim Kolar 1997-2004, GPL Open Source Software.\n"
+	 );
+ build_request(argv[optind]);
+ /* print bench info */
+ printf("\nBenchmarking: ");
+ switch(method)
+ {
+	 case METHOD_GET:
+	 default:
+		 printf("GET");break;
+	 case METHOD_OPTIONS:
+		 printf("OPTIONS");break;
+	 case METHOD_HEAD:
+		 printf("HEAD");break;
+	 case METHOD_TRACE:
+		 printf("TRACE");break;
+ }
+ printf(" %s",argv[optind]);
+ switch(http10)
+ {
+	 case 0: printf(" (using HTTP/0.9)");break;
+	 case 2: printf(" (using HTTP/1.1)");break;
+ }
+ printf("\n");
+ if(clients==1) printf("1 client");
+ else
+   printf("%d clients",clients);
+
+ printf(", running %d sec", benchtime);
+ if(force) printf(", early socket close");
+ if(proxyhost!=NULL) printf(", via proxy server %s:%d",proxyhost,proxyport);
+ if(force_reload) printf(", forcing reload");
+ printf(".\n");
+ return bench();
+}
+
+void build_request(const char *url)
+{
+  char tmp[10];
+  int i;
+
+  bzero(host,MAXHOSTNAMELEN);
+  bzero(request,REQUEST_SIZE);
+
+  if(force_reload && proxyhost!=NULL && http10<1) http10=1;
+  if(method==METHOD_HEAD && http10<1) http10=1;
+  if(method==METHOD_OPTIONS && http10<2) http10=2;
+  if(method==METHOD_TRACE && http10<2) http10=2;
+
+  switch(method)
+  {
+	  default:
+	  case METHOD_GET: strcpy(request,"GET");break;
+	  case METHOD_HEAD: strcpy(request,"HEAD");break;
+	  case METHOD_OPTIONS: strcpy(request,"OPTIONS");break;
+	  case METHOD_TRACE: strcpy(request,"TRACE");break;
+  }
+		  
+  strcat(request," ");
+
+  if(NULL==strstr(url,"://"))
+  {
+	  fprintf(stderr, "\n%s: is not a valid URL.\n",url);
+	  exit(2);
+  }
+  if(strlen(url)>1500)
+  {
+         fprintf(stderr,"URL is too long.\n");
+	 exit(2);
+  }
+  if(proxyhost==NULL)
+	   if (0!=strncasecmp("http://",url,7)) 
+	   { fprintf(stderr,"\nOnly HTTP protocol is directly supported, set --proxy for others.\n");
+             exit(2);
+           }
+  /* protocol/host delimiter */
+  i=strstr(url,"://")-url+3;
+  /* printf("%d\n",i); */
+
+  if(strchr(url+i,'/')==NULL) {
+                                fprintf(stderr,"\nInvalid URL syntax - hostname don't ends with '/'.\n");
+                                exit(2);
+                              }
+  if(proxyhost==NULL)
+  {
+   /* get port from hostname */
+   if(index(url+i,':')!=NULL &&
+      index(url+i,':')<index(url+i,'/'))
+   {
+	   strncpy(host,url+i,strchr(url+i,':')-url-i);
+	   bzero(tmp,10);
+	   strncpy(tmp,index(url+i,':')+1,strchr(url+i,'/')-index(url+i,':')-1);
+	   /* printf("tmp=%s\n",tmp); */
+	   proxyport=atoi(tmp);
+	   if(proxyport==0) proxyport=80;
+   } else
+   {
+     strncpy(host,url+i,strcspn(url+i,"/"));
+   }
+   // printf("Host=%s\n",host);
+   strcat(request+strlen(request),url+i+strcspn(url+i,"/"));
+  } else
+  {
+   // printf("ProxyHost=%s\nProxyPort=%d\n",proxyhost,proxyport);
+   strcat(request,url);
+  }
+  if(http10==1)
+	  strcat(request," HTTP/1.0");
+  else if (http10==2)
+	  strcat(request," HTTP/1.1");
+  strcat(request,"\r\n");
+  if(http10>0)
+	  strcat(request,"User-Agent: WebBench "PROGRAM_VERSION"\r\n");
+  if(proxyhost==NULL && http10>0)
+  {
+	  strcat(request,"Host: ");
+	  strcat(request,host);
+	  strcat(request,"\r\n");
+  }
+  if(force_reload && proxyhost!=NULL)
+  {
+	  strcat(request,"Pragma: no-cache\r\n");
+  }
+  if(http10>1)
+	  strcat(request,"Connection: close\r\n");
+  /* add empty line at end */
+  if(http10>0) strcat(request,"\r\n"); 
+  // printf("Req=%s\n",request);
+}
+
+/* vraci system rc error kod */
+static int bench(void)
+{
+  int i,j,k;	
+  pid_t pid=0;
+  FILE *f;
+#if ENABLE_NTYCO == 0
+  /* check avaibility of target server */
+  i=Socket(proxyhost==NULL?host:proxyhost,proxyport);
+  if(i<0) { 
+	   fprintf(stderr,"\nConnect to server failed. Aborting benchmark.\n");
+           return 1;
+         }
+  close(i);
+#endif
+  /* create pipe */
+  if(pipe(mypipe))
+  {
+	  perror("pipe failed.");
+	  return 3;
+  }
+
+  /* not needed, since we have alarm() in childrens */
+  /* wait 4 next system clock tick */
+  /*
+  cas=time(NULL);
+  while(time(NULL)==cas)
+        sched_yield();
+  */
+  printf("bench enter\n");
+
+  /* fork childs */
+  for(i=0;i<clients;i++)
+  {
+	   pid=fork();
+	   if(pid <= (pid_t) 0)
+	   {
+		   /* child process or error*/
+	           sleep(1); /* make childs faster */
+		   break;
+	   }
+  }
+
+  if( pid< (pid_t) 0)
+  {
+          fprintf(stderr,"problems forking worker no. %d\n",i);
+	  perror("fork failed.");
+	  return 3;
+  }
+
+  if(pid== (pid_t) 0)
+  {
+    /* I am a child */
+    if(proxyhost==NULL)
+      benchcore(host,proxyport,request);
+         else
+      benchcore(proxyhost,proxyport,request);
+
+         /* write results to pipe */
+	 f=fdopen(mypipe[1],"w");
+	 if(f==NULL)
+	 {
+		 perror("open pipe for writing failed.");
+		 return 3;
+	 }
+	 /* fprintf(stderr,"Child - %d %d\n",speed,failed); */
+	 fprintf(f,"%d %d %d\n",speed,failed,bytes);
+	 fclose(f);
+	 return 0;
+  } else
+  {
+	  f=fdopen(mypipe[0],"r");
+	  if(f==NULL) 
+	  {
+		  perror("open pipe for reading failed.");
+		  return 3;
+	  }
+	  setvbuf(f,NULL,_IONBF,0);
+	  speed=0;
+          failed=0;
+          bytes=0;
+
+	  while(1)
+	  {
+		  pid=fscanf(f,"%d %d %d",&i,&j,&k);
+		  if(pid<2)
+                  {
+                       fprintf(stderr,"Some of our childrens died.\n");
+                       break;
+                  }
+		  speed+=i;
+		  failed+=j;
+		  bytes+=k;
+		  /* fprintf(stderr,"*Knock* %d %d read=%d\n",speed,failed,pid); */
+		  if(--clients==0) break;
+	  }
+	  fclose(f);
+
+  printf("\nSpeed=%d pages/min, %d bytes/sec.\nRequests: %d susceed, %d failed.\n",
+		  (int)((speed+failed)/(benchtime/60.0f)),
+		  (int)(bytes/(float)benchtime),
+		  speed,
+		  failed);
+  }
+  return i;
+}
+
+struct serv_host {
+	const char *host;
+	const int port;
+	const char *req;
+};
+
+int testok(char *response) {
+	if (response == NULL) return -1;
+	
+	char* p;
+	char* q;
+	
+	p = strchr(response, '\n');
+	*p = '\0';
+	if(NULL != (q = strstr(response, "200 OK")))
+		return 1;
+	else
+		return 0;
+}
+
+int send_buffer(int sockfd, char *buffer, int length) {
+
+	int idx = 0;
+	while (idx < length) {
+		
+		int count = 0;
+		if ((idx+MAX_BUFFER_LENGTH) < length) {
+			count = send(sockfd, buffer+idx, MAX_BUFFER_LENGTH, 0);
+		} else {
+			count = send(sockfd, buffer+idx, length-idx, 0);
+		}
+		if (count < 0) break;
+
+		idx += count;
+	}
+
+	return idx;
+
+}
+
+int recv_buffer(int sockfd, char *buffer, int length, int *ret) {
+
+	int idx = 0;
+
+	while (1) {
+		
+		int count = recv(sockfd, buffer+idx, MAX_BUFFER_LENGTH, 0);
+		//printf("sockfd : %d, recv_buffer --> %d\n", sockfd, count);
+		if (count < 0) {
+			*ret = -1;
+			break;
+		} else if (count == 0) {
+			*ret = 0;
+			close(sockfd);
+			break;
+		} else {
+			//printf("sockfd : %d, recv_buffer --> %d\n", sockfd, count);
+			idx += count;
+		}
+	}
+	if (idx > 0) *ret = 1;
+	
+	return idx;
+}
+
+
+void httprequest_commit(void *arg) {
+	char buf[1500] = {0};
+	struct serv_host *shost = (struct serv_host *)arg;
+
+	int slen = strlen(shost->req);
+	
+	int idx = 0;
+	for (idx = 0;idx < (nreqs / sockets);idx ++) {
+		
+		int s = Socket(shost->host, shost->port);
+		if (s < 0) {
+			failed ++;
+			continue ;
+		}
+		
+		if (slen != send(s, shost->req, slen, 0)) {
+			failed ++;
+			close(s);
+			continue;
+		}
+
+		memset(buf, 0, 1500);
+#if ENABLE_NTYCO
+		int ret = 0;
+		int rlen = recv_buffer(s, buf, 1500, &ret);
+#else
+		int rlen = recv(s, buf, 1500, 0);
+#endif
+		if (ret < 0) {
+			failed ++;
+		} else {
+			
+			if (testok(buf)) {
+				speed ++;
+				
+			} else {
+				failed ++;
+				
+			}
+			bytes += rlen;
+		}
+	}
+
+	return ;
+}
+
+
+void benchcore(const char *host,const int port,const char *req)
+{
+ int rlen;
+ char buf[1500];
+ int s,i;
+ struct sigaction sa;
+
+ /* setup alarm signal handler */
+ sa.sa_handler=alarm_handler;
+ sa.sa_flags=0;
+ if(sigaction(SIGALRM,&sa,NULL))
+    exit(3);
+ alarm(benchtime);
+ 
+#if ENABLE_NTYCO
+
+	//int i = 0;
+	nty_coroutine *co = NULL;
+	struct serv_host shost = {host, port, req};
+
+	for (i = 0;i < sockets / clients;i ++) {
+		
+		nty_coroutine_create(&co, httprequest_commit, &shost);
+	}
+	nty_schedule_run();
+
+#else
+
+ rlen=strlen(req);
+ nexttry:while(1)
+ {
+    if(timerexpired)
+    {
+       if(failed>0)
+       {
+          /* fprintf(stderr,"Correcting failed by signal\n"); */
+          failed--;
+       }
+       return;
+    }
+    s=Socket(host,port);                          
+    if(s<0) { failed++;continue;} 
+    if(rlen!=write(s, req, rlen)) {failed++;close(s);continue;}
+	
+#if ENABLE_NTYCO
+	if(http10==0) 
+		close(s);
+#else
+    if(http10==0) 
+	    if(shutdown(s,1)) { failed++;close(s);continue;}
+	
+#endif
+    if(force==0) 
+    {
+            /* read all available data from socket */
+	    while(1)
+	    {
+              if(timerexpired) break; 
+	      i=read(s,buf,1500);
+              /* fprintf(stderr,"%d\n",i); */
+	      if(i<0) 
+              { 
+                 failed++;
+                 close(s);
+                 goto nexttry;
+              }
+	       else
+		       if(i==0) break;
+		       else
+			       bytes+=i;
+	    }
+    }
+    if(close(s)) {failed++;continue;}
+    speed++;
+ }
+#endif
+}
+
+
+
+
+
+

NtyCo/sample/nty_client.c

@@ -0,0 +1,111 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+
+#include "nty_coroutine.h"
+
+#include <arpa/inet.h>
+
+
+
+#define NTY_SERVER_IPADDR		"127.0.0.1"
+#define NTY_SERVER_PORT			9096
+
+int init_client(void) {
+
+	int clientfd = nty_socket(AF_INET, SOCK_STREAM, 0);
+	if (clientfd <= 0) {
+		printf("socket failed\n");
+		return -1;
+	}
+
+	struct sockaddr_in serveraddr = {0};
+	serveraddr.sin_family = AF_INET;
+	serveraddr.sin_port = htons(NTY_SERVER_PORT);
+	serveraddr.sin_addr.s_addr = inet_addr(NTY_SERVER_IPADDR);
+
+	int result = nty_connect(clientfd, (struct sockaddr *)&serveraddr, sizeof(serveraddr));
+	if (result != 0) {
+		printf("connect failed\n");
+		return -2;
+	}
+
+	return clientfd;
+	
+}
+
+void client(void *arg) {
+
+	int clientfd = init_client();
+	char *buffer = "ntyco_client\r\n";
+
+	while (1) {
+
+		int length = nty_send(clientfd, buffer, strlen(buffer), 0);
+		printf("echo length : %d\n", length);
+
+		sleep(1);
+	}
+
+}
+
+
+
+int main(int argc, char *argv[]) {
+	nty_coroutine *co = NULL;
+
+	nty_coroutine_create(&co, client, NULL);
+	
+	nty_schedule_run(); //run
+
+	return 0;
+}
+
+
+
+
+
+
+

NtyCo/sample/nty_http_epoll.c

@@ -0,0 +1,745 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+
+
+#include <arpa/inet.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/sysinfo.h>
+#include <sys/syscall.h>
+
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/sem.h>
+
+#include <errno.h>
+
+#include <fcntl.h>
+#include <sys/epoll.h>
+#include <sys/poll.h>
+
+#define __USE_GNU
+
+#include <sched.h>
+#include <string.h>
+#include <pthread.h>
+
+#include <ctype.h>
+
+#define MAX_CLIENT_NUM			1000000
+#define TOTALFDS			16
+
+typedef struct _shm_area {
+	int totalfds[TOTALFDS];
+	char cpu_lb[TOTALFDS];
+	//mtx : default 0
+	// 1 : lock -- del epoll
+	// 2 : lock -- del complete
+	// 3 : unlock -- add
+	// 0 : unlock -- add complete
+	int accept_mtx;
+} shm_area;
+
+static shm_area *global_shmaddr = NULL;
+static int global_shmid = -1;
+
+int cpu_size = 0;
+int accept_disable = 1000;
+int enable_accept = 1;
+pid_t self_id = 0;
+
+
+
+unsigned long cmpxchg(void *addr, unsigned long _old, unsigned long _new, int size) {
+	unsigned long prev;
+	volatile unsigned int *_ptr = (volatile unsigned int *)(addr);
+
+	switch (size) {
+		case 1: {
+			__asm__ volatile (
+		        "lock; cmpxchgb %b1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 2: {
+			__asm__ volatile (
+		        "lock; cmpxchgw %w1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 4: {
+			__asm__ volatile (
+		        "lock; cmpxchg %1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+	}
+
+	return prev;
+}
+
+int atomic_add(volatile int *value, int add)
+{
+    __asm__ volatile (
+
+         "lock;"
+    "    addl  %0, %1;   "
+
+    : "+r" (add) : "m" (*value) : "cc", "memory");
+
+    return add;
+}
+
+int atomic_sub(volatile int *value, int sub)
+{
+    __asm__ volatile (
+
+         "lock;"
+    "    subl  %0, %1;   "
+
+    : "+r" (sub) : "m" (*value) : "cc", "memory");
+
+    return sub;
+}
+
+
+
+
+
+#define ISspace(x) isspace((int)(x))
+
+#define SERVER_STRING "Server: ntyco_httpd/0.1.0\r\n"
+#define STDIN   0
+#define STDOUT  1
+#define STDERR  2
+
+#define ENABLE_NTYCO	0
+
+#if ENABLE_NTYCO
+
+#define socket 	nty_socket 
+#define accept 	nty_accept 
+#define recv(a, b, c, d) 	nty_recv(a, b, c, d)
+#define send(a, b, c, d) 	nty_send(a, b, c, d) 
+
+#define MAX_BUFFER_LENGTH	1024
+
+
+
+#endif
+
+#define INC_COUNTFD  do {				\
+	atomic_add(&global_shmaddr->totalfds[self_id % cpu_size], 1);	\
+} while (0)
+
+
+#define DEC_COUNTFD  do {				\
+	atomic_sub(&global_shmaddr->totalfds[self_id % cpu_size], 1);	\
+} while (0)
+
+int get_countfd(void) {	
+	return global_shmaddr->totalfds[self_id % cpu_size];	
+}
+
+int max_countfd(void) {
+
+	int count = -1;
+	int i = 0;
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count < global_shmaddr->totalfds[i]) {
+			count = global_shmaddr->totalfds[i];
+		}
+	}
+	return count;
+}
+
+int min_countfd(void) {
+
+	int count = 0xffffffff;
+	int i = 0;
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count > global_shmaddr->totalfds[i]) {
+			count = global_shmaddr->totalfds[i];
+		}
+	}
+	return count;
+}
+
+int compare_out_countfd(void) {
+
+	int current = get_countfd();
+	int min = min_countfd();
+
+	if ((current * 7 / 8) > min) {
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
+int compare_in_countfd(void) {
+
+	int current = get_countfd();
+	int max = max_countfd();
+
+	if ((current * 8 / 7) < max) {
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
+void print_countfds(void) {
+
+	int i = 0;
+	for (i = 0;i < cpu_size;i ++) {
+		printf("%5d : %5d ", i, global_shmaddr->totalfds[i]);
+	}
+	printf("\n");
+}
+
+void lock_accept(void) {
+	
+	global_shmaddr->cpu_lb[self_id % cpu_size] = 1;
+
+	int count = 0xffffffff;
+	int i = 0;
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count > global_shmaddr->totalfds[i]) {
+			count = global_shmaddr->totalfds[i];
+		}
+	}
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count == global_shmaddr->totalfds[i]) {
+			global_shmaddr->cpu_lb[i] = 3;
+		}
+	}
+	
+}
+
+char read_accept(void) {
+	return global_shmaddr->cpu_lb[self_id % cpu_size];
+}
+
+void write_accept(char state) { //0, 1, 2, 3		
+	global_shmaddr->cpu_lb[self_id % cpu_size] = state;
+}
+
+int lock(void) {
+
+	return cmpxchg(&global_shmaddr->accept_mtx, 0, 1, 4); //zero:success, non-zero:failed
+
+}
+
+void unlock(void) {
+
+	global_shmaddr->accept_mtx = 0;
+
+}
+
+
+void accept_request(int fd, int epfd);
+void bad_request(int);
+void cat(int);
+void cannot_execute(int);
+void error_die(const char *);
+void execute_cgi(int, const char *, const char *, const char *);
+int get_line(int, char *, int);
+void headers(int);
+void not_found(int);
+void serve_file(int);
+int startup(u_short *);
+void unimplemented(int);
+
+
+ssize_t nsend(int fd, const void *buf, size_t len, int flags) {
+	
+	int sent = 0;
+
+	int ret = send(fd, ((char*)buf)+sent, len-sent, flags);
+	if (ret == 0) return ret;
+	if (ret > 0) sent += ret;
+
+	while (sent < len) {
+#if 0
+        struct pollfd fds;
+		fds.fd = fd;
+		fds.events = POLLIN | POLLERR | POLLHUP;
+        poll(&fds, 1, -1);
+#endif
+		ret = send(fd, ((char*)buf)+sent, len-sent, flags);
+		//printf("send --> len : %d\n", ret);
+		if (ret <= 0) {	
+			if (errno == EAGAIN) continue;
+			
+			break;
+		}
+		sent += ret;
+	}
+
+	if (ret <= 0 && sent == 0) return ret;
+	
+	return sent;
+}
+
+static int ntySetNonblock(int fd) {
+	int flags;
+
+	flags = fcntl(fd, F_GETFL, 0);
+	if (flags < 0) return flags;
+	flags |= O_NONBLOCK;
+	if (fcntl(fd, F_SETFL, flags) < 0) return -1;
+	return 0;
+}
+
+static int ntySetReUseAddr(int fd) {
+	int reuse = 1;
+	return setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuse, sizeof(reuse));
+}
+
+
+/**********************************************************************/
+/* A request has caused a call to accept() on the server port to
+ * return.  Process the request appropriately.
+ * Parameters: the socket connected to the client */
+/**********************************************************************/
+void accept_request(int fd, int epfd)
+{
+    int client = fd;
+	
+    char buf[1024];
+    size_t numchars;
+    char method[16];
+    char url[32];
+    char path[64];
+    size_t i, j;
+    struct stat st;
+    int cgi = 0;      /* becomes true if server decides this is a CGI
+                       * program */
+    char *query_string = NULL;
+
+    numchars = recv(client, buf, sizeof(buf), 0);
+    if (numchars == -1) {
+        //printf("recv errno : %d\n", errno);
+		if (errno == ECONNRESET) {
+			close(client);
+
+			struct epoll_event ev;
+			ev.events = EPOLLIN  | EPOLLET;
+			ev.data.fd = client;
+			epoll_ctl(epfd, EPOLL_CTL_DEL, client, &ev);
+
+			DEC_COUNTFD;
+		}
+    } else if (numchars == 0) {
+        close(client);
+
+		struct epoll_event ev;
+		ev.events = EPOLLIN  | EPOLLET;
+		ev.data.fd = client;
+		epoll_ctl(epfd, EPOLL_CTL_DEL, client, &ev);
+
+		DEC_COUNTFD;
+    } else {
+        //printf("recv numchars : %ld\n", numchars);
+    }
+    //serve_file(client, path);
+
+}
+
+#if 0
+#define HTML_PAGE 		"<HTML> \
+						<TITLE>Index</TITLE> \
+						<BODY> \
+						<P>Welcome to J. David's webserver.\
+						<H1>CGI demo \
+						<FORM ACTION=\"color.cgi\" METHOD=\"POST\">\
+						Enter a color: <INPUT TYPE=\"text\" NAME=\"color\">\
+						<INPUT TYPE=\"submit\">\
+						</FORM>\
+						</BODY>\
+						</HTML>"
+
+#else
+#define HTML_PAGE       "<!DOCTYPE html> \
+<html> \
+<head> \
+<title>Welcome to nginx!</title> \
+<style> \
+    body { \
+        width: 35em;\
+        margin: 0 auto;\
+        font-family: Tahoma, Verdana, Arial, sans-serif;\
+    }\
+</style>\
+</head>\
+<body>\
+<h1>Welcome to nginx!</h1>\
+<p>If you see this page, the nginx web server is successfully installed and\
+working. Further configuration is required.</p>\
+\
+<p>For online documentation and support please refer to\
+<a href=\"http://nginx.org/\">nginx.org</a>.<br/>\
+Commercial support is available at\
+<a href=\"http://nginx.com/\">nginx.com</a>.</p>\
+\
+<p><em>Thank you for using nginx.</em></p>\
+</body>\
+</html>"
+#endif
+
+void cat(int client)
+{
+
+    //char buf[1024] =  HTML_PAGE;
+
+	int ret = send(client, HTML_PAGE, strlen(HTML_PAGE), 0);
+    if (ret == -1) {
+        printf("send : errno : %d\n", errno);
+    }  else {
+        printf("cat send ret : %d\n", ret);
+    }
+	
+}
+
+void error_die(const char *sc)
+{
+    printf("%s\n", sc);
+    exit(1);
+}
+
+void headers(int client)
+{
+    char buf[1024] = {0};
+	char content[128] = {0};
+
+	sprintf(buf, "HTTP/1.0 200 OK\r\n");
+	strcat(buf, SERVER_STRING);
+	strcat(buf, "Content-Type: text/html\r\n");
+#if 0
+	strcat(buf, "Transfer-Encoding: chunked\r\n");
+#else 
+	sprintf(content, "Content-Length: %ld\r\n", strlen(HTML_PAGE));
+	strcat(buf, content);
+#endif 
+	strcat(buf, "\r\n");
+
+    strcat(buf, HTML_PAGE);
+	
+	int ret = send(client, buf, strlen(buf), 0);
+    if (ret == -1) {
+        printf("send : errno : %d\n", errno);
+    } else {
+        //printf("headers send ret : %d\n", ret);
+    }
+    
+
+}
+
+/**********************************************************************/
+/* Send a regular file to the client.  Use headers, and report
+ * errors to client if they occur.
+ * Parameters: a pointer to a file structure produced from the socket
+ *              file descriptor
+ *             the name of the file to serve */
+/**********************************************************************/
+void serve_file(int client)
+{
+    headers(client);
+    //cat(client);
+   
+}
+
+#define MAX_EPOLLSIZE		10240
+
+void server(void *arg) {
+
+	int fd = *(int *)arg;
+
+	int epfd = epoll_create(1);
+    struct epoll_event events[MAX_EPOLLSIZE];
+
+    struct epoll_event ev;
+	ev.events = EPOLLIN;
+	ev.data.fd = fd;
+	epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev);
+	int acc = 1; //
+
+	while (1) {
+#if 0
+		char lock = read_accept();
+		if (lock == 1) { //lock
+			ev.events = EPOLLIN;
+			ev.data.fd = fd;
+			epoll_ctl(epfd, EPOLL_CTL_DEL, fd, &ev);
+
+			write_accept(2); //add complete
+			
+		} else if (acc == 0 && lock == 3) {
+			ev.events = EPOLLIN;
+			ev.data.fd = fd;
+			epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev);
+			
+			write_accept(0);
+		}
+#endif
+        int nready = epoll_wait(epfd, events, MAX_EPOLLSIZE, 1);
+		if (nready == -1) {
+			if (errno == EINTR) {
+				printf("errno : EINTR\n");
+				continue;
+			}
+			continue;
+		} 
+
+		int i = 0;
+		for (i = 0;i < nready;i ++) {
+			int sockfd = events[i].data.fd;
+			if (sockfd == fd) {
+
+				struct sockaddr_in client_addr;
+				memset(&client_addr, 0, sizeof(struct sockaddr_in));
+				socklen_t client_len = sizeof(client_addr);
+#if 0
+				if (get_countfd() % accept_disable == 999) {
+					lock_accept();
+					acc = 0;
+				}
+				char lock = read_accept();
+				if (lock != 0 && lock != 3) continue;
+#endif
+				if (lock()) {
+					continue;
+				}
+				int clientfd = accept(sockfd, (struct sockaddr*)&client_addr, &client_len);
+				unlock();
+
+				if (clientfd < 0) {
+					if (errno == EAGAIN) {
+						//printf("accept : EAGAIN\n");
+						continue;
+					} else if (errno == ECONNABORTED) {
+						printf("accept : ECONNABORTED\n");
+						
+					} else if (errno == EMFILE || errno == ENFILE) {
+						printf("accept : EMFILE || ENFILE\n");
+					}
+					return ;
+				}
+
+				INC_COUNTFD;
+
+				ntySetNonblock(clientfd);
+                ntySetReUseAddr(clientfd);
+
+				struct epoll_event ev;
+				ev.events = EPOLLIN | EPOLLET | EPOLLONESHOT;
+				ev.data.fd = clientfd;
+				epoll_ctl(epfd, EPOLL_CTL_ADD, clientfd, &ev);
+
+			}
+			else if (events[i].events & EPOLLIN) {
+
+				accept_request(sockfd, epfd);
+#if 1
+				//struct epoll_event ev;
+				ev.events = EPOLLOUT  | EPOLLET | EPOLLONESHOT;
+				ev.data.fd = sockfd;
+				epoll_ctl(epfd, EPOLL_CTL_MOD, sockfd, &ev);
+				
+				//close(sockfd);
+#endif
+			}       
+            else if (events[i].events & EPOLLOUT) {
+
+				serve_file(sockfd);
+
+				//close(sockfd);
+
+				//struct epoll_event ev;
+				ev.events = EPOLLIN | EPOLLHUP | EPOLLRDHUP | EPOLLERR;
+				ev.data.fd = sockfd;
+				epoll_ctl(epfd, EPOLL_CTL_MOD, sockfd, &ev);
+
+				DEC_COUNTFD;
+				
+			}
+			else if (events[i].events & (EPOLLHUP|EPOLLRDHUP|EPOLLERR)) {
+
+				close(sockfd);
+
+				printf(" EPOLLHUP | EPOLLRDHUP\n");
+				//struct epoll_event ev;
+				ev.events = EPOLLIN  | EPOLLET | EPOLLONESHOT;
+				ev.data.fd = sockfd;
+				epoll_ctl(epfd, EPOLL_CTL_DEL, sockfd, &ev);
+				
+				DEC_COUNTFD;
+			}
+		}
+
+//		print_countfds();
+	}
+
+}
+
+int mulcore_entry(int begin_port) {
+
+	int i = 0;
+	unsigned short base_port = begin_port;
+
+	unsigned short *port = calloc(1, sizeof(unsigned short));
+	*port = base_port;
+	server(port);
+
+}
+
+int process_bind(int fd) {
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+
+	self_id = syscall(__NR_gettid);
+	//printf("selfid --> %d\n", self_id);
+
+	cpu_set_t mask;
+
+	CPU_ZERO(&mask);
+	CPU_SET(self_id % num, &mask);
+
+	sched_setaffinity(0, sizeof(mask), &mask);
+    server(&fd);
+	//mulcore_entry(9000);
+
+}
+
+int init_shmvalue(int num) {
+	
+	global_shmid = shmget(IPC_PRIVATE, sizeof(shm_area), IPC_CREAT|0600);
+	if (global_shmid < 0) {
+		perror("shmget failed\n");
+		return -1;
+	}
+	global_shmaddr = (shm_area*)shmat(global_shmid, NULL, 0);
+	if (global_shmaddr == (shm_area*)-1) {
+		perror("shmat addr error");
+		return -1;
+	}
+	memset(global_shmaddr->totalfds, 0, TOTALFDS * sizeof(int));
+	memset(global_shmaddr->cpu_lb, 0, TOTALFDS * sizeof(char));
+	global_shmaddr->accept_mtx = 0;
+}
+
+
+
+
+int main(int argc, char *argv[]) {
+
+    short port = 9000;
+	struct sockaddr_in client_name;
+	socklen_t  client_name_len = sizeof(client_name);
+
+	int fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) return -1;
+
+    ntySetNonblock(fd);
+    ntySetReUseAddr(fd);
+
+
+	struct sockaddr_in local, remote;
+	local.sin_family = AF_INET;
+	local.sin_port = htons(port);
+	local.sin_addr.s_addr = INADDR_ANY;
+	bind(fd, (struct sockaddr*)&local, sizeof(struct sockaddr_in));
+
+	listen(fd, 20);
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+	cpu_size = num;
+	init_shmvalue(num);
+	
+	int i = 0;
+	pid_t pid=0;
+	for(i = 0;i < num;i ++) {
+		pid=fork();
+		if(pid <= (pid_t) 0)
+		{
+		   usleep(1);
+		   break;
+		}
+  	}
+#if 1
+	if (pid > 0) {
+		printf("ntyco_httpd server running ...\n");
+		getchar();
+	} else if (pid == 0) {
+		process_bind(fd);
+	}
+#else
+
+
+	INC_COUNTFD;
+
+	INC_COUNTFD;
+	print_countfds();
+
+
+#endif
+}
+

NtyCo/sample/nty_http_server.c

@@ -0,0 +1,634 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+/* J. David's webserver */
+/* This is a simple webserver.
+ * Created November 1999 by J. David Blackstone.
+ * CSE 4344 (Network concepts), Prof. Zeigler
+ * University of Texas at Arlington
+ */
+/* This program compiles for Sparc Solaris 2.6.
+ * To compile for Linux:
+ *  1) Comment out the #include <pthread.h> line.
+ *  2) Comment out the line that defines the variable newthread.
+ *  3) Comment out the two lines that run pthread_create().
+ *  4) Uncomment the line that runs accept_request().
+ *  5) Remove -lsocket from the Makefile.
+ */
+#include <stdio.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <strings.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <pthread.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#define ISspace(x) isspace((int)(x))
+
+#define SERVER_STRING "Server: ntyco_httpd/0.1.0\r\n"
+#define STDIN   0
+#define STDOUT  1
+#define STDERR  2
+
+#define ENABLE_NTYCO	1
+
+#if ENABLE_NTYCO
+#include "nty_coroutine.h" 
+
+#define socket 	nty_socket 
+#define accept 	nty_accept 
+#define recv(a, b, c, d) 	nty_recv(a, b, c, d)
+#define send(a, b, c, d) 	nty_send(a, b, c, d) 
+
+#define MAX_BUFFER_LENGTH	1024
+
+#endif
+
+void accept_request(void *);
+void bad_request(int);
+void cat(int, FILE *);
+void cannot_execute(int);
+void error_die(const char *);
+void execute_cgi(int, const char *, const char *, const char *);
+int get_line(int, char *, int);
+void headers(int, const char *);
+void not_found(int);
+void serve_file(int, const char *);
+int startup(u_short *);
+void unimplemented(int);
+
+int readline(char* allbuf,int level,char* linebuf){    
+	int len = strlen(allbuf);    
+
+	for (;level < len; ++level)    {        
+		if(allbuf[level]=='\r' && allbuf[level+1]=='\n')            
+			return level+2;        
+		else            
+			*(linebuf++) = allbuf[level];    
+	}    
+
+	return -1;
+}
+
+
+/**********************************************************************/
+/* A request has caused a call to accept() on the server port to
+ * return.  Process the request appropriately.
+ * Parameters: the socket connected to the client */
+/**********************************************************************/
+void accept_request(void *arg)
+{
+    int *pclient = (int*)arg;
+	int client = *pclient;
+	
+    char buf[1024];
+    size_t numchars;
+    char method[255];
+    char url[255];
+    char path[512];
+    size_t i, j;
+    struct stat st;
+    int cgi = 0;      /* becomes true if server decides this is a CGI
+                       * program */
+    char *query_string = NULL;
+
+    numchars = nty_recv(client, buf, sizeof(buf), 0);
+
+    i = 0; j = 0;
+    while (!ISspace(buf[i]) && (i < sizeof(method) - 1))
+    {
+        method[i] = buf[i];
+        i++;
+    }
+    j=i;
+    method[i] = '\0';
+
+    if (strcasecmp(method, "GET") && strcasecmp(method, "POST"))
+    {
+        unimplemented(client);
+        return;
+    }
+
+    if (strcasecmp(method, "POST") == 0)
+        cgi = 1;
+
+    i = 0;
+    while (ISspace(buf[j]) && (j < numchars))
+        j++;
+    while (!ISspace(buf[j]) && (i < sizeof(url) - 1) && (j < numchars))
+    {
+        url[i] = buf[j];
+        i++; j++;
+    }
+    url[i] = '\0';
+
+    if (strcasecmp(method, "GET") == 0)
+    {
+        query_string = url;
+        while ((*query_string != '?') && (*query_string != '\0'))
+            query_string++;
+        if (*query_string == '?')
+        {
+            cgi = 1;
+            *query_string = '\0';
+            query_string++;
+        }
+    }
+
+    sprintf(path, "htdocs%s", url);
+    if (path[strlen(path) - 1] == '/')
+        strcat(path, "index.html");
+    if (stat(path, &st) == -1) {
+        while ((numchars > 0) && strcmp("\n", buf))  /* read & discard headers */
+            numchars = get_line(client, buf, sizeof(buf));
+        not_found(client);
+    }
+    else
+    {
+        if ((st.st_mode & S_IFMT) == S_IFDIR)
+            strcat(path, "/index.html");
+        if ((st.st_mode & S_IXUSR) ||
+                (st.st_mode & S_IXGRP) ||
+                (st.st_mode & S_IXOTH)    )
+            cgi = 1;
+
+		cgi = 0;
+		
+        if (!cgi)
+            serve_file(client, path);
+        else
+            execute_cgi(client, path, method, query_string);
+    }
+
+    nty_close(client);
+	free(pclient);
+}
+
+
+/**********************************************************************/
+/* Inform the client that a request it has made has a problem.
+ * Parameters: client socket */
+/**********************************************************************/
+void bad_request(int client)
+{
+    char buf[1024];
+
+    sprintf(buf, "HTTP/1.0 400 BAD REQUEST\r\n");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "Content-type: text/html\r\n");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "<P>Your browser sent a bad request, ");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "such as a POST without a Content-Length.\r\n");
+    send(client, buf, sizeof(buf), 0);
+}
+
+/**********************************************************************/
+/* Put the entire contents of a file out on a socket.  This function
+ * is named after the UNIX "cat" command, because it might have been
+ * easier just to do something like pipe, fork, and exec("cat").
+ * Parameters: the client socket descriptor
+ *             FILE pointer for the file to cat */
+/**********************************************************************/
+void cat(int client, FILE *resource)
+{
+    char buf[1024];
+
+    char *ch = fgets(buf, sizeof(buf), resource);
+    while (!feof(resource))
+    {
+        send(client, buf, strlen(buf), 0);
+        ch = fgets(buf, sizeof(buf), resource);
+    }
+}
+
+/**********************************************************************/
+/* Inform the client that a CGI script could not be executed.
+ * Parameter: the client socket descriptor. */
+/**********************************************************************/
+void cannot_execute(int client)
+{
+    char buf[1024];
+
+    sprintf(buf, "HTTP/1.0 500 Internal Server Error\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<P>Error prohibited CGI execution.\r\n");
+    send(client, buf, strlen(buf), 0);
+}
+
+/**********************************************************************/
+/* Print out an error message with perror() (for system errors; based
+ * on value of errno, which indicates system call errors) and exit the
+ * program indicating an error. */
+/**********************************************************************/
+void error_die(const char *sc)
+{
+    perror(sc);
+    exit(1);
+}
+
+/**********************************************************************/
+/* Execute a CGI script.  Will need to set environment variables as
+ * appropriate.
+ * Parameters: client socket descriptor
+ *             path to the CGI script */
+/**********************************************************************/
+void execute_cgi(int client, const char *path,
+        const char *method, const char *query_string)
+{
+    char buf[1024];
+    int cgi_output[2];
+    int cgi_input[2];
+    pid_t pid;
+    int status;
+    int i;
+    char c;
+    int numchars = 1;
+    int content_length = -1;
+
+    buf[0] = 'A'; buf[1] = '\0';
+    if (strcasecmp(method, "GET") == 0)
+        while ((numchars > 0) && strcmp("\n", buf))  /* read & discard headers */
+            numchars = get_line(client, buf, sizeof(buf));
+    else if (strcasecmp(method, "POST") == 0) /*POST*/
+    {
+        numchars = get_line(client, buf, sizeof(buf));
+        while ((numchars > 0) && strcmp("\n", buf))
+        {
+            buf[15] = '\0';
+            if (strcasecmp(buf, "Content-Length:") == 0)
+                content_length = atoi(&(buf[16]));
+            numchars = get_line(client, buf, sizeof(buf));
+        }
+        if (content_length == -1) {
+            bad_request(client);
+            return;
+        }
+    }
+    else/*HEAD or other*/
+    {
+    }
+
+
+    if (pipe(cgi_output) < 0) {
+        cannot_execute(client);
+        return;
+    }
+    if (pipe(cgi_input) < 0) {
+        cannot_execute(client);
+        return;
+    }
+
+    if ( (pid = fork()) < 0 ) {
+        cannot_execute(client);
+        return;
+    }
+    sprintf(buf, "HTTP/1.0 200 OK\r\n");
+    send(client, buf, strlen(buf), 0);
+    if (pid == 0)  /* child: CGI script */
+    {
+        char meth_env[255];
+        char query_env[255];
+        char length_env[255];
+
+        dup2(cgi_output[1], STDOUT);
+        dup2(cgi_input[0], STDIN);
+        close(cgi_output[0]);
+        close(cgi_input[1]);
+        sprintf(meth_env, "REQUEST_METHOD=%s", method);
+        putenv(meth_env);
+        if (strcasecmp(method, "GET") == 0) {
+            sprintf(query_env, "QUERY_STRING=%s", query_string);
+            putenv(query_env);
+        }
+        else {   /* POST */
+            sprintf(length_env, "CONTENT_LENGTH=%d", content_length);
+            putenv(length_env);
+        }
+        execl(path, "", NULL);
+        exit(0);
+    } else {    /* parent */
+        close(cgi_output[1]);
+        close(cgi_input[0]);
+        if (strcasecmp(method, "POST") == 0)
+            for (i = 0; i < content_length; i++) {
+                recv(client, &c, 1, 0);
+                int len = write(cgi_input[1], &c, 1);
+            }
+        while (read(cgi_output[0], &c, 1) > 0)
+            send(client, &c, 1, 0);
+
+        close(cgi_output[0]);
+        close(cgi_input[1]);
+        waitpid(pid, &status, 0);
+    }
+}
+
+/**********************************************************************/
+/* Get a line from a socket, whether the line ends in a newline,
+ * carriage return, or a CRLF combination.  Terminates the string read
+ * with a null character.  If no newline indicator is found before the
+ * end of the buffer, the string is terminated with a null.  If any of
+ * the above three line terminators is read, the last character of the
+ * string will be a linefeed and the string will be terminated with a
+ * null character.
+ * Parameters: the socket descriptor
+ *             the buffer to save the data in
+ *             the size of the buffer
+ * Returns: the number of bytes stored (excluding null) */
+/**********************************************************************/
+int get_line(int sock, char *buf, int size)
+{
+    int i = 0;
+    char c = '\0';
+    int n;
+
+    while ((i < size - 1) && (c != '\n'))
+    {
+
+		n = recv(sock, &c, 1, 0);
+
+        if (n > 0)
+        {
+            if (c == '\r')
+            {
+                n = recv(sock, &c, 1, MSG_PEEK);
+
+                if ((n > 0) && (c == '\n'))
+                    recv(sock, &c, 1, 0);
+                else
+                    c = '\n';
+            }
+            buf[i] = c;
+            i++;
+        }
+        else
+            c = '\n';
+    }
+    buf[i] = '\0';
+
+    return(i);
+}
+
+/**********************************************************************/
+/* Return the informational HTTP headers about a file. */
+/* Parameters: the socket to print the headers on
+ *             the name of the file */
+/**********************************************************************/
+void headers(int client, const char *filename)
+{
+    char buf[1024];
+    (void)filename;  /* could use filename to determine file type */
+
+    strcpy(buf, "HTTP/1.0 200 OK\r\n");
+    send(client, buf, strlen(buf), 0);
+    strcpy(buf, SERVER_STRING);
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-Type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    strcpy(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+}
+
+/**********************************************************************/
+/* Give a client a 404 not found status message. */
+/**********************************************************************/
+void not_found(int client)
+{
+    char buf[1024];
+
+    sprintf(buf, "HTTP/1.0 404 NOT FOUND\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, SERVER_STRING);
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-Type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<HTML><TITLE>Not Found</TITLE>\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<BODY><P>The server could not fulfill\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "your request because the resource specified\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "is unavailable or nonexistent.\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "</BODY></HTML>\r\n");
+    send(client, buf, strlen(buf), 0);
+}
+
+/**********************************************************************/
+/* Send a regular file to the client.  Use headers, and report
+ * errors to client if they occur.
+ * Parameters: a pointer to a file structure produced from the socket
+ *              file descriptor
+ *             the name of the file to serve */
+/**********************************************************************/
+void serve_file(int client, const char *filename)
+{
+    FILE *resource = NULL;
+    int numchars = 1;
+    char buf[1024];
+	
+#if (ENABLE_NTYCO == 0)
+    buf[0] = 'A'; buf[1] = '\0';
+    while ((numchars > 0) && strcmp("\n", buf))  /* read & discard headers */
+        numchars = get_line(client, buf, sizeof(buf));
+#endif
+
+    resource = fopen(filename, "r");
+    if (resource == NULL)
+        not_found(client);
+    else
+    {
+        headers(client, filename);
+        cat(client, resource);
+    }
+    fclose(resource);
+}
+
+/**********************************************************************/
+/* This function starts the process of listening for web connections
+ * on a specified port.  If the port is 0, then dynamically allocate a
+ * port and modify the original port variable to reflect the actual
+ * port.
+ * Parameters: pointer to variable containing the port to connect on
+ * Returns: the socket */
+/**********************************************************************/
+int startup(u_short *port)
+{
+    int httpd = 0;
+    int on = 1;
+    struct sockaddr_in name;
+
+    httpd = socket(PF_INET, SOCK_STREAM, 0);
+    if (httpd == -1)
+        error_die("socket");
+    memset(&name, 0, sizeof(name));
+    name.sin_family = AF_INET;
+    name.sin_port = htons(*port);
+    name.sin_addr.s_addr = htonl(INADDR_ANY);
+    if ((setsockopt(httpd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) < 0)  
+    {  
+        error_die("setsockopt failed");
+    }
+    if (bind(httpd, (struct sockaddr *)&name, sizeof(name)) < 0)
+        error_die("bind");
+    if (*port == 0)  /* if dynamically allocating a port */
+    {
+        socklen_t namelen = sizeof(name);
+        if (getsockname(httpd, (struct sockaddr *)&name, &namelen) == -1)
+            error_die("getsockname");
+        *port = ntohs(name.sin_port);
+    }
+    if (listen(httpd, 5) < 0)
+        error_die("listen");
+
+	
+    return(httpd);
+}
+
+/**********************************************************************/
+/* Inform the client that the requested web method has not been
+ * implemented.
+ * Parameter: the client socket */
+/**********************************************************************/
+void unimplemented(int client)
+{
+    char buf[1024];
+
+    sprintf(buf, "HTTP/1.0 501 Method Not Implemented\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, SERVER_STRING);
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-Type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<HTML><HEAD><TITLE>Method Not Implemented\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "</TITLE></HEAD>\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<BODY><P>HTTP request method not supported.\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "</BODY></HTML>\r\n");
+    send(client, buf, strlen(buf), 0);
+}
+
+/**********************************************************************/
+
+
+void server(void *arg)
+{
+    int server_sock = -1;
+    u_short port = 4000;
+    struct sockaddr_in client_name;
+    socklen_t  client_name_len = sizeof(client_name);
+
+    server_sock = startup(&port);
+    printf("httpd running on port %d\n", port);
+
+    while (1)
+    {
+    	int *client_sock = (int *)malloc(sizeof(int));
+        *client_sock = accept(server_sock,
+                (struct sockaddr *)&client_name,
+                &client_name_len);
+        if (*client_sock == -1)
+            error_die("accept");
+        /* accept_request(&client_sock); */
+#if 1 //ENABLE_NTYCO 
+
+		printf(" %d.%d.%d.%d:%d clientfd:%d --> New Client Connected \n", 
+			*(unsigned char*)(&client_name.sin_addr.s_addr), *((unsigned char*)(&client_name.sin_addr.s_addr)+1),													
+			*((unsigned char*)(&client_name.sin_addr.s_addr)+2), *((unsigned char*)(&client_name.sin_addr.s_addr)+3),													
+			client_name.sin_port, *client_sock);
+
+		nty_coroutine *read_co;
+		nty_coroutine_create(&read_co, accept_request, client_sock);
+
+#else
+    	pthread_t newthread;
+        if (pthread_create(&newthread , NULL, (void *)accept_request, (void *)(intptr_t)client_sock) != 0)
+            perror("pthread_create");
+#endif
+    }
+
+    nty_close(server_sock);
+
+    return ;
+}
+
+
+int main(int argc, char *argv[]) {
+	
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, server, 0); ////////no run
+	
+
+	nty_schedule_run(); //run
+
+	return 0;
+}
+
+
+
+
+
+
+
+

NtyCo/sample/nty_http_server_mulcore.c

@@ -0,0 +1,891 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+
+
+#include <arpa/inet.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/sysinfo.h>
+#include <sys/syscall.h>
+
+#include <sys/stat.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/sem.h>
+
+
+#define __USE_GNU
+
+#include <sched.h>
+#include <string.h>
+#include <pthread.h>
+
+#include <ctype.h>
+
+#include "nty_coroutine.h"
+
+static int global_shmid = -1;
+static int global_semid = -1;
+
+#define TIME_SUB_MS(tv1, tv2)  ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
+
+
+typedef struct _shm_area {
+	int total;
+	int accept_lock;
+	struct timeval tv_begin;
+} shm_area;
+
+static shm_area *global_shmaddr = NULL;
+
+
+#define MAX_CLIENT_NUM			1000000
+
+
+struct timeval* get_shm_timevalue(void);
+void set_shm_timevalue(struct timeval *tv);
+
+
+int add_shmvalue(void);
+int sub_shmvalue(void);
+
+int init_shmvalue(void);
+
+int lock_accept_mutex(void);
+int unlock_accept_mutex(void);
+
+
+
+
+unsigned long cmpxchg(void *addr, unsigned long _old, unsigned long _new, int size) {
+	unsigned long prev;
+	volatile unsigned int *_ptr = (volatile unsigned int *)(addr);
+
+	switch (size) {
+		case 1: {
+			__asm__ volatile (
+		        "lock; cmpxchgb %b1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 2: {
+			__asm__ volatile (
+		        "lock; cmpxchgw %w1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 4: {
+			__asm__ volatile (
+		        "lock; cmpxchg %1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+	}
+
+	return prev;
+}
+
+
+#define ISspace(x) isspace((int)(x))
+
+#define SERVER_STRING "Server: ntyco_httpd/0.1.0\r\n"
+#define STDIN   0
+#define STDOUT  1
+#define STDERR  2
+
+#define ENABLE_NTYCO	1
+
+#if ENABLE_NTYCO
+
+#define socket 	nty_socket 
+#define accept 	nty_accept 
+#define recv(a, b, c, d) 	nty_recv(a, b, c, d)
+#define send(a, b, c, d) 	nty_send(a, b, c, d) 
+#define close(a)	nty_close(a)
+
+#define MAX_BUFFER_LENGTH	1024
+
+#endif
+
+void accept_request(void *);
+void bad_request(int);
+void cat(int, FILE *);
+void cannot_execute(int);
+void error_die(const char *);
+void execute_cgi(int, const char *, const char *, const char *);
+int get_line(int, char *, int);
+void headers(int, const char *);
+void not_found(int);
+void serve_file(int, const char *);
+int startup(u_short *);
+void unimplemented(int);
+
+int readline(char* allbuf, int level, char* linebuf) {    
+	int len = strlen(allbuf);    
+
+	for (;level < len; ++level)    {        
+		if(allbuf[level]=='\r' && allbuf[level+1]=='\n')            
+			return level+2;        
+		else            
+			*(linebuf++) = allbuf[level];    
+	}    
+
+	return -1;
+}
+
+
+/**********************************************************************/
+/* A request has caused a call to accept() on the server port to
+ * return.  Process the request appropriately.
+ * Parameters: the socket connected to the client */
+/**********************************************************************/
+void accept_request(void *arg)
+{
+    int *pclient = (int*)arg;
+	int client = *pclient;
+	
+    char buf[1024];
+    size_t numchars;
+    char method[16];
+    char url[32];
+    char path[64];
+    size_t i, j;
+    struct stat st;
+    int cgi = 0;      /* becomes true if server decides this is a CGI
+                       * program */
+    char *query_string = NULL;
+
+    numchars = recv(client, buf, sizeof(buf), 0);
+
+    i = 0; j = 0;
+    while (!ISspace(buf[i]) && (i < sizeof(method) - 1))
+    {
+        method[i] = buf[i];
+        i++;
+    }
+    j=i;
+    method[i] = '\0';
+
+    if (strcasecmp(method, "GET") && strcasecmp(method, "POST"))
+    {
+        unimplemented(client);
+        return;
+    }
+
+    if (strcasecmp(method, "POST") == 0)
+        cgi = 1;
+
+    i = 0;
+    while (ISspace(buf[j]) && (j < numchars))
+        j++;
+    while (!ISspace(buf[j]) && (i < sizeof(url) - 1) && (j < numchars))
+    {
+        url[i] = buf[j];
+        i++; j++;
+    }
+    url[i] = '\0';
+
+    if (strcasecmp(method, "GET") == 0)
+    {
+        query_string = url;
+        while ((*query_string != '?') && (*query_string != '\0'))
+            query_string++;
+        if (*query_string == '?')
+        {
+            cgi = 1;
+            *query_string = '\0';
+            query_string++;
+        }
+    }
+
+    sprintf(path, "htdocs%s", url);
+    if (path[strlen(path) - 1] == '/')
+        strcat(path, "index.html");
+    if (stat(path, &st) == -1) {
+        while ((numchars > 0) && strcmp("\n", buf))  /* read & discard headers */
+            numchars = get_line(client, buf, sizeof(buf));
+        not_found(client);
+    }
+    else
+    {
+        if ((st.st_mode & S_IFMT) == S_IFDIR)
+            strcat(path, "/index.html");
+        if ((st.st_mode & S_IXUSR) ||
+                (st.st_mode & S_IXGRP) ||
+                (st.st_mode & S_IXOTH)    )
+            cgi = 1;
+
+		cgi = 0;
+
+        if (!cgi)
+            serve_file(client, path);
+        else
+            execute_cgi(client, path, method, query_string);
+    }
+
+    close(client);
+	free(pclient);
+}
+
+
+/**********************************************************************/
+/* Inform the client that a request it has made has a problem.
+ * Parameters: client socket */
+/**********************************************************************/
+void bad_request(int client)
+{
+    char buf[1024];
+
+    sprintf(buf, "HTTP/1.0 400 BAD REQUEST\r\n");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "Content-type: text/html\r\n");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "<P>Your browser sent a bad request, ");
+    send(client, buf, sizeof(buf), 0);
+    sprintf(buf, "such as a POST without a Content-Length.\r\n");
+    send(client, buf, sizeof(buf), 0);
+}
+
+/**********************************************************************/
+/* Put the entire contents of a file out on a socket.  This function
+ * is named after the UNIX "cat" command, because it might have been
+ * easier just to do something like pipe, fork, and exec("cat").
+ * Parameters: the client socket descriptor
+ *             FILE pointer for the file to cat */
+/**********************************************************************/
+
+#define HTML_PAGE 		"<HTML> \
+						<TITLE>Index</TITLE> \
+						<BODY> \
+						<P>Welcome to J. David's webserver.\
+						<H1>CGI demo \
+						<FORM ACTION=\"color.cgi\" METHOD=\"POST\">\
+						Enter a color: <INPUT TYPE=\"text\" NAME=\"color\">\
+						<INPUT TYPE=\"submit\">\
+						</FORM>\
+						</BODY>\
+						</HTML>"
+
+void cat(int client, FILE *resource)
+{
+#if ENABLE_NTYCO
+    char buf[1024] =  HTML_PAGE;
+
+	send(client, buf, strlen(buf), 0);
+
+#else
+	char buf[1024] = {0};
+    fgets(buf, sizeof(buf), resource);
+    while (!feof(resource))
+    {
+        send(client, buf, strlen(buf), 0);
+        fgets(buf, sizeof(buf), resource);
+    }
+#endif
+}
+
+/**********************************************************************/
+/* Inform the client that a CGI script could not be executed.
+ * Parameter: the client socket descriptor. */
+/**********************************************************************/
+void cannot_execute(int client)
+{
+    char buf[1024];
+
+    sprintf(buf, "HTTP/1.0 500 Internal Server Error\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<P>Error prohibited CGI execution.\r\n");
+    send(client, buf, strlen(buf), 0);
+}
+
+/**********************************************************************/
+/* Print out an error message with perror() (for system errors; based
+ * on value of errno, which indicates system call errors) and exit the
+ * program indicating an error. */
+/**********************************************************************/
+void error_die(const char *sc)
+{
+    perror(sc);
+    exit(1);
+}
+
+/**********************************************************************/
+/* Execute a CGI script.  Will need to set environment variables as
+ * appropriate.
+ * Parameters: client socket descriptor
+ *             path to the CGI script */
+/**********************************************************************/
+void execute_cgi(int client, const char *path,
+        const char *method, const char *query_string)
+{
+    char buf[1024];
+    int cgi_output[2];
+    int cgi_input[2];
+    pid_t pid;
+    int status;
+    int i;
+    char c;
+    int numchars = 1;
+    int content_length = -1;
+
+    buf[0] = 'A'; buf[1] = '\0';
+    if (strcasecmp(method, "GET") == 0)
+        while ((numchars > 0) && strcmp("\n", buf))  /* read & discard headers */
+            numchars = get_line(client, buf, sizeof(buf));
+    else if (strcasecmp(method, "POST") == 0) /*POST*/
+    {
+        numchars = get_line(client, buf, sizeof(buf));
+        while ((numchars > 0) && strcmp("\n", buf))
+        {
+            buf[15] = '\0';
+            if (strcasecmp(buf, "Content-Length:") == 0)
+                content_length = atoi(&(buf[16]));
+            numchars = get_line(client, buf, sizeof(buf));
+        }
+        if (content_length == -1) {
+            bad_request(client);
+            return;
+        }
+    }
+    else/*HEAD or other*/
+    {
+    }
+
+
+    if (pipe(cgi_output) < 0) {
+        cannot_execute(client);
+        return;
+    }
+    if (pipe(cgi_input) < 0) {
+        cannot_execute(client);
+        return;
+    }
+
+    if ( (pid = fork()) < 0 ) {
+        cannot_execute(client);
+        return;
+    }
+    sprintf(buf, "HTTP/1.0 200 OK\r\n");
+    send(client, buf, strlen(buf), 0);
+    if (pid == 0)  /* child: CGI script */
+    {
+        char meth_env[255];
+        char query_env[255];
+        char length_env[255];
+
+        dup2(cgi_output[1], STDOUT);
+        dup2(cgi_input[0], STDIN);
+        close(cgi_output[0]);
+        close(cgi_input[1]);
+        sprintf(meth_env, "REQUEST_METHOD=%s", method);
+        putenv(meth_env);
+        if (strcasecmp(method, "GET") == 0) {
+            sprintf(query_env, "QUERY_STRING=%s", query_string);
+            putenv(query_env);
+        }
+        else {   /* POST */
+            sprintf(length_env, "CONTENT_LENGTH=%d", content_length);
+            putenv(length_env);
+        }
+        execl(path, "", NULL);
+        exit(0);
+    } else {    /* parent */
+        close(cgi_output[1]);
+        close(cgi_input[0]);
+        if (strcasecmp(method, "POST") == 0)
+            for (i = 0; i < content_length; i++) {
+
+				recv(client, &c, 1, 0);
+
+                int len = write(cgi_input[1], &c, 1);
+            }
+        while (read(cgi_output[0], &c, 1) > 0)
+
+			send(client, &c, 1, 0);
+
+
+        close(cgi_output[0]);
+        close(cgi_input[1]);
+        waitpid(pid, &status, 0);
+    }
+}
+
+/**********************************************************************/
+/* Get a line from a socket, whether the line ends in a newline,
+ * carriage return, or a CRLF combination.  Terminates the string read
+ * with a null character.  If no newline indicator is found before the
+ * end of the buffer, the string is terminated with a null.  If any of
+ * the above three line terminators is read, the last character of the
+ * string will be a linefeed and the string will be terminated with a
+ * null character.
+ * Parameters: the socket descriptor
+ *             the buffer to save the data in
+ *             the size of the buffer
+ * Returns: the number of bytes stored (excluding null) */
+/**********************************************************************/
+int get_line(int sock, char *buf, int size)
+{
+    int i = 0;
+    char c = '\0';
+    int n;
+
+    while ((i < size - 1) && (c != '\n'))
+    {
+
+		n = recv(sock, &c, 1, 0);
+
+        if (n > 0)
+        {
+            if (c == '\r')
+            {
+
+                n = recv(sock, &c, 1, MSG_PEEK);
+
+                if ((n > 0) && (c == '\n'))
+
+					recv(sock, &c, 1, 0);
+
+                else
+                    c = '\n';
+            }
+            buf[i] = c;
+            i++;
+        }
+        else
+            c = '\n';
+    }
+    buf[i] = '\0';
+
+    return(i);
+}
+
+/**********************************************************************/
+/* Return the informational HTTP headers about a file. */
+/* Parameters: the socket to print the headers on
+ *             the name of the file */
+/**********************************************************************/
+void headers(int client, const char *filename)
+{
+    char buf[1024] = {0};
+    (void)filename;  /* could use filename to determine file type */
+	
+#if ENABLE_NTYCO
+   
+
+	sprintf(buf, "HTTP/1.0 200 OK\r\n");
+	strcat(buf, SERVER_STRING);
+	strcat(buf, "Content-Type: text/html\r\n");
+	strcat(buf, "\r\n");
+
+	send(client, buf, strlen(buf), 0);
+	
+#else
+	strcpy(buf, "HTTP/1.0 200 OK\r\n");
+	send(client, buf, strlen(buf), 0);
+	strcpy(buf, SERVER_STRING);
+	send(client, buf, strlen(buf), 0);
+	sprintf(buf, "Content-Type: text/html\r\n");
+	send(client, buf, strlen(buf), 0);
+	strcpy(buf, "\r\n");
+	send(client, buf, strlen(buf), 0);
+	
+#endif
+}
+
+/**********************************************************************/
+/* Give a client a 404 not found status message. */
+/**********************************************************************/
+void not_found(int client)
+{
+    char buf[1024];
+#if ENABLE_NTYCO
+
+	sprintf(buf, "HTTP/1.0 404 NOT FOUND\r\n");
+	strcat(buf, SERVER_STRING);
+	strcat(buf, "Content-Type: text/html\r\n");
+	strcat(buf, "\r\n");
+	strcat(buf, "<HTML><TITLE>Not Found</TITLE>\r\n");
+	strcat(buf, "<BODY><P>The server could not fulfill\r\n");
+	strcat(buf, "your request because the resource specified\r\n");
+	strcat(buf, "is unavailable or nonexistent.\r\n");
+	strcat(buf, "</BODY></HTML>\r\n");
+
+	send(client, buf, strlen(buf), 0);
+
+#else
+    sprintf(buf, "HTTP/1.0 404 NOT FOUND\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, SERVER_STRING);
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-Type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<HTML><TITLE>Not Found</TITLE>\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<BODY><P>The server could not fulfill\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "your request because the resource specified\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "is unavailable or nonexistent.\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "</BODY></HTML>\r\n");
+    send(client, buf, strlen(buf), 0);
+#endif
+}
+
+/**********************************************************************/
+/* Send a regular file to the client.  Use headers, and report
+ * errors to client if they occur.
+ * Parameters: a pointer to a file structure produced from the socket
+ *              file descriptor
+ *             the name of the file to serve */
+/**********************************************************************/
+void serve_file(int client, const char *filename)
+{
+    FILE *resource = NULL;
+    int numchars = 1;
+    char buf[1024];
+	
+#if (ENABLE_NTYCO == 0)
+    buf[0] = 'A'; buf[1] = '\0';
+    while ((numchars > 0) && strcmp("\n", buf))  /* read & discard headers */
+        numchars = get_line(client, buf, sizeof(buf));
+	resource = fopen(filename, "r");
+    if (resource == NULL)
+        not_found(client);
+    else
+    {
+        headers(client, filename);
+        cat(client, resource);
+    }
+    fclose(resource);
+#else
+	headers(client, filename);
+    cat(client, resource);
+#endif
+
+    
+}
+
+/**********************************************************************/
+/* This function starts the process of listening for web connections
+ * on a specified port.  If the port is 0, then dynamically allocate a
+ * port and modify the original port variable to reflect the actual
+ * port.
+ * Parameters: pointer to variable containing the port to connect on
+ * Returns: the socket */
+/**********************************************************************/
+int startup(u_short *port)
+{
+    int httpd = 0;
+    int on = 1;
+    struct sockaddr_in name;
+
+    httpd = socket(PF_INET, SOCK_STREAM, 0);
+    if (httpd == -1)
+        error_die("socket");
+    memset(&name, 0, sizeof(name));
+    name.sin_family = AF_INET;
+    name.sin_port = htons(*port);
+    name.sin_addr.s_addr = htonl(INADDR_ANY);
+    if ((setsockopt(httpd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) < 0)  
+    {  
+        error_die("setsockopt failed");
+    }
+    if (bind(httpd, (struct sockaddr *)&name, sizeof(name)) < 0)
+        error_die("bind");
+    if (*port == 0)  /* if dynamically allocating a port */
+    {
+        socklen_t namelen = sizeof(name);
+        if (getsockname(httpd, (struct sockaddr *)&name, &namelen) == -1)
+            error_die("getsockname");
+        *port = ntohs(name.sin_port);
+    }
+    if (listen(httpd, 5) < 0)
+        error_die("listen");
+
+	
+    return(httpd);
+}
+
+/**********************************************************************/
+/* Inform the client that the requested web method has not been
+ * implemented.
+ * Parameter: the client socket */
+/**********************************************************************/
+void unimplemented(int client)
+{
+    char buf[1024];
+#if ENABLE_NTYCO
+    sprintf(buf, "HTTP/1.0 501 Method Not Implemented\r\n");
+	strcat(buf, SERVER_STRING);
+	strcat(buf, "Content-Type: text/html\r\n");
+	strcat(buf, "\r\n");
+	strcat(buf, "<HTML><HEAD><TITLE>Method Not Implemented\r\n");
+	strcat(buf, "</TITLE></HEAD>\r\n");
+	strcat(buf, "<BODY><P>HTTP request method not supported.\r\n");
+	strcat(buf, "</BODY></HTML>\r\n");
+
+	send(client, buf, strlen(buf), 0);
+#else
+	sprintf(buf, "HTTP/1.0 501 Method Not Implemented\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, SERVER_STRING);
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "Content-Type: text/html\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<HTML><HEAD><TITLE>Method Not Implemented\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "</TITLE></HEAD>\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "<BODY><P>HTTP request method not supported.\r\n");
+    send(client, buf, strlen(buf), 0);
+    sprintf(buf, "</BODY></HTML>\r\n");
+    send(client, buf, strlen(buf), 0);
+#endif
+}
+
+void server(void *arg) {
+
+	unsigned short port = *(unsigned short *)arg;
+	struct sockaddr_in client_name;
+	socklen_t  client_name_len = sizeof(client_name);
+	free(arg);
+
+	int fd = nty_socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) return ;
+
+	struct sockaddr_in local, remote;
+	local.sin_family = AF_INET;
+	local.sin_port = htons(port);
+	local.sin_addr.s_addr = INADDR_ANY;
+	bind(fd, (struct sockaddr*)&local, sizeof(struct sockaddr_in));
+
+	listen(fd, 20);
+
+	while (1) {
+		int *client_sock = (int *)malloc(sizeof(int));
+        *client_sock = accept(fd,
+                (struct sockaddr *)&client_name,
+                &client_name_len);
+        if (*client_sock == -1)
+            error_die("accept");
+        /* accept_request(&client_sock); */
+#if 0
+		printf(" %d.%d.%d.%d:%d clientfd:%d --> New Client Connected \n", 
+			*(unsigned char*)(&client_name.sin_addr.s_addr), *((unsigned char*)(&client_name.sin_addr.s_addr)+1),													
+			*((unsigned char*)(&client_name.sin_addr.s_addr)+2), *((unsigned char*)(&client_name.sin_addr.s_addr)+3),													
+			client_name.sin_port, *client_sock);
+#endif
+		nty_coroutine *read_co;
+		nty_coroutine_create(&read_co, accept_request, client_sock);
+	}
+
+}
+
+int mulcore_entry(int begin_port) {
+
+	nty_coroutine *co = NULL;
+
+	int i = 0;
+	unsigned short base_port = begin_port;
+	
+	unsigned short *port = calloc(1, sizeof(unsigned short));
+	*port = base_port + i;
+	nty_coroutine_create(&co, server, port); ////////no run
+	
+
+	nty_schedule_run(); //run
+
+}
+
+int process_bind(void) {
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+
+	pid_t self_id = syscall(__NR_gettid);
+	//printf("selfid --> %d\n", self_id);
+
+	cpu_set_t mask;
+
+	CPU_ZERO(&mask);
+	CPU_SET(self_id % num, &mask);
+
+	sched_setaffinity(0, sizeof(mask), &mask);
+
+	mulcore_entry(9096);
+
+}
+
+
+struct timeval* get_shm_timevalue(void) {
+
+	return global_shmaddr ? &global_shmaddr->tv_begin : NULL;
+
+}
+
+void set_shm_timevalue(struct timeval *tv) {
+	if (global_shmaddr == NULL || tv == NULL) {
+		return ;
+	}
+
+	memcpy(&global_shmaddr->tv_begin, tv, sizeof(struct timeval));
+	
+}
+
+int add_shmvalue(void) {
+	int value = 0;
+	int ret = -1;
+
+	do {
+
+		value = global_shmaddr->total;
+		ret = cmpxchg(&global_shmaddr->total, (unsigned long)value, (unsigned long)(value+1), 4);
+
+
+	} while (ret != value);
+
+	return global_shmaddr->total;
+}
+
+int sub_shmvalue(void) {
+	int value = 0;
+	int ret = -1;
+
+	do {
+
+		value = global_shmaddr->total;
+		ret = cmpxchg(&global_shmaddr->total, (unsigned long)value, (unsigned long)(value-1), 4);
+
+
+	} while (ret != value);
+
+	return global_shmaddr->total;
+}
+
+int lock_accept_mutex(void) {
+
+	int ret = cmpxchg(&global_shmaddr->accept_lock, (unsigned long)0, (unsigned long)1, 4);
+	
+	return ret; //success 0, failed 1.
+}
+
+int unlock_accept_mutex(void) {
+
+	int ret = cmpxchg(&global_shmaddr->accept_lock, (unsigned long)1, (unsigned long)0, 4);
+	
+	return ret; //unlock 1, lock 0
+}
+
+
+int init_shmvalue(void) {
+	
+	global_shmid = shmget(IPC_PRIVATE, sizeof(shm_area), IPC_CREAT|0600);
+	if (global_shmid < 0) {
+		perror("shmget failed\n");
+		return -1;
+	}
+	global_shmaddr = (shm_area*)shmat(global_shmid, NULL, 0);
+	if (global_shmaddr == (shm_area*)-1) {
+		perror("shmat addr error");
+		return -1;
+	}
+	global_shmaddr->total = 0;
+	gettimeofday(&global_shmaddr->tv_begin, NULL);
+
+}
+
+
+
+int main(int argc, char *argv[]) {
+
+	int ret = init_shmvalue();
+	if (ret == -1) {
+		printf("init share memory failed\n");
+		return -1;
+	}
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+	int i = 0;
+	pid_t pid=0;
+	for(i = 0;i < num;i ++) {
+		pid=fork();
+		if(pid <= (pid_t) 0)
+		{
+		   usleep(1);
+		   break;
+		}
+  	}
+
+	if (pid > 0) {
+		printf("ntyco_httpd server running ...\n");
+		getchar();
+	} else if (pid == 0) {
+		process_bind();
+	}
+
+}
+

NtyCo/sample/nty_mysql_client.c

@@ -0,0 +1,49 @@
+
+
+
+
+
+#include "nty_coroutine.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <mysql.h>
+
+
+void func (void *arg) {
+
+	
+	MYSQL* m_mysql = mysql_init(NULL);
+	if (!m_mysql) {
+		printf("mysql_init failed\n");
+		return ;
+	}
+
+	if (!mysql_real_connect(m_mysql, 
+                "192.168.233.133", "king", "123456",
+                "KING_DB", 3306,
+                NULL, CLIENT_FOUND_ROWS)) {
+		printf("mysql_real_connect failed: %s\n", mysql_error(m_mysql));
+		return ;
+	} else{
+		printf("mysql_real_connect success\n");
+	}
+
+	
+}
+
+int main() {
+#if 1
+	init_hook();
+
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, func, NULL);
+	nty_schedule_run(); //run
+#else
+
+	func(NULL);
+
+#endif
+}
+
+

NtyCo/sample/nty_mysql_oper.c

@@ -0,0 +1,315 @@
+
+
+
+
+#include "nty_coroutine.h"
+
+
+#include <mysql.h>
+
+
+#define KING_DB_SERVER_IP		"192.168.233.133"
+#define KING_DB_SERVER_PORT		3306
+
+#define KING_DB_USERNAME		"king"
+#define KING_DB_PASSWORD		"123456"
+
+#define KING_DB_DEFAULTDB		"KING_DB"
+
+
+#define SQL_INSERT_TBL_USER		"INSERT TBL_USER(U_NAME, U_GENDER) VALUES('King', 'man');"
+#define SQL_SELECT_TBL_USER		"SELECT * FROM TBL_USER;"
+
+#define SQL_DELETE_TBL_USER		"CALL PROC_DELETE_USER('King')"
+#define SQL_INSERT_IMG_USER		"INSERT TBL_USER(U_NAME, U_GENDER, U_IMG) VALUES('King', 'man', ?);"
+
+#define SQL_SELECT_IMG_USER		"SELECT U_IMG FROM TBL_USER WHERE U_NAME='King';"
+
+
+#define FILE_IMAGE_LENGTH		(64*1024)
+// C U R D --> 
+// 
+
+int king_mysql_select(MYSQL *handle) { //
+
+	// mysql_real_query --> sql
+	if (mysql_real_query(handle, SQL_SELECT_TBL_USER, strlen(SQL_SELECT_TBL_USER))) {
+		printf("mysql_real_query : %s\n", mysql_error(handle));
+		return -1;
+	}
+	
+
+	// store --> 
+	MYSQL_RES *res = mysql_store_result(handle);
+	if (res == NULL) {
+		printf("mysql_store_result : %s\n", mysql_error(handle));
+		return -2;
+	}
+
+	// rows / fields
+	int rows = mysql_num_rows(res);
+	printf("rows: %d\n", rows);
+	
+	int fields = mysql_num_fields(res);
+	printf("fields: %d\n", fields);
+
+	// fetch
+	MYSQL_ROW row;
+	while ((row = mysql_fetch_row(res))) {
+
+		int i = 0;
+		for (i = 0;i < fields;i ++) {
+			printf("%s\t", row[i]);
+		}
+		printf("\n");
+		
+	}
+
+	mysql_free_result(res);
+
+	return 0;
+}
+
+
+// filename : path + file name
+// buffer : store image data
+
+int read_image(char *filename, char *buffer) {
+
+	if (filename == NULL || buffer == NULL) return -1;
+	
+	FILE *fp = fopen(filename, "rb"); //
+	if (fp == NULL) {
+		printf("fopen failed\n");
+		return -2;
+	}
+
+	// file size
+	fseek(fp, 0, SEEK_END);
+	int length = ftell(fp); // file size
+	fseek(fp, 0, SEEK_SET);
+
+	int size = fread(buffer, 1, length, fp);
+	if (size != length) {
+		printf("fread failed: %d\n", size);
+		return -3;
+	}
+
+	fclose(fp);
+
+	return size;
+
+}
+
+// filename :
+// buffer : 
+// length :
+
+int write_image(char *filename, char *buffer, int length) {
+
+	if (filename == NULL || buffer == NULL || length <= 0) return -1;
+
+	FILE *fp = fopen(filename, "wb+"); //
+	if (fp == NULL) {
+		printf("fopen failed\n");
+		return -2;
+	}
+
+	int size = fwrite(buffer, 1, length, fp);
+	if (size != length) {
+		printf("fwrite failed: %d\n", size);
+		return -3;
+	}
+
+	fclose(fp);
+
+	return size;
+}
+
+int mysql_write(MYSQL *handle, char *buffer, int length) {
+
+	if (handle == NULL || buffer == NULL || length <= 0) return -1;
+
+	MYSQL_STMT *stmt = mysql_stmt_init(handle);
+	int ret = mysql_stmt_prepare(stmt, SQL_INSERT_IMG_USER, strlen(SQL_INSERT_IMG_USER));
+	if (ret) {
+		printf("mysql_stmt_prepare : %s\n", mysql_error(handle));
+		return -2;
+	}
+
+	MYSQL_BIND param = {0};
+	param.buffer_type  = MYSQL_TYPE_LONG_BLOB;
+	param.buffer = NULL;
+	param.is_null = 0;
+	param.length = NULL;
+
+	ret = mysql_stmt_bind_param(stmt, &param);
+	if (ret) {
+		printf("mysql_stmt_bind_param : %s\n", mysql_error(handle));
+		return -3;
+	}
+
+	ret = mysql_stmt_send_long_data(stmt, 0, buffer, length);
+	if (ret) {
+		printf("mysql_stmt_send_long_data : %s\n", mysql_error(handle));
+		return -4;
+	}
+
+	ret = mysql_stmt_execute(stmt);
+	if (ret) {
+		printf("mysql_stmt_execute : %s\n", mysql_error(handle));
+		return -5;
+	}
+
+	ret = mysql_stmt_close(stmt);
+	if (ret) {
+		printf("mysql_stmt_close : %s\n", mysql_error(handle));
+		return -6;
+	}
+	
+
+	return ret;
+}
+
+int mysql_read(MYSQL *handle, char *buffer, int length) {
+
+	if (handle == NULL || buffer == NULL || length <= 0) return -1;
+
+	MYSQL_STMT *stmt = mysql_stmt_init(handle);
+	int ret = mysql_stmt_prepare(stmt, SQL_SELECT_IMG_USER, strlen(SQL_SELECT_IMG_USER));
+	if (ret) {
+		printf("mysql_stmt_prepare : %s\n", mysql_error(handle));
+		return -2;
+	}
+
+	
+	MYSQL_BIND result = {0};
+	
+	result.buffer_type  = MYSQL_TYPE_LONG_BLOB;
+	unsigned long total_length = 0;
+	result.length = &total_length;
+
+	ret = mysql_stmt_bind_result(stmt, &result);
+	if (ret) {
+		printf("mysql_stmt_bind_result : %s\n", mysql_error(handle));
+		return -3;
+	}
+
+	ret = mysql_stmt_execute(stmt);
+	if (ret) {
+		printf("mysql_stmt_execute : %s\n", mysql_error(handle));
+		return -4;
+	}
+
+	ret = mysql_stmt_store_result(stmt);
+	if (ret) {
+		printf("mysql_stmt_store_result : %s\n", mysql_error(handle));
+		return -5;
+	}
+
+
+	while (1) {
+
+		ret = mysql_stmt_fetch(stmt);
+		if (ret != 0 && ret != MYSQL_DATA_TRUNCATED) break; // 
+
+		int start = 0;
+		while (start < (int)total_length) {
+			result.buffer = buffer + start;
+			result.buffer_length = 1;
+			mysql_stmt_fetch_column(stmt, &result, 0, start);
+			start += result.buffer_length;
+		}
+	}
+
+	mysql_stmt_close(stmt);
+
+	return total_length;
+
+}
+
+void coroutine_func(void *arg) {
+
+	MYSQL mysql;
+
+	printf("coroutine_func\n");
+	if (NULL == mysql_init(&mysql)) {
+		printf("mysql_init : %s\n", mysql_error(&mysql));
+		return ;
+	}
+
+	if (!mysql_real_connect(&mysql, KING_DB_SERVER_IP, KING_DB_USERNAME, KING_DB_PASSWORD, 
+		KING_DB_DEFAULTDB, KING_DB_SERVER_PORT, NULL, 0)) {
+
+		printf("mysql_real_connect : %s\n", mysql_error(&mysql));
+		goto Exit;
+	}
+
+	// mysql --> insert 
+	printf("case 1 : mysql --> insert \n");
+#if 1
+	if (mysql_real_query(&mysql, SQL_INSERT_TBL_USER, strlen(SQL_INSERT_TBL_USER))) {
+		printf("mysql_real_query : %s\n", mysql_error(&mysql));
+		goto Exit;
+	}
+#endif
+
+	king_mysql_select(&mysql);
+
+	// mysql --> delete 
+
+	printf("case 2 : mysql --> delete \n");
+#if 1
+	if (mysql_real_query(&mysql, SQL_DELETE_TBL_USER, strlen(SQL_DELETE_TBL_USER))) {
+		printf("mysql_real_query : %s\n", mysql_error(&mysql));
+		goto Exit;
+	}
+#endif
+	
+	king_mysql_select(&mysql);
+
+
+
+	printf("case 3 : mysql --> read image and write mysql\n");
+	
+	char buffer[FILE_IMAGE_LENGTH] = {0};
+	int length = read_image("0voice.jpg", buffer);
+	if (length < 0) goto Exit;
+	
+	mysql_write(&mysql, buffer, length); /// 
+
+
+	printf("case 4 : mysql --> read mysql and write image\n");
+	
+	memset(buffer, 0, FILE_IMAGE_LENGTH);
+	length = mysql_read(&mysql, buffer, FILE_IMAGE_LENGTH);
+
+	write_image("a.jpg", buffer, length);
+
+Exit:
+	mysql_close(&mysql);
+
+	return ;
+
+}
+
+
+int main() {
+#if 1
+	//init_hook();
+
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, coroutine_func, NULL);
+	nty_schedule_run(); //run
+#else
+
+	coroutine_func(NULL);
+
+#endif
+}
+
+
+
+
+
+

NtyCo/sample/nty_rediscli.c

@@ -0,0 +1,108 @@
+
+
+
+
+#include "nty_coroutine.h"
+
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <hiredis.h>
+
+
+void coroutine_func(void *arg) {
+    unsigned int j, isunix = 0;
+    redisContext *c;
+    redisReply *reply;
+    const char *hostname = "192.168.233.133";
+    int port = 6379;
+
+    struct timeval timeout = { 1, 500000 }; // 1.5 seconds
+    if (isunix) {
+        c = redisConnectUnixWithTimeout(hostname, timeout);
+    } else {
+        c = redisConnectWithTimeout(hostname, port, timeout);
+    }
+    if (c == NULL || c->err) {
+        if (c) {
+            printf("Connection error: %s\n", c->errstr);
+            redisFree(c);
+        } else {
+            printf("Connection error: can't allocate redis context\n");
+        }
+        exit(1);
+    }
+
+    /* PING server */
+    reply = redisCommand(c,"PING");
+    printf("PING: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Set a key */
+    reply = redisCommand(c,"SET %s %s", "foo", "hello world");
+    printf("SET: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Set a key using binary safe API */
+    reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5);
+    printf("SET (binary API): %s\n", reply->str);
+    freeReplyObject(reply);
+
+    /* Try a GET and two INCR */
+    reply = redisCommand(c,"GET foo");
+    printf("GET foo: %s\n", reply->str);
+    freeReplyObject(reply);
+
+    reply = redisCommand(c,"INCR counter");
+    printf("INCR counter: %lld\n", reply->integer);
+    freeReplyObject(reply);
+    /* again ... */
+    reply = redisCommand(c,"INCR counter");
+    printf("INCR counter: %lld\n", reply->integer);
+    freeReplyObject(reply);
+
+    /* Create a list of numbers, from 0 to 9 */
+    reply = redisCommand(c,"DEL mylist");
+    freeReplyObject(reply);
+    for (j = 0; j < 10; j++) {
+        char buf[64];
+
+        snprintf(buf,64,"%u",j);
+        reply = redisCommand(c,"LPUSH mylist element-%s", buf);
+        freeReplyObject(reply);
+    }
+
+    /* Let's check what we have inside the list */
+    reply = redisCommand(c,"LRANGE mylist 0 -1");
+    if (reply->type == REDIS_REPLY_ARRAY) {
+        for (j = 0; j < reply->elements; j++) {
+            printf("%u) %s\n", j, reply->element[j]->str);
+        }
+    }
+    freeReplyObject(reply);
+
+    /* Disconnects and frees the context */
+    redisFree(c);
+
+    return ;
+}
+
+
+int main(int argc, char **argv) {
+
+	//init_hook();
+
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, coroutine_func, NULL);
+	nty_schedule_run(); //run
+
+
+
+}
+
+
+
+
+
+

NtyCo/sample/nty_server.c

@@ -0,0 +1,150 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+ 
+
+#include "nty_coroutine.h"
+
+#include <arpa/inet.h>
+
+#define MAX_CLIENT_NUM			1000000
+#define TIME_SUB_MS(tv1, tv2)  ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
+
+
+void server_reader(void *arg) {
+	int fd = *(int *)arg;
+	free(arg);
+	int ret = 0;
+
+ 
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN;
+
+	while (1) {
+		
+		char buf[1024] = {0};
+		ret = nty_recv(fd, buf, 1024, 0);
+		if (ret > 0) {
+			if(fd > MAX_CLIENT_NUM) 
+			printf("read from server: %.*s\n", ret, buf);
+
+			ret = nty_send(fd, buf, strlen(buf), 0);
+			if (ret == -1) {
+				nty_close(fd);
+				break;
+			}
+		} else if (ret == 0) {	
+			nty_close(fd);
+			break;
+		}
+
+	}
+}
+
+
+void server(void *arg) {
+
+	unsigned short port = *(unsigned short *)arg;
+	free(arg);
+
+	int fd = nty_socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) return ;
+
+	struct sockaddr_in local, remote;
+	local.sin_family = AF_INET;
+	local.sin_port = htons(port);
+	local.sin_addr.s_addr = INADDR_ANY;
+	bind(fd, (struct sockaddr*)&local, sizeof(struct sockaddr_in));
+
+	listen(fd, 20);
+	printf("listen port : %d\n", port);
+
+	
+	struct timeval tv_begin;
+	gettimeofday(&tv_begin, NULL);
+
+	while (1) {
+		socklen_t len = sizeof(struct sockaddr_in);
+		int cli_fd = nty_accept(fd, (struct sockaddr*)&remote, &len);
+		if (cli_fd % 1000 == 999) {
+
+			struct timeval tv_cur;
+			memcpy(&tv_cur, &tv_begin, sizeof(struct timeval));
+			
+			gettimeofday(&tv_begin, NULL);
+			int time_used = TIME_SUB_MS(tv_begin, tv_cur);
+			
+			printf("client fd : %d, time_used: %d\n", cli_fd, time_used);
+		}
+		printf("new client comming\n");
+
+		nty_coroutine *read_co;
+		int *arg = malloc(sizeof(int));
+		*arg = cli_fd;
+		nty_coroutine_create(&read_co, server_reader, arg);
+
+	}
+	
+}
+
+
+
+int main(int argc, char *argv[]) {
+	nty_coroutine *co = NULL;
+
+	int i = 0;
+	unsigned short base_port = 9096;
+	for (i = 0;i < 100;i ++) {
+		unsigned short *port = calloc(1, sizeof(unsigned short));
+		*port = base_port + i;
+		nty_coroutine_create(&co, server, port); ////////no run
+	}
+
+	nty_schedule_run(); //run
+
+	return 0;
+}
+
+
+

NtyCo/sample/nty_server_mulcore.c

@@ -0,0 +1,351 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+
+
+#include <arpa/inet.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/sysinfo.h>
+#include <sys/syscall.h>
+
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/sem.h>
+
+
+#define __USE_GNU
+
+#include <sched.h>
+#include <string.h>
+#include <pthread.h>
+
+#include <ctype.h>
+
+#include "nty_coroutine.h"
+
+static int global_shmid = -1;
+static int global_semid = -1;
+
+#define TIME_SUB_MS(tv1, tv2)  ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
+
+
+typedef struct _shm_area {
+	int total;
+	int lock;
+	struct timeval tv_begin;
+} shm_area;
+
+static shm_area *global_shmaddr = NULL;
+
+
+#define MAX_CLIENT_NUM			1000000
+
+
+struct timeval* get_shm_timevalue(void);
+void set_shm_timevalue(struct timeval *tv);
+
+
+int add_shmvalue(void);
+int sub_shmvalue(void);
+
+int init_shmvalue(void);
+
+
+unsigned long cmpxchg(void *addr, unsigned long _old, unsigned long _new, int size) {
+	unsigned long prev;
+	volatile unsigned int *_ptr = (volatile unsigned int *)(addr);
+
+	switch (size) {
+		case 1: {
+			__asm__ volatile (
+		        "lock; cmpxchgb %b1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 2: {
+			__asm__ volatile (
+		        "lock; cmpxchgw %w1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 4: {
+			__asm__ volatile (
+		        "lock; cmpxchg %1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+	}
+
+	return prev;
+}
+
+#define SERVER_STRING "Server: ntyco\r\n"
+
+
+int headers(char *buffer)
+{
+    strcat(buffer, "HTTP/1.0 200 OK\r\n");
+    strcat(buffer, SERVER_STRING);
+    strcat(buffer, "Content-Type: text/html\r\n");
+    strcat(buffer, "\r\n");
+
+	return strlen(buffer);
+}
+
+int body(char *buffer) {
+	strcat(buffer, "<html><h1> coroutine --> ntyco<h1></html>");
+
+	return strlen(buffer);
+}
+
+void server_reader(void *arg) {
+	int fd = *(int *)arg;
+	int ret = 0;
+
+
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN;
+
+	while (1) {
+		
+		char buf[1024] = {0};
+		ret = nty_recv(fd, buf, 1024, 0);
+		if (ret > 0) {
+			if(fd > MAX_CLIENT_NUM) 
+				printf("read from server: %.*s\n", ret, buf);
+
+			memset(buf, 0, 1024);
+			int hlen = headers(buf);
+			int blen = body(buf+hlen);
+
+			ret = nty_send(fd, buf, strlen(buf), 0);
+			if (ret == -1) {
+				nty_close(fd);
+				sub_shmvalue();
+				
+				break;
+			}
+		} else if (ret == 0) {	
+			nty_close(fd);
+			
+			sub_shmvalue();
+			
+			break;
+		}
+	}
+}
+
+
+void server(void *arg) {
+
+	unsigned short port = *(unsigned short *)arg;
+	free(arg);
+
+	int fd = nty_socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) return ;
+
+	struct sockaddr_in local, remote;
+	local.sin_family = AF_INET;
+	local.sin_port = htons(port);
+	local.sin_addr.s_addr = INADDR_ANY;
+	bind(fd, (struct sockaddr*)&local, sizeof(struct sockaddr_in));
+
+	listen(fd, 20);
+	printf("listen port : %d\n", port);
+
+	while (1) {
+		socklen_t len = sizeof(struct sockaddr_in);
+		int cli_fd = nty_accept(fd, (struct sockaddr*)&remote, &len);
+
+		printf("new client comming\n");
+		
+		nty_coroutine *read_co;
+		nty_coroutine_create(&read_co, server_reader, &cli_fd);
+
+		int value = add_shmvalue();
+		if (value % 1000 == 999) {
+			struct timeval *tv_begin = get_shm_timevalue();
+
+			struct timeval tv_cur;
+			memcpy(&tv_cur, tv_begin, sizeof(struct timeval));	
+			gettimeofday(tv_begin, NULL);
+
+			int time_used = TIME_SUB_MS((*tv_begin), tv_cur);
+			
+			printf("client sum: %d, time_used: %d\n", value, time_used);
+		}
+
+	}
+	
+}
+
+int mulcore_entry(int begin_port) {
+
+	nty_coroutine *co = NULL;
+
+	int i = 0;
+	unsigned short base_port = begin_port;
+	//for (i = 0;i < 10;i ++) {
+	unsigned short *port = calloc(1, sizeof(unsigned short));
+	*port = base_port + i;
+	nty_coroutine_create(&co, server, port); ////////no run
+	//}
+
+	nty_schedule_run(); //run
+
+}
+
+int process_bind(void) {
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+
+	pid_t self_id = syscall(__NR_gettid);
+	//printf("selfid --> %d\n", self_id);
+
+	cpu_set_t mask;
+
+	CPU_ZERO(&mask);
+	CPU_SET(self_id % num, &mask);
+
+	sched_setaffinity(0, sizeof(mask), &mask);
+
+	mulcore_entry(9096);
+
+}
+
+
+struct timeval* get_shm_timevalue(void) {
+
+	return global_shmaddr ? &global_shmaddr->tv_begin : NULL;
+
+}
+
+void set_shm_timevalue(struct timeval *tv) {
+	if (global_shmaddr == NULL || tv == NULL) {
+		return ;
+	}
+
+	memcpy(&global_shmaddr->tv_begin, tv, sizeof(struct timeval));
+	
+}
+
+int add_shmvalue(void) {
+	int value = 0;
+	int ret = -1;
+
+	do {
+
+		value = global_shmaddr->total;
+		ret = cmpxchg(&global_shmaddr->total, (unsigned long)value, (unsigned long)(value+1), 4);
+
+
+	} while (ret != value);
+
+	return global_shmaddr->total;
+}
+
+int sub_shmvalue(void) {
+	int value = 0;
+	int ret = -1;
+
+	do {
+
+		value = global_shmaddr->total;
+		ret = cmpxchg(&global_shmaddr->total, (unsigned long)value, (unsigned long)(value-1), 4);
+
+
+	} while (ret != value);
+
+	return global_shmaddr->total;
+}
+
+
+int init_shmvalue(void) {
+	
+	global_shmid = shmget(IPC_PRIVATE, sizeof(shm_area), IPC_CREAT|0600);
+	if (global_shmid < 0) {
+		perror("shmget failed\n");
+		return -1;
+	}
+	global_shmaddr = (shm_area*)shmat(global_shmid, NULL, 0);
+	if (global_shmaddr == (shm_area*)-1) {
+		perror("shmat addr error");
+		return -1;
+	}
+	global_shmaddr->total = 0;
+	gettimeofday(&global_shmaddr->tv_begin, NULL);
+
+}
+
+
+
+int main(int argc, char *argv[]) {
+
+	int ret = init_shmvalue();
+	if (ret == -1) {
+		printf("init share memory failed\n");
+		return -1;
+	}
+	
+	fork();
+	fork();
+	//fork();
+	//fork();
+	//fork();
+
+	process_bind();
+

NtyCo/sample/nty_websocket_server.c

@@ -0,0 +1,377 @@
+
+#include "nty_coroutine.h"
+
+#include <arpa/inet.h>
+
+#include <openssl/sha.h>
+#include <openssl/pem.h>
+#include <openssl/bio.h>
+#include <openssl/evp.h>
+
+
+#define MAX_CLIENT_NUM			1000000
+#define TIME_SUB_MS(tv1, tv2)  ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
+
+#define NTY_WEBSOCKET_SERVER_PORT	9096
+#define GUID 					"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+#define MAX_BUFFER_LENGTH			1024
+
+struct _nty_ophdr {
+
+	unsigned char opcode:4,
+		 rsv3:1,
+		 rsv2:1,
+		 rsv1:1,
+		 fin:1;
+	unsigned char payload_length:7,
+		mask:1;
+
+} __attribute__ ((packed));
+
+struct _nty_websocket_head_126 {
+	unsigned short payload_length;
+	char mask_key[4];
+	unsigned char data[8];
+} __attribute__ ((packed));
+
+struct _nty_websocket_head_127 {
+
+	unsigned long long payload_length;
+	char mask_key[4];
+
+	unsigned char data[8];
+	
+} __attribute__ ((packed));
+
+#define UNION_HEADER(type1, type2) 	\
+	struct {						\
+		struct type1 hdr;			\
+		struct type2 len;			\
+	}
+
+
+typedef struct _nty_websocket_head_127 nty_websocket_head_127;
+typedef struct _nty_websocket_head_126 nty_websocket_head_126;
+typedef struct _nty_ophdr nty_ophdr;
+
+
+static int ntySetNonblock(int fd) {
+	int flags;
+
+	flags = fcntl(fd, F_GETFL, 0);
+	if (flags < 0) return flags;
+	flags |= O_NONBLOCK;
+	if (fcntl(fd, F_SETFL, flags) < 0) return -1;
+	return 0;
+}
+
+static int ntySetBlock(int fd)  
+{
+    int flags = fcntl(fd, F_GETFL, 0);
+	if (flags < 0) return flags; 
+    flags &=~O_NONBLOCK;  
+    if (fcntl(fd, F_SETFL, flags) < 0) return -1;
+
+	return 0;
+} 
+
+
+int base64_encode(char *in_str, int in_len, char *out_str) {    
+	BIO *b64, *bio;    
+	BUF_MEM *bptr = NULL;    
+	size_t size = 0;    
+
+	if (in_str == NULL || out_str == NULL)        
+		return -1;    
+
+	b64 = BIO_new(BIO_f_base64());    
+	bio = BIO_new(BIO_s_mem());    
+	bio = BIO_push(b64, bio);
+	
+	BIO_write(bio, in_str, in_len);    
+	BIO_flush(bio);    
+
+	BIO_get_mem_ptr(bio, &bptr);    
+	memcpy(out_str, bptr->data, bptr->length);    
+	out_str[bptr->length-1] = '\0';    
+	size = bptr->length;    
+
+	BIO_free_all(bio);    
+	return size;
+}
+
+
+int readline(char* allbuf,int level,char* linebuf){    
+	int len = strlen(allbuf);    
+
+	for (;level < len; ++level)    {        
+		if(allbuf[level]=='\r' && allbuf[level+1]=='\n')            
+			return level+2;        
+		else            
+			*(linebuf++) = allbuf[level];    
+	}    
+
+	return -1;
+}
+
+void umask(char *data,int len,char *mask){    
+	int i;    
+	for (i = 0;i < len;i ++)        
+		*(data+i) ^= *(mask+(i%4));
+}
+
+
+char* decode_packet(unsigned char *stream, char *mask, int length, int *ret) {
+
+	nty_ophdr *hdr =  (nty_ophdr*)stream;
+	unsigned char *data = stream + sizeof(nty_ophdr);
+	int size = 0;
+	int start = 0;
+	//char mask[4] = {0};
+	int i = 0;
+
+	if ((hdr->mask & 0x7F) == 126) {
+
+		nty_websocket_head_126 *hdr126 = (nty_websocket_head_126*)data;
+		size = hdr126->payload_length;
+		
+		for (i = 0;i < 4;i ++) {
+			mask[i] = hdr126->mask_key[i];
+		}
+		
+		start = 8;
+		
+	} else if ((hdr->mask & 0x7F) == 127) {
+
+		nty_websocket_head_127 *hdr127 = (nty_websocket_head_127*)data;
+		size = hdr127->payload_length;
+		
+		for (i = 0;i < 4;i ++) {
+			mask[i] = hdr127->mask_key[i];
+		}
+		
+		start = 14;
+
+	} else {
+		size = hdr->payload_length;
+
+		memcpy(mask, data, 4);
+		start = 6;
+	}
+
+	*ret = size;
+	umask(stream+start, size, mask);
+
+	return stream + start;
+	
+}
+
+int encode_packet(char *buffer,char *mask, char *stream, int length) {
+
+	nty_ophdr head = {0};
+	head.fin = 1;
+	head.opcode = 1;
+	int size = 0;
+
+	if (length < 126) {
+		head.payload_length = length;
+		memcpy(buffer, &head, sizeof(nty_ophdr));
+		size = 2;
+	} else if (length < 0xffff) {
+		nty_websocket_head_126 hdr = {0};
+		hdr.payload_length = length;
+		memcpy(hdr.mask_key, mask, 4);
+
+		memcpy(buffer, &head, sizeof(nty_ophdr));
+		memcpy(buffer+sizeof(nty_ophdr), &hdr, sizeof(nty_websocket_head_126));
+		size = sizeof(nty_websocket_head_126);
+		
+	} else {
+		
+		nty_websocket_head_127 hdr = {0};
+		hdr.payload_length = length;
+		memcpy(hdr.mask_key, mask, 4);
+		
+		memcpy(buffer, &head, sizeof(nty_ophdr));
+		memcpy(buffer+sizeof(nty_ophdr), &hdr, sizeof(nty_websocket_head_127));
+
+		size = sizeof(nty_websocket_head_127);
+		
+	}
+
+	memcpy(buffer+2, stream, length);
+
+	return length + 2;
+}
+
+
+
+void server_reader(void *arg) {
+	int fd = *(int *)arg;
+	int length = 0;
+	int ret = 0;
+
+	struct pollfd fds;
+	fds.fd = fd;
+	fds.events = POLLIN;
+
+	while (1) {
+		
+		char stream[MAX_BUFFER_LENGTH] = {0};
+		length = nty_recv(fd, stream, MAX_BUFFER_LENGTH, 0);
+		if (length > 0) {
+			if(fd > MAX_CLIENT_NUM) 
+				printf("read from server: %.*s\n", length, stream);
+
+			char mask[4] = {0};
+			char *data = decode_packet(stream, mask, length, &ret);
+
+			printf(" data : %s , length : %d\n", data, ret);
+			
+#if 1					
+			char buffer[MAX_BUFFER_LENGTH+14] = {0};
+			ret = encode_packet(buffer, mask, data, ret);
+			ret = nty_send(fd, buffer, ret, 0);
+#endif	
+		} else if (length == 0) {	
+			nty_close(fd);
+			break;
+		}
+
+	}
+}
+
+
+int handshake(int cli_fd) {    
+	int level = 0;     
+	char buffer[MAX_BUFFER_LENGTH];    
+	char linebuf[128];    //Sec-WebSocket-Accept    
+	char sec_accept[32] = {0};    //sha1 data    
+	unsigned char sha1_data[SHA_DIGEST_LENGTH+1] = {0};    //reponse head buffer    
+	//char head[MAX_BUFFER_LENGTH] = {0};    
+
+#if 1
+	if (read(cli_fd, buffer, sizeof(buffer))<=0)        
+		perror("read");
+#else
+
+	int ret = 0;
+	int length = recv_buffer(cli_fd, buffer, MAX_BUFFER_LENGTH, &ret);
+	if (ret < 0) perror("read");
+
+#endif
+	printf("request\n");    
+	printf("%s\n",buffer);   
+	
+	do {        
+		memset(linebuf, 0, sizeof(linebuf));        
+		level = readline(buffer,level,linebuf); 
+
+		if (strstr(linebuf,"Sec-WebSocket-Key") != NULL)        {   
+			
+			strcat(linebuf, GUID);    
+			
+			SHA1((unsigned char*)&linebuf+19,strlen(linebuf+19),(unsigned char*)&sha1_data);  
+
+			memset(buffer, 0, MAX_BUFFER_LENGTH);
+			base64_encode(sha1_data,strlen(sha1_data),sec_accept);           
+			sprintf(buffer, "HTTP/1.1 101 Switching Protocols\r\n" \
+				"Upgrade: websocket\r\n" \
+				"Connection: Upgrade\r\n" \
+				"Sec-WebSocket-Accept: %s\r\n" \
+				"\r\n", sec_accept);  
+			
+
+			printf("response\n");       
+			printf("%s",buffer);            
+#if 1
+			if (write(cli_fd, buffer, strlen(buffer)) < 0)                
+				perror("write");            
+#else
+
+			length = send_buffer(cli_fd, head, strlen(head));
+			assert(length == strlen(head));
+
+#endif
+			printf("accept : %s\n", sec_accept);
+			break;        
+		}    
+
+	}while((buffer[level]!='\r' || buffer[level+1]!='\n') && level!=-1);    
+
+	return 0;
+
+}
+
+
+int init_server(void) {
+
+	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
+	if (sockfd < 0) {
+		printf("socket failed\n");
+		return -1;
+	}
+
+	struct sockaddr_in addr;
+	addr.sin_family = AF_INET;
+	addr.sin_port = htons(NTY_WEBSOCKET_SERVER_PORT);
+	addr.sin_addr.s_addr = INADDR_ANY;
+
+	if (bind(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) {
+		printf("bind failed\n");
+		return -2;
+	}
+
+	if (listen(sockfd, 5) < 0) {
+		printf("listen failed\n");
+		return -3;
+	}
+
+	return sockfd;
+	
+}
+
+void server(void *arg) {
+
+	int sockfd = init_server();
+
+	struct timeval tv_begin;
+	gettimeofday(&tv_begin, NULL);
+
+	while (1) {
+		socklen_t len = sizeof(struct sockaddr_in);
+		struct sockaddr_in remote;
+		
+		int cli_fd = nty_accept(sockfd, (struct sockaddr*)&remote, &len);
+		if (cli_fd % 1000 == 999) {
+
+			struct timeval tv_cur;
+			memcpy(&tv_cur, &tv_begin, sizeof(struct timeval));
+			
+			gettimeofday(&tv_begin, NULL);
+			int time_used = TIME_SUB_MS(tv_begin, tv_cur);
+			
+			printf("client fd : %d, time_used: %d\n", cli_fd, time_used);
+		}
+		printf("new client comming\n");
+		ntySetBlock(cli_fd);
+		handshake(cli_fd);
+		ntySetNonblock(cli_fd);
+
+		nty_coroutine *read_co = NULL;
+		nty_coroutine_create(&read_co, server_reader, &cli_fd);
+
+	}
+	
+}
+
+int main() {
+
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, server, NULL); 
+
+	nty_schedule_run();
+}
+
+
+

NtyCo/sample/ntyco_httpd.c

@@ -0,0 +1,536 @@
+/*
+ *  Author : WangBoJing , email : 1989wangbojing@gmail.com
+ * 
+ *  Copyright Statement:
+ *  --------------------
+ *  This software is protected by Copyright and the information contained
+ *  herein is confidential. The software may not be copied and the information
+ *  contained herein may not be used or disclosed except with the written
+ *  permission of Author. (C) 2017
+ * 
+ *
+
+****       *****                                      *****
+  ***        *                                       **    ***
+  ***        *         *                            *       **
+  * **       *         *                           **        **
+  * **       *         *                          **          *
+  *  **      *        **                          **          *
+  *  **      *       ***                          **
+  *   **     *    ***********    *****    *****  **                   ****
+  *   **     *        **           **      **    **                 **    **
+  *    **    *        **           **      *     **                 *      **
+  *    **    *        **            *      *     **                **      **
+  *     **   *        **            **     *     **                *        **
+  *     **   *        **             *    *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *      **  *        **             **   *      **               **        **
+  *       ** *        **              *  *       **               **        **
+  *       ** *        **              ** *        **          *   **        **
+  *        ***        **               * *        **          *   **        **
+  *        ***        **     *         **          *         *     **      **
+  *         **        **     *         **          **       *      **      **
+  *         **         **   *          *            **     *        **    **
+*****        *          ****           *              *****           ****
+                                       *
+                                      *
+                                  *****
+                                  ****
+
+
+
+ *
+ */
+
+
+
+
+#include <arpa/inet.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/sysinfo.h>
+#include <sys/syscall.h>
+
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/sem.h>
+
+#include <errno.h>
+
+#include <fcntl.h>
+#include <sys/epoll.h>
+#include <sys/poll.h>
+
+#define __USE_GNU
+
+#include <sched.h>
+#include <string.h>
+#include <pthread.h>
+
+#include <ctype.h>
+
+#include "nty_coroutine.h"
+
+
+#define MAX_CLIENT_NUM			1000000
+#define TOTALFDS			16
+
+
+typedef struct _shm_area {
+	int totalfds[TOTALFDS];
+	char cpu_lb[TOTALFDS];
+	//mtx : default 0
+	// 1 : lock -- del epoll
+	// 2 : lock -- del complete
+	// 3 : unlock -- add
+	// 0 : unlock -- add complete
+	int accept_mtx;
+} shm_area;
+
+static shm_area *global_shmaddr = NULL;
+static int global_shmid = -1;
+
+int cpu_size = 0;
+int accept_disable = 1000;
+int enable_accept = 1;
+pid_t self_id = 0;
+
+
+
+unsigned long cmpxchg(void *addr, unsigned long _old, unsigned long _new, int size) {
+	unsigned long prev;
+	volatile unsigned int *_ptr = (volatile unsigned int *)(addr);
+
+	switch (size) {
+		case 1: {
+			__asm__ volatile (
+		        "lock; cmpxchgb %b1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 2: {
+			__asm__ volatile (
+		        "lock; cmpxchgw %w1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+		case 4: {
+			__asm__ volatile (
+		        "lock; cmpxchg %1, %2"
+		        : "=a" (prev)
+		        : "r" (_new), "m" (*_ptr), "0" (_old)
+		        : "memory");
+			break;
+		}
+	}
+
+	return prev;
+}
+
+
+int atomic_add(volatile int *value, int add)
+{
+    __asm__ volatile (
+
+         "lock;"
+    "    addl  %0, %1;   "
+
+    : "+r" (add) : "m" (*value) : "cc", "memory");
+
+    return add;
+}
+
+int atomic_sub(volatile int *value, int sub)
+{
+    __asm__ volatile (
+
+         "lock;"
+    "    subl  %0, %1;   "
+
+    : "+r" (sub) : "m" (*value) : "cc", "memory");
+
+    return sub;
+}
+
+
+#define ISspace(x) isspace((int)(x))
+
+#define SERVER_STRING "Server: ntyco_httpd/0.1.0\r\n"
+#define STDIN   0
+#define STDOUT  1
+#define STDERR  2
+
+#define ENABLE_NTYCO	0
+
+#if ENABLE_NTYCO
+
+#define socket 	nty_socket 
+#define accept 	nty_accept 
+#define recv(a, b, c, d) 	nty_recv(a, b, c, d)
+#define send(a, b, c, d) 	nty_send(a, b, c, d) 
+
+#define MAX_BUFFER_LENGTH	1024
+
+
+
+#endif
+
+#define INC_COUNTFD  do {				\
+	atomic_add(&global_shmaddr->totalfds[self_id % cpu_size], 1);	\
+} while (0)
+
+
+#define DEC_COUNTFD  do {				\
+	atomic_sub(&global_shmaddr->totalfds[self_id % cpu_size], 1);	\
+} while (0)
+
+int get_countfd(void) {	
+	return global_shmaddr->totalfds[self_id % cpu_size];	
+}
+
+int max_countfd(void) {
+
+	int count = -1;
+	int i = 0;
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count < global_shmaddr->totalfds[i]) {
+			count = global_shmaddr->totalfds[i];
+		}
+	}
+	return count;
+}
+
+int min_countfd(void) {
+
+	int count = 0xffffffff;
+	int i = 0;
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count > global_shmaddr->totalfds[i]) {
+			count = global_shmaddr->totalfds[i];
+		}
+	}
+	return count;
+}
+
+int compare_out_countfd(void) {
+
+	int current = get_countfd();
+	int min = min_countfd();
+
+	if ((current * 7 / 8) > min) {
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
+int compare_in_countfd(void) {
+
+	int current = get_countfd();
+	int max = max_countfd();
+
+	if ((current * 8 / 7) < max) {
+		return 1;
+	} else {
+		return 0;
+	}
+}
+
+void print_countfds(void) {
+
+	int i = 0;
+	for (i = 0;i < cpu_size;i ++) {
+		printf("%5d : %5d ", i, global_shmaddr->totalfds[i]);
+	}
+	printf("\n");
+}
+
+void lock_accept(void) {
+	
+	global_shmaddr->cpu_lb[self_id % cpu_size] = 1;
+
+	int count = 0xffffffff;
+	int i = 0;
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count > global_shmaddr->totalfds[i]) {
+			count = global_shmaddr->totalfds[i];
+		}
+	}
+
+	for (i = 0;i < cpu_size;i ++) {
+		if (count == global_shmaddr->totalfds[i]) {
+			global_shmaddr->cpu_lb[i] = 3;
+		}
+	}
+	
+}
+
+char read_accept(void) {
+	return global_shmaddr->cpu_lb[self_id % cpu_size];
+}
+
+void write_accept(char state) { //0, 1, 2, 3		
+	global_shmaddr->cpu_lb[self_id % cpu_size] = state;
+}
+
+int lock(void) {
+
+	return cmpxchg(&global_shmaddr->accept_mtx, 0, 1, 4); //zero:success, non-zero:failed
+
+}
+
+void unlock(void) {
+
+	global_shmaddr->accept_mtx = 0;
+
+}
+
+
+void accept_request(void *arg);
+void bad_request(int);
+void cat(int);
+void cannot_execute(int);
+void error_die(const char *);
+void execute_cgi(int, const char *, const char *, const char *);
+int get_line(int, char *, int);
+void headers(int);
+void not_found(int);
+void serve_file(int);
+int startup(u_short *);
+void unimplemented(int);
+
+
+void accept_request(void* arg)
+{
+    int client = *(int*)arg;
+	
+    char buf[1024];
+    size_t numchars;
+    char method[16];
+    char url[32];
+    char path[64];
+    size_t i, j;
+    struct stat st;
+    int cgi = 0;      /* becomes true if server decides this is a CGI
+                       * program */
+    char *query_string = NULL;
+
+	while (1) {
+		
+		numchars = nty_recv(client, buf, sizeof(buf), 0);
+		if (numchars > 0) {
+			serve_file(client);
+		} else if (numchars == 0) {	
+			nty_close(client);
+			DEC_COUNTFD;
+			break;
+		} else if (numchars == -1) {
+			if (errno == EAGAIN) {
+				continue;
+			} 
+			nty_close(client);
+			DEC_COUNTFD;
+			break;
+		}
+	}
+
+}
+
+
+#define HTML_PAGE       "<!DOCTYPE html> \
+<html> \
+<head> \
+<title>Welcome to nginx!</title> \
+<style> \
+    body { \
+        width: 35em;\
+        margin: 0 auto;\
+        font-family: Tahoma, Verdana, Arial, sans-serif;\
+    }\
+</style>\
+</head>\
+<body>\
+<h1>Welcome to nginx!</h1>\
+<p>If you see this page, the nginx web server is successfully installed and\
+working. Further configuration is required.</p>\
+\
+<p>For online documentation and support please refer to\
+<a href=\"http://nginx.org/\">nginx.org</a>.<br/>\
+Commercial support is available at\
+<a href=\"http://nginx.com/\">nginx.com</a>.</p>\
+\
+<p><em>Thank you for using nginx.</em></p>\
+</body>\
+</html>"
+
+
+void error_die(const char *sc)
+{
+    printf("%s\n", sc);
+    exit(1);
+}
+
+void headers(int client)
+{
+    char buf[1024] = {0};
+	char content[128] = {0};
+
+	sprintf(buf, "HTTP/1.0 200 OK\r\n");
+	strcat(buf, SERVER_STRING);
+	strcat(buf, "Content-Type: text/html\r\n");
+#if 0
+	strcat(buf, "Transfer-Encoding: chunked\r\n");
+#else 
+	sprintf(content, "Content-Length: %ld\r\n", strlen(HTML_PAGE));
+	strcat(buf, content);
+#endif 
+	strcat(buf, "\r\n");
+
+    strcat(buf, HTML_PAGE);
+	
+	int ret = nty_send(client, buf, strlen(buf), 0);
+    if (ret == -1) {
+        printf("send : errno : %d\n", errno);
+    } else {
+        //printf("headers send ret : %d\n", ret);
+    }
+    
+
+}
+
+void serve_file(int client)
+{
+    headers(client);
+}
+
+#define MAX_EPOLLSIZE		10240
+
+void server(void *arg) {
+
+	int fd = *(int *)arg;
+	
+	while (1) {
+
+		struct sockaddr_in client_addr;
+		memset(&client_addr, 0, sizeof(struct sockaddr_in));
+		socklen_t client_len = sizeof(client_addr);
+
+		if (lock()) {
+			nty_coroutine_sleep(0);
+			continue;
+		}
+		
+		int clientfd = nty_accept(fd, (struct sockaddr*)&client_addr, &client_len);
+		unlock();
+
+		if (clientfd < 0) {
+			return ;
+		}
+
+		INC_COUNTFD;
+
+		nty_coroutine *read_co;
+		int *client_sock = (int *)malloc(sizeof(int));
+		*client_sock = clientfd;
+		nty_coroutine_create(&read_co, accept_request, client_sock);
+
+		print_countfds();
+	}
+
+}
+
+int process_bind(int fd) {
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+
+	self_id = syscall(__NR_gettid);
+	//printf("selfid --> %d\n", self_id);
+
+	cpu_set_t mask;
+
+	CPU_ZERO(&mask);
+	CPU_SET(self_id % num, &mask);
+
+	sched_setaffinity(0, sizeof(mask), &mask);
+
+	nty_coroutine *co = NULL;
+	nty_coroutine_create(&co, server, &fd);
+
+	nty_schedule_run();
+}
+
+int init_shmvalue(int num) {
+	
+	global_shmid = shmget(IPC_PRIVATE, sizeof(shm_area), IPC_CREAT|0600);
+	if (global_shmid < 0) {
+		perror("shmget failed\n");
+		return -1;
+	}
+	global_shmaddr = (shm_area*)shmat(global_shmid, NULL, 0);
+	if (global_shmaddr == (shm_area*)-1) {
+		perror("shmat addr error");
+		return -1;
+	}
+	memset(global_shmaddr->totalfds, 0, TOTALFDS * sizeof(int));
+	memset(global_shmaddr->cpu_lb, 0, TOTALFDS * sizeof(char));
+	global_shmaddr->accept_mtx = 0;
+}
+
+
+int main(int argc, char *argv[]) {
+
+    short port = 9001;
+	struct sockaddr_in client_name;
+	socklen_t  client_name_len = sizeof(client_name);
+
+	int fd = nty_socket(AF_INET, SOCK_STREAM, 0);
+	if (fd < 0) return -1;
+
+	struct sockaddr_in local, remote;
+	local.sin_family = AF_INET;
+	local.sin_port = htons(port);
+	local.sin_addr.s_addr = INADDR_ANY;
+	bind(fd, (struct sockaddr*)&local, sizeof(struct sockaddr_in));
+
+	listen(fd, 5);
+
+	int num = sysconf(_SC_NPROCESSORS_CONF);
+	cpu_size = num;
+	init_shmvalue(num);
+	
+	int i = 0;
+	pid_t pid = 0;
+	for(i = 0;i < num;i ++) {
+		pid = fork();
+		if(pid <= (pid_t) 0)
+		{
+		   usleep(1);
+		   break;
+		}
+  	}
+
+	if (pid > 0) {
+		printf("ntyco_httpd server running ...\n");
+		getchar();
+	} else if (pid == 0) {
+		process_bind(fd);
+	}
+
+}
+
+
+
+
+
+

NtyCo/websocket/websocket_client.html

@@ -0,0 +1,52 @@
+<button onclick="svc_connectPlatform()"> connect</button>
+<button onclick="svc_send('hello, websocket')"> send</button>
+<script>
+
+    function svc_connectPlatform() {
+        //alert("");
+        var wsServer = 'ws://192.168.189.128:9096/';
+        try {
+            svc_websocket = new WebSocket(wsServer);
+        } catch (evt) {
+            console.log("new WebSocket error:" + evt.data);
+            svc_websocket = null;
+            if (typeof(connCb) != "undefined" && connCb != null)
+                connCb("-1", "connect error!");
+            return;
+        }
+        //alert("");
+        svc_websocket.onopen = svc_onOpen;
+        svc_websocket.onclose = svc_onClose;
+        svc_websocket.onmessage = svc_onMessage;
+        svc_websocket.onerror = svc_onError;
+    }
+
+
+    function svc_onOpen(evt) {
+		console.log("Connected to WebSocket server.");
+    }
+
+
+    function svc_onClose(evt) {
+        console.log("Disconnected");
+    }
+
+
+    function svc_onMessage(evt) {
+        alert('Retrieved data from server: ' + evt.data);
+    }
+
+
+    function svc_onError(evt) {
+        console.log('Error occured: ' + evt.data);
+    }
+
+
+    function svc_send(msg) {
+        if (svc_websocket.readyState == WebSocket.OPEN) {
+            svc_websocket.send(msg);
+        } else {
+            console.log("send failed. websocket not open. please check.");
+        }
+    }
+</script>

README.md

@@ -1,71 +1,38 @@
 # 9.1 Kvstore
 
-#### 目前KV存储没有持久化，down 之后 就清空。
+## 环境安装
-1. 全量持久化。
+```shell
-save() -> 全保存数据集。
+sudo apt install libxml2 libxml2-dev
-2. 增量持久化。
+```
-每执行一条，把命令写入文件里。下次允许把日志重新运行。
 
-#### 目前KV存储 key value 都是调用的 malloc， 都是系统的内存分配。
+## 需求
-1. 会出现大量的内存碎片，实现一个内存池，管理内存数据。
+- ntyco需要作为kvstore的submodule,通过git clone一次下载。 **完成**。
-2. 对比有内存池和没有内存池的性能差异，以及开源内存池 jemalloc 的性能差别。
+- README需要包含编译步骤，测试方案与可行性，性能数据。
-`sudo apt-get install -y libjemalloc-dev` 
+- 增量持久化需要包含完整的指令。
+- 全量持久化保存数据集。 
+- 持久化的性能数据。
 
-测试：
+- 特殊字符，可以解决redis的resp协议。
+```
+简单字符串
++OK\r\n
+错误
+-ERR message\r\n
+整数
+:1000\r\n
+批量字符串
+$6\r\nfoobar\r\n
+数组 
+*2\r\n$3\r\nfoo\r\n$3\r\nbar\r\n
+```
 
-虚拟内存的占用情况 htop。插入百万条数据集(KV*100w，set 200w del 100w delete 200w set 100w)。
+- 实现配置文件，把日志级别，端口ip，主从模式，持久化方案。 **完成**。
-
+- 持久化落盘用io_uring，加载配置文件用mmap。
-未运行状态 VIRT 24.8M RES 4004
+- 主从同步的性能，开启与关闭性能做到。
-![alt text](img/未运行状态.png)
+- 主从同步60ew条,出现的coredump。
-
+- 主从同步用ebpf实现。
-一次执行100条，执行60000次。
+- 内存池测试qps与虚拟内存，物理内存。
-
+- 实现一个内存泄露检测组件。
-malloc：\
---> time_used=192898 ms, ops=6000000, qps=31104\
-VIRT 377M RES 367M 
-
-jemalloc：\
---> time_used=196866 ms, ops=6000000, qps=30477\
-VIRT 433M RES 377M --> 11096 
-
-自实现内存池：\ 
---> time_used=180338 ms, ops=6000000, qps=33270\
-VIRT 721M  RES 628M 
-
---> time_used=170471 ms, ops=6000000, qps=35196\
-VIRT 567M  RES 558M
-
-
-
-性能分析：
-1. key : 
-    1. A_0: mp_alloc (3+8B) -> 16B,  malloc (3B) -> 8B
-    2. A_1000: mp_alloc (6+8B) -> 16B, malloc (3B) -> 8B 
-    2. A_100000 : mp_alloc(8+8B) -> 16B, malloc (8B) -> 16B
-
-内存初始状态 25B, malloc (115-25B) = 90B, mp_alloc(193-25B) = 168B，平均浪费接近50%。
-
-#### 目前的kv存储是单指令的，一条指令一交互。
-从单挑指令实现批量处理。
-
-对于同一个连接，目前缺少对以下情况的处理：
-1. 收到半包
-2. 收到多条消息
-
-对不同连接，隔离他们的消息到不同的缓冲区？要不先不考虑不同的连接的情况？Nytoco代码不是很熟。
-
-如果read->execute->send->read同步循环，不会出现缓冲区被覆盖的情况。但是可能会有半包或者两个命令都抵达了，才触发事件。所以应用层需要解析到完整请求才执行下一步。
-
-
-#### 目前KV存储是一个单点服务，实现一个主从同步的功能。
-主从同步：启动两个KVstore。在a里set，在b里get。
-
-#### 目前的Key和Value都是单个单词的，实现支持特殊字符。
-如：空格回车换行。支持插入博客。
-
-这里需要在协议中定义消息长度而不是特殊字符做分隔来确定哪些数据是key哪些是value。
-
-还要修改底层，不要用strlen和strcpy，用memcpy。
 
 
 ### 面试题

common/config.c

@@ -0,0 +1,237 @@
+#include "config.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <strings.h>      // strcasecmp
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+
+static xmlNodePtr find_child(xmlNodePtr parent, const char *name)
+{
+    if (!parent || !name) return NULL;
+
+    for (xmlNodePtr n = parent->children; n; n = n->next)
+    {
+        if (n->type == XML_ELEMENT_NODE &&
+            xmlStrcmp(n->name, BAD_CAST name) == 0)
+        {
+            return n;
+        }
+    }
+    return NULL;
+}
+
+static void set_default_config(AppConfig *cfg)
+{
+    memset(cfg, 0, sizeof(*cfg));
+    strncpy(cfg->ip, "127.0.0.1", sizeof(cfg->ip) - 1);
+    cfg->ip[sizeof(cfg->ip) - 1] = '\0';
+    strncpy(cfg->master_ip, "127.0.0.1", sizeof(cfg->master_ip) - 1);
+    cfg->master_ip[sizeof(cfg->master_ip) - 1] = '\0';
+
+    cfg->port        = 8888;
+    cfg->log_level   = LOG_LEVEL_INFO;
+    cfg->mode        = MODE_MASTER;
+    cfg->master_port = 8888;
+    cfg->persistence = PERSIST_NONE;
+    cfg->allocator   = ALLOC_JEMALLOC;
+}
+
+/* ---- 字符串转枚举 ---- */
+
+static void parse_log_level(const char *s, LogLevel *out)
+{
+    if (!s || !out) return;
+
+    if (!strcasecmp(s, "DEBUG"))      *out = LOG_LEVEL_DEBUG;
+    else if (!strcasecmp(s, "INFO"))  *out = LOG_LEVEL_INFO;
+    else if (!strcasecmp(s, "ERROR")) *out = LOG_LEVEL_ERROR;
+    /* 不认识就保持默认 */
+}
+
+static void parse_server_mode(const char *s, ServerMode *out)
+{
+    if (!s || !out) return;
+    if (!strcasecmp(s, "master")) *out = MODE_MASTER;
+    else if (!strcasecmp(s, "slave")) *out = MODE_SLAVE;
+}
+
+static void parse_persistence(const char *s, PersistenceType *out)
+{
+    if (!s || !out) return;
+    if (!strcasecmp(s, "incremental")) *out = PERSIST_INCREMENTAL;
+    else if (!strcasecmp(s, "none")) *out = PERSIST_NONE;
+}
+
+static void parse_allocator(const char *s, AllocatorType *out)
+{
+    if (!s || !out) return;
+    if (!strcasecmp(s, "jemalloc")) *out = ALLOC_JEMALLOC;
+    else if (!strcasecmp(s, "malloc")) *out = ALLOC_MALLOC;
+    else *out = ALLOC_OTHER;
+}
+
+/* ---- 对外的枚举转字符串工具 ---- */
+
+const char *log_level_to_string(LogLevel lvl)
+{
+    switch (lvl) {
+    case LOG_LEVEL_DEBUG: return "DEBUG";
+    case LOG_LEVEL_INFO:  return "INFO";
+    case LOG_LEVEL_ERROR: return "ERROR";
+    default:              return "UNKNOWN";
+    }
+}
+
+const char *server_mode_to_string(ServerMode mode)
+{
+    switch (mode) {
+    case MODE_MASTER: return "master";
+    case MODE_SLAVE:  return "slave";
+    default:          return "unknown";
+    }
+}
+
+const char *persistence_to_string(PersistenceType p)
+{
+    switch (p) {
+    case PERSIST_INCREMENTAL: return "incremental";
+    case PERSIST_NONE:        return "none";
+    default:                  return "unknown";
+    }
+}
+
+const char *allocator_to_string(AllocatorType a)
+{
+    switch (a) {
+    case ALLOC_JEMALLOC: return "jemalloc";
+    case ALLOC_MALLOC:   return "malloc";
+    case ALLOC_OTHER:    return "other";
+    default:             return "unknown";
+    }
+}
+
+/* ---- 主函数：从 XML 加载配置 ---- */
+
+int config_load(const char *filename, AppConfig *out_cfg)
+{
+    if (!filename || !out_cfg) return -1;
+
+    set_default_config(out_cfg);
+
+    xmlDocPtr doc = xmlReadFile(filename, "UTF-8", XML_PARSE_NOBLANKS);
+    if (!doc) {
+        fprintf(stderr, "config_load: failed to read file %s\n", filename);
+        return -1;
+    }
+
+    xmlNodePtr root = xmlDocGetRootElement(doc);
+    if (!root || xmlStrcmp(root->name, BAD_CAST "config") != 0) {
+        fprintf(stderr, "config_load: root element is not <config>\n");
+        xmlFreeDoc(doc);
+        return -1;
+    }
+
+    /* server 部分 */
+    xmlNodePtr server = find_child(root, "server");
+    if (server) {
+        /* ip */
+        xmlNodePtr ip_node = find_child(server, "ip");
+        if (ip_node) {
+            xmlChar *txt = xmlNodeGetContent(ip_node);
+            if (txt) {
+                strncpy(out_cfg->ip, (char *)txt, sizeof(out_cfg->ip) - 1);
+                out_cfg->ip[sizeof(out_cfg->ip) - 1] = '\0';
+                xmlFree(txt);
+            }
+        }
+
+        /* port */
+        xmlNodePtr port_node = find_child(server, "port");
+        if (port_node) {
+            xmlChar *txt = xmlNodeGetContent(port_node);
+            if (txt) {
+                out_cfg->port = atoi((char *)txt);
+                xmlFree(txt);
+            }
+        }
+
+        /* mode: master / slave */
+        xmlNodePtr mode_node = find_child(server, "mode");
+        if (mode_node) {
+            xmlChar *txt = xmlNodeGetContent(mode_node);
+            if (txt) {
+                parse_server_mode((char *)txt, &out_cfg->mode);
+                xmlFree(txt);
+            }
+        }
+
+        /* master (always read if present) */
+        xmlNodePtr master = find_child(server, "master");
+        if (master) {
+            xmlNodePtr mip_node = find_child(master, "ip");
+            if (mip_node) {
+                xmlChar *txt = xmlNodeGetContent(mip_node);
+                if (txt) {
+                    strncpy(out_cfg->master_ip, (char *)txt, sizeof(out_cfg->master_ip) - 1);
+                    out_cfg->master_ip[sizeof(out_cfg->master_ip) - 1] = '\0';
+                    xmlFree(txt);
+                }
+            }
+
+            xmlNodePtr mport_node = find_child(master, "port");
+            if (mport_node) {
+                xmlChar *txt = xmlNodeGetContent(port_node);
+                if (txt) {
+                    out_cfg->master_port = atoi((char *)txt);
+                    xmlFree(txt);
+                }
+            }
+        }
+    }
+
+    /* log 部分 */
+    xmlNodePtr log = find_child(root, "log");
+    if (log) {
+        xmlNodePtr lvl_node = find_child(log, "level");
+        if (lvl_node) {
+            xmlChar *txt = xmlNodeGetContent(lvl_node);
+            if (txt) {
+                parse_log_level((char *)txt, &out_cfg->log_level);
+                xmlFree(txt);
+            }
+        }
+    }
+
+    /* persistence 部分 */
+    xmlNodePtr pers = find_child(root, "persistence");
+    if (pers) {
+        xmlNodePtr type_node = find_child(pers, "type");
+        if (type_node) {
+            xmlChar *txt = xmlNodeGetContent(type_node);
+            if (txt) {
+                parse_persistence((char *)txt, &out_cfg->persistence);
+                xmlFree(txt);
+            }
+        }
+    }
+
+    /* memory 部分 */
+    xmlNodePtr mem = find_child(root, "memory");
+    if (mem) {
+        xmlNodePtr alloc_node = find_child(mem, "allocator");
+        if (alloc_node) {
+            xmlChar *txt = xmlNodeGetContent(alloc_node);
+            if (txt) {
+                parse_allocator((char *)txt, &out_cfg->allocator);
+                xmlFree(txt);
+            }
+        }
+    }
+
+    xmlFreeDoc(doc);
+    /* xmlCleanupParser() 建议在程序退出时 main 里统一调用 */
+
+    return 0;
+}

common/config.h

@@ -0,0 +1,52 @@
+#ifndef __CONFIG_H__
+#define __CONFIG_H__
+
+#include <stddef.h>
+
+typedef enum {
+    LOG_LEVEL_DEBUG,
+    LOG_LEVEL_INFO,
+    LOG_LEVEL_ERROR
+} LogLevel;
+
+typedef enum {
+    MODE_MASTER,
+    MODE_SLAVE
+} ServerMode;
+
+typedef enum {
+    PERSIST_INCREMENTAL,
+    PERSIST_NONE
+} PersistenceType;
+
+typedef enum {
+    ALLOC_JEMALLOC,
+    ALLOC_MALLOC,
+    ALLOC_OTHER
+} AllocatorType;
+
+typedef struct {
+    char            ip[64];
+    int             port;
+    ServerMode      mode;
+
+    char            master_ip[64];  // slave 才需要
+    int             master_port;    // slave 才需要
+
+    LogLevel        log_level;
+    PersistenceType persistence;
+    AllocatorType   allocator;
+} AppConfig;
+
+/**
+ * 从 XML 文件加载配置
+ * 返回 0 表示成功；非 0 表示失败
+ */
+int config_load(const char *filename, AppConfig *out_cfg);
+
+const char *log_level_to_string(LogLevel lvl);
+const char *server_mode_to_string(ServerMode mode);
+const char *persistence_to_string(PersistenceType p);
+const char *allocator_to_string(AllocatorType a);
+
+#endif /* CONFIG_H */

config/config.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config>
+    <server>
+        <ip>127.0.0.1</ip>
+        <port>8888</port>
+        <mode>master</mode> <!-- master / slave -->
+
+        <!-- 仅当 mode=slave 时使用 -->
+        <master>
+            <ip>192.168.10.129</ip>
+            <port>8888</port>
+        </master>
+    </server>
+
+    <log>
+        <level>INFO</level> <!-- DEBUG / INFO / ERROR -->
+    </log>
+
+    <persistence>
+        <type>none</type> <!-- incremental / none -->
+    </persistence>
+
+    <memory>
+        <allocator>jemalloc</allocator> <!-- jemalloc / malloc / other -->
+    </memory>
+</config>

kvs_array_bin.c

@@ -54,7 +54,7 @@ int kvs_array_find_index(kvs_array_t *inst, const void *key, uint32_t key_len) {
 }
 
 /*
- * return: <0 error; 0 success; 1 exist
+ * @return: <0 error; 0 success; 1 exist
  */
 int kvs_array_set_bin(kvs_array_t *inst,
                       const void *key, uint32_t key_len,
@@ -111,7 +111,7 @@ int kvs_array_set_bin(kvs_array_t *inst,
 }
 
 /*
- * @return: NULL notexist, NOTNULL exist
+ * @return: NULL notexist, NOTNULL exist。out_value_len 是长度。
  */
 void *kvs_array_get_bin(kvs_array_t *inst, const void *key, uint32_t key_len, uint32_t *out_value_len) {
     if (out_value_len) *out_value_len = 0;
@@ -126,7 +126,7 @@ void *kvs_array_get_bin(kvs_array_t *inst, const void *key, uint32_t key_len, ui
 }
 
 /*
- * @return < 0, error;  =0,  success; >0, no exist
+ * @return <0 error; =0 success; >0 no exist
  */
 int kvs_array_del_bin(kvs_array_t *inst, const void *key, uint32_t key_len) {
     if (!inst || !inst->table || !key || key_len == 0) return -1;
@@ -153,9 +153,8 @@ int kvs_array_del_bin(kvs_array_t *inst, const void *key, uint32_t key_len) {
 }
 
 /*
- * @return : < 0, error; =0, success; >0, no exist 
+ * @return < 0 error; =0 success; >0 no exist 
  */
-
 int kvs_array_mod_bin(kvs_array_t *inst,
                       const void *key, uint32_t key_len,
                       const void *value, uint32_t value_len) {
@@ -184,7 +183,7 @@ int kvs_array_mod_bin(kvs_array_t *inst,
 }
 
 /*
- * @return 0: exist, 1: no exist
+ * @return =0 exist, =1 no exist
  */
 int kvs_array_exist_bin(kvs_array_t *inst, const void *key, uint32_t key_len) {
     if (!inst || !inst->table || !key || key_len == 0) return -1;

kvs_hash_bin.c

@@ -113,7 +113,7 @@ void kvs_hash_destroy(kvs_hash_t *hash) {
 
 // mp
 /*
- * @return: <0 error; =0 success; >0 exist
+ * @return: <0 error; 0 success; 1 exist
  */
 int kvs_hash_set_bin(kvs_hash_t *hash, const void *key, uint32_t key_len, const void *value, uint32_t value_len) {
 	if (!hash || !hash->nodes || !key || key_len == 0) return -1;
@@ -140,7 +140,7 @@ int kvs_hash_set_bin(kvs_hash_t *hash, const void *key, uint32_t key_len, const
 }
 
 /*
- * get 返回：value 指针（由 hash 持有），并通过 out_value_len 返回长度
+ * @return: NULL notexist, NOTNULL exist。out_value_len 是长度。
  */
 void *kvs_hash_get_bin(kvs_hash_t *hash, const void *key, uint32_t key_len, uint32_t *out_value_len) {
 	if (!hash || !hash->nodes || !key || key_len == 0 || !out_value_len) return NULL;
@@ -210,7 +210,7 @@ int kvs_hash_count(kvs_hash_t *hash) {
 }
 
 /*
- * @return < 0, error;  =0,  success; >0, no exist
+ * @return <0 error; =0 success; >0 no exist
  */
 int kvs_hash_del_bin(kvs_hash_t *hash, const void *key, uint32_t key_len) {
 	if (!hash || !key || key_len == 0) return -1;
@@ -258,7 +258,7 @@ int kvs_hash_del_bin(kvs_hash_t *hash, const void *key, uint32_t key_len) {
 }
 
 /*
- * @return 0 exist, 1 no exist
+ * @return =0 exist, =1 no exist
  */
 int kvs_hash_exist_bin(kvs_hash_t *hash, const void *key, uint32_t key_len) {
 	uint32_t vlen = 0;

kvs_protocol_resp.c

@@ -0,0 +1,550 @@
+#include "kvs_protocol_resp.h"
+
+
+#if ENABLE_ARRAY
+extern kvs_array_t global_array;
+#endif
+
+#if ENABLE_RBTREE
+extern kvs_rbtree_t global_rbtree;
+#endif
+
+#if ENABLE_HASH
+extern kvs_hash_t global_hash;
+#endif
+
+static int need(const uint8_t *p, const uint8_t *end, size_t n) {
+    return (p + n <= end) ? 0 : -1;
+}
+
+/* find "\r\n", return \r */
+static int find_crlf(const uint8_t *p, const uint8_t *end, const uint8_t **line_end) {
+    const uint8_t *q = p;
+    size_t lim = (size_t)(end - p);
+    if (lim > (size_t)RESP_MAX_LINE) lim = (size_t)RESP_MAX_LINE;
+
+    const uint8_t *stop = p + lim;
+    while (q + 1 < stop) {
+        if (q[0] == '\r' && q[1] == '\n') {
+            *line_end = q;
+            return 0;
+        }
+        q++;
+    }
+    return -1;
+}
+
+/* 解析有符号整数 [p, line_end) */
+static int parse_i64(const uint8_t *p, const uint8_t *line_end, int64_t *out) {
+    if (!p || !line_end || p >= line_end) return -1;
+
+    int neg = 0;
+    if (*p == '-') {
+        neg = 1;
+        p++;
+        if (p >= line_end) return -1;
+    }
+
+    int64_t x = 0;
+    for (const uint8_t *q = p; q < line_end; q++) {
+        if (*q < '0' || *q > '9') return -1;
+        int digit = (int)(*q - '0');
+        /* overflow-safe-ish for typical Redis sizes */
+        if (x > (INT64_MAX - digit) / 10) return -1;
+        x = x * 10 + digit;
+    }
+    *out = neg ? -x : x;
+    return 0;
+}
+
+// 字符串比对
+static int ascii_casecmp(const uint8_t *a, uint32_t alen, const char *b) {
+    size_t blen = strlen(b);
+    if (alen != (uint32_t)blen) return -1;
+    for (uint32_t i = 0; i < alen; i++) {
+        uint8_t ca = a[i];
+        uint8_t cb = (uint8_t)b[i];
+        if (ca >= 'A' && ca <= 'Z') ca = (uint8_t)(ca + 32);
+        if (cb >= 'A' && cb <= 'Z') cb = (uint8_t)(cb + 32);
+        if (ca != cb) return -1;
+    }
+    return 0;
+}
+
+/* ----------------- RESP2 parser (one command) ----------------- */
+
+/* 解析批量字符串：$<长度>\r\n<字节>\r\n
+ * 成功时，将切片指向 <字节> 部分，并前进 *pp。
+ * 返回值：-1 错误，0 需要更多数据，1 成功
+ */
+static int parse_bulk(const uint8_t **pp, const uint8_t *end, resp_slice_t *out) {
+    const uint8_t *p = *pp;
+    if (need(p, end, 1) < 0) return 0;
+    if (*p != '$') return -1;
+    p++;
+
+    const uint8_t *le = NULL;
+    // 寻找 <长度> 的末尾
+    if (find_crlf(p, end, &le) < 0) return 0; /* need more */
+    int64_t n64 = 0;
+    // 解析 <长度> 
+    if (parse_i64(p, le, &n64) < 0) return -1;
+    p = le + 2; /* 跳过 CRLF */
+
+    if (n64 < 0) {
+        /* nil bulk ($-1) 无效. */
+        return -1;
+    }
+    /* 大于 RESP_MAX_BULK 也无效 */
+    if ((uint64_t)n64 > (uint64_t)RESP_MAX_BULK) return -1;
+
+    uint32_t n = (uint32_t)n64;
+    if (need(p, end, (size_t)n + 2) < 0) return 0; /* need more */
+    
+    // byte = <字节> 的首位
+    const uint8_t *bytes = p;
+    p += n;
+    // <字节>+n 不是\r\n 解析错误
+    if (p[0] != '\r' || p[1] != '\n') return -1;
+    // 跳过\r\n
+    p += 2;
+
+    out->ptr = bytes;
+    out->len = n;
+    // 移动指针
+    *pp = p;
+    return 1;
+}
+
+/* 解析数组头: *<n>\r\n */
+static int parse_array_len(const uint8_t **pp, const uint8_t *end, int64_t *out_n) {
+    const uint8_t *p = *pp;
+    if (need(p, end, 1) < 0) return 0;
+    if (*p != '*') return -1;
+    p++;
+
+    const uint8_t *le = NULL;
+    if (find_crlf(p, end, &le) < 0) return 0;
+    int64_t n64 = 0;
+    if (parse_i64(p, le, &n64) < 0) return -1;
+    p = le + 2;
+    *pp = p;
+    *out_n = n64;
+    return 1;
+}
+
+/* 解析简单命令 */
+static int parse_inline(const uint8_t *buf, int len, resp_cmd_t *out_cmd) {
+    const uint8_t *p = buf;
+    const uint8_t *end = buf + (size_t)len;
+
+    const uint8_t *le = NULL;
+    if (find_crlf(p, end, &le) < 0) return 0; /* need more */
+
+    /* split [p, le) by spaces/tabs */
+    out_cmd->argc = 0;
+
+    const uint8_t *s = p;
+    while (s < le) {
+        while (s < le && (*s == ' ' || *s == '\t')) s++;
+        if (s >= le) break;
+
+        const uint8_t *t = s;
+        while (t < le && *t != ' ' && *t != '\t') t++;
+
+        if (out_cmd->argc >= RESP_MAX_ARGC) return -1;
+        out_cmd->argv[out_cmd->argc].ptr = s;
+        out_cmd->argv[out_cmd->argc].len = (uint32_t)(t - s);
+        out_cmd->argc++;
+
+        s = t;
+    }
+
+    if (out_cmd->argc == 0) return -1;
+    return (int)((le + 2) - buf);
+}
+
+int resp_parse_one_cmd(const uint8_t *buf, int len, resp_cmd_t *out_cmd) {
+    if (!buf || len <= 0 || !out_cmd) return -1;
+
+    memset(out_cmd, 0, sizeof(*out_cmd));
+
+    const uint8_t *p = buf;
+    const uint8_t *end = buf + (size_t)len;
+
+    if (need(p, end, 1) < 0) return 0;
+
+    if (*p != '*') {
+        /* inline */
+        return parse_inline(buf, len, out_cmd);
+    }
+
+    /* multi bulk */
+    int64_t n64 = 0;
+    int r = parse_array_len(&p, end, &n64);
+    if (r == 0) return 0;
+    if (r < 0) return -1;
+    if (n64 <= 0 || n64 > (int64_t)RESP_MAX_ARGC) return -1;
+
+    out_cmd->argc = (uint32_t)n64;
+
+    /* scan + parse each bulk string */
+    for (uint32_t i = 0; i < out_cmd->argc; i++) {
+        resp_slice_t sl = {0};
+        int rr = parse_bulk(&p, end, &sl);
+        if (rr == 0) return 0;
+        if (rr < 0) return -1;
+        out_cmd->argv[i] = sl;
+    }
+
+    return (int)(p - buf);
+}
+
+
+
+
+/* ----------------- RESP2 builder ----------------- */
+
+static int write_bytes(uint8_t **pp, const uint8_t *end, const void *src, size_t n) {
+    uint8_t *p = *pp;
+    if ((size_t)(end - p) < n) return -1;
+    memcpy(p, src, n);
+    p += n;
+    *pp = p;
+    return 0;
+}
+
+static int write_crlf(uint8_t **pp, const uint8_t *end) {
+    return write_bytes(pp, end, "\r\n", 2);
+}
+
+static int write_i64_ascii(uint8_t **pp, const uint8_t *end, int64_t x) {
+    char tmp[64];
+    int n = snprintf(tmp, sizeof(tmp), "%lld", (long long)x);
+    if (n <= 0) return -1;
+    return write_bytes(pp, end, tmp, (size_t)n);
+}
+
+static int write_u32_ascii(uint8_t **pp, const uint8_t *end, uint32_t x) {
+    char tmp[32];
+    int n = snprintf(tmp, sizeof(tmp), "%u", (unsigned)x);
+    if (n <= 0) return -1;
+    return write_bytes(pp, end, tmp, (size_t)n);
+}
+
+int resp_build_value(const resp_value_t *v, uint8_t *out, size_t cap) {
+    if (!v || !out || cap == 0) return -1;
+
+    uint8_t *p = out;
+    const uint8_t *end = out + cap;
+
+    switch (v->type) {
+    case RESP_T_SIMPLE_STR:
+        if (write_bytes(&p, end, "+", 1) < 0) return -1;
+        if (write_bytes(&p, end, v->bulk.ptr, v->bulk.len) < 0) return -1;
+        if (write_crlf(&p, end) < 0) return -1;
+        break;
+
+    case RESP_T_ERROR:
+        if (write_bytes(&p, end, "-", 1) < 0) return -1;
+        if (write_bytes(&p, end, v->bulk.ptr, v->bulk.len) < 0) return -1;
+        if (write_crlf(&p, end) < 0) return -1;
+        break;
+
+    case RESP_T_INTEGER:
+        if (write_bytes(&p, end, ":", 1) < 0) return -1;
+        if (write_i64_ascii(&p, end, v->i64) < 0) return -1;
+        if (write_crlf(&p, end) < 0) return -1;
+        break;
+
+    case RESP_T_NIL:
+        if (write_bytes(&p, end, "$-1\r\n", 5) < 0) return -1;
+        break;
+
+    case RESP_T_BULK_STR:
+        if (write_bytes(&p, end, "$", 1) < 0) return -1;
+        if (write_u32_ascii(&p, end, v->bulk.len) < 0) return -1;
+        if (write_crlf(&p, end) < 0) return -1;
+        if (v->bulk.len > 0 && v->bulk.ptr) {
+            if (write_bytes(&p, end, v->bulk.ptr, v->bulk.len) < 0) return -1;
+        }
+        if (write_crlf(&p, end) < 0) return -1;
+        break;
+
+    default:
+        return -1;
+    }
+
+    return (int)(p - out);
+}
+
+/* helpers */
+resp_value_t resp_simple(const char *s) {
+    resp_value_t v;
+    v.type = RESP_T_SIMPLE_STR;
+    v.i64 = 0;
+    v.bulk.ptr = (const uint8_t*)s;
+    v.bulk.len = (uint32_t)strlen(s);
+    return v;
+}
+
+resp_value_t resp_error(const char *s) {
+    resp_value_t v;
+    v.type = RESP_T_ERROR;
+    v.i64 = 0;
+    v.bulk.ptr = (const uint8_t*)s;
+    v.bulk.len = (uint32_t)strlen(s);
+    return v;
+}
+
+resp_value_t resp_int(int64_t x) {
+    resp_value_t v;
+    v.type = RESP_T_INTEGER;
+    v.i64 = x;
+    v.bulk.ptr = NULL;
+    v.bulk.len = 0;
+    return v;
+}
+
+resp_value_t resp_bulk(const uint8_t *p, uint32_t n) {
+    resp_value_t v;
+    v.type = RESP_T_BULK_STR;
+    v.i64 = 0;
+    v.bulk.ptr = p;
+    v.bulk.len = n;
+    return v;
+}
+
+resp_value_t resp_nil(void) {
+    resp_value_t v;
+    v.type = RESP_T_NIL;
+    v.i64 = 0;
+    v.bulk.ptr = NULL;
+    v.bulk.len = 0;
+    return v;
+}
+
+/* ----------------- dispatcher (minimal) ----------------- */
+
+static int expect_argv(const resp_cmd_t *cmd, uint32_t n) {
+    return (cmd && cmd->argc == n) ? 0 : -1;
+}
+
+
+const char *command[] = {
+	"SET", "GET", "DEL", "MOD", "EXIST",
+	"RSET", "RGET", "RDEL", "RMOD", "REXIST",
+	"HSET", "HGET", "HDEL", "HMOD", "HEXIST",
+	"SAVE", "PSYNC"
+};
+
+
+/**
+ * 输入：cmd
+ * 输出：out_value
+ * 返回：-1 失败，参数错误，0 成功
+ */
+int resp_dispatch(const resp_cmd_t *cmd, resp_value_t *out_value) {
+    if (!cmd || !out_value) return -1;
+    if (cmd->argc == 0 || cmd->argv[0].ptr == NULL) {
+        *out_value = resp_error("ERR empty command");
+        return 0;
+    }
+
+    const uint8_t *cptr = cmd->argv[0].ptr;
+    uint32_t clen = cmd->argv[0].len;
+
+    kvs_cmd_t op = KVS_CMD_COUNT;
+    for(kvs_cmd_t i = KVS_CMD_START; i < KVS_CMD_COUNT; ++ i){
+        if(ascii_casecmp(cptr, clen, command[i]) == 0){
+            op = i;
+            break;
+        }
+    }
+
+    
+    if (op == KVS_CMD_COUNT) {
+        *out_value = resp_error("ERR unknown command");
+        return -1;
+    }
+
+	switch (op) {
+#if ENABLE_ARRAY
+    case KVS_CMD_SET: {
+        if (cmd->argc != 3) { *out_value = resp_error("ERR wrong number of arguments for 'set'"); return -1; }
+        // <0 error; 0 success; 1 exist
+        int r = kvs_array_set_bin(&global_array,
+                                 cmd->argv[1].ptr, cmd->argv[1].len,
+                                 cmd->argv[2].ptr, cmd->argv[2].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        else if (r == 1) { *out_value = resp_error("ERR key has exist"); return 0; }
+        *out_value = resp_simple("OK");
+        return 0;
+    }
+
+    case KVS_CMD_GET: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'get'"); return -1; }
+        uint32_t vlen = 0;
+        // NULL not exist, NOTNULL exist
+        const char *v = kvs_array_get_bin(&global_array, cmd->argv[1].ptr, cmd->argv[1].len, &vlen);
+        if (!v) { *out_value = resp_nil(); return 0; }
+        *out_value = resp_bulk((const uint8_t*)v, vlen);
+        return 0;
+    }
+
+    case KVS_CMD_DEL: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'del'"); return -1; }
+        // <0 error; =0 success; >0 no exist
+        int r = kvs_array_del_bin(&global_array, cmd->argv[1].ptr, cmd->argv[1].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        // r == 0, del 1; r > 0, del 0.
+        *out_value = resp_int((r == 0) ? 1 : 0);
+        return 0;
+    }
+
+    case KVS_CMD_MOD: {
+        if (cmd->argc != 3) { *out_value = resp_error("ERR wrong number of arguments for 'mod'"); return -1; }
+        // <0 error; =0 success; >0 no exist 
+        int r = kvs_array_mod_bin(&global_array,
+                                 cmd->argv[1].ptr, cmd->argv[1].len,
+                                 cmd->argv[2].ptr, cmd->argv[2].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        if (r == 0) *out_value = resp_simple("OK");
+        else *out_value = resp_error("ERR no such key");
+        return 0;
+    }
+
+    case KVS_CMD_EXIST: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'exist'"); return -1; }
+        // =0 exist, =1 no exist
+        int r = kvs_array_exist_bin(&global_array, cmd->argv[1].ptr, cmd->argv[1].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_int((r == 0) ? 1 : 0);
+        return 0;
+    }
+#endif
+
+#if ENABLE_RBTREE
+	case KVS_CMD_RSET: {
+        if (cmd->argc != 3) { *out_value = resp_error("ERR wrong number of arguments for 'rset'"); return 0; }
+        // <0 error; 0 success; 1 exist
+        int r = kvs_rbtree_set(&global_rbtree,
+                              cmd->argv[1].ptr, cmd->argv[1].len,
+                              cmd->argv[2].ptr, cmd->argv[2].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_simple("OK");
+        return 0;
+    }
+
+    case KVS_CMD_RGET: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'rget'"); return 0; }
+        uint32_t vlen = 0;
+        // NULL notexist, NOTNULL exist。out_value_len 是长度。
+        const char *v = kvs_rbtree_get(&global_rbtree, cmd->argv[1].ptr, cmd->argv[1].len, &vlen);
+        if (!v) { *out_value = resp_nil(); return 0; }
+        *out_value = resp_bulk((const uint8_t*)v, vlen);
+        return 0;
+    }
+
+    case KVS_CMD_RDEL: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'rdel'"); return 0; }
+        // <0 error; =0 success; >0 no exist
+        int r = kvs_rbtree_del(&global_rbtree, cmd->argv[1].ptr, cmd->argv[1].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_int((r == 0) ? 1 : 0);
+        return 0;
+    }
+
+    case KVS_CMD_RMOD: {
+        if (cmd->argc != 3) { *out_value = resp_error("ERR wrong number of arguments for 'rmod'"); return 0; }
+        // < 0 error; =0 success; >0 no exist 
+        int r = kvs_rbtree_mod(&global_rbtree,
+                              cmd->argv[1].ptr, cmd->argv[1].len,
+                              cmd->argv[2].ptr, cmd->argv[2].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        if (r == 0) *out_value = resp_simple("OK");
+        else *out_value = resp_error("ERR no such key");
+        return 0;
+    }
+
+    case KVS_CMD_REXIST: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'rexist'"); return 0; }
+        // =0 exist, =1 no exist
+        int r = kvs_rbtree_exist(&global_rbtree, cmd->argv[1].ptr, cmd->argv[1].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_int((r == 0) ? 1 : 0);
+        return 0;
+    }
+#endif
+
+#if ENABLE_HASH
+	case KVS_CMD_HSET: {
+        if (cmd->argc != 3) { *out_value = resp_error("ERR wrong number of arguments for 'hset'"); return 0; }
+        //  <0 error; 0 success; 1 exist
+        int r = kvs_hash_set_bin(&global_hash,
+                                cmd->argv[1].ptr, cmd->argv[1].len,
+                                cmd->argv[2].ptr, cmd->argv[2].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_simple("OK");
+        return 0;
+    }
+
+    case KVS_CMD_HGET: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'hget'"); return 0; }
+        uint32_t vlen = 0;
+        // NULL notexist, NOTNULL exist。out_value_len 是长度。
+        const char *v = kvs_hash_get_bin(&global_hash, cmd->argv[1].ptr, cmd->argv[1].len, &vlen);
+        if (!v) { *out_value = resp_nil(); return 0; }
+        *out_value = resp_bulk((const uint8_t*)v, vlen);
+        return 0;
+    }
+
+    case KVS_CMD_HDEL: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'hdel'"); return 0; }
+        // <0 error; =0 success; >0 no exist
+        int r = kvs_hash_del_bin(&global_hash, cmd->argv[1].ptr, cmd->argv[1].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_int((r == 0) ? 1 : 0);
+        return 0;
+    }
+
+    case KVS_CMD_HMOD: {
+        if (cmd->argc != 3) { *out_value = resp_error("ERR wrong number of arguments for 'hmod'"); return 0; }
+        // <0 error; =0 success; >0 no exist
+        int r = kvs_hash_mod_bin(&global_hash,
+                                cmd->argv[1].ptr, cmd->argv[1].len,
+                                cmd->argv[2].ptr, cmd->argv[2].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        if (r == 0) *out_value = resp_simple("OK");
+        else *out_value = resp_error("ERR no such key");
+        return 0;
+    }
+
+    case KVS_CMD_HEXIST: {
+        if (cmd->argc != 2) { *out_value = resp_error("ERR wrong number of arguments for 'hexist'"); return 0; }
+        // =0 exist, =1 no exist
+        int r = kvs_hash_exist_bin(&global_hash, cmd->argv[1].ptr, cmd->argv[1].len);
+        if (r < 0) { *out_value = resp_error("ERR internal error"); return 0; }
+        *out_value = resp_int((r == 0) ? 1 : 0);
+        return 0;
+    }
+
+#endif
+    /* ---------------- misc ---------------- */
+    case KVS_CMD_SAVE: {
+        if (cmd->argc != 1) { *out_value = resp_error("ERR wrong number of arguments for 'save'"); return 0; }
+        int r = kvs_save_to_file();
+        if (r < 0) { *out_value = resp_error("ERR save failed"); return 0; }
+        *out_value = resp_simple("OK");
+        return 0;
+    }
+	case KVS_CMD_PSYNC:
+		*out_value = resp_simple("OK");
+		return 0;
+	default:
+		break;
+	}
+
+    *out_value = resp_error("ERR unknown command");
+    return 0;
+}

kvs_protocol_resp.h

@@ -0,0 +1,90 @@
+#ifndef __RESP_H__
+#define __RESP_H__
+
+#include <stdint.h>
+#include <stddef.h>
+#include "kvstore.h"
+
+#define RESP_MAX_ARGC  16
+#define RESP_MAX_LINE  1024
+#define RESP_MAX_BULK  (16u * 1024u * 1024u) /* 16MB default */
+
+/**
+ * SET 
+ */
+
+typedef enum {
+	KVS_CMD_START = 0,
+	// array
+	KVS_CMD_SET = KVS_CMD_START,
+	KVS_CMD_GET,
+	KVS_CMD_DEL,
+	KVS_CMD_MOD,
+	KVS_CMD_EXIST,
+	// rbtree
+	KVS_CMD_RSET,
+	KVS_CMD_RGET,
+	KVS_CMD_RDEL,
+	KVS_CMD_RMOD,
+	KVS_CMD_REXIST,
+	// hash
+	KVS_CMD_HSET,
+	KVS_CMD_HGET,
+	KVS_CMD_HDEL,
+	KVS_CMD_HMOD,
+	KVS_CMD_HEXIST,
+	
+	KVS_CMD_SAVE,
+	KVS_CMD_PSYNC,
+	KVS_CMD_COUNT,
+}kvs_cmd_t;
+
+typedef struct resp_slice_s {
+    const uint8_t *ptr;
+    uint32_t len;
+} resp_slice_t;
+
+typedef struct resp_cmd_s {
+    uint32_t argc;
+    resp_slice_t argv[RESP_MAX_ARGC]; /* argv[0] is command name */
+} resp_cmd_t;
+
+typedef enum resp_type_e {
+    RESP_T_SIMPLE_STR,
+    RESP_T_ERROR,
+    RESP_T_INTEGER,
+    RESP_T_BULK_STR,
+    RESP_T_NIL,          /* nil bulk string ($-1\r\n) */
+} resp_type_t;
+
+typedef struct resp_value_s {
+    resp_type_t type;
+    int64_t i64;         /* for integer */
+    resp_slice_t bulk;   /* for simple/error/bulk string (bytes) */
+} resp_value_t;
+
+/* ----------------- parsing ----------------- */
+/* Parse one RESP command from [buf, buf+len).
+ * return: -1 protocol error, 0 need more data, >0 bytes consumed
+ */
+int resp_parse_one_cmd(const uint8_t *buf, int len, resp_cmd_t *out_cmd);
+
+/* ----------------- response building ----------------- */
+/* Build RESP value into out buffer. return bytes written, -1 on error/capacity. */
+int resp_build_value(const resp_value_t *v, uint8_t *out, size_t cap);
+
+/* Convenience helpers */
+resp_value_t resp_simple(const char *s); /* +s\r\n */
+resp_value_t resp_error(const char *s);  /* -s\r\n */
+resp_value_t resp_int(int64_t x);        /* :x\r\n */
+resp_value_t resp_bulk(const uint8_t *p, uint32_t n); /* $n\r\n...\r\n */
+resp_value_t resp_nil(void);             /* $-1\r\n */
+
+/* ----------------- dispatcher ----------------- */
+/* Execute one command and return a RESP value.
+ * NOTE: output value may reference storage memory (for GET), so build response
+ * promptly after this call.
+ */
+int resp_dispatch(const resp_cmd_t *cmd, resp_value_t *out_value);
+
+#endif

kvs_rbtree_bin.c

@@ -364,7 +364,7 @@ void kvs_rbtree_destroy(kvs_rbtree_t *inst) {
 }
 
 /*
- * return: <0 error; 0 success; 1 exist
+ * @return: <0 error; 0 success; 1 exist
  */
 int kvs_rbtree_set(kvs_rbtree_t *inst, const void *key, uint32_t key_len, const void *value, uint32_t value_len) {
 
@@ -398,7 +398,7 @@ int kvs_rbtree_set(kvs_rbtree_t *inst, const void *key, uint32_t key_len, const
 
 
 /*
- * @return: NULL notexist, NOTNULL exist
+ * @return: NULL notexist, NOTNULL exist。out_value_len 是长度。
  */
 void* kvs_rbtree_get(kvs_rbtree_t *inst, const void *key, uint32_t key_len, uint32_t *out_valuelen)  {
 	if (!inst || !key || key_len == 0 || !out_valuelen) return NULL;
@@ -412,7 +412,7 @@ void* kvs_rbtree_get(kvs_rbtree_t *inst, const void *key, uint32_t key_len, uint
 }
 
 /*
- * @return < 0, error;  =0,  success; >0, no exist
+ * @return <0 error; =0 success; >0 no exist
  */
 int kvs_rbtree_del(rbtree *inst, const void *key, uint32_t key_len) {
 
@@ -433,9 +433,8 @@ int kvs_rbtree_del(rbtree *inst, const void *key, uint32_t key_len) {
 }
 
 /*
- * @return : < 0, error; =0, success; >0, no exist 
+ * @return < 0 error; =0 success; >0 no exist 
  */
-
 int kvs_rbtree_mod(kvs_rbtree_t *inst, const void *key, uint32_t key_len, const void *value, uint32_t value_len) {
 
 	if (!inst || !key || key_len==0 || !value) return -1;
@@ -457,7 +456,7 @@ int kvs_rbtree_mod(kvs_rbtree_t *inst, const void *key, uint32_t key_len, const
 }
 
 /*
- * @return 0: exist, 1: no exist
+ * @return =0 exist, =1 no exist
  */
 int kvs_rbtree_exist(kvs_rbtree_t *inst, const void *key, uint32_t key_len) {
 

kvs_rw_tools.c

@@ -1,5 +1,6 @@
 #include "kvstore.h"
 #include "kvs_rw_tools.h"
+#include "mem_pool/mem_pool.h"
 #include <arpa/inet.h>
 #include <unistd.h>
 
@@ -144,3 +145,353 @@ int kvs_read_file(FILE *fp, void *buf, size_t n){
     }
     return 0;
 }
+
+
+
+// return: -1 fail, 0 half, >0 consumed
+int kvs_parse_one_cmd(const uint8_t *request, int request_length, kvs_req_t *req_out){
+	if (!request || request_length <= 0 || !req_out) return -1;
+	
+	req_out->op = KVS_CMD_COUNT;
+    req_out->argc = 0;
+    req_out->args = NULL;
+
+	const uint8_t *p = request;
+    const uint8_t *end = request + (size_t)request_length;
+
+	// OP + ARGC 
+    if (kvs_need(p, end, 2)) {
+        return 0; // NEED_MORE
+    }
+
+	uint8_t op = 0, argc = 0;
+	if (kvs_read_u8(&p, end, &op) < 0) return -1;
+	if (kvs_read_u8(&p, end, &argc) < 0) return -1;
+
+	if (argc > KVS_MAX_ARGC) return -1;
+
+	// 先扫描一遍确认整条命令数据都在 buffer 里
+	const uint8_t *scan = p;
+	uint32_t lens[KVS_MAX_ARGC];
+	if (argc > 0) {
+		for (uint8_t i = 0; i < argc; i++) {
+			if (kvs_need(scan, end, 4)) {
+				return 0; // NEED_MORE
+			}
+			uint32_t alen = 0;
+			if (kvs_read_u32(&scan, end, &alen) < 0) return -1;
+
+			// 防御：单个参数长度限制
+			if (alen > KVS_MAX_ARGLEN) return -1;
+
+			// 防御：scan + alen 越界 / 半包
+			if (kvs_need(scan, end, (size_t)alen)) {
+				return 0; // NEED_MORE
+			}
+			lens[i] = alen;
+			scan += alen;
+        }
+	}
+
+	size_t total_len = (size_t)(scan - request);
+    if (total_len > KVS_MAX_CMD_BYTES) return -1;
+
+	req_out->op = op;
+    req_out->argc = argc;
+
+	if (argc == 0) {
+        return (int)total_len;
+    }
+
+	kvs_arg_t *args = (kvs_arg_t *)kvs_malloc((size_t)argc * sizeof(kvs_arg_t));
+    if (!args) {
+        kvs_free_request(req_out);
+        return -1;
+    }
+	memset(args, 0, (size_t)argc * sizeof(kvs_arg_t));
+
+	for (uint8_t i = 0; i < argc; i++) {
+        uint32_t alen = 0;
+        if (kvs_read_u32(&p, end, &alen) < 0) {
+            kvs_free(args);
+            kvs_free_request(req_out);
+            return -1;
+        }
+
+        // alen 与 lens[i] 应当一致（扫描时读过），不一致说明解析器/输入异常
+        if (alen != lens[i]) {
+            kvs_free(args);
+            kvs_free_request(req_out);
+            return -1;
+        }
+
+        args[i].len = alen;
+        args[i].data = p;     // 直接指向输入 buffer（零拷贝）
+        p += alen;
+    }
+
+	
+    req_out->args = args;
+
+    return (int)(p - request);
+}
+
+void kvs_free_request(kvs_req_t *req) {
+    if (!req) return;
+    if (req->args) {
+        kvs_free(req->args);
+        req->args = NULL;
+    }
+    req->op = KVS_CMD_COUNT;
+    req->argc = 0;
+}
+
+/**
+ * 输入：req
+ * 输出：rsp
+ * 返回：-1 失败，参数错误，0 成功
+ */
+ int kvs_execute_one_cmd(const kvs_req_t *req, kvs_rsp_t *rsp_out) {
+	if(!req || !rsp_out) return -1;
+	rsp_out->op = req->op;
+	rsp_out->status = KVS_STATUS_ERROR;
+	rsp_out->data = NULL;
+	rsp_out->dlen = 0;
+
+	int argc = req->argc;
+	kvs_cmd_t op = req->op;
+	kvs_arg_t *argv = req->args;
+
+	uint32_t key_len = 0;
+	const void *key = NULL;
+	uint32_t value_len = 0;
+	const void *val = NULL;
+
+	if(argc == 1){
+		key_len = argv[0].len;
+		key = argv[0].data;
+	}else if(argc == 2){
+		key_len = argv[0].len;
+		key = argv[0].data;
+		value_len = argv[1].len;
+		val = argv[1].data;
+	}
+
+	// 基本参数校验（按你原有命令语义）
+	switch (op) {
+	case KVS_CMD_SET:
+	case KVS_CMD_MOD:
+	case KVS_CMD_RSET:
+	case KVS_CMD_RMOD:
+	case KVS_CMD_HSET:
+	case KVS_CMD_HMOD:
+		if (argc != 2 || !key || !val) { rsp_out->status = KVS_STATUS_BADREQ; return -1; }
+		break;
+	case KVS_CMD_GET:
+	case KVS_CMD_DEL:
+	case KVS_CMD_EXIST:
+	case KVS_CMD_RGET:
+	case KVS_CMD_RDEL:
+	case KVS_CMD_REXIST:
+	case KVS_CMD_HGET:
+	case KVS_CMD_HDEL:
+	case KVS_CMD_HEXIST:
+	case KVS_CMD_PSYNC:
+		if (argc != 1 || !key) { rsp_out->status = KVS_STATUS_BADREQ; return -1; }
+		break;
+	case KVS_CMD_SAVE:
+		if(argc != 0) { rsp_out->status = KVS_STATUS_BADREQ; return -1; }
+		break;
+	default:
+		rsp_out->status = KVS_STATUS_BADREQ;
+		return -1;
+	}
+
+	int ret = 0;
+	const char *result = NULL;
+
+	switch (op) {
+#if ENABLE_ARRAY
+	case KVS_CMD_SET:
+		ret = kvs_array_set_bin(&global_array, key, key_len, val, value_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_EXIST;
+		return 0;
+
+	case KVS_CMD_GET:
+		result = kvs_array_get_bin(&global_array, key, key_len, &value_len);
+		if (!result) { rsp_out->status = KVS_STATUS_NO_EXIST; return 0; }
+		rsp_out->status = KVS_STATUS_OK;
+		rsp_out->data = (uint8_t*)result;
+		rsp_out->dlen = (uint32_t)value_len;
+		return 0;
+
+	case KVS_CMD_DEL:
+		ret = kvs_array_del_bin(&global_array, key, key_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_NO_EXIST;
+		return 0;
+
+	case KVS_CMD_MOD:
+		ret = kvs_array_mod_bin(&global_array, key, key_len, val, value_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_NO_EXIST;
+		return 0;
+
+	case KVS_CMD_EXIST:
+		ret = kvs_array_exist_bin(&global_array, key, key_len);
+		rsp_out->status = (ret == 0) ? KVS_STATUS_EXIST : KVS_STATUS_NO_EXIST;
+		return 0;
+#endif
+
+#if ENABLE_RBTREE
+	case KVS_CMD_RSET:
+		ret = kvs_rbtree_set(&global_rbtree, key, key_len, val, value_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_EXIST;
+		return 0;
+
+	case KVS_CMD_RGET:
+		result = kvs_rbtree_get(&global_rbtree, key, key_len, &value_len);
+		if (!result) { rsp_out->status = KVS_STATUS_NO_EXIST; return 0; }
+		rsp_out->status = KVS_STATUS_OK;
+		rsp_out->data = (uint8_t*)result;
+		rsp_out->dlen = (uint32_t)value_len;
+		return 0;
+
+	case KVS_CMD_RDEL:
+		ret = kvs_rbtree_del(&global_rbtree, key, key_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_NO_EXIST;
+		return 0;
+
+	case KVS_CMD_RMOD:
+		ret = kvs_rbtree_mod(&global_rbtree, key, key_len, val, value_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_NO_EXIST;
+		return 0;
+
+	case KVS_CMD_REXIST:
+		ret = kvs_rbtree_exist(&global_rbtree, key, key_len);
+		rsp_out->status = (ret == 0) ? KVS_STATUS_EXIST : KVS_STATUS_NO_EXIST;
+		return 0;
+#endif
+
+#if ENABLE_HASH
+	case KVS_CMD_HSET:
+		ret = kvs_hash_set_bin(&global_hash, key, key_len, val, value_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_EXIST;
+		return 0;
+
+	case KVS_CMD_HGET:
+		result = kvs_hash_get_bin(&global_hash, key, key_len, &value_len);
+		if (!result) { rsp_out->status = KVS_STATUS_NO_EXIST; return 0; }
+		rsp_out->status = KVS_STATUS_OK;
+		rsp_out->data = (uint8_t*)result;
+		rsp_out->dlen = (uint32_t)value_len;
+		return 0;
+
+	case KVS_CMD_HDEL:
+		ret = kvs_hash_del_bin(&global_hash, key, key_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_NO_EXIST;
+		return 0;
+
+	case KVS_CMD_HMOD:
+		ret = kvs_hash_mod_bin(&global_hash, key, key_len, val, value_len);
+		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
+		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_NO_EXIST;
+		return 0;
+
+	case KVS_CMD_HEXIST:
+		ret = kvs_hash_exist_bin(&global_hash, key, key_len);
+		rsp_out->status = (ret == 0) ? KVS_STATUS_EXIST : KVS_STATUS_NO_EXIST;
+		return 0;
+#endif
+	case KVS_CMD_SAVE:
+		ret = kvs_save_to_file();
+		if(ret == 0) rsp_out->status = KVS_STATUS_OK;
+		else rsp_out->status = KVS_STATUS_ERROR;
+		return 0;
+	case KVS_CMD_PSYNC:
+		rsp_out->op = req->op;
+		rsp_out->status = KVS_STATUS_OK;
+		return 0;
+	default:
+		rsp_out->status = KVS_STATUS_BADREQ;
+		return -1;
+	}
+
+	return -1;
+}
+
+/**
+ * 构建单条响应
+ * 返回：-1 失败，>=0 响应长度
+ */
+int kvs_build_one_rsp(const kvs_rsp_t *results, uint8_t *response, size_t response_cap){
+	if (!results || !response) return -1;
+
+	const uint8_t *end = response + response_cap;
+    uint8_t *p = response;
+
+    // 计算所需长度：1 + 1 + 4 + dlen
+    // 注意防止 size_t 溢出
+    size_t need = 1u + 1u + 4u + (size_t)results->dlen;
+    if (need > response_cap) return -1;
+
+    if (kvs_write_u8(&p, end, (uint8_t)results->op) < 0) return -1;
+    if (kvs_write_u8(&p, end, results->status) < 0) return -1;
+    if (kvs_write_u32(&p, end, results->dlen) < 0) return -1;
+
+    if (results->dlen > 0) {
+        if (!results->data) return -1; // 有长度却没指针，视为错误
+        if (kvs_need(p, end, (size_t)results->dlen) < 0) return -1;
+        memcpy(p, results->data, results->dlen);
+        p += results->dlen;
+    }
+
+    return (int)(p - response);
+}
+
+int kvs_save_to_file(){
+	int ret = 0;
+	int rc = 0;
+	#if ENABLE_ARRAY
+	rc = kvs_array_save(&global_array, KVS_ARRAY_FILE);
+	if(rc < 0){
+		printf("kvs_engine_array save error\n");
+		ret = -1;
+	}
+	#endif
+
+	#if ENABLE_RBTREE
+	rc = kvs_rbtree_save(&global_rbtree, KVS_RBTREE_FILE);
+	if(rc < 0){
+		printf("kvs_engine_rbtree save error\n");
+		ret = -1;
+	}
+	#endif
+
+	#if ENABLE_HASH
+	rc = kvs_hash_save(&global_hash, KVS_HASH_FILE);
+	if(rc < 0){
+		printf("kvs_engine_hash save error\n");
+		ret = -1;
+	}
+	#endif
+
+	ksv_clear_log(global_cmd_log_fd);
+
+	return ret;
+}

kvs_rw_tools.h

@@ -1,5 +1,5 @@
-#ifndef __KVS_RW_TOOLS_H__
+#ifndef __KVS_PROTOCOL_H__
-#define __KVS_RW_TOOLS_H__
+#define __KVS_PROTOCOL_H__
 
 #include <stdint.h>
 #include <stdio.h>

kvstore.c

@@ -12,6 +12,8 @@
 #include <errno.h>
 #include <unistd.h>
 #include <arpa/inet.h>
+#include <libxml/parser.h>
+#include "common/config.h"
 
 #if ENABLE_ARRAY
 extern kvs_array_t global_array;
@@ -29,370 +31,13 @@ extern kvs_hash_t global_hash;
 extern mp_pool_t global_mempool;
 #endif
 
-#define KVS_SERVER_NOT_INIT	0
+AppConfig cfg;
-#define KVS_SERVER_MASETER 	1
+
-#define KVS_SERVER_SLAVE	2
+ServerMode mode = MODE_MASTER;
-int kvs_server_role = KVS_SERVER_NOT_INIT;
 
 int global_cmd_log_fd = -1;
 
-const char *command[] = {
-	"SET", "GET", "DEL", "MOD", "EXIST",
-	"RSET", "RGET", "RDEL", "RMOD", "REXIST",
-	"HSET", "HGET", "HDEL", "HMOD", "HEXIST"
-};
 
-const char *response[] = {
-
-};
-
-// return: -1 fail, 0 half, >0 consumed
-int kvs_parse_one_cmd(const uint8_t *request, int request_length, kvs_req_t *req_out){
-	if (!request || request_length <= 0 || !req_out) return -1;
-	
-	req_out->op = KVS_CMD_COUNT;
-    req_out->argc = 0;
-    req_out->args = NULL;
-
-	const uint8_t *p = request;
-    const uint8_t *end = request + (size_t)request_length;
-
-	// OP + ARGC 
-    if (kvs_need(p, end, 2)) {
-        return 0; // NEED_MORE
-    }
-
-	uint8_t op = 0, argc = 0;
-	if (kvs_read_u8(&p, end, &op) < 0) return -1;
-	if (kvs_read_u8(&p, end, &argc) < 0) return -1;
-
-	if (argc > KVS_MAX_ARGC) return -1;
-
-	// 先扫描一遍确认整条命令数据都在 buffer 里
-	const uint8_t *scan = p;
-	uint32_t lens[KVS_MAX_ARGC];
-	if (argc > 0) {
-		for (uint8_t i = 0; i < argc; i++) {
-			if (kvs_need(scan, end, 4)) {
-				return 0; // NEED_MORE
-			}
-			uint32_t alen = 0;
-			if (kvs_read_u32(&scan, end, &alen) < 0) return -1;
-
-			// 防御：单个参数长度限制
-			if (alen > KVS_MAX_ARGLEN) return -1;
-
-			// 防御：scan + alen 越界 / 半包
-			if (kvs_need(scan, end, (size_t)alen)) {
-				return 0; // NEED_MORE
-			}
-			lens[i] = alen;
-			scan += alen;
-        }
-	}
-
-	size_t total_len = (size_t)(scan - request);
-    if (total_len > KVS_MAX_CMD_BYTES) return -1;
-
-	req_out->op = op;
-    req_out->argc = argc;
-
-	if (argc == 0) {
-        return (int)total_len;
-    }
-
-	kvs_arg_t *args = (kvs_arg_t *)kvs_malloc((size_t)argc * sizeof(kvs_arg_t));
-    if (!args) {
-        kvs_free_request(req_out);
-        return -1;
-    }
-	memset(args, 0, (size_t)argc * sizeof(kvs_arg_t));
-
-	for (uint8_t i = 0; i < argc; i++) {
-        uint32_t alen = 0;
-        if (kvs_read_u32(&p, end, &alen) < 0) {
-            kvs_free(args);
-            kvs_free_request(req_out);
-            return -1;
-        }
-
-        // alen 与 lens[i] 应当一致（扫描时读过），不一致说明解析器/输入异常
-        if (alen != lens[i]) {
-            kvs_free(args);
-            kvs_free_request(req_out);
-            return -1;
-        }
-
-        args[i].len = alen;
-        args[i].data = p;     // 直接指向输入 buffer（零拷贝）
-        p += alen;
-    }
-
-	
-    req_out->args = args;
-
-    return (int)(p - request);
-}
-
-void kvs_free_request(kvs_req_t *req) {
-    if (!req) return;
-    if (req->args) {
-        kvs_free(req->args);
-        req->args = NULL;
-    }
-    req->op = KVS_CMD_COUNT;
-    req->argc = 0;
-}
-
-/**
- * 输入：req
- * 输出：rsp
- * 返回：-1 失败，参数错误，0 成功
- */
- int kvs_execute_one_cmd(const kvs_req_t *req, kvs_rsp_t *rsp_out) {
-	if(!req || !rsp_out) return -1;
-	rsp_out->op = req->op;
-	rsp_out->status = KVS_STATUS_ERROR;
-	rsp_out->data = NULL;
-	rsp_out->dlen = 0;
-
-	int argc = req->argc;
-	kvs_cmd_t op = req->op;
-	kvs_arg_t *argv = req->args;
-
-	uint32_t key_len = 0;
-	const void *key = NULL;
-	uint32_t value_len = 0;
-	const void *val = NULL;
-
-	if(argc == 1){
-		key_len = argv[0].len;
-		key = argv[0].data;
-	}else if(argc == 2){
-		key_len = argv[0].len;
-		key = argv[0].data;
-		value_len = argv[1].len;
-		val = argv[1].data;
-	}
-
-	// 基本参数校验（按你原有命令语义）
-	switch (op) {
-	case KVS_CMD_SET:
-	case KVS_CMD_MOD:
-	case KVS_CMD_RSET:
-	case KVS_CMD_RMOD:
-	case KVS_CMD_HSET:
-	case KVS_CMD_HMOD:
-		if (argc != 2 || !key || !val) { rsp_out->status = KVS_STATUS_BADREQ; return -1; }
-		break;
-	case KVS_CMD_GET:
-	case KVS_CMD_DEL:
-	case KVS_CMD_EXIST:
-	case KVS_CMD_RGET:
-	case KVS_CMD_RDEL:
-	case KVS_CMD_REXIST:
-	case KVS_CMD_HGET:
-	case KVS_CMD_HDEL:
-	case KVS_CMD_HEXIST:
-	case KVS_CMD_PSYNC:
-		if (argc != 1 || !key) { rsp_out->status = KVS_STATUS_BADREQ; return -1; }
-		break;
-	case KVS_CMD_SAVE:
-		if(argc != 0) { rsp_out->status = KVS_STATUS_BADREQ; return -1; }
-		break;
-	default:
-		rsp_out->status = KVS_STATUS_BADREQ;
-		return -1;
-	}
-
-	int ret = 0;
-	const char *result = NULL;
-
-	switch (op) {
-#if ENABLE_ARRAY
-	case KVS_CMD_SET:
-		ret = kvs_array_set_bin(&global_array, key, key_len, val, value_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_EXIST;
-		return 0;
-
-	case KVS_CMD_GET:
-		result = kvs_array_get_bin(&global_array, key, key_len, &value_len);
-		if (!result) { rsp_out->status = KVS_STATUS_NO_EXIST; return 0; }
-		rsp_out->status = KVS_STATUS_OK;
-		rsp_out->data = (uint8_t*)result;
-		rsp_out->dlen = (uint32_t)value_len;
-		return 0;
-
-	case KVS_CMD_DEL:
-		ret = kvs_array_del_bin(&global_array, key, key_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_NO_EXIST;
-		return 0;
-
-	case KVS_CMD_MOD:
-		ret = kvs_array_mod_bin(&global_array, key, key_len, val, value_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_NO_EXIST;
-		return 0;
-
-	case KVS_CMD_EXIST:
-		ret = kvs_array_exist_bin(&global_array, key, key_len);
-		rsp_out->status = (ret == 0) ? KVS_STATUS_EXIST : KVS_STATUS_NO_EXIST;
-		return 0;
-#endif
-
-#if ENABLE_RBTREE
-	case KVS_CMD_RSET:
-		ret = kvs_rbtree_set(&global_rbtree, key, key_len, val, value_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_EXIST;
-		return 0;
-
-	case KVS_CMD_RGET:
-		result = kvs_rbtree_get(&global_rbtree, key, key_len, &value_len);
-		if (!result) { rsp_out->status = KVS_STATUS_NO_EXIST; return 0; }
-		rsp_out->status = KVS_STATUS_OK;
-		rsp_out->data = (uint8_t*)result;
-		rsp_out->dlen = (uint32_t)value_len;
-		return 0;
-
-	case KVS_CMD_RDEL:
-		ret = kvs_rbtree_del(&global_rbtree, key, key_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_NO_EXIST;
-		return 0;
-
-	case KVS_CMD_RMOD:
-		ret = kvs_rbtree_mod(&global_rbtree, key, key_len, val, value_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_NO_EXIST;
-		return 0;
-
-	case KVS_CMD_REXIST:
-		ret = kvs_rbtree_exist(&global_rbtree, key, key_len);
-		rsp_out->status = (ret == 0) ? KVS_STATUS_EXIST : KVS_STATUS_NO_EXIST;
-		return 0;
-#endif
-
-#if ENABLE_HASH
-	case KVS_CMD_HSET:
-		ret = kvs_hash_set_bin(&global_hash, key, key_len, val, value_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_EXIST;
-		return 0;
-
-	case KVS_CMD_HGET:
-		result = kvs_hash_get_bin(&global_hash, key, key_len, &value_len);
-		if (!result) { rsp_out->status = KVS_STATUS_NO_EXIST; return 0; }
-		rsp_out->status = KVS_STATUS_OK;
-		rsp_out->data = (uint8_t*)result;
-		rsp_out->dlen = (uint32_t)value_len;
-		return 0;
-
-	case KVS_CMD_HDEL:
-		ret = kvs_hash_del_bin(&global_hash, key, key_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_NO_EXIST;
-		return 0;
-
-	case KVS_CMD_HMOD:
-		ret = kvs_hash_mod_bin(&global_hash, key, key_len, val, value_len);
-		if (ret < 0) rsp_out->status = KVS_STATUS_ERROR;
-		else if (ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_NO_EXIST;
-		return 0;
-
-	case KVS_CMD_HEXIST:
-		ret = kvs_hash_exist_bin(&global_hash, key, key_len);
-		rsp_out->status = (ret == 0) ? KVS_STATUS_EXIST : KVS_STATUS_NO_EXIST;
-		return 0;
-#endif
-	case KVS_CMD_SAVE:
-		ret = kvs_save_to_file();
-		if(ret == 0) rsp_out->status = KVS_STATUS_OK;
-		else rsp_out->status = KVS_STATUS_ERROR;
-		return 0;
-	case KVS_CMD_PSYNC:
-		rsp_out->op = req->op;
-		rsp_out->status = KVS_STATUS_OK;
-		return 0;
-	default:
-		rsp_out->status = KVS_STATUS_BADREQ;
-		return -1;
-	}
-
-	return -1;
-}
-
-/**
- * 构建单条响应
- * 返回：-1 失败，>=0 响应长度
- */
-int kvs_build_one_rsp(const kvs_rsp_t *results, uint8_t *response, size_t response_cap){
-	if (!results || !response) return -1;
-
-	const uint8_t *end = response + response_cap;
-    uint8_t *p = response;
-
-    // 计算所需长度：1 + 1 + 4 + dlen
-    // 注意防止 size_t 溢出
-    size_t need = 1u + 1u + 4u + (size_t)results->dlen;
-    if (need > response_cap) return -1;
-
-    if (kvs_write_u8(&p, end, (uint8_t)results->op) < 0) return -1;
-    if (kvs_write_u8(&p, end, results->status) < 0) return -1;
-    if (kvs_write_u32(&p, end, results->dlen) < 0) return -1;
-
-    if (results->dlen > 0) {
-        if (!results->data) return -1; // 有长度却没指针，视为错误
-        if (kvs_need(p, end, (size_t)results->dlen) < 0) return -1;
-        memcpy(p, results->data, results->dlen);
-        p += results->dlen;
-    }
-
-    return (int)(p - response);
-}
-
-int kvs_save_to_file(){
-	int ret = 0;
-	int rc = 0;
-	#if ENABLE_ARRAY
-	rc = kvs_array_save(&global_array, KVS_ARRAY_FILE);
-	if(rc < 0){
-		printf("kvs_engine_array save error\n");
-		ret = -1;
-	}
-	#endif
-
-	#if ENABLE_RBTREE
-	rc = kvs_rbtree_save(&global_rbtree, KVS_RBTREE_FILE);
-	if(rc < 0){
-		printf("kvs_engine_rbtree save error\n");
-		ret = -1;
-	}
-	#endif
-
-	#if ENABLE_HASH
-	rc = kvs_hash_save(&global_hash, KVS_HASH_FILE);
-	if(rc < 0){
-		printf("kvs_engine_hash save error\n");
-		ret = -1;
-	}
-	#endif
-
-	ksv_clear_log(global_cmd_log_fd);
-
-	return ret;
-}
 
 int is_update_cmd(kvs_cmd_t op){
 	if(op == KVS_CMD_SET || op == KVS_CMD_RSET || op == KVS_CMD_HSET
@@ -676,38 +321,47 @@ void dest_memory_pool(void){
 #endif 
 }
 
-/** 
+int init_config(AppConfig *cfg){
- * ./ksvtore [role] [port]
+	xmlInitParser();
- * ./kvstore master 8888 
+	
- * 
+	if (config_load("config/config.xml", cfg) != 0) {
- * ./ksvtore [role] [port] [masterip] [masterport]
+        fprintf(stderr, "Failed to load config/config.xml\n");
- * ./kvstore slave  7000 192.168.10.129 8888
+        xmlCleanupParser();
- */ 
+        return -1;
-int main(int argc, char *argv[]) {
-	printf("memory type: ");
-	if(MEMORY_SELECT_MALLOC == MEMORY_USE_DEFAULT){
-		printf("default\n");
-	}else if(MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC){
-		printf("my memory pool\n");
-	}else if(MEMORY_SELECT_MALLOC == MEMORY_USE_JEMALLOC){
-		printf("jemalloc\n");
     }
 
-	if (argc < 3) return -1;
+	printf("=== Config ===\n");
+    printf("IP          : %s\n", cfg->ip);
+    printf("Port        : %d\n", cfg->port);
+    printf("Log level   : %s\n", log_level_to_string(cfg->log_level));
+	printf("Master IP   : %s\n", cfg->master_ip);
+    printf("Master Port : %d\n", cfg->master_port);
+    printf("Mode        : %s\n", server_mode_to_string(cfg->mode));
+    printf("Persistence : %s\n", persistence_to_string(cfg->persistence));
+    printf("Allocator   : %s\n", allocator_to_string(cfg->allocator));
+	printf("=== Config ===\n");
 	
-	char *role = argv[1];
+	xmlCleanupParser();
-	int port = atoi(argv[2]);
+	return 0;
+}
+
+
+int main(int argc, char *argv[]) {
+	if(-1 == init_config(&cfg)){
+		printf("Init Config error");
+		return -1;
+	} 
+	
+	int port = cfg.port;
+	mode = cfg.mode;
 	
 	char *master_ip = NULL;
 	int master_port = -1;
 
-	if(strcmp(role, "master") == 0){
+	if(mode == MODE_SLAVE){
-		kvs_server_role = KVS_SERVER_SLAVE;
+		master_ip = cfg.master_ip;
-	}else if(strcmp(role, "slave") == 0){
+		master_port = cfg.master_port;
-		kvs_server_role = KVS_SERVER_MASETER;
+	}else if(mode == MODE_MASTER){
-		if(argc < 5) return -1;
-		master_ip = argv[3];
-		master_port = atoi(argv[4]);
 		
 	}