lian/ldb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

实现内存池的测试用例、继承jemalloc和自实现内存池。

Browse Source
This commit is contained in:
1iaan
2026-01-09 20:09:05 +08:00
parent 4b4e06b33d
commit 1adb24482b
16 changed files with 664 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,5 +1,6 @@
NtyCo/
.vscode/
img/
*.db
*.copy

4
Makefile
View File

@@ -1,8 +1,8 @@
CC = gcc
FLAGS = -I ./NtyCo/core/ -L ./NtyCo/ -lntyco -lpthread -luring -ldl
FLAGS = -I ./NtyCo/core/ -L ./NtyCo/ -lntyco -lpthread -luring -ldl -ljemalloc
# SRCS = kvstore.c ntyco.c proactor.c reactor.c kvs_array.c kvs_rbtree.c kvs_hash.c kvs_rw_tools.c
SRCS = kvstore.c ntyco.c proactor.c reactor.c kvs_array_bin.c kvs_rbtree_bin.c kvs_hash_bin.c kvs_rw_tools.c kvs_cmd_log.c
SRCS = kvstore.c ntyco.c proactor.c reactor.c kvs_array_bin.c kvs_rbtree_bin.c kvs_hash_bin.c kvs_rw_tools.c kvs_cmd_log.c ./mem_pool/mem_pool.c
TESTCASE_SRCS = testcase.c
TARGET = kvstore
SUBDIR = ./NtyCo/

18
README.md
View File

@@ -9,9 +9,27 @@ save() -> 全保存数据集。
#### 目前KV存储 key value 都是调用的 malloc， 都是系统的内存分配。
1. 会出现大量的内存碎片，实现一个内存池，管理内存数据。
2. 对比有内存池和没有内存池的性能差异，以及开源内存池 jemalloc 的性能差别。
`sudo apt-get install -y libjemalloc-dev`
3. 性能测：
1. qps 有没有内存池。
![alt text](img/无内存池.png)
2.虚拟内存的占用情况 htop。插入百万条数据集(KV*100w，set 200w del 100w delete 200w set 100w)。
![alt text](img/未运行状态.png)
![alt text](img/200w插入100w删除.png)
![alt text](img/100w插入200w删除.png)
![alt text](img/无内存池200w条插入100w删除.png)
无内存池: --> time_used=1488789 ms, ops=6000000, qps=4030
![alt text](img/mpalloc200w插入100w删除.png)
自实现内存池: --> time_used=1439444 ms, ops=6000000, qps=4168
性能分析：
1. key :
1. A_0: mp_alloc (3+8B) -> 16B, malloc (3B) -> 8B
2. A_1000: mp_alloc (6+8B) -> 16B, malloc (3B) -> 8B
2. A_100000 : mp_alloc(8+8B) -> 16B, malloc (8B) -> 16B
内存初始状态 25B, malloc (115-25B) = 90B, mp_alloc(193-25B) = 168B，平均浪费接近50%。
#### 目前的kv存储是单指令的，一条指令一交互。
从单挑指令实现批量处理。

280
hash.c
View File

@@ -1,228 +1,228 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <pthread.h>
// #include <stdio.h>
// #include <string.h>
// #include <stdlib.h>
// #include <pthread.h>
#define MAX_KEY_LEN 128
#define MAX_VALUE_LEN 512
#define MAX_TABLE_SIZE 1024
// #define MAX_KEY_LEN 128
// #define MAX_VALUE_LEN 512
// #define MAX_TABLE_SIZE 1024
typedef struct hashnode_s {
// typedef struct hashnode_s {
char key[MAX_KEY_LEN];
char value[MAX_VALUE_LEN];
// char key[MAX_KEY_LEN];
// char value[MAX_VALUE_LEN];
struct hashnode_s *next;
// struct hashnode_s *next;
} hashnode_t;
// } hashnode_t;
typedef struct hashtable_s {
// typedef struct hashtable_s {
hashnode_t **nodes; //* change **,
// hashnode_t **nodes; //* change **,
int max_slots;
int count;
// int max_slots;
// int count;
pthread_mutex_t lock;
// pthread_mutex_t lock;
} hashtable_t;
// } hashtable_t;
hashtable_t hash;
// hashtable_t hash;
//Connection
// 'C' + 'o' + 'n'
static int _hash(char *key, int size) {
// //Connection
// // 'C' + 'o' + 'n'
// static int _hash(char *key, int size) {
if (!key) return -1;
// if (!key) return -1;
int sum = 0;
int i = 0;
// int sum = 0;
// int i = 0;
while (key[i] != 0) {
sum += key[i];
i ++;
}
// while (key[i] != 0) {
// sum += key[i];
// i ++;
// }
return sum % size;
// return sum % size;
}
// }
hashnode_t *_create_node(char *key, char *value) {
// hashnode_t *_create_node(char *key, char *value) {
hashnode_t *node = (hashnode_t*)malloc(sizeof(hashnode_t));
if (!node) return NULL;
// hashnode_t *node = (hashnode_t*)malloc(sizeof(hashnode_t));
// if (!node) return NULL;
strncpy(node->key, key, MAX_KEY_LEN);
strncpy(node->value, value, MAX_VALUE_LEN);
node->next = NULL;
// strncpy(node->key, key, MAX_KEY_LEN);
// strncpy(node->value, value, MAX_VALUE_LEN);
// node->next = NULL;
return node;
}
// return node;
// }
//
int init_hashtable(hashtable_t *hash) {
// //
// int init_hashtable(hashtable_t *hash) {
if (!hash) return -1;
// if (!hash) return -1;
hash->nodes = (hashnode_t**)malloc(sizeof(hashnode_t*) * MAX_TABLE_SIZE);
if (!hash->nodes) return -1;
// hash->nodes = (hashnode_t**)malloc(sizeof(hashnode_t*) * MAX_TABLE_SIZE);
// if (!hash->nodes) return -1;
hash->max_slots = MAX_TABLE_SIZE;
hash->count = 0;
// hash->max_slots = MAX_TABLE_SIZE;
// hash->count = 0;
pthread_mutex_init(&hash->lock, NULL);
// pthread_mutex_init(&hash->lock, NULL);
return 0;
}
// return 0;
// }
//
void dest_hashtable(hashtable_t *hash) {
// //
// void dest_hashtable(hashtable_t *hash) {
if (!hash) return;
// if (!hash) return;
int i = 0;
for (i = 0;i < hash->max_slots;i ++) {
hashnode_t *node = hash->nodes[i];
// int i = 0;
// for (i = 0;i < hash->max_slots;i ++) {
// hashnode_t *node = hash->nodes[i];
while (node != NULL) { // error
// while (node != NULL) { // error
hashnode_t *tmp = node;
node = node->next;
hash->nodes[i] = node;
// hashnode_t *tmp = node;
// node = node->next;
// hash->nodes[i] = node;
free(tmp);
// free(tmp);
}
}
// }
// }
free(hash->nodes);
// free(hash->nodes);
}
// }
// mp
int put_kv_hashtable(hashtable_t *hash, char *key, char *value) {
// // mp
// int put_kv_hashtable(hashtable_t *hash, char *key, char *value) {
if (!hash || !key || !value) return -1;
// if (!hash || !key || !value) return -1;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
pthread_mutex_lock(&hash->lock);
// pthread_mutex_lock(&hash->lock);
hashnode_t *node = hash->nodes[idx];
#if 1
while (node != NULL) {
if (strcmp(node->key, key) == 0) { // exist
pthread_mutex_unlock(&hash->lock);
return 1;
}
node = node->next;
}
#endif
// hashnode_t *node = hash->nodes[idx];
// #if 1
// while (node != NULL) {
// if (strcmp(node->key, key) == 0) { // exist
// pthread_mutex_unlock(&hash->lock);
// return 1;
// }
// node = node->next;
// }
// #endif
hashnode_t *new_node = _create_node(key, value);
new_node->next = hash->nodes[idx];
hash->nodes[idx] = new_node;
// hashnode_t *new_node = _create_node(key, value);
// new_node->next = hash->nodes[idx];
// hash->nodes[idx] = new_node;
hash->count ++;
// hash->count ++;
pthread_mutex_unlock(&hash->lock);
// pthread_mutex_unlock(&hash->lock);
return 0;
}
// return 0;
// }
char * get_kv_hashtable(hashtable_t *hash, char *key) {
// char * get_kv_hashtable(hashtable_t *hash, char *key) {
if (!hash || !key) return NULL;
// if (!hash || !key) return NULL;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
pthread_mutex_lock(&hash->lock);
hashnode_t *node = hash->nodes[idx];
// pthread_mutex_lock(&hash->lock);
// hashnode_t *node = hash->nodes[idx];
while (node != NULL) {
// while (node != NULL) {
if (strcmp(node->key, key) == 0) {
pthread_mutex_unlock(&hash->lock);
return node->value;
}
// if (strcmp(node->key, key) == 0) {
// pthread_mutex_unlock(&hash->lock);
// return node->value;
// }
node = node->next;
}
// node = node->next;
// }
pthread_mutex_unlock(&hash->lock);
// pthread_mutex_unlock(&hash->lock);
return NULL;
// return NULL;
}
// }
int count_kv_hashtable(hashtable_t *hash) {
return hash->count;
}
// int count_kv_hashtable(hashtable_t *hash) {
// return hash->count;
// }
int delete_kv_hashtable(hashtable_t *hash, char *key) {
if (!hash || !key) return -2;
// int delete_kv_hashtable(hashtable_t *hash, char *key) {
// if (!hash || !key) return -2;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
pthread_mutex_lock(&hash->lock);
hashnode_t *head = hash->nodes[idx];
if (head == NULL) return -1; // noexist
// head node
if (strcmp(head->key, key) == 0) {
hashnode_t *tmp = head->next;
hash->nodes[idx] = tmp;
// pthread_mutex_lock(&hash->lock);
// hashnode_t *head = hash->nodes[idx];
// if (head == NULL) return -1; // noexist
// // head node
// if (strcmp(head->key, key) == 0) {
// hashnode_t *tmp = head->next;
// hash->nodes[idx] = tmp;
free(head);
hash->count --;
pthread_mutex_unlock(&hash->lock);
// free(head);
// hash->count --;
// pthread_mutex_unlock(&hash->lock);
return 0;
}
// return 0;
// }
hashnode_t *cur = head;
while (cur->next != NULL) {
if (strcmp(cur->next->key, key) == 0) break; // search node
// hashnode_t *cur = head;
// while (cur->next != NULL) {
// if (strcmp(cur->next->key, key) == 0) break; // search node
cur = cur->next;
}
// cur = cur->next;
// }
if (cur->next == NULL) {
// if (cur->next == NULL) {
pthread_mutex_unlock(&hash->lock);
return -1;
}
// pthread_mutex_unlock(&hash->lock);
// return -1;
// }
hashnode_t *tmp = cur->next;
cur->next = tmp->next;
free(tmp);
hash->count --;
// hashnode_t *tmp = cur->next;
// cur->next = tmp->next;
// free(tmp);
// hash->count --;
pthread_mutex_unlock(&hash->lock);
// pthread_mutex_unlock(&hash->lock);
return 0;
}
// return 0;
// }
int exist_kv_hashtable(hashtable_t *hash, char *key) {
// int exist_kv_hashtable(hashtable_t *hash, char *key) {
char *value = get_kv_hashtable(hash, key);
if (value) return 1;
else return 0;
// char *value = get_kv_hashtable(hash, key);
// if (value) return 1;
// else return 0;
}
// }

1
kvs_array.c
View File

@@ -2,6 +2,7 @@
#include "kvstore.h"
#include "kvs_rw_tools.h"
#include "mem_pool/mem_pool.h"
#include <arpa/inet.h>

1
kvs_array_bin.c
View File

@@ -2,6 +2,7 @@
#include "kvstore.h"
#include "kvs_rw_tools.h"
#include "mem_pool/mem_pool.h"
#include <arpa/inet.h>

5
kvs_cmd_log.c
View File

@@ -1,5 +1,6 @@
#include "kvstore.h"
#include "kvs_rw_tools.h"
#include "mem_pool/mem_pool.h"
#include <arpa/inet.h>
#include <fcntl.h>
#include <unistd.h>
@@ -14,6 +15,7 @@ int init_cmd_log(const char *file, int *logfd){
}
int destroy_cmd_log(int logfd){
fsync(logfd);
close(logfd);
return 0;
}
@@ -35,9 +37,6 @@ int kvs_save_cmd_to_logfile(const uint8_t *cmd, size_t len, int logfd){
if (write_full(logfd, cmd, len) < 0)
return -4;
if (fsync(logfd) < 0)
return -5;
return 0;
}

367
kvs_hash.c
View File

@@ -1,285 +1,286 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <pthread.h>
// #include <stdio.h>
// #include <string.h>
// #include <stdlib.h>
// #include <pthread.h>
#include "kvstore.h"
// #include "mem_pool/mem_pool.h"
// #include "kvstore.h"
// Key, Value -->
// Modify
// // Key, Value -->
// // Modify
kvs_hash_t global_hash;
// kvs_hash_t global_hash;
//Connection
// 'C' + 'o' + 'n'
static int _hash(char *key, int size) {
// //Connection
// // 'C' + 'o' + 'n'
// static int _hash(char *key, int size) {
if (!key) return -1;
// if (!key) return -1;
int sum = 0;
int i = 0;
// int sum = 0;
// int i = 0;
while (key[i] != 0) {
sum += key[i];
i ++;
}
// while (key[i] != 0) {
// sum += key[i];
// i ++;
// }
return sum % size;
// return sum % size;
}
// }
hashnode_t *_create_node(char *key, char *value) {
// hashnode_t *_create_node(char *key, char *value) {
hashnode_t *node = (hashnode_t*)kvs_malloc(sizeof(hashnode_t));
if (!node) return NULL;
// hashnode_t *node = (hashnode_t*)kvs_malloc(sizeof(hashnode_t));
// if (!node) return NULL;
#if ENABLE_KEY_POINTER
char *kcopy = kvs_malloc(strlen(key) + 1);
if (kcopy == NULL) return NULL;
memset(kcopy, 0, strlen(key) + 1);
strncpy(kcopy, key, strlen(key));
// #if ENABLE_KEY_POINTER
// char *kcopy = kvs_malloc(strlen(key) + 1);
// if (kcopy == NULL) return NULL;
// memset(kcopy, 0, strlen(key) + 1);
// strncpy(kcopy, key, strlen(key));
node->key = kcopy;
// node->key = kcopy;
char *kvalue = kvs_malloc(strlen(value) + 1);
if (kvalue == NULL) {
kvs_free(kvalue);
return NULL;
}
memset(kvalue, 0, strlen(value) + 1);
strncpy(kvalue, value, strlen(value));
// char *kvalue = kvs_malloc(strlen(value) + 1);
// if (kvalue == NULL) {
// kvs_free(kvalue);
// return NULL;
// }
// memset(kvalue, 0, strlen(value) + 1);
// strncpy(kvalue, value, strlen(value));
node->value = kvalue;
// node->value = kvalue;
#else
strncpy(node->key, key, MAX_KEY_LEN);
strncpy(node->value, value, MAX_VALUE_LEN);
#endif
node->next = NULL;
// #else
// strncpy(node->key, key, MAX_KEY_LEN);
// strncpy(node->value, value, MAX_VALUE_LEN);
// #endif
// node->next = NULL;
return node;
}
// return node;
// }
//
int kvs_hash_create(kvs_hash_t *hash) {
// //
// int kvs_hash_create(kvs_hash_t *hash) {
if (!hash) return -1;
// if (!hash) return -1;
hash->nodes = (hashnode_t**)kvs_malloc(sizeof(hashnode_t*) * MAX_TABLE_SIZE);
if (!hash->nodes) return -1;
// hash->nodes = (hashnode_t**)kvs_malloc(sizeof(hashnode_t*) * MAX_TABLE_SIZE);
// if (!hash->nodes) return -1;
hash->max_slots = MAX_TABLE_SIZE;
hash->count = 0;
// hash->max_slots = MAX_TABLE_SIZE;
// hash->count = 0;
return 0;
}
// return 0;
// }
//
void kvs_hash_destroy(kvs_hash_t *hash) {
// //
// void kvs_hash_destroy(kvs_hash_t *hash) {
if (!hash) return;
// if (!hash) return;
int i = 0;
for (i = 0;i < hash->max_slots;i ++) {
hashnode_t *node = hash->nodes[i];
// int i = 0;
// for (i = 0;i < hash->max_slots;i ++) {
// hashnode_t *node = hash->nodes[i];
while (node != NULL) { // error
// while (node != NULL) { // error
hashnode_t *tmp = node;
node = node->next;
hash->nodes[i] = node;
// hashnode_t *tmp = node;
// node = node->next;
// hash->nodes[i] = node;
kvs_free(tmp);
// kvs_free(tmp);
}
}
// }
// }
kvs_free(hash->nodes);
// kvs_free(hash->nodes);
}
// }
// 5 + 2
// // 5 + 2
// mp
int kvs_hash_set(kvs_hash_t *hash, char *key, char *value) {
// // mp
// int kvs_hash_set(kvs_hash_t *hash, char *key, char *value) {
if (!hash || !key || !value) return -1;
// if (!hash || !key || !value) return -1;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
hashnode_t *node = hash->nodes[idx];
#if 1
while (node != NULL) {
if (strcmp(node->key, key) == 0) { // exist
return 1;
}
node = node->next;
}
#endif
// hashnode_t *node = hash->nodes[idx];
// #if 1
// while (node != NULL) {
// if (strcmp(node->key, key) == 0) { // exist
// return 1;
// }
// node = node->next;
// }
// #endif
hashnode_t *new_node = _create_node(key, value);
new_node->next = hash->nodes[idx];
hash->nodes[idx] = new_node;
// hashnode_t *new_node = _create_node(key, value);
// new_node->next = hash->nodes[idx];
// hash->nodes[idx] = new_node;
hash->count ++;
// hash->count ++;
return 0;
}
// return 0;
// }
char * kvs_hash_get(kvs_hash_t *hash, char *key) {
// char * kvs_hash_get(kvs_hash_t *hash, char *key) {
if (!hash || !key) return NULL;
// if (!hash || !key) return NULL;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
hashnode_t *node = hash->nodes[idx];
// hashnode_t *node = hash->nodes[idx];
while (node != NULL) {
// while (node != NULL) {
if (strcmp(node->key, key) == 0) {
return node->value;
}
// if (strcmp(node->key, key) == 0) {
// return node->value;
// }
node = node->next;
}
// node = node->next;
// }
return NULL;
// return NULL;
}
// }
int kvs_hash_mod(kvs_hash_t *hash, char *key, char *value) {
// int kvs_hash_mod(kvs_hash_t *hash, char *key, char *value) {
if (!hash || !key) return -1;
// if (!hash || !key) return -1;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
hashnode_t *node = hash->nodes[idx];
// hashnode_t *node = hash->nodes[idx];
while (node != NULL) {
// while (node != NULL) {
if (strcmp(node->key, key) == 0) {
break;
}
// if (strcmp(node->key, key) == 0) {
// break;
// }
node = node->next;
}
// node = node->next;
// }
if (node == NULL) {
return 1;
}
// if (node == NULL) {
// return 1;
// }
// node -->
kvs_free(node->value);
// // node -->
// kvs_free(node->value);
char *kvalue = kvs_malloc(strlen(value) + 1);
if (kvalue == NULL) return -2;
memset(kvalue, 0, strlen(value) + 1);
strncpy(kvalue, value, strlen(value));
// char *kvalue = kvs_malloc(strlen(value) + 1);
// if (kvalue == NULL) return -2;
// memset(kvalue, 0, strlen(value) + 1);
// strncpy(kvalue, value, strlen(value));
node->value = kvalue;
// node->value = kvalue;
return 0;
}
// return 0;
// }
int kvs_hash_count(kvs_hash_t *hash) {
return hash->count;
}
// int kvs_hash_count(kvs_hash_t *hash) {
// return hash->count;
// }
int kvs_hash_del(kvs_hash_t *hash, char *key) {
if (!hash || !key) return -2;
// int kvs_hash_del(kvs_hash_t *hash, char *key) {
// if (!hash || !key) return -2;
int idx = _hash(key, MAX_TABLE_SIZE);
// int idx = _hash(key, MAX_TABLE_SIZE);
hashnode_t *head = hash->nodes[idx];
if (head == NULL) return -1; // noexist
// head node
if (strcmp(head->key, key) == 0) {
hashnode_t *tmp = head->next;
hash->nodes[idx] = tmp;
// hashnode_t *head = hash->nodes[idx];
// if (head == NULL) return -1; // noexist
// // head node
// if (strcmp(head->key, key) == 0) {
// hashnode_t *tmp = head->next;
// hash->nodes[idx] = tmp;
kvs_free(head);
hash->count --;
// kvs_free(head);
// hash->count --;
return 0;
}
// return 0;
// }
hashnode_t *cur = head;
while (cur->next != NULL) {
if (strcmp(cur->next->key, key) == 0) break; // search node
// hashnode_t *cur = head;
// while (cur->next != NULL) {
// if (strcmp(cur->next->key, key) == 0) break; // search node
cur = cur->next;
}
// cur = cur->next;
// }
if (cur->next == NULL) {
// if (cur->next == NULL) {
return -1;
}
// return -1;
// }
hashnode_t *tmp = cur->next;
cur->next = tmp->next;
#if ENABLE_KEY_POINTER
kvs_free(tmp->key);
kvs_free(tmp->value);
#endif
kvs_free(tmp);
// hashnode_t *tmp = cur->next;
// cur->next = tmp->next;
// #if ENABLE_KEY_POINTER
// kvs_free(tmp->key);
// kvs_free(tmp->value);
// #endif
// kvs_free(tmp);
hash->count --;
// hash->count --;
return 0;
}
// return 0;
// }
int kvs_hash_exist(kvs_hash_t *hash, char *key) {
// int kvs_hash_exist(kvs_hash_t *hash, char *key) {
char *value = kvs_hash_get(hash, key);
if (!value) return 1;
// char *value = kvs_hash_get(hash, key);
// if (!value) return 1;
return 0;
// return 0;
}
// }
#if 0
int main() {
// #if 0
// int main() {
kvs_hash_create(&hash);
// kvs_hash_create(&hash);
kvs_hash_set(&hash, "Teacher1", "King");
kvs_hash_set(&hash, "Teacher2", "Darren");
kvs_hash_set(&hash, "Teacher3", "Mark");
kvs_hash_set(&hash, "Teacher4", "Vico");
kvs_hash_set(&hash, "Teacher5", "Nick");
// kvs_hash_set(&hash, "Teacher1", "King");
// kvs_hash_set(&hash, "Teacher2", "Darren");
// kvs_hash_set(&hash, "Teacher3", "Mark");
// kvs_hash_set(&hash, "Teacher4", "Vico");
// kvs_hash_set(&hash, "Teacher5", "Nick");
char *value1 = kvs_hash_get(&hash, "Teacher1");
printf("Teacher1 : %s\n", value1);
// char *value1 = kvs_hash_get(&hash, "Teacher1");
// printf("Teacher1 : %s\n", value1);
int ret = kvs_hash_mod(&hash, "Teacher1", "King1");
printf("mode Teacher1 ret : %d\n", ret);
// int ret = kvs_hash_mod(&hash, "Teacher1", "King1");
// printf("mode Teacher1 ret : %d\n", ret);
char *value2 = kvs_hash_get(&hash, "Teacher1");
printf("Teacher2 : %s\n", value1);
// char *value2 = kvs_hash_get(&hash, "Teacher1");
// printf("Teacher2 : %s\n", value1);
ret = kvs_hash_del(&hash, "Teacher1");
printf("delete Teacher1 ret : %d\n", ret);
// ret = kvs_hash_del(&hash, "Teacher1");
// printf("delete Teacher1 ret : %d\n", ret);
ret = kvs_hash_exist(&hash, "Teacher1");
printf("Exist Teacher1 ret : %d\n", ret);
// ret = kvs_hash_exist(&hash, "Teacher1");
// printf("Exist Teacher1 ret : %d\n", ret);
kvs_hash_destroy(&hash);
// kvs_hash_destroy(&hash);
return 0;
}
// return 0;
// }
#endif
// #endif

1
kvs_hash_bin.c
View File

@@ -9,6 +9,7 @@
#include "kvstore.h"
#include "kvs_rw_tools.h"
#include "mem_pool/mem_pool.h"
#include <arpa/inet.h>
// Key, Value -->

1
kvs_rbtree_bin.c
View File

@@ -8,6 +8,7 @@
#include "kvstore.h"
#include "kvs_rw_tools.h"
#include "mem_pool/mem_pool.h"
#include <arpa/inet.h>
int kvs_keycmp(const uint8_t *a, uint32_t alen,

28
kvstore.c
View File

@@ -1,9 +1,9 @@
#include "kvstore.h"
#include "kvs_rw_tools.h"
#include "mem_pool/mem_pool.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
@@ -21,17 +21,12 @@ extern kvs_rbtree_t global_rbtree;
extern kvs_hash_t global_hash;
#endif
#if MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
extern mp_pool_t global_mempool;
#endif
int global_cmd_log_fd = -1;
void *kvs_malloc(size_t size) {
return malloc(size);
}
void kvs_free(void *ptr) {
return free(ptr);
}
const char *command[] = {
"SET", "GET", "DEL", "MOD", "EXIST",
"RSET", "RGET", "RDEL", "RMOD", "REXIST",
@@ -504,7 +499,17 @@ void dest_kvengine(void) {
destroy_cmd_log(global_cmd_log_fd);
}
void init_memory_pool(void){
#if MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
mp_init(&global_mempool);
#endif
}
void dest_memory_pool(void){
#if MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
mp_destroy(&global_mempool);
#endif
}
int main(int argc, char *argv[]) {
@@ -512,6 +517,7 @@ int main(int argc, char *argv[]) {
int port = atoi(argv[1]);
init_memory_pool();
init_kvengine();
@@ -524,7 +530,7 @@ int main(int argc, char *argv[]) {
#endif
dest_kvengine();
dest_memory_pool();
}

4
kvstore.h
View File

@@ -270,10 +270,6 @@ int kvs_hash_exist(kvs_hash_t *hash, char *key);
#endif
void *kvs_malloc(size_t size);
void kvs_free(void *ptr);
#endif

164
mem_pool/mem_pool.c Normal file
View File

@@ -0,0 +1,164 @@
#include "mem_pool.h"
#include <stddef.h>
#include <stdlib.h>
#define JEMALLOC_NO_DEMANGLE
#include <jemalloc/jemalloc.h>
void *kvs_malloc(size_t size) {
#if MEMORY_SELECT_MALLOC == MEMORY_USE_DEFAULT
return malloc(size);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
return mp_alloc(size);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_JEMALLOC
return je_malloc(size);
#endif
}
void kvs_free(void *ptr) {
#if MEMORY_SELECT_MALLOC == MEMORY_USE_DEFAULT
free(ptr);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
mp_free(ptr);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_JEMALLOC
je_free(ptr);
#endif
}
#if MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
mp_pool_t global_mempool = {0};
#endif
// >= x 的第一个2^n
uint32_t mp_ceil_pow2_u32(uint32_t x) {
if (x <= 1) return 1;
x--;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
return x + 1;
}
// return cid
uint16_t mp_class_id_for(size_t size) {
if (size < 8) size = 8;
if (size > (1u << MP_MAX_SHIFT)) return MP_LARGE_CLASS;
uint32_t s = (uint32_t)size;
uint32_t p2 = mp_ceil_pow2_u32(s); /* 8/16/32... */
/* 计算 log2(p2) */
uint16_t shift = 0;
while ((1u << shift) < p2) shift++;
return (uint16_t)(shift - MP_MIN_SHIFT);
}
size_t mp_user_size_from_class(uint16_t cid) {
return (size_t)1u << (cid + MP_MIN_SHIFT);
}
int mp_add_page(mp_pool_t* p, void* mem) {
mp_page_t* pg = (mp_page_t*)malloc(sizeof(mp_page_t));
if (!pg) return -1;
pg->mem = mem;
pg->next = p->pages;
p->pages = pg;
return 0;
}
int mp_grow(mp_pool_t* p, uint16_t cid) {
void* page = malloc(MP_PAGE_SIZE);
if (!page) return -1;
if (mp_add_page(p, page) != 0) { free(page); return -1; }
// cid对应的空间大小
const size_t user_sz = mp_user_size_from_class(cid);
const size_t block_sz = sizeof(mp_blk_hdr_t) + user_sz;
const size_t n = MP_PAGE_SIZE / block_sz;
if (n == 0) return -1;
uint8_t* cur = (uint8_t*)page;
for (size_t i = 0; i < n; ++i) {
mp_blk_hdr_t* h = (mp_blk_hdr_t*)cur;
h->magic = MP_MAGIC;
h->class_id = cid;
mp_free_node_t* user = (mp_free_node_t*)(h + 1);
user->next = p->buckets[cid].free_list;
p->buckets[cid].free_list = user;
cur += block_sz;
}
return 0;
}
void mp_init(mp_pool_t* p){
if (!p) return;
for (int i = 0; i < MP_NBUCKETS; ++i) p->buckets[i].free_list = NULL;
p->pages = NULL;
}
void mp_destroy(mp_pool_t* p){
if (!p) return;
mp_page_t* pg = p->pages;
while (pg) {
mp_page_t* nxt = pg->next;
free(pg->mem);
free(pg);
pg = nxt;
}
p->pages = NULL;
for (int i = 0; i < MP_NBUCKETS; ++i) p->buckets[i].free_list = NULL;
}
void* mp_alloc(size_t size){
mp_pool_t* p = &global_mempool;
if (!p) return NULL;
if (size == 0) size = 1;
uint16_t cid = mp_class_id_for(size);
if (cid == MP_LARGE_CLASS) {
mp_blk_hdr_t* h = (mp_blk_hdr_t*)malloc(sizeof(mp_blk_hdr_t) + size);
if (!h) return NULL;
h->magic = MP_MAGIC;
h->class_id = MP_LARGE_CLASS;
return (void*)(h + 1);
}
mp_free_node_t* node = p->buckets[cid].free_list;
if (!node) {
if (mp_grow(p, cid) != 0) return NULL;
node = p->buckets[cid].free_list;
if (!node) return NULL;
}
p->buckets[cid].free_list = node->next;
return (void*)node;
}
void mp_free(void* ptr){
mp_pool_t* p = &global_mempool;
if (!p || !ptr) return;
mp_blk_hdr_t* h = ((mp_blk_hdr_t*)ptr) - 1;
if (h->magic != MP_MAGIC) {
/* 非法指针：极简处理（你也可以 assert/abort） */
return;
}
if (h->class_id == MP_LARGE_CLASS) {
/* 大对象：直接 free 整块（从 header 起） */
free(h);
return;
}
if (!p) return;
uint16_t cid = h->class_id;
if (cid >= MP_NBUCKETS) return; /* 极简：不做更多校验 */
mp_free_node_t* node = (mp_free_node_t*)ptr;
node->next = p->buckets[cid].free_list;
p->buckets[cid].free_list = node;
}

57
mem_pool/mem_pool.h Normal file
View File

@@ -0,0 +1,57 @@
#ifndef __MEM_POOL_H__
#define __MEM_POOL_H__
#define MEMORY_USE_DEFAULT 0
#define MEMORY_USE_MYMALLOC 1
#define MEMORY_USE_JEMALLOC 2
#define MEMORY_SELECT_MALLOC MEMORY_USE_MYMALLOC
#include <stddef.h>
#include <stdint.h>
void *kvs_malloc(size_t size);
void kvs_free(void *ptr);
#define MP_PAGE_SIZE 4096
#define MP_MIN_SHIFT 3
#define MP_MAX_SHIFT 12
#define MP_NBUCKETS (MP_MAX_SHIFT - MP_MIN_SHIFT + 1)
#define MP_MAGIC 0xB10C0A11u
#define MP_LARGE_CLASS 0xFFFFu
typedef struct mp_free_node {
struct mp_free_node* next;
} mp_free_node_t;
typedef struct mp_blk_hdr {
uint32_t magic;
uint16_t class_id; /* 0..MP_NBUCKETS-1, or MP_LARGE_CLASS */
uint16_t reserved;
} mp_blk_hdr_t;
typedef struct mp_page {
void* mem;
struct mp_page* next;
} mp_page_t;
typedef struct mp_bucket {
mp_free_node_t* free_list; /* 指向用户区 */
} mp_bucket_t;
typedef struct mp_pool {
mp_bucket_t buckets[MP_NBUCKETS];
mp_page_t* pages; /* 所有申请过的页 */
} mp_pool_t;
/* 初始化/销毁 */
void mp_init(mp_pool_t* p);
void mp_destroy(mp_pool_t* p);
/* 分配/释放 */
void* mp_alloc(size_t size);
void mp_free(void* ptr);
#endif

4
test/test_client.h
View File

@@ -19,8 +19,8 @@
#define KVS_BATCH_MAX 64
#define TIME_SUB_MS(tv1, tv2) ((tv1.tv_sec - tv2.tv_sec) * 1000 + (tv1.tv_usec - tv2.tv_usec) / 1000)
#define PRESP print_response
// #define PRESP
// #define PRESP print_response
#define PRESP
typedef enum {

164
test/testcase.c
View File

@@ -68,7 +68,7 @@ void testcase(int connfd, uint8_t op, const char* key, uint32_t key_len, const c
void array_testcase_1w(int connfd) {
int count = 1;
int count = 1000;
int i = 0;
struct timeval tv_begin;
@@ -98,7 +98,7 @@ void array_testcase_1w(int connfd) {
void rbtree_testcase_1w(int connfd) {
int count = 1;
int count = 1000;
int i = 0;
struct timeval tv_begin;
@@ -128,7 +128,7 @@ void rbtree_testcase_1w(int connfd) {
void hash_testcase_1w(int connfd) {
int count = 1;
int count = 1000;
int i = 0;
struct timeval tv_begin;
@@ -156,94 +156,6 @@ void hash_testcase_1w(int connfd) {
}
// void do_batch_SET_example(int fd)
// {
// kvs_batch_t batch;
// kvs_batch_init(&batch);
// char key[10]={0}, val[10]={0};
// // 组 batch（最多 64 条）
// for(int i = 0;i < 48; ++ i){
// int len = sprintf(key, "k%d", i);
// len = sprintf(val, "v%d", i);
// kvs_batch_add(&batch, KVS_CMD_SET, key, val);
// }
// // 一次性发送
// kvs_batch_send(fd, &batch);
// // 一次性 recv + parse
// uint8_t recvbuf[BATCH_SIZE];
// kvs_response_t rsps[KVS_BATCH_MAX];
// int nrsp = kvs_batch_recv_parse(fd, &batch, rsps, recvbuf, sizeof(recvbuf));
// // 打印/处理
// for (int i = 0; i < nrsp; i++) {
// int len = sprintf(key, "SET%d", i);
// PRESP(key, &rsps[i]);
// }
// }
// void do_batch_GET_example(int fd)
// {
// kvs_batch_t batch;
// kvs_batch_init(&batch);
// char key[10]={0}, val[10]={0};
// // 组 batch（最多 64 条）
// for(int i = 0;i < 48; ++ i){
// int len = sprintf(key, "k%d", i);
// kvs_batch_add(&batch, KVS_CMD_GET, key, NULL);
// }
// // 一次性发送
// kvs_batch_send(fd, &batch);
// // 一次性 recv + parse
// uint8_t recvbuf[BATCH_SIZE];
// kvs_response_t rsps[KVS_BATCH_MAX];
// int nrsp = kvs_batch_recv_parse(fd, &batch, rsps, recvbuf, sizeof(recvbuf));
// // 打印/处理
// for (int i = 0; i < nrsp; i++) {
// int len = sprintf(key, "GET%d", i);
// PRESP(key, &rsps[i]);
// }
// }
// void do_batch_DEL_example(int fd)
// {
// kvs_batch_t batch;
// kvs_batch_init(&batch);
// char key[10]={0}, val[10]={0};
// // 组 batch（最多 64 条）
// for(int i = 0;i < 48; ++ i){
// int len = sprintf(key, "k%d", i);
// kvs_batch_add(&batch, KVS_CMD_DEL, key, NULL);
// }
// // 一次性发送
// kvs_batch_send(fd, &batch);
// // 一次性 recv + parse
// uint8_t recvbuf[BATCH_SIZE];
// kvs_response_t rsps[KVS_BATCH_MAX];
// int nrsp = kvs_batch_recv_parse(fd, &batch, rsps, recvbuf, sizeof(recvbuf));
// // 打印/处理
// for (int i = 0; i < nrsp; i++) {
// int len = sprintf(key, "DEL%d", i);
// PRESP(key, &rsps[i]);
// }
// }
void do_batch_test(int fd, int op, const char *key, const char *value){
kvs_batch_t batch;
kvs_batch_init(&batch);
@@ -282,6 +194,74 @@ void save(int connfd){
testcase(connfd, KVS_CMD_SAVE, NULL, 0, NULL, 0, KVS_STATUS_OK, NULL, 0, "SAVE");
}
void testcase_add2_del1_then_add1_del2_100w(int connfd) {
const int N = 1000000;
// 如果你有 KVS_CMD_ADD 就用 ADD；没有就用 SET（但 SET 会覆盖，意义不同）
// 这里按你说的“会返回 EXIST”，所以我用 KVS_CMD_ADD 来写。
// 若你实际命令叫 KVS_CMD_SET 且语义是 ADD，请自行替换。
const char *valA = "va";
const char *valB = "vb";
const char *valC = "vc";
char keyA[64], keyB[64], keyC[64];
struct timeval tv_begin, tv_end;
gettimeofday(&tv_begin, NULL);
// ---------------- Phase 1: ADD 两条 DEL 一条 (100w) ----------------
// 每轮：ADD A_i, ADD B_i, DEL A_i -> 留下 B_i
for (int i = 0; i < N; i++) {
int klenA = snprintf(keyA, sizeof(keyA), "A_%d", i);
int klenB = snprintf(keyB, sizeof(keyB), "B_%d", i);
testcase(connfd, KVS_CMD_RSET, keyA, klenA, valA, 2, KVS_STATUS_OK, NULL, 0, "P1 ADD A_i");
testcase(connfd, KVS_CMD_RSET, keyB, klenB, valB, 2, KVS_STATUS_OK, NULL, 0, "P1 ADD B_i");
testcase(connfd, KVS_CMD_RDEL, keyA, klenA, NULL, 0, KVS_STATUS_OK, NULL, 0, "P1 DEL A_i");
if(i%10000 == 0) printf("i:%d\n", i);
}
printf("phase 1 end\n");
// ---------------- Phase 2: ADD 一条 DEL 两条 (100w) ----------------
// 每轮：ADD C_i, DEL B_i, DEL C_i -> 每轮净删一个 B_i
for (int i = 0; i < N; i++) {
int klenC = snprintf(keyC, sizeof(keyC), "C_%d", i);
int klenB = snprintf(keyB, sizeof(keyB), "B_%d", i);
testcase(connfd, KVS_CMD_RSET, keyC, klenC, valC, 2, KVS_STATUS_OK, NULL, 0, "P2 ADD C_i");
testcase(connfd, KVS_CMD_RDEL, keyB, klenB, NULL, 0, KVS_STATUS_OK, NULL, 0, "P2 DEL B_i");
testcase(connfd, KVS_CMD_RDEL, keyC, klenC, NULL, 0, KVS_STATUS_OK, NULL, 0, "P2 DEL C_i");
if(i%10000 == 0) printf("i:%d\n", i);
}
printf("phase 2 end\n");
// for (int j = 0; j < 5; j++) {
// int idx = (j == 0) ? 0 : (j == 1) ? (N/2) : (N-1);
// int klenA = snprintf(keyA, sizeof(keyA), "A_%d", idx);
// int klenB = snprintf(keyB, sizeof(keyB), "B_%d", idx);
// int klenC = snprintf(keyC, sizeof(keyC), "C_%d", idx);
// testcase(connfd, KVS_CMD_EXIST, keyA, klenA, NULL, 0, KVS_STATUS_NO_EXIST, NULL, 0, "FINAL A not exist");
// testcase(connfd, KVS_CMD_EXIST, keyB, klenB, NULL, 0, KVS_STATUS_NO_EXIST, NULL, 0, "FINAL B not exist");
// testcase(connfd, KVS_CMD_EXIST, keyC, klenC, NULL, 0, KVS_STATUS_NO_EXIST, NULL, 0, "FINAL C not exist");
// }
gettimeofday(&tv_end, NULL);
int time_used = TIME_SUB_MS(tv_end, tv_begin);
// 统计：Phase1 每轮3 ops，Phase2 每轮3 ops
long long ops = (long long)N * 3 + (long long)N * 3;
long long qps = (time_used > 0) ? (ops * 1000 / time_used) : 0;
printf("ADD2-DEL1 then ADD1-DEL2 (N=%d) --> time_used=%d ms, ops=%lld, qps=%lld\n",
N, time_used, ops, qps);
}
int main(int argc, char *argv[]) {
if (argc != 4) {
@@ -301,6 +281,8 @@ int main(int argc, char *argv[]) {
rbtree_testcase_1w(connfd);
}else if(mode == 2){
hash_testcase_1w(connfd);
}else if(mode == 3){
testcase_add2_del1_then_add1_del2_100w(connfd);
}else if(mode == 10){
do_batch_test(connfd, KVS_CMD_SET, "array_set", "array_val");
}else if(mode == 11){