lian/ldb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

内存探测组件，预留热插拔。

Browse Source
This commit is contained in:
1iaan
2026-01-26 13:07:10 +00:00
parent 9e757ece87
commit c99867b342
9 changed files with 187 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -7,3 +7,4 @@
kvstore
testcase
mem_leak

9
Makefile
View File

@@ -9,8 +9,8 @@ COMMON_SRCS = ./common/config.c ./diskuring/diskuring.c
SRCS = $(NET_SRCS) $(KV_SRCS) $(MEM_SRCS) $(COMMON_SRCS)
INC = -I./NtyCo/core/ -I/usr/include/libxml2
LIBDIRS = -L./NtyCo/ -L./memory/ -L./diskuring/
INC = -I./NtyCo/core/ -I/usr/include/libxml2 -I./
LIBDIRS = -L./NtyCo/
LIBS = -lntyco -lpthread -luring -ldl -lxml2 -ljemalloc
TARGET = kvstore
@@ -41,7 +41,8 @@ $(TEST_REDIS): $(TEST_REDIS_SRCS)
%.o: %.c
$(CC) $(CFLAGS) $(INC) -c $^ -g -o $@
clean:
clean: clmem
rm -rf $(OBJS) $(TARGET) $(TEST_REDIS)
clmem:
rm -rf mem_leak/*

29
common/config.c
View File

@@ -57,6 +57,7 @@ static void set_default_config(AppConfig *cfg)
cfg->master_port = 8888;
cfg->persistence = PERSIST_NONE;
cfg->allocator = ALLOC_JEMALLOC;
cfg->leak_mode = MEMLEAK_DETECT_OFF;
}
/* ---- 字符串转枚举 ---- */
@@ -90,9 +91,18 @@ static void parse_allocator(const char *s, AllocatorType *out)
if (!s || !out) return;
if (!strcasecmp(s, "jemalloc")) *out = ALLOC_JEMALLOC;
else if (!strcasecmp(s, "malloc")) *out = ALLOC_MALLOC;
else if (!strcasecmp(s, "mypool")) *out = ALLOC_MYPOOL;
else *out = ALLOC_OTHER;
}
static void parse_leakage(const char *s, MemLeakDetectMode *out)
{
if (!s || !out) return;
if (!strcasecmp(s, "enable")) *out = MEMLEAK_DETECT_ON;
else if (!strcasecmp(s, "disable")) *out = MEMLEAK_DETECT_OFF;
else *out = MEMLEAK_DETECT_OFF;
}
static int read_file_mmap(const char *filename, void **out_addr, size_t *out_len, int *out_fd) {
if (!filename || !out_addr || !out_len || !out_fd) return -1;
@@ -174,11 +184,21 @@ const char *allocator_to_string(AllocatorType a)
switch (a) {
case ALLOC_JEMALLOC: return "jemalloc";
case ALLOC_MALLOC: return "malloc";
case ALLOC_MYPOOL: return "mypool";
case ALLOC_OTHER: return "other";
default: return "unknown";
}
}
const char *leakage_to_string(MemLeakDetectMode a)
{
switch (a) {
case MEMLEAK_DETECT_ON: return "enable";
case MEMLEAK_DETECT_OFF: return "disable";
default: return "unknown";
}
}
/* ---- 主函数：从 XML 加载配置 ---- */
/* server 部分 */
@@ -333,6 +353,15 @@ void memory_load(xmlNodePtr *root, AppConfig *out_cfg){
xmlFree(txt);
}
}
xmlNodePtr leakage_node = find_child(mem, "leakage");
if (leakage_node) {
xmlChar *txt = xmlNodeGetContent(leakage_node);
if (txt) {
parse_leakage((char *)txt, &out_cfg->leak_mode);
xmlFree(txt);
}
}
}
}

19
common/config.h
View File

@@ -2,6 +2,7 @@
#define __CONFIG_H__
#include <stddef.h>
#include <memory/alloc_dispatch.h>
typedef enum {
LOG_LEVEL_DEBUG,
@@ -19,11 +20,17 @@ typedef enum {
PERSIST_NONE
} PersistenceType;
typedef enum {
ALLOC_JEMALLOC,
ALLOC_MALLOC,
ALLOC_OTHER
} AllocatorType;
// typedef enum {
// ALLOC_JEMALLOC,
// ALLOC_MALLOC,
// ALLOC_MYPOOL,
// ALLOC_OTHER
// } AllocatorType;
// typedef enum {
// MEMLEAK_DETECT_OFF = 0, // 关闭检测
// MEMLEAK_DETECT_ON = 1 // 开启检测
// } MemLeakDetectMode;
typedef struct {
char ip[64];
@@ -43,6 +50,7 @@ typedef struct {
char hash_file[256];
AllocatorType allocator;
MemLeakDetectMode leak_mode;
} AppConfig;
/**
@@ -55,5 +63,6 @@ const char *log_level_to_string(LogLevel lvl);
const char *server_mode_to_string(ServerMode mode);
const char *persistence_to_string(PersistenceType p);
const char *allocator_to_string(AllocatorType a);
const char *leakage_to_string(MemLeakDetectMode a);
#endif /* CONFIG_H */

3
config/config.xml
View File

@@ -27,6 +27,7 @@
</persistence>
<memory>
<allocator>jemalloc</allocator> <!-- jemalloc / malloc / other -->
<allocator>mypool</allocator> <!-- jemalloc / malloc / mypool -->
<leakage>enable</leakage> <!-- enable/disable-->
</memory>
</config>

2
diskuring/diskuring.c
View File

@@ -1,5 +1,5 @@
#include "diskuring.h"
#include "../memory/alloc_dispatch.h"
#include "memory/alloc_dispatch.h"
#include <poll.h>
#include <sys/eventfd.h>

14
kvstore.c
View File

@@ -353,6 +353,7 @@ int init_kvengine(void) {
kvs_replay_log(global_cmd_log_fd);
}
printf("kvengine init complete\n");
return 0;
}
@@ -370,10 +371,14 @@ void dest_kvengine(void) {
destroy_cmd_log(global_cmd_log_fd);
}
void init_memory_pool(void){
void init_memory_pool(AppConfig *cfg){
#if MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
mp_create(&global_mempool);
#endif
kvs_set_memleak_detect(cfg->leak_mode);
kvs_set_alloc_type(cfg->allocator);
printf("mempool init complete\n");
}
void dest_memory_pool(void){
@@ -440,7 +445,9 @@ int init_config(AppConfig *cfg){
printf("|—— Persist-hash : %s\n", cfg->hash_file);
printf("Log level : %s\n", log_level_to_string(cfg->log_level));
printf("Allocator : %s\n", allocator_to_string(cfg->allocator));
printf("Memory : \n");
printf("|——Allocator : %s\n", allocator_to_string(cfg->allocator));
printf("|——MemLeakDetectMode : %s\n", leakage_to_string(cfg->leak_mode));
printf("=============== Config ===============\n");
xmlCleanupParser();
@@ -476,10 +483,9 @@ int main(int argc, char *argv[]) {
init_memory_pool();
init_memory_pool(&global_cfg);
init_kvengine();
#if (NETWORK_SELECT == NETWORK_REACTOR)
reactor_start(port, kvs_protocol, master_ip, master_port); //
#elif (NETWORK_SELECT == NETWORK_PROACTOR)

111
memory/alloc_dispatch.c
View File

@@ -1,30 +1,111 @@
#include "alloc_dispatch.h"
#include "mempool.h"
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <stdatomic.h>
#include <jemalloc/jemalloc.h>
mp_pool_t global_mempool;
void *kvs_malloc(size_t size) {
#if MEMORY_SELECT_MALLOC == MEMORY_USE_DEFAULT
return malloc(size);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
return mp_alloc(size);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_JEMALLOC
return je_malloc(size);
#endif
static atomic_int g_memory_mode = ATOMIC_VAR_INIT(MEMLEAK_DETECT_OFF);
static atomic_int g_memleak_detect_mode = ATOMIC_VAR_INIT(MEMLEAK_DETECT_OFF);
// 设置内存池类型
void kvs_set_alloc_type(AllocatorType mode) {
atomic_store(&g_memory_mode, mode);
}
void kvs_free(void *ptr) {
#if MEMORY_SELECT_MALLOC == MEMORY_USE_DEFAULT
// 获取当前内存池类型
AllocatorType kvs_get_alloc_type(void) {
return atomic_load(&g_memory_mode);
}
// 设置内存泄漏检测模式
void kvs_set_memleak_detect(MemLeakDetectMode mode) {
atomic_store(&g_memleak_detect_mode, mode);
}
// 获取当前内存泄漏检测模式
MemLeakDetectMode kvs_get_memleak_detect(void) {
return atomic_load(&g_memleak_detect_mode);
}
void *kvs_malloc_impl(size_t size){
switch (atomic_load(&g_memleak_detect_mode)){
case ALLOC_MALLOC:
return malloc(size);
case ALLOC_MYPOOL:
return mp_alloc(&global_mempool, size);
case ALLOC_JEMALLOC:
return je_malloc(size);
case ALLOC_OTHER:
default:
return malloc(size);
}
return NULL;
}
void kvs_free_impl(void *ptr) {
switch (atomic_load(&g_memleak_detect_mode)){
case ALLOC_MALLOC:
free(ptr);
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_MYMALLOC
int ret = mp_free(ptr);
break;
case ALLOC_MYPOOL:{
int ret = mp_free(&global_mempool, ptr);
if(ret == MEMPOOL_DOUBLE_FREE){
printf("double free %p\n", ptr);
}
#elif MEMORY_SELECT_MALLOC == MEMORY_USE_JEMALLOC
je_free(ptr);
#endif
break;
}
case ALLOC_JEMALLOC:
je_free(ptr);
break;
case ALLOC_OTHER:
default:
free(ptr);
break;
}
}
void *nMalloc(size_t size, const char * filename, const char *func, int line){
void *ptr = kvs_malloc_impl(size);
if(atomic_load(&g_memleak_detect_mode) == MEMLEAK_DETECT_ON) {
char buff[128];
snprintf(buff, 128, "./mem_leak/%p.mem", ptr);
FILE* fp = fopen(buff, "w");
if(!fp){
kvs_free(ptr);
return NULL;
}
fprintf(fp, "[+] [%s : %s : %d] %p: %ld malloc\n", filename, func, line, ptr, size);
fflush(fp);
fclose(fp);
}
return ptr;
}
void nFree(void *ptr, const char * filename, const char *func, int line){
if(!ptr) {
return ;
}
if(atomic_load(&g_memleak_detect_mode) == MEMLEAK_DETECT_ON) {
char buff[128];
snprintf(buff, 128, "./mem_leak/%p.mem", ptr);
if(unlink(buff) < 0) {
return ;
}
}
kvs_free_impl(ptr);
}

28
memory/alloc_dispatch.h
View File

@@ -5,14 +5,36 @@
#define MEMORY_USE_MYMALLOC 1
#define MEMORY_USE_JEMALLOC 2
#define MEMORY_SELECT_MALLOC MEMORY_USE_MYMALLOC
#include <stddef.h>
#include <stdint.h>
#include <stdatomic.h>
#include "mempool.h"
void *kvs_malloc(size_t size);
void kvs_free(void *ptr);
typedef enum {
ALLOC_JEMALLOC,
ALLOC_MALLOC,
ALLOC_MYPOOL,
ALLOC_OTHER
} AllocatorType;
void kvs_set_alloc_type(AllocatorType mode);
AllocatorType kvs_get_alloc_type(void);
typedef enum {
MEMLEAK_DETECT_OFF = 0, // 关闭检测
MEMLEAK_DETECT_ON = 1 // 开启检测
} MemLeakDetectMode;
void kvs_set_memleak_detect(MemLeakDetectMode mode);
MemLeakDetectMode kvs_get_memleak_detect(void);
void *kvs_malloc_impl(size_t size);
void kvs_free_impl(void *ptr);
void *nMalloc(size_t size, const char * filename, const char *func, int line);
void nFree(void *ptr, const char * filename, const char *func, int line);
#define kvs_malloc(size) nMalloc (size , __FILE__, __func__, __LINE__)
#define kvs_free(ptr) nFree (ptr , __FILE__, __func__, __LINE__)
#endif