lian/ldb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ebpf的主从同步实现，QPS测试与内存池QPS测试。

Browse Source
This commit is contained in:
1iaan
2026-01-30 16:00:06 +00:00
parent 2bdb48d63d
commit fbdcff6878
23 changed files with 599 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
ebpf/c/replica.bpf.c
View File

@@ -1,103 +1,80 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/* Copyright (c) 2020 Facebook */
#include <linux/bpf.h>
#include <linux/ptrace.h>
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include "replica.h"
char LICENSE[] SEC("license") = "Dual BSD/GPL";
int my_pid = 0;
struct {
__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(int));
} channel SEC(".maps");
__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
__uint(key_size, sizeof(int));
__uint(value_size, sizeof(int));
} events SEC(".maps");
SEC("uprobe/kvs_create_snapshot_async")
int uprobe_create_snapshot_async(struct pt_regs *ctx)
/* __completed_cmd(const uint8_t *cmd, size_t len, unsigned long long seq); */
SEC("uprobe//home/lian/share/9.1-kvstore/kvstore:__completed_cmd")
int BPF_KPROBE(handle_completed_cmd,
const __u8 *cmd, size_t len, __u64 seq)
{
struct event ev;
__builtin_memset(&ev, 0, sizeof(ev));
struct replica_event evt = {};
__u32 copy_len;
const char *ip;
__u32 port;
evt.type = EVENT_COMPLETED_CMD;
evt.complete.seq = seq;
ev.type = EVENT_CREATE_SNAPSHOT_ASYNC;
copy_len = len;
if (copy_len > MAX_CMD_LEN)
copy_len = MAX_CMD_LEN;
ip = (const char *)PT_REGS_PARM1(ctx);
port = (__u32)PT_REGS_PARM2(ctx);
evt.complete.len = copy_len;
bpf_probe_read_user_str(ev.data.sync.ip,
sizeof(ev.data.sync.ip),
ip);
ev.data.sync.port = port;
if (cmd)
bpf_probe_read_user(evt.complete.cmd, copy_len, cmd);
bpf_perf_event_output(ctx, &channel,
bpf_perf_event_output(ctx, &events,
BPF_F_CURRENT_CPU,
&ev, sizeof(ev));
&evt, sizeof(evt));
return 0;
}
SEC("uprobe/__compeleted_cmd")
int uprobe_completed_cmd(struct pt_regs *ctx)
/* __ssync(const uint8_t *ip, uint32_t ip_len, int port, unsigned long long seq); */
SEC("uprobe//home/lian/share/9.1-kvstore/kvstore:__ssync")
int BPF_KPROBE(handle_ssync,
const __u8 *ip, __u32 ip_len, int port, __u64 seq)
{
struct event ev;
__builtin_memset(&ev, 0, sizeof(ev));
struct replica_event evt = {};
const __u8 *cmd;
__u32 len;
evt.type = EVENT_SSYNC;
evt.sync.seq = seq;
evt.sync.port = port;
ev.type = EVENT_COMPLETED_CMD;
__u32 copy_len = ip_len;
if (copy_len > sizeof(evt.sync.ip))
copy_len = sizeof(evt.sync.ip);
cmd = (const __u8 *)PT_REGS_PARM1(ctx);
len = (__u32)PT_REGS_PARM2(ctx);
if (ip)
bpf_probe_read_user(evt.sync.ip, copy_len, ip);
if (len > sizeof(ev.data.cmd.cmd))
len = sizeof(ev.data.cmd.cmd);
ev.data.cmd.len = len;
bpf_probe_read_user(ev.data.cmd.cmd, len, cmd);
bpf_perf_event_output(ctx, &channel,
bpf_perf_event_output(ctx, &events,
BPF_F_CURRENT_CPU,
&ev, sizeof(ev));
&evt, sizeof(evt));
return 0;
}
SEC("uprobe/__create_snapshot_ok")
int uprobe_create_snapshot_ok(struct pt_regs *ctx)
/* __sready(void); */
SEC("uprobe//home/lian/share/9.1-kvstore/kvstore:__sready")
int BPF_KPROBE(handle_sready)
{
struct event ev;
__builtin_memset(&ev, 0, sizeof(ev));
const char *array_file;
const char *rbtree_file;
const char *hash_file;
struct replica_event evt = {};
ev.type = EVENT_CREATE_SNAPSHOT_OK;
evt.type = EVENT_SREADY;
array_file = (const char *)PT_REGS_PARM1(ctx);
rbtree_file = (const char *)PT_REGS_PARM2(ctx);
hash_file = (const char *)PT_REGS_PARM3(ctx);
bpf_probe_read_user_str(ev.data.ok.array_file,
sizeof(ev.data.ok.array_file),
array_file);
bpf_probe_read_user_str(ev.data.ok.rbtree_file,
sizeof(ev.data.ok.rbtree_file),
rbtree_file);
bpf_probe_read_user_str(ev.data.ok.hash_file,
sizeof(ev.data.ok.hash_file),
hash_file);
bpf_perf_event_output(ctx, &channel,
bpf_perf_event_output(ctx, &events,
BPF_F_CURRENT_CPU,
&ev, sizeof(ev));
&evt, sizeof(evt));
return 0;
}

359
ebpf/c/replica.c
View File

@@ -2,6 +2,8 @@
/* Copyright (c) 2020 Facebook */
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/resource.h>
#include <bpf/libbpf.h>
#include "replica.skel.h"
@@ -12,10 +14,15 @@
#include "replica.h"
typedef enum {
OFFLINE = 0,
ONLINE = 1,
}replica_state_e ;
struct cmd_node {
__u64 seq;
__u32 len;
uint8_t *cmd;
size_t len;
struct cmd_node *next;
};
@@ -25,164 +32,184 @@ struct pending_queue {
int count;
};
static void queue_init(struct pending_queue *q) {
q->head = q->tail = NULL;
q->count = 0;
}
/* ================= 全局状态 ================= */
static void queue_push(struct pending_queue *q, const uint8_t *cmd, size_t len) {
struct cmd_node *node = malloc(sizeof(*node));
if (!node) return;
node->cmd = malloc(len);
if (!node->cmd) { free(node); return; }
memcpy(node->cmd, cmd, len);
node->len = len;
node->next = NULL;
if (q->tail) q->tail->next = node;
else q->head = node;
q->tail = node;
q->count++;
}
static replica_state_e state = OFFLINE;
static int sockfd = -1;
static void queue_send_and_clear(struct pending_queue *q, int sock) {
struct cmd_node *node = q->head;
int sent_count = 0;
while (node) {
if (send(sock, node->cmd, node->len, 0) > 0) {
sent_count++;
}
struct cmd_node *tmp = node;
node = node->next;
free(tmp->cmd);
free(tmp);
}
if (sent_count > 0) {
printf("[QUEUE] Sent %d commands to slave\n", sent_count);
}
queue_init(q);
}
static char peer_ip[MAX_IP_LEN];
static int peer_port;
static __u64 peer_seq;
static void queue_free(struct pending_queue *q) {
struct cmd_node *node = q->head;
while (node) {
struct cmd_node *tmp = node;
node = node->next;
free(tmp->cmd);
free(tmp);
}
queue_init(q);
}
static struct pending_queue pending = {
.head = NULL,
.tail = NULL,
.count = 0,
};
static int send_file(int sock, const char *path) {
FILE *fp = fopen(path, "rb");
if (!fp) {
printf("[ERROR] Failed to open file: %s\n", path);
return -1;
}
char buf[4096];
size_t n, total = 0;
while ((n = fread(buf, 1, sizeof(buf), fp)) > 0) {
if (send(sock, buf, n, 0) < 0) {
fclose(fp);
printf("[ERROR] Failed to send file: %s (sent %zu bytes)\n", path, total);
return -1;
}
total += n;
}
fclose(fp);
printf("[FILE] Sent %s (%zu bytes)\n", path, total);
return 0;
}
// 全局状态（单 Slave 简化）
static enum state current_state = NOSLAVE;
static char slave_ip[16] = {0};
static int slave_port = 0;
static char array_file[128] = {0};
static char rbtree_file[128] = {0};
static char hash_file[128] = {0};
static struct pending_queue pending;
static int slave_sock = -1; // 连接 Slave 的 socket
// 连接 Slave
static int connect_slave() {
int sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock < 0) return -1;
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(slave_port);
inet_pton(AF_INET, slave_ip, &addr.sin_addr);
if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
close(sock);
return -1;
}
return sock;
}
static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
struct event *ev = (struct event *)data;
switch (ev->type) {
case EVENT_CREATE_SNAPSHOT_ASYNC:
printf("[EVENT] Type: CREATE_SNAPSHOT_ASYNC\n");
printf("[EVENT] Slave IP: %s, Port: %u\n", ev->data.sync.ip, ev->data.sync.port);
if (current_state == NOSLAVE) {
current_state = START;
strncpy(slave_ip, ev->data.sync.ip, sizeof(slave_ip));
slave_port = ev->data.sync.port;
queue_init(&pending);
slave_sock = connect_slave(); // 连接 Slave
if (slave_sock < 0) {
printf("Failed to connect to Slave %s:%d\n", slave_ip, slave_port);
current_state = NOSLAVE;
}
}
break;
case EVENT_COMPLETED_CMD:
printf("[EVENT] Type: COMPLETED_CMD\n");
printf("[EVENT] Command length: %llu bytes\n", ev->data.cmd.len);
if (current_state != NOSLAVE) {
queue_push(&pending, ev->data.cmd.cmd, ev->data.cmd.len);
}
break;
case EVENT_CREATE_SNAPSHOT_OK:
printf("[EVENT] Type: CREATE_SNAPSHOT_OK\n");
printf("[EVENT] Array file: %s\n", ev->data.ok.array_file);
printf("[EVENT] RBTree file: %s\n", ev->data.ok.rbtree_file);
printf("[EVENT] Hash file: %s\n", ev->data.ok.hash_file);
if (current_state == START) {
current_state = DONE;
strncpy(array_file, ev->data.ok.array_file, sizeof(array_file));
strncpy(rbtree_file, ev->data.ok.rbtree_file, sizeof(rbtree_file));
strncpy(hash_file, ev->data.ok.hash_file, sizeof(hash_file));
}
break;
}
}
static void lost_event(void *ctx, int cpu, __u64 cnt) {
printf("Lost %llu events\n", cnt);
}
static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
/* ================= pending 队列操作 ================= */
static void pending_free()
{
return vfprintf(stderr, format, args);
struct pending_queue *q = &pending;
struct cmd_node *cur = q->head;
while (cur) {
struct cmd_node *tmp = cur;
cur = cur->next;
free(tmp->cmd);
free(tmp);
}
q->head = q->tail = NULL;
q->count = 0;
}
static void pending_push(__u64 seq, __u32 len, const uint8_t *cmd)
{
struct cmd_node *node = malloc(sizeof(*node));
if (!node)
return;
node->cmd = malloc(len);
if (!node->cmd) {
free(node);
return;
}
memcpy(node->cmd, cmd, len);
node->seq = seq;
node->len = len;
node->next = NULL;
if (!pending.tail) {
pending.head = pending.tail = node;
} else {
pending.tail->next = node;
pending.tail = node;
}
pending.count++;
}
static void pending_gc(__u64 min_seq)
{
struct cmd_node *cur = pending.head;
int n = pending.count;
while (cur && cur->seq < min_seq) {
struct cmd_node *tmp = cur;
cur = cur->next;
free(tmp->cmd);
free(tmp);
pending.count--;
}
printf("gc:%d\n", n-pending.count);
pending.head = cur;
if (!cur)
pending.tail = NULL;
}
static void pending_send_one(struct cmd_node *node){
int rt = send(sockfd, node->cmd, node->len, 0);
printf("send seq:%lld, rt=%d\n", node->seq, rt);
}
static void pending_send_all(void)
{
struct cmd_node *cur = pending.head;
while (cur) {
pending_send_one(cur);
cur = cur->next;
}
}
/* ================= 网络逻辑 ================= */
static void try_connect(void)
{
if(sockfd > 0){
close(sockfd);
sockfd = -1;
}
struct sockaddr_in addr = {};
int i = 0;
addr.sin_family = AF_INET;
addr.sin_port = htons(peer_port);
inet_pton(AF_INET, peer_ip, &addr.sin_addr);
for(i = 0;i < 10; ++ i){
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0) {
perror("socket");
return;
}
printf("connect try %d...\n", i + 1);
if (connect(sockfd, (struct sockaddr *)&addr, sizeof(addr)) == 0) {
printf("connect success: %s:%d\n", peer_ip, peer_port);
state = ONLINE;
pending_send_all();
return;
}
perror("connect");
close(sockfd);
sockfd = -1;
sleep(1);
}
printf("connect failed after 10 retries\n");
}
/* ================= perf buffer 回调 ================= */
static void handle_event(void *ctx, int cpu, void *data, __u32 size)
{
struct replica_event *evt = data;
switch (evt->type) {
case EVENT_SSYNC:
strncpy(peer_ip, evt->sync.ip, sizeof(peer_ip));
peer_port = evt->sync.port;
peer_seq = evt->sync.seq;
printf("SSYNC [seq:%lld], [%s:%d]\n", peer_seq, peer_ip, peer_port);
state = OFFLINE;
pending_gc(peer_seq);
break;
case EVENT_COMPLETED_CMD:
// printf("CMD [seq:%lld], cmd:\n[\n%s]\n", evt->complete.seq, evt->complete.cmd);
pending_push(evt->complete.seq,
evt->complete.len,
evt->complete.cmd);
if (state == ONLINE && pending.tail) {
struct cmd_node *n = pending.tail;
pending_send_one(n);
}
break;
case EVENT_SREADY:
printf("SREADY \n");
if (state == OFFLINE)
try_connect();
break;
}
}
/* ================= main ================= */
int main(int argc, char **argv)
{
struct replica_bpf *skel;
struct perf_buffer *pb = NULL;
int err;
/* Set up libbpf errors and debug info callback */
libbpf_set_print(libbpf_print_fn);
/* Open BPF application */
skel = replica_bpf__open();
if (!skel) {
@@ -190,9 +217,6 @@ int main(int argc, char **argv)
return 1;
}
/* ensure BPF program only handles write() syscalls from our process */
skel->bss->my_pid = getpid();
/* Load & verify BPF programs */
err = replica_bpf__load(skel);
if (err) {
@@ -211,47 +235,22 @@ int main(int argc, char **argv)
"to see output of the BPF programs.\n");
struct perf_buffer *pb = perf_buffer__new(bpf_map__fd(skel->maps.channel), 8, handle_event, lost_event, NULL, NULL);
pb = perf_buffer__new(bpf_map__fd(skel->maps.events), 8,
handle_event, NULL, NULL, NULL);
if(!pb){
goto cleanup;
}
#if 0
while(1){
perf_buffer__poll(pb, 1000);
}
#else
while (1) {
perf_buffer__poll(pb, 1000); // 处理事件
// 循环中检查状态并发送
if (current_state == DONE && slave_sock >= 0) {
// 发送快照文件
if (send_file(slave_sock, array_file) == 0 &&
send_file(slave_sock, rbtree_file) == 0 &&
send_file(slave_sock, hash_file) == 0) {
current_state = ONLINE;
printf("Snapshot sent, state to ONLINE\n");
} else {
printf("Failed to send snapshot\n");
current_state = NOSLAVE;
close(slave_sock);
slave_sock = -1;
}
}
if (current_state == ONLINE && slave_sock >= 0) {
// 发送 pending
queue_send_and_clear(&pending, slave_sock);
}
}
#endif
perf_buffer__free(pb);
cleanup:
queue_free(&pending);
if (slave_sock >= 0) close(slave_sock);
pending_free();
if (sockfd >= 0) close(sockfd);
replica_bpf__destroy(skel);
return -err;
}

51
ebpf/c/replica.h
View File

@@ -1,36 +1,37 @@
#ifndef __REPLICA_H__
#define __REPLICA_H__
#define MAX_CMD_LEN 256
#define MAX_IP_LEN 64
enum event_type {
EVENT_CREATE_SNAPSHOT_ASYNC,
EVENT_CREATE_SNAPSHOT_OK,
EVENT_CREATE_SNAPSHOT_READY,
EVENT_COMPLETED_CMD
EVENT_COMPLETED_CMD,
EVENT_SSYNC,
EVENT_SREADY,
};
struct event {
enum event_type type;
struct complete_cmd_evt {
__u64 seq;
__u32 len;
__u8 cmd[MAX_CMD_LEN];
};
struct sync_evt {
__u64 seq;
char ip[MAX_IP_LEN];
__s32 port;
};
struct replica_event {
__u32 type;
__u32 _pad;
union {
struct {
char ip[16];
__u32 port;
} sync;
struct {
__u8 cmd[256];
__u64 len;
} cmd;
struct {
char array_file[128];
char rbtree_file[128];
char hash_file[128];
} ok;
} data;
struct complete_cmd_evt complete;
struct sync_evt sync;
};
};
enum state {
NOSLAVE,
PREPARING,
ONLINE
};
#endif