ebpf的主从同步实现，QPS测试与内存池QPS测试。
This commit is contained in:
1iaan
359
ebpf/c/replica.c
359
ebpf/c/replica.c
@@ -2,6 +2,8 @@
|
||||
/* Copyright (c) 2020 Facebook */
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/resource.h>
|
||||
#include <bpf/libbpf.h>
|
||||
#include "replica.skel.h"
|
||||
@@ -12,10 +14,15 @@
|
||||
|
||||
#include "replica.h"
|
||||
|
||||
typedef enum {
|
||||
OFFLINE = 0,
|
||||
ONLINE = 1,
|
||||
}replica_state_e ;
|
||||
|
||||
struct cmd_node {
|
||||
__u64 seq;
|
||||
__u32 len;
|
||||
uint8_t *cmd;
|
||||
size_t len;
|
||||
struct cmd_node *next;
|
||||
};
|
||||
|
||||
@@ -25,164 +32,184 @@ struct pending_queue {
|
||||
int count;
|
||||
};
|
||||
|
||||
static void queue_init(struct pending_queue *q) {
|
||||
q->head = q->tail = NULL;
|
||||
q->count = 0;
|
||||
}
|
||||
/* ================= 全局状态 ================= */
|
||||
|
||||
static void queue_push(struct pending_queue *q, const uint8_t *cmd, size_t len) {
|
||||
struct cmd_node *node = malloc(sizeof(*node));
|
||||
if (!node) return;
|
||||
node->cmd = malloc(len);
|
||||
if (!node->cmd) { free(node); return; }
|
||||
memcpy(node->cmd, cmd, len);
|
||||
node->len = len;
|
||||
node->next = NULL;
|
||||
if (q->tail) q->tail->next = node;
|
||||
else q->head = node;
|
||||
q->tail = node;
|
||||
q->count++;
|
||||
}
|
||||
static replica_state_e state = OFFLINE;
|
||||
static int sockfd = -1;
|
||||
|
||||
static void queue_send_and_clear(struct pending_queue *q, int sock) {
|
||||
struct cmd_node *node = q->head;
|
||||
int sent_count = 0;
|
||||
while (node) {
|
||||
if (send(sock, node->cmd, node->len, 0) > 0) {
|
||||
sent_count++;
|
||||
}
|
||||
struct cmd_node *tmp = node;
|
||||
node = node->next;
|
||||
free(tmp->cmd);
|
||||
free(tmp);
|
||||
}
|
||||
if (sent_count > 0) {
|
||||
printf("[QUEUE] Sent %d commands to slave\n", sent_count);
|
||||
}
|
||||
queue_init(q);
|
||||
}
|
||||
static char peer_ip[MAX_IP_LEN];
|
||||
static int peer_port;
|
||||
static __u64 peer_seq;
|
||||
|
||||
static void queue_free(struct pending_queue *q) {
|
||||
struct cmd_node *node = q->head;
|
||||
while (node) {
|
||||
struct cmd_node *tmp = node;
|
||||
node = node->next;
|
||||
free(tmp->cmd);
|
||||
free(tmp);
|
||||
}
|
||||
queue_init(q);
|
||||
}
|
||||
static struct pending_queue pending = {
|
||||
.head = NULL,
|
||||
.tail = NULL,
|
||||
.count = 0,
|
||||
};
|
||||
|
||||
static int send_file(int sock, const char *path) {
|
||||
FILE *fp = fopen(path, "rb");
|
||||
if (!fp) {
|
||||
printf("[ERROR] Failed to open file: %s\n", path);
|
||||
return -1;
|
||||
}
|
||||
|
||||
char buf[4096];
|
||||
size_t n, total = 0;
|
||||
while ((n = fread(buf, 1, sizeof(buf), fp)) > 0) {
|
||||
if (send(sock, buf, n, 0) < 0) {
|
||||
fclose(fp);
|
||||
printf("[ERROR] Failed to send file: %s (sent %zu bytes)\n", path, total);
|
||||
return -1;
|
||||
}
|
||||
total += n;
|
||||
}
|
||||
fclose(fp);
|
||||
printf("[FILE] Sent %s (%zu bytes)\n", path, total);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 全局状态（单 Slave 简化）
|
||||
static enum state current_state = NOSLAVE;
|
||||
static char slave_ip[16] = {0};
|
||||
static int slave_port = 0;
|
||||
static char array_file[128] = {0};
|
||||
static char rbtree_file[128] = {0};
|
||||
static char hash_file[128] = {0};
|
||||
static struct pending_queue pending;
|
||||
static int slave_sock = -1; // 连接 Slave 的 socket
|
||||
|
||||
// 连接 Slave
|
||||
static int connect_slave() {
|
||||
int sock = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (sock < 0) return -1;
|
||||
struct sockaddr_in addr;
|
||||
memset(&addr, 0, sizeof(addr));
|
||||
addr.sin_family = AF_INET;
|
||||
addr.sin_port = htons(slave_port);
|
||||
inet_pton(AF_INET, slave_ip, &addr.sin_addr);
|
||||
if (connect(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
|
||||
close(sock);
|
||||
return -1;
|
||||
}
|
||||
return sock;
|
||||
}
|
||||
|
||||
static void handle_event(void *ctx, int cpu, void *data, __u32 size) {
|
||||
struct event *ev = (struct event *)data;
|
||||
|
||||
switch (ev->type) {
|
||||
case EVENT_CREATE_SNAPSHOT_ASYNC:
|
||||
printf("[EVENT] Type: CREATE_SNAPSHOT_ASYNC\n");
|
||||
printf("[EVENT] Slave IP: %s, Port: %u\n", ev->data.sync.ip, ev->data.sync.port);
|
||||
|
||||
if (current_state == NOSLAVE) {
|
||||
current_state = START;
|
||||
strncpy(slave_ip, ev->data.sync.ip, sizeof(slave_ip));
|
||||
slave_port = ev->data.sync.port;
|
||||
queue_init(&pending);
|
||||
slave_sock = connect_slave(); // 连接 Slave
|
||||
if (slave_sock < 0) {
|
||||
printf("Failed to connect to Slave %s:%d\n", slave_ip, slave_port);
|
||||
current_state = NOSLAVE;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case EVENT_COMPLETED_CMD:
|
||||
printf("[EVENT] Type: COMPLETED_CMD\n");
|
||||
printf("[EVENT] Command length: %llu bytes\n", ev->data.cmd.len);
|
||||
|
||||
if (current_state != NOSLAVE) {
|
||||
queue_push(&pending, ev->data.cmd.cmd, ev->data.cmd.len);
|
||||
}
|
||||
break;
|
||||
case EVENT_CREATE_SNAPSHOT_OK:
|
||||
printf("[EVENT] Type: CREATE_SNAPSHOT_OK\n");
|
||||
printf("[EVENT] Array file: %s\n", ev->data.ok.array_file);
|
||||
printf("[EVENT] RBTree file: %s\n", ev->data.ok.rbtree_file);
|
||||
printf("[EVENT] Hash file: %s\n", ev->data.ok.hash_file);
|
||||
|
||||
if (current_state == START) {
|
||||
current_state = DONE;
|
||||
strncpy(array_file, ev->data.ok.array_file, sizeof(array_file));
|
||||
strncpy(rbtree_file, ev->data.ok.rbtree_file, sizeof(rbtree_file));
|
||||
strncpy(hash_file, ev->data.ok.hash_file, sizeof(hash_file));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void lost_event(void *ctx, int cpu, __u64 cnt) {
|
||||
printf("Lost %llu events\n", cnt);
|
||||
}
|
||||
|
||||
static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
|
||||
/* ================= pending 队列操作 ================= */
|
||||
static void pending_free()
|
||||
{
|
||||
return vfprintf(stderr, format, args);
|
||||
struct pending_queue *q = &pending;
|
||||
struct cmd_node *cur = q->head;
|
||||
while (cur) {
|
||||
struct cmd_node *tmp = cur;
|
||||
cur = cur->next;
|
||||
free(tmp->cmd);
|
||||
free(tmp);
|
||||
}
|
||||
q->head = q->tail = NULL;
|
||||
q->count = 0;
|
||||
}
|
||||
|
||||
static void pending_push(__u64 seq, __u32 len, const uint8_t *cmd)
|
||||
{
|
||||
struct cmd_node *node = malloc(sizeof(*node));
|
||||
if (!node)
|
||||
return;
|
||||
|
||||
node->cmd = malloc(len);
|
||||
if (!node->cmd) {
|
||||
free(node);
|
||||
return;
|
||||
}
|
||||
|
||||
memcpy(node->cmd, cmd, len);
|
||||
node->seq = seq;
|
||||
node->len = len;
|
||||
node->next = NULL;
|
||||
|
||||
if (!pending.tail) {
|
||||
pending.head = pending.tail = node;
|
||||
} else {
|
||||
pending.tail->next = node;
|
||||
pending.tail = node;
|
||||
}
|
||||
|
||||
pending.count++;
|
||||
}
|
||||
|
||||
static void pending_gc(__u64 min_seq)
|
||||
{
|
||||
struct cmd_node *cur = pending.head;
|
||||
|
||||
int n = pending.count;
|
||||
while (cur && cur->seq < min_seq) {
|
||||
struct cmd_node *tmp = cur;
|
||||
cur = cur->next;
|
||||
|
||||
free(tmp->cmd);
|
||||
free(tmp);
|
||||
pending.count--;
|
||||
}
|
||||
|
||||
printf("gc:%d\n", n-pending.count);
|
||||
|
||||
pending.head = cur;
|
||||
if (!cur)
|
||||
pending.tail = NULL;
|
||||
}
|
||||
|
||||
static void pending_send_one(struct cmd_node *node){
|
||||
int rt = send(sockfd, node->cmd, node->len, 0);
|
||||
printf("send seq:%lld, rt=%d\n", node->seq, rt);
|
||||
}
|
||||
|
||||
static void pending_send_all(void)
|
||||
{
|
||||
struct cmd_node *cur = pending.head;
|
||||
|
||||
while (cur) {
|
||||
pending_send_one(cur);
|
||||
cur = cur->next;
|
||||
}
|
||||
}
|
||||
|
||||
/* ================= 网络逻辑 ================= */
|
||||
static void try_connect(void)
|
||||
{
|
||||
if(sockfd > 0){
|
||||
close(sockfd);
|
||||
sockfd = -1;
|
||||
}
|
||||
|
||||
struct sockaddr_in addr = {};
|
||||
int i = 0;
|
||||
|
||||
addr.sin_family = AF_INET;
|
||||
addr.sin_port = htons(peer_port);
|
||||
inet_pton(AF_INET, peer_ip, &addr.sin_addr);
|
||||
|
||||
for(i = 0;i < 10; ++ i){
|
||||
sockfd = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (sockfd < 0) {
|
||||
perror("socket");
|
||||
return;
|
||||
}
|
||||
|
||||
printf("connect try %d...\n", i + 1);
|
||||
if (connect(sockfd, (struct sockaddr *)&addr, sizeof(addr)) == 0) {
|
||||
printf("connect success: %s:%d\n", peer_ip, peer_port);
|
||||
state = ONLINE;
|
||||
pending_send_all();
|
||||
return;
|
||||
}
|
||||
|
||||
perror("connect");
|
||||
close(sockfd);
|
||||
sockfd = -1;
|
||||
|
||||
sleep(1);
|
||||
}
|
||||
|
||||
printf("connect failed after 10 retries\n");
|
||||
}
|
||||
|
||||
|
||||
/* ================= perf buffer 回调 ================= */
|
||||
static void handle_event(void *ctx, int cpu, void *data, __u32 size)
|
||||
{
|
||||
struct replica_event *evt = data;
|
||||
|
||||
switch (evt->type) {
|
||||
|
||||
case EVENT_SSYNC:
|
||||
strncpy(peer_ip, evt->sync.ip, sizeof(peer_ip));
|
||||
peer_port = evt->sync.port;
|
||||
peer_seq = evt->sync.seq;
|
||||
printf("SSYNC [seq:%lld], [%s:%d]\n", peer_seq, peer_ip, peer_port);
|
||||
|
||||
state = OFFLINE;
|
||||
pending_gc(peer_seq);
|
||||
break;
|
||||
|
||||
case EVENT_COMPLETED_CMD:
|
||||
// printf("CMD [seq:%lld], cmd:\n[\n%s]\n", evt->complete.seq, evt->complete.cmd);
|
||||
pending_push(evt->complete.seq,
|
||||
evt->complete.len,
|
||||
evt->complete.cmd);
|
||||
|
||||
if (state == ONLINE && pending.tail) {
|
||||
struct cmd_node *n = pending.tail;
|
||||
pending_send_one(n);
|
||||
}
|
||||
break;
|
||||
|
||||
case EVENT_SREADY:
|
||||
printf("SREADY \n");
|
||||
if (state == OFFLINE)
|
||||
try_connect();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* ================= main ================= */
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
struct replica_bpf *skel;
|
||||
struct perf_buffer *pb = NULL;
|
||||
int err;
|
||||
|
||||
/* Set up libbpf errors and debug info callback */
|
||||
libbpf_set_print(libbpf_print_fn);
|
||||
|
||||
/* Open BPF application */
|
||||
skel = replica_bpf__open();
|
||||
if (!skel) {
|
||||
@@ -190,9 +217,6 @@ int main(int argc, char **argv)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ensure BPF program only handles write() syscalls from our process */
|
||||
skel->bss->my_pid = getpid();
|
||||
|
||||
/* Load & verify BPF programs */
|
||||
err = replica_bpf__load(skel);
|
||||
if (err) {
|
||||
@@ -211,47 +235,22 @@ int main(int argc, char **argv)
|
||||
"to see output of the BPF programs.\n");
|
||||
|
||||
|
||||
struct perf_buffer *pb = perf_buffer__new(bpf_map__fd(skel->maps.channel), 8, handle_event, lost_event, NULL, NULL);
|
||||
pb = perf_buffer__new(bpf_map__fd(skel->maps.events), 8,
|
||||
handle_event, NULL, NULL, NULL);
|
||||
|
||||
if(!pb){
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
#if 0
|
||||
while(1){
|
||||
perf_buffer__poll(pb, 1000);
|
||||
}
|
||||
#else
|
||||
while (1) {
|
||||
perf_buffer__poll(pb, 1000); // 处理事件
|
||||
|
||||
// 循环中检查状态并发送
|
||||
if (current_state == DONE && slave_sock >= 0) {
|
||||
// 发送快照文件
|
||||
if (send_file(slave_sock, array_file) == 0 &&
|
||||
send_file(slave_sock, rbtree_file) == 0 &&
|
||||
send_file(slave_sock, hash_file) == 0) {
|
||||
current_state = ONLINE;
|
||||
printf("Snapshot sent, state to ONLINE\n");
|
||||
} else {
|
||||
printf("Failed to send snapshot\n");
|
||||
current_state = NOSLAVE;
|
||||
close(slave_sock);
|
||||
slave_sock = -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (current_state == ONLINE && slave_sock >= 0) {
|
||||
// 发送 pending
|
||||
queue_send_and_clear(&pending, slave_sock);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
perf_buffer__free(pb);
|
||||
|
||||
cleanup:
|
||||
queue_free(&pending);
|
||||
if (slave_sock >= 0) close(slave_sock);
|
||||
pending_free();
|
||||
if (sockfd >= 0) close(sockfd);
|
||||
replica_bpf__destroy(skel);
|
||||
return -err;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user