From 5dafb129754e0c44495d2b2413d2bb593f65cb91 Mon Sep 17 00:00:00 2001 From: lec-bit Date: Wed, 23 Oct 2024 17:08:57 +0800 Subject: [PATCH 1/6] g_inner_map_mng.inner_map support persist in restart Signed-off-by: lec-bit --- .../deserialization_to_bpf_map.c | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c index 7d98bdbe5..b9b303148 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c @@ -278,6 +278,18 @@ static int get_map_fd_info(unsigned int id, int *map_fd, struct bpf_map_info *in return ret; } +static int bpf_get_map_id_by_fd(int map_fd) +{ + struct bpf_map_info info = {}; + __u32 info_len = sizeof(info); + + if (bpf_obj_get_info_by_fd(map_fd, &info, &info_len) < 0) { + LOG_ERR("bpf_obj_get_info_by_fd failed"); + return -1; + } + return info.id; +} + static int free_outter_map_entry(struct op_context *ctx, void *outter_key) { int key = *(int *)outter_key; @@ -337,13 +349,16 @@ static int copy_sfield_to_map(struct op_context *ctx, int o_index, const Protobu *(uintptr_t *)value = (size_t)o_index; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { + LOG_ERR("bpf_map_update_elem failed\n"); free_outter_map_entry(ctx, &o_index); return ret; } inner_fd = outter_key_to_inner_fd(ctx, o_index); - if (inner_fd < 0) + if (inner_fd < 0) { + LOG_ERR("outter_key_to_inner_fd\n"); return inner_fd; + } strcpy_s(ctx->inner_map_object, ctx->inner_info->value_size, save_value); ret = bpf_map_update_elem(inner_fd, &key, ctx->inner_map_object, BPF_ANY); @@ -363,13 +378,16 @@ static int copy_msg_field_to_map(struct op_context *ctx, int o_index, const Prot *(uintptr_t *)value = (size_t)o_index; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { + LOG_ERR("bpf_map_update_elem failed\n"); free_outter_map_entry(ctx, &o_index); return ret; } inner_fd = outter_key_to_inner_fd(ctx, o_index); - if (inner_fd < 0) + if (inner_fd < 0) { + LOG_ERR("outter_key_to_inner_fd failed\n"); return inner_fd; + } memcpy_s(&new_ctx, sizeof(new_ctx), ctx, sizeof(*ctx)); @@ -476,6 +494,7 @@ static int repeat_field_handle(struct op_context *ctx, const ProtobufCFieldDescr *(uintptr_t *)value = (size_t)outter_key; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { + LOG_ERR("bpf_map_update_elem failed\n"); free_outter_map_entry(ctx, &outter_key); return ret; } @@ -1727,8 +1746,18 @@ int inner_map_mng_persist() p->allocated_cnt = g_inner_map_mng.allocated_cnt; p->used_cnt = g_inner_map_mng.used_cnt; p->max_allocated_idx = g_inner_map_mng.max_allocated_idx; + + /* Since map_fd cannot be used universally in different processes, + map_fd needs to be converted into map_id and stored, and then + converted into map_fd in the corresponding process when the + configuration is restored. + When storing map_fd into outer_map, the kernel update interface will + perform special processing on maps of types BPF_MAP_TYPE_ARRAY_OF_MAPS + and BPF_MAP_TYPE_HASH_OF_MAPS. The input parameter is map_fd, but the + kernel will convert it into a bpf_map pointer for storage, so it will + not be affected.*/ for (i = 0; i <= g_inner_map_mng.max_allocated_idx; i++) { - p->inner_map_stat[i].map_fd = g_inner_map_mng.inner_maps[i].map_fd; + p->inner_map_stat[i].map_fd = bpf_get_map_id_by_fd(g_inner_map_mng.inner_maps[i].map_fd); p->inner_map_stat[i].used = g_inner_map_mng.inner_maps[i].used; p->inner_map_stat[i].allocated = g_inner_map_mng.inner_maps[i].allocated; } @@ -1746,20 +1775,35 @@ int inner_map_mng_persist() return 0; } -void inner_map_mng_restore_by_persist_stat(struct persist_info *p, struct inner_map_stat *stat) +int inner_map_mng_restore_by_persist_stat(struct persist_info *p, struct inner_map_stat *stat) { memcpy(g_inner_map_mng.inner_maps, stat, sizeof(struct inner_map_stat) * (p->max_allocated_idx + 1)); + // What is recorded in g_inner_map_mng.inner_maps[i].map_fd is map_id. + // Use map_id to get the fd of the inner_map in this process and refresh + // it to inner_maps for use. + for (int i = 0; i < p->max_allocated_idx; i++) { + if (g_inner_map_mng.inner_maps[i].allocated) { + int map_fd = bpf_map_get_fd_by_id(g_inner_map_mng.inner_maps[i].map_fd); + if (map_fd < 0) { + LOG_ERR("bpf_map_get_fd_by_id failed:%d\n", map_fd); + return map_fd; + } + g_inner_map_mng.inner_maps[i].map_fd = map_fd; + } + } + g_inner_map_mng.used_cnt = p->used_cnt; g_inner_map_mng.allocated_cnt = p->allocated_cnt; g_inner_map_mng.max_allocated_idx = p->max_allocated_idx; - return; + return 0; } int inner_map_restore(bool restore) { int size; int read_size; + int ret; FILE *f = NULL; struct persist_info p; struct inner_map_stat *stat = NULL; @@ -1796,7 +1840,13 @@ int inner_map_restore(bool restore) return 0; } - inner_map_mng_restore_by_persist_stat(&p, stat); + ret = inner_map_mng_restore_by_persist_stat(&p, stat); + if (ret < 0) { + LOG_ERR("inner_map_restore persist_stat failed, ret/%d\n", ret); + fclose(f); + free(stat); + return -1; + } free(stat); fclose(f); return 0; From 4da118935604d0de81cd1aaa660a19605b816791 Mon Sep 17 00:00:00 2001 From: lec-bit Date: Wed, 23 Oct 2024 17:55:10 +0800 Subject: [PATCH 2/6] update log Signed-off-by: lec-bit --- .../deserialization_to_bpf_map.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c index b9b303148..673db6df8 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c @@ -349,14 +349,14 @@ static int copy_sfield_to_map(struct op_context *ctx, int o_index, const Protobu *(uintptr_t *)value = (size_t)o_index; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { - LOG_ERR("bpf_map_update_elem failed\n"); + LOG_ERR("copy_sfield_to_map bpf_map_update_elem failed, ret:%d\n", ret); free_outter_map_entry(ctx, &o_index); return ret; } inner_fd = outter_key_to_inner_fd(ctx, o_index); if (inner_fd < 0) { - LOG_ERR("outter_key_to_inner_fd\n"); + LOG_ERR("copy_sfield_to_map outter_key_to_inner_fd failed, inner_fd:%d\n", inner_fd); return inner_fd; } @@ -378,14 +378,14 @@ static int copy_msg_field_to_map(struct op_context *ctx, int o_index, const Prot *(uintptr_t *)value = (size_t)o_index; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { - LOG_ERR("bpf_map_update_elem failed\n"); + LOG_ERR("copy_msg_field_to_map bpf_map_update_elem failed, ret:%d\n", ret); free_outter_map_entry(ctx, &o_index); return ret; } inner_fd = outter_key_to_inner_fd(ctx, o_index); if (inner_fd < 0) { - LOG_ERR("outter_key_to_inner_fd failed\n"); + LOG_ERR("copy_msg_field_to_map outter_key_to_inner_fd failed, inner_fd:%d\n", inner_fd); return inner_fd; } @@ -494,7 +494,7 @@ static int repeat_field_handle(struct op_context *ctx, const ProtobufCFieldDescr *(uintptr_t *)value = (size_t)outter_key; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { - LOG_ERR("bpf_map_update_elem failed\n"); + LOG_ERR("repeat_field_handle bpf_map_update_elem failed, ret:%d\n", ret); free_outter_map_entry(ctx, &outter_key); return ret; } @@ -1786,7 +1786,7 @@ int inner_map_mng_restore_by_persist_stat(struct persist_info *p, struct inner_m if (g_inner_map_mng.inner_maps[i].allocated) { int map_fd = bpf_map_get_fd_by_id(g_inner_map_mng.inner_maps[i].map_fd); if (map_fd < 0) { - LOG_ERR("bpf_map_get_fd_by_id failed:%d\n", map_fd); + LOG_ERR("restore_by_persist_stat bpf_map_get_fd_by_id failed, i:[%d] map_fd:[%d]\n", i, map_fd); return map_fd; } g_inner_map_mng.inner_maps[i].map_fd = map_fd; @@ -1842,7 +1842,7 @@ int inner_map_restore(bool restore) ret = inner_map_mng_restore_by_persist_stat(&p, stat); if (ret < 0) { - LOG_ERR("inner_map_restore persist_stat failed, ret/%d\n", ret); + LOG_ERR("inner_map_restore persist_stat failed, ret:%d\n", ret); fclose(f); free(stat); return -1; From dce4ca014e67f7f7d9a047cae89e3c3c0cbd65e0 Mon Sep 17 00:00:00 2001 From: lec-bit Date: Wed, 23 Oct 2024 21:22:13 +0800 Subject: [PATCH 3/6] update log Signed-off-by: lec-bit --- .../deserialization_to_bpf_map.c | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c index 673db6df8..06200399d 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c @@ -283,9 +283,10 @@ static int bpf_get_map_id_by_fd(int map_fd) struct bpf_map_info info = {}; __u32 info_len = sizeof(info); - if (bpf_obj_get_info_by_fd(map_fd, &info, &info_len) < 0) { - LOG_ERR("bpf_obj_get_info_by_fd failed"); - return -1; + int ret = bpf_obj_get_info_by_fd(map_fd, &info, &info_len); + if (ret < 0) { + LOG_ERR("bpf_obj_get_info_by_fd failed, map_fd:%d ret:%d ERRNO:%d", map_fd, ret, errno); + return ret; } return info.id; } @@ -349,14 +350,14 @@ static int copy_sfield_to_map(struct op_context *ctx, int o_index, const Protobu *(uintptr_t *)value = (size_t)o_index; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { - LOG_ERR("copy_sfield_to_map bpf_map_update_elem failed, ret:%d\n", ret); + LOG_ERR("copy_sfield_to_map bpf_map_update_elem failed, ret:%d ERRNO:%d\n", ret, errno); free_outter_map_entry(ctx, &o_index); return ret; } inner_fd = outter_key_to_inner_fd(ctx, o_index); if (inner_fd < 0) { - LOG_ERR("copy_sfield_to_map outter_key_to_inner_fd failed, inner_fd:%d\n", inner_fd); + LOG_ERR("copy_sfield_to_map outter_key_to_inner_fd failed, inner_fd:%d ERRNO:%d\n", inner_fd, errno); return inner_fd; } @@ -378,14 +379,14 @@ static int copy_msg_field_to_map(struct op_context *ctx, int o_index, const Prot *(uintptr_t *)value = (size_t)o_index; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { - LOG_ERR("copy_msg_field_to_map bpf_map_update_elem failed, ret:%d\n", ret); + LOG_ERR("copy_msg_field_to_map bpf_map_update_elem failed, ret:%d ERRNO:%d\n", ret, errno); free_outter_map_entry(ctx, &o_index); return ret; } inner_fd = outter_key_to_inner_fd(ctx, o_index); if (inner_fd < 0) { - LOG_ERR("copy_msg_field_to_map outter_key_to_inner_fd failed, inner_fd:%d\n", inner_fd); + LOG_ERR("copy_msg_field_to_map outter_key_to_inner_fd failed, inner_fd:%d ERRNO:%d\n", inner_fd, errno); return inner_fd; } @@ -494,7 +495,7 @@ static int repeat_field_handle(struct op_context *ctx, const ProtobufCFieldDescr *(uintptr_t *)value = (size_t)outter_key; ret = bpf_map_update_elem(ctx->curr_fd, ctx->key, ctx->value, BPF_ANY); if (ret) { - LOG_ERR("repeat_field_handle bpf_map_update_elem failed, ret:%d\n", ret); + LOG_ERR("repeat_field_handle bpf_map_update_elem failed, ret:%d ERRNO:%d\n", ret, errno); free_outter_map_entry(ctx, &outter_key); return ret; } @@ -1841,15 +1842,9 @@ int inner_map_restore(bool restore) } ret = inner_map_mng_restore_by_persist_stat(&p, stat); - if (ret < 0) { - LOG_ERR("inner_map_restore persist_stat failed, ret:%d\n", ret); - fclose(f); - free(stat); - return -1; - } free(stat); fclose(f); - return 0; + return ret; } int deserial_init(bool restore) From c616c0ea8c68c2c6e1b628fbaf5b3bb6ae1a056f Mon Sep 17 00:00:00 2001 From: lec-bit Date: Wed, 23 Oct 2024 22:19:00 +0800 Subject: [PATCH 4/6] update log Signed-off-by: lec-bit --- .../deserialization_to_bpf_map.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c index 06200399d..3673e3ba8 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c @@ -285,7 +285,7 @@ static int bpf_get_map_id_by_fd(int map_fd) int ret = bpf_obj_get_info_by_fd(map_fd, &info, &info_len); if (ret < 0) { - LOG_ERR("bpf_obj_get_info_by_fd failed, map_fd:%d ret:%d ERRNO:%d", map_fd, ret, errno); + LOG_ERR("bpf_obj_get_info_by_fd failed, map_fd:%d ret:%d ERRNO:%d\n", map_fd, ret, errno); return ret; } return info.id; @@ -1616,7 +1616,10 @@ void inner_map_batch_delete() void deserial_uninit(bool persist) { if (persist) - inner_map_mng_persist(); + if (inner_map_mng_persist() < 0) { + LOG_ERR("inner_map_mng_persist failed\n"); + remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); + } else remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); @@ -1758,7 +1761,11 @@ int inner_map_mng_persist() kernel will convert it into a bpf_map pointer for storage, so it will not be affected.*/ for (i = 0; i <= g_inner_map_mng.max_allocated_idx; i++) { - p->inner_map_stat[i].map_fd = bpf_get_map_id_by_fd(g_inner_map_mng.inner_maps[i].map_fd); + int map_fd = bpf_get_map_id_by_fd(g_inner_map_mng.inner_maps[i].map_fd); + if (map_fd < 0) { + return map_fd; + } + p->inner_map_stat[i].map_fd = map_fd; p->inner_map_stat[i].used = g_inner_map_mng.inner_maps[i].used; p->inner_map_stat[i].allocated = g_inner_map_mng.inner_maps[i].allocated; } From f48c7168c9bdebf522ce017165cbd546ef120897 Mon Sep 17 00:00:00 2001 From: lec-bit Date: Wed, 23 Oct 2024 22:36:38 +0800 Subject: [PATCH 5/6] update log and ret Signed-off-by: lec-bit --- .../deserialization_to_bpf_map.c | 14 ++++++-------- .../deserialization_to_bpf_map.h | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c index 3673e3ba8..0fdc639c4 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c @@ -1613,15 +1613,11 @@ void inner_map_batch_delete() return; } -void deserial_uninit(bool persist) +int deserial_uninit(bool persist) { + int ret = 0; if (persist) - if (inner_map_mng_persist() < 0) { - LOG_ERR("inner_map_mng_persist failed\n"); - remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); - } - else - remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); + ret = inner_map_mng_persist(); for (int i = 1; i <= g_inner_map_mng.max_allocated_idx; i++) { g_inner_map_mng.inner_maps[i].allocated = 0; @@ -1639,7 +1635,9 @@ void deserial_uninit(bool persist) close(g_inner_map_mng.inner_fd); close(g_inner_map_mng.outter_fd); - return; + if (!persist || ret < 0) + remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); + return ret; } int inner_map_scaleup() diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.h b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.h index 657c16c0c..76f257738 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.h +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.h @@ -33,7 +33,7 @@ void deserial_free_elem_list(struct element_list_node *head); int deserial_delete_elem(void *key, const void *msg_desciptor); int deserial_init(bool restore); -void deserial_uninit(bool persist); +int deserial_uninit(bool persist); int inner_map_mng_persist(); #endif /* __DESERIALIZATION_TO_BPF_MAP_H__ */ From 1902392d868369bf5f06e75df9ec277d3f35f878 Mon Sep 17 00:00:00 2001 From: lec-bit Date: Wed, 23 Oct 2024 23:06:21 +0800 Subject: [PATCH 6/6] update Signed-off-by: lec-bit --- .../deserialization_to_bpf_map.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c index 0fdc639c4..2d03b506b 100644 --- a/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c +++ b/bpf/deserialization_to_bpf_map/deserialization_to_bpf_map.c @@ -1616,6 +1616,7 @@ void inner_map_batch_delete() int deserial_uninit(bool persist) { int ret = 0; + remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); if (persist) ret = inner_map_mng_persist(); @@ -1635,8 +1636,6 @@ int deserial_uninit(bool persist) close(g_inner_map_mng.inner_fd); close(g_inner_map_mng.outter_fd); - if (!persist || ret < 0) - remove(MAP_IN_MAP_MNG_PERSIST_FILE_PATH); return ret; } @@ -1730,7 +1729,7 @@ int inner_map_elastic_scaling() int inner_map_mng_persist() { - int i, size; + int i, size, map_fd; FILE *f = NULL; struct persist_info *p = NULL; @@ -1759,13 +1758,15 @@ int inner_map_mng_persist() kernel will convert it into a bpf_map pointer for storage, so it will not be affected.*/ for (i = 0; i <= g_inner_map_mng.max_allocated_idx; i++) { - int map_fd = bpf_get_map_id_by_fd(g_inner_map_mng.inner_maps[i].map_fd); - if (map_fd < 0) { - return map_fd; + if (g_inner_map_mng.inner_maps[i].allocated) { + map_fd = bpf_get_map_id_by_fd(g_inner_map_mng.inner_maps[i].map_fd); + if (map_fd < 0) { + return map_fd; + } + p->inner_map_stat[i].map_fd = map_fd; + p->inner_map_stat[i].used = g_inner_map_mng.inner_maps[i].used; + p->inner_map_stat[i].allocated = g_inner_map_mng.inner_maps[i].allocated; } - p->inner_map_stat[i].map_fd = map_fd; - p->inner_map_stat[i].used = g_inner_map_mng.inner_maps[i].used; - p->inner_map_stat[i].allocated = g_inner_map_mng.inner_maps[i].allocated; } f = fopen(MAP_IN_MAP_MNG_PERSIST_FILE_PATH, "wb");