Skip to content

Commit 3ffe630

Browse files
spredolacspredolac
committed
[EASY] Encapsulate better, do not pass hpa_shard when hooks are enough, move shard independent actions to hpa_utils
1 parent f5f0f06 commit 3ffe630

File tree

10 files changed

+174
-175
lines changed

10 files changed

+174
-175
lines changed

‎Makefile.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
124124
$(srcroot)src/hook.c \
125125
$(srcroot)src/hpa.c \
126126
$(srcroot)src/hpa_hooks.c \
127+
$(srcroot)src/hpa_utils.c \
127128
$(srcroot)src/hpdata.c \
128129
$(srcroot)src/inspect.c \
129130
$(srcroot)src/large.c \

‎include/jemalloc/internal/hpa_utils.h‎

Lines changed: 59 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,20 @@
22
#define JEMALLOC_INTERNAL_HPA_UTILS_H
33

44
#include "jemalloc/internal/hpa.h"
5+
#include "jemalloc/internal/extent.h"
56

67
#define HPA_MIN_VAR_VEC_SIZE 8
8+
/*
9+
* This is used for jemalloc internal tuning and may change in the future based
10+
* on production traffic.
11+
*
12+
* This value protects two things:
13+
* 1. Stack size
14+
* 2. Number of huge pages that are being purged in a batch as we do not
15+
* allow allocations while making madvise syscall.
16+
*/
17+
#define HPA_PURGE_BATCH_MAX 16
18+
719
#ifdef JEMALLOC_HAVE_PROCESS_MADVISE
820
typedef struct iovec hpa_io_vector_t;
921
#else
@@ -13,27 +25,35 @@ typedef struct {
1325
} hpa_io_vector_t;
1426
#endif
1527

28+
static inline size_t
29+
hpa_process_madvise_max_iovec_len(void) {
30+
assert(
31+
opt_process_madvise_max_batch <= PROCESS_MADVISE_MAX_BATCH_LIMIT);
32+
return opt_process_madvise_max_batch == 0
33+
? HPA_MIN_VAR_VEC_SIZE
34+
: opt_process_madvise_max_batch;
35+
}
36+
1637
/* Actually invoke hooks. If we fail vectorized, use single purges */
1738
static void
1839
hpa_try_vectorized_purge(
19-
hpa_shard_t *shard, hpa_io_vector_t *vec, size_t vlen, size_t nbytes) {
40+
hpa_hooks_t *hooks, hpa_io_vector_t *vec, size_t vlen, size_t nbytes) {
2041
bool success = opt_process_madvise_max_batch > 0
21-
&& !shard->central->hooks.vectorized_purge(vec, vlen, nbytes);
42+
&& !hooks->vectorized_purge(vec, vlen, nbytes);
2243
if (!success) {
2344
/* On failure, it is safe to purge again (potential perf
24-
* penalty) If kernel can tell exactly which regions
25-
* failed, we could avoid that penalty.
26-
*/
45+
* penalty) If kernel can tell exactly which regions
46+
* failed, we could avoid that penalty.
47+
*/
2748
for (size_t i = 0; i < vlen; ++i) {
28-
shard->central->hooks.purge(
29-
vec[i].iov_base, vec[i].iov_len);
49+
hooks->purge(vec[i].iov_base, vec[i].iov_len);
3050
}
3151
}
3252
}
3353

3454
/*
35-
* This struct accumulates the regions for process_madvise.
36-
* It invokes the hook when batch limit is reached
55+
* This structure accumulates the regions for process_madvise. It invokes the
56+
* hook when batch limit is reached.
3757
*/
3858
typedef struct {
3959
hpa_io_vector_t *vp;
@@ -51,16 +71,16 @@ hpa_range_accum_init(hpa_range_accum_t *ra, hpa_io_vector_t *v, size_t sz) {
5171
}
5272

5373
static inline void
54-
hpa_range_accum_flush(hpa_range_accum_t *ra, hpa_shard_t *shard) {
74+
hpa_range_accum_flush(hpa_range_accum_t *ra, hpa_hooks_t *hooks) {
5575
assert(ra->total_bytes > 0 && ra->cur > 0);
56-
hpa_try_vectorized_purge(shard, ra->vp, ra->cur, ra->total_bytes);
76+
hpa_try_vectorized_purge(hooks, ra->vp, ra->cur, ra->total_bytes);
5777
ra->cur = 0;
5878
ra->total_bytes = 0;
5979
}
6080

6181
static inline void
6282
hpa_range_accum_add(
63-
hpa_range_accum_t *ra, void *addr, size_t sz, hpa_shard_t *shard) {
83+
hpa_range_accum_t *ra, void *addr, size_t sz, hpa_hooks_t *hooks) {
6484
assert(ra->cur < ra->capacity);
6585

6686
ra->vp[ra->cur].iov_base = addr;
@@ -69,14 +89,14 @@ hpa_range_accum_add(
6989
ra->cur++;
7090

7191
if (ra->cur == ra->capacity) {
72-
hpa_range_accum_flush(ra, shard);
92+
hpa_range_accum_flush(ra, hooks);
7393
}
7494
}
7595

7696
static inline void
77-
hpa_range_accum_finish(hpa_range_accum_t *ra, hpa_shard_t *shard) {
97+
hpa_range_accum_finish(hpa_range_accum_t *ra, hpa_hooks_t *hooks) {
7898
if (ra->cur > 0) {
79-
hpa_range_accum_flush(ra, shard);
99+
hpa_range_accum_flush(ra, hooks);
80100
}
81101
}
82102

@@ -114,4 +134,28 @@ struct hpa_purge_batch_s {
114134
size_t npurged_hp_total;
115135
};
116136

137+
static inline bool
138+
hpa_batch_full(hpa_purge_batch_t *b) {
139+
/* It's okay for ranges to go above */
140+
return b->npurged_hp_total == b->max_hp
141+
|| b->item_cnt == b->items_capacity
142+
|| b->nranges >= b->range_watermark;
143+
}
144+
145+
static inline void
146+
hpa_batch_pass_start(hpa_purge_batch_t *b) {
147+
b->item_cnt = 0;
148+
b->nranges = 0;
149+
b->ndirty_in_batch = 0;
150+
}
151+
152+
static inline bool
153+
hpa_batch_empty(hpa_purge_batch_t *b) {
154+
return b->item_cnt == 0;
155+
}
156+
157+
/* Purge pages in a batch using given hooks */
158+
void hpa_purge_batch(
159+
hpa_hooks_t *hooks, hpa_purge_item_t *batch, size_t batch_sz);
160+
117161
#endif /* JEMALLOC_INTERNAL_HPA_UTILS_H */

‎msvc/projects/vc2015/jemalloc/jemalloc.vcxproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<ClCompile Include="..\..\..\..\src\hook.c" />
6363
<ClCompile Include="..\..\..\..\src\hpa.c" />
6464
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
65+
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
6566
<ClCompile Include="..\..\..\..\src\hpdata.c" />
6667
<ClCompile Include="..\..\..\..\src\inspect.c" />
6768
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
@@ -380,4 +381,4 @@
380381
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
381382
<ImportGroup Label="ExtensionTargets">
382383
</ImportGroup>
383-
</Project>
384+
</Project>

‎msvc/projects/vc2017/jemalloc/jemalloc.vcxproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<ClCompile Include="..\..\..\..\src\hook.c" />
6363
<ClCompile Include="..\..\..\..\src\hpa.c" />
6464
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
65+
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
6566
<ClCompile Include="..\..\..\..\src\hpdata.c" />
6667
<ClCompile Include="..\..\..\..\src\inspect.c" />
6768
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
@@ -379,4 +380,4 @@
379380
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
380381
<ImportGroup Label="ExtensionTargets">
381382
</ImportGroup>
382-
</Project>
383+
</Project>

‎msvc/projects/vc2019/jemalloc/jemalloc.vcxproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<ClCompile Include="..\..\..\..\src\hook.c" />
6363
<ClCompile Include="..\..\..\..\src\hpa.c" />
6464
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
65+
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
6566
<ClCompile Include="..\..\..\..\src\hpdata.c" />
6667
<ClCompile Include="..\..\..\..\src\inspect.c" />
6768
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
@@ -379,4 +380,4 @@
379380
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
380381
<ImportGroup Label="ExtensionTargets">
381382
</ImportGroup>
382-
</Project>
383+
</Project>

‎msvc/projects/vc2022/jemalloc/jemalloc.vcxproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
<ClCompile Include="..\..\..\..\src\hook.c" />
6363
<ClCompile Include="..\..\..\..\src\hpa.c" />
6464
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
65+
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
6566
<ClCompile Include="..\..\..\..\src\hpdata.c" />
6667
<ClCompile Include="..\..\..\..\src\inspect.c" />
6768
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
@@ -379,4 +380,4 @@
379380
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
380381
<ImportGroup Label="ExtensionTargets">
381382
</ImportGroup>
382-
</Project>
383+
</Project>

‎src/hpa.c‎

Lines changed: 2 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -472,70 +472,6 @@ hpa_shard_has_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
472472
return to_hugify != NULL || hpa_should_purge(tsdn, shard);
473473
}
474474

475-
/*
476-
* This is used for jemalloc internal tuning and may change in the
477-
* future based on production traffic.
478-
*
479-
* This value protects two things:
480-
* 1. Stack size
481-
* 2. Number of huge pages that are being purged in a batch as
482-
* we do not allow allocations while making madvise syscall.
483-
*/
484-
#define HPA_PURGE_BATCH_MAX_DEFAULT 16
485-
486-
#ifndef JEMALLOC_JET
487-
# define HPA_PURGE_BATCH_MAX HPA_PURGE_BATCH_MAX_DEFAULT
488-
#else
489-
size_t hpa_purge_max_batch_size_for_test = HPA_PURGE_BATCH_MAX_DEFAULT;
490-
size_t
491-
hpa_purge_max_batch_size_for_test_set(size_t new_size) {
492-
size_t old_size = hpa_purge_max_batch_size_for_test;
493-
hpa_purge_max_batch_size_for_test = new_size;
494-
return old_size;
495-
}
496-
# define HPA_PURGE_BATCH_MAX hpa_purge_max_batch_size_for_test
497-
#endif
498-
499-
static inline size_t
500-
hpa_process_madvise_max_iovec_len(void) {
501-
assert(
502-
opt_process_madvise_max_batch <= PROCESS_MADVISE_MAX_BATCH_LIMIT);
503-
return opt_process_madvise_max_batch == 0
504-
? HPA_MIN_VAR_VEC_SIZE
505-
: opt_process_madvise_max_batch;
506-
}
507-
508-
static inline void
509-
hpa_purge_actual_unlocked(
510-
hpa_shard_t *shard, hpa_purge_item_t *batch, size_t batch_sz) {
511-
assert(batch_sz > 0);
512-
513-
size_t len = hpa_process_madvise_max_iovec_len();
514-
VARIABLE_ARRAY(hpa_io_vector_t, vec, len);
515-
516-
hpa_range_accum_t accum;
517-
hpa_range_accum_init(&accum, vec, len);
518-
519-
for (size_t i = 0; i < batch_sz; ++i) {
520-
/* Actually do the purging, now that the lock is dropped. */
521-
if (batch[i].dehugify) {
522-
shard->central->hooks.dehugify(
523-
hpdata_addr_get(batch[i].hp), HUGEPAGE);
524-
}
525-
void *purge_addr;
526-
size_t purge_size;
527-
size_t total_purged_on_one_hp = 0;
528-
while (hpdata_purge_next(
529-
batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) {
530-
total_purged_on_one_hp += purge_size;
531-
assert(total_purged_on_one_hp <= HUGEPAGE);
532-
hpa_range_accum_add(
533-
&accum, purge_addr, purge_size, shard);
534-
}
535-
}
536-
hpa_range_accum_finish(&accum, shard);
537-
}
538-
539475
static inline bool
540476
hpa_needs_dehugify(hpa_shard_t *shard, const hpdata_t *ps) {
541477
return hpa_is_hugify_lazy(shard) && hpdata_huge_get(ps)
@@ -622,26 +558,6 @@ hpa_purge_finish_hp(
622558
psset_update_end(&shard->psset, hp_item->hp);
623559
}
624560

625-
static inline bool
626-
hpa_batch_full(hpa_purge_batch_t *b) {
627-
/* It's okay for ranges to go above */
628-
return b->npurged_hp_total == b->max_hp
629-
|| b->item_cnt == b->items_capacity
630-
|| b->nranges >= b->range_watermark;
631-
}
632-
633-
static inline void
634-
hpa_batch_pass_start(hpa_purge_batch_t *b) {
635-
b->item_cnt = 0;
636-
b->nranges = 0;
637-
b->ndirty_in_batch = 0;
638-
}
639-
640-
static inline bool
641-
hpa_batch_empty(hpa_purge_batch_t *b) {
642-
return b->item_cnt == 0;
643-
}
644-
645561
/* Returns number of huge pages purged. */
646562
static inline size_t
647563
hpa_purge(tsdn_t *tsdn, hpa_shard_t *shard, size_t max_hp) {
@@ -677,8 +593,9 @@ hpa_purge(tsdn_t *tsdn, hpa_shard_t *shard, size_t max_hp) {
677593
if (hpa_batch_empty(&batch)) {
678594
break;
679595
}
596+
hpa_hooks_t *hooks = &shard->central->hooks;
680597
malloc_mutex_unlock(tsdn, &shard->mtx);
681-
hpa_purge_actual_unlocked(shard, batch.items, batch.item_cnt);
598+
hpa_purge_batch(hooks, batch.items, batch.item_cnt);
682599
malloc_mutex_lock(tsdn, &shard->mtx);
683600

684601
/* The shard updates */

‎src/hpa_utils.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#include "jemalloc/internal/jemalloc_preamble.h"
2+
#include "jemalloc/internal/jemalloc_internal_includes.h"
3+
4+
#include "jemalloc/internal/hpa_utils.h"
5+
6+
void
7+
hpa_purge_batch(hpa_hooks_t *hooks, hpa_purge_item_t *batch, size_t batch_sz) {
8+
assert(batch_sz > 0);
9+
10+
size_t len = hpa_process_madvise_max_iovec_len();
11+
VARIABLE_ARRAY(hpa_io_vector_t, vec, len);
12+
13+
hpa_range_accum_t accum;
14+
hpa_range_accum_init(&accum, vec, len);
15+
16+
for (size_t i = 0; i < batch_sz; ++i) {
17+
/* Actually do the purging, now that the lock is dropped. */
18+
if (batch[i].dehugify) {
19+
hooks->dehugify(hpdata_addr_get(batch[i].hp), HUGEPAGE);
20+
}
21+
void *purge_addr;
22+
size_t purge_size;
23+
size_t total_purged_on_one_hp = 0;
24+
while (hpdata_purge_next(
25+
batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) {
26+
total_purged_on_one_hp += purge_size;
27+
assert(total_purged_on_one_hp <= HUGEPAGE);
28+
hpa_range_accum_add(
29+
&accum, purge_addr, purge_size, hooks);
30+
}
31+
}
32+
hpa_range_accum_finish(&accum, hooks);
33+
}

0 commit comments

Comments
 (0)