Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store hash in the probed_indices array in common/raw_hashtable.h to avoid its recomputation. #4726

Merged
merged 5 commits into from
Dec 31, 2024
Merged

Store hash in the probed_indices array in common/raw_hashtable.h to avoid its recomputation. #4726

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
aacdd6a
Store hash in probed_indices array to avoid its recomputation.
goldvitaly Dec 21, 2024
0ac670f
Store hash in probed_indices array to avoid its recomputation.
goldvitaly Dec 21, 2024
85b81a5
Keep only one `InsertIntoEmpty` signature and change emplace_back to
goldvitaly Dec 21, 2024
f51c91f
Merge branch 'carbon-language:trunk' into trunk
goldvitaly Dec 27, 2024
1b0e675
Update common/raw_hashtable.h
chandlerc Dec 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 14 additions & 21 deletions common/raw_hashtable.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,16 +570,13 @@ class BaseImpl {
auto CopySlotsFrom(const BaseImpl& arg) -> void;
auto MoveFrom(BaseImpl&& arg, Storage* small_storage) -> void;

template <typename LookupKeyT>
auto InsertIntoEmpty(LookupKeyT lookup_key, KeyContextT key_context)
-> EntryT*;
auto InsertIntoEmpty(HashCode hash) -> EntryT*;

static auto ComputeNextAllocSize(ssize_t old_alloc_size) -> ssize_t;
static auto GrowthThresholdForAllocSize(ssize_t alloc_size) -> ssize_t;

auto GrowToNextAllocSize(KeyContextT key_context) -> void;
template <typename LookupKeyT>
auto GrowAndInsert(LookupKeyT lookup_key, KeyContextT key_context) -> EntryT*;
auto GrowAndInsert(HashCode hash, KeyContextT key_context) -> EntryT*;

ViewImplT view_impl_;
int growth_budget_;
Expand Down Expand Up @@ -974,7 +971,7 @@ auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::InsertImpl(
// empty slot. Without the growth budget we'll have to completely rehash and
// so we can just bail here.
if (LLVM_UNLIKELY(growth_budget_ == 0)) {
return {GrowAndInsert(lookup_key, key_context), true};
return {GrowAndInsert(hash, key_context), true};
}

--growth_budget_;
Expand Down Expand Up @@ -1029,8 +1026,9 @@ BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::GrowToAllocSizeImpl(
for (ssize_t byte_index : present_matched_range) {
++count;
ssize_t index = group_index + byte_index;
EntryT* new_entry =
InsertIntoEmpty(old_entries[index].key(), key_context);
HashCode hash =
key_context.HashKey(old_entries[index].key(), ComputeSeed());
EntryT* new_entry = InsertIntoEmpty(hash);
new_entry->MoveFrom(std::move(old_entries[index]));
}
}
Expand Down Expand Up @@ -1291,11 +1289,8 @@ auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::MoveFrom(
// these are true, typically just after growth, we can dramatically simplify the
// insert position search.
template <typename InputKeyT, typename InputValueT, typename InputKeyContextT>
template <typename LookupKeyT>
[[clang::noinline]] auto
BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::InsertIntoEmpty(
LookupKeyT lookup_key, KeyContextT key_context) -> EntryT* {
HashCode hash = key_context.HashKey(lookup_key, ComputeSeed());
auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::InsertIntoEmpty(
HashCode hash) -> EntryT* {
auto [hash_index, tag] = hash.ExtractIndexAndTag<7>();
uint8_t* local_metadata = metadata();
EntryT* local_entries = entries();
Expand Down Expand Up @@ -1375,7 +1370,7 @@ auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::GrowToNextAllocSize(
// the group walk rather than after the group walk. In practice, between the
// statistical rareness and using a large small size buffer here on the stack,
// we can handle this most efficiently with temporary, additional storage.
llvm::SmallVector<ssize_t, 128> probed_indices;
llvm::SmallVector<std::pair<ssize_t, HashCode>, 128> probed_indices;

// Create locals for the old state of the table.
ssize_t old_size = alloc_size();
Expand Down Expand Up @@ -1449,7 +1444,7 @@ auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::GrowToNextAllocSize(
ssize_t old_hash_index = hash.ExtractIndexAndTag<7>().first &
ComputeProbeMaskFromSize(old_size);
if (LLVM_UNLIKELY(old_hash_index != group_index)) {
probed_indices.push_back(old_index);
probed_indices.push_back({old_index, hash});
if constexpr (MetadataGroup::FastByteClear) {
low_g.ClearByte(byte_index);
high_g.ClearByte(byte_index);
Expand Down Expand Up @@ -1510,9 +1505,8 @@ auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::GrowToNextAllocSize(

// We then need to do a normal insertion for anything that was probed before
// growth, but we know we'll find an empty slot, so leverage that.
for (ssize_t old_index : probed_indices) {
EntryT* new_entry =
InsertIntoEmpty(old_entries[old_index].key(), key_context);
for (auto [old_index, hash] : probed_indices) {
EntryT* new_entry = InsertIntoEmpty(hash);
new_entry->MoveFrom(std::move(old_entries[old_index]));
}
CARBON_DCHECK(count ==
Expand All @@ -1538,16 +1532,15 @@ auto BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::GrowToNextAllocSize(
// that this function can be directly called and the result returned from
// `InsertImpl`.
template <typename InputKeyT, typename InputValueT, typename InputKeyContextT>
template <typename LookupKeyT>
[[clang::noinline]] auto
BaseImpl<InputKeyT, InputValueT, InputKeyContextT>::GrowAndInsert(
LookupKeyT lookup_key, KeyContextT key_context) -> EntryT* {
HashCode hash, KeyContextT key_context) -> EntryT* {
GrowToNextAllocSize(key_context);

// And insert the lookup_key into an index in the newly grown map and return
// that index for use.
--growth_budget_;
return InsertIntoEmpty(lookup_key, key_context);
return InsertIntoEmpty(hash);
}

template <typename InputBaseT, ssize_t SmallSize>
Expand Down
Loading