Skip to content

Commit

Permalink
Put idfMap calculation behind a memoization layer.
Browse files Browse the repository at this point in the history
Memoization checks that the card count hasn't changed by 10% or more since the last time we ran.

This leaves a "good enough" idfMap in most cases.

Part of #694.
  • Loading branch information
jkomoros committed Sep 1, 2024
1 parent 3c6df55 commit 4312d72
Showing 1 changed file with 16 additions and 1 deletion.
17 changes: 16 additions & 1 deletion src/nlp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1605,8 +1605,23 @@ type IDFMap = {
maxIDF: number
};

//TODO: put this behind memoization logic that cheats if cards hasn't changed length by much.
let memoizedIDFMap: IDFMap = {idf: {}, maxIDF: 0};
let memoizedIDMapCardCount = 0;
let memoizedIDFMapNgramSize = 0;

const idfMapForCards = (cards : ProcessedCards, ngramSize: number) : IDFMap => {
const cardCount = Object.keys(cards).length;
//Check if the card count is greater than or equal to card count and within 10% of the last time we calculated the idf map
const cardCountCloseEnough = cardCount >= memoizedIDMapCardCount && cardCount <= memoizedIDMapCardCount * 1.1;
if (cardCountCloseEnough && ngramSize == memoizedIDFMapNgramSize) return memoizedIDFMap;
const result = calcIDFMapForCards(cards, ngramSize);
memoizedIDFMap = result;
memoizedIDFMapNgramSize = ngramSize;
memoizedIDMapCardCount = cardCount;
return result;
};

const calcIDFMapForCards = (cards : ProcessedCards, ngramSize: number) : IDFMap => {
if (!cards || Object.keys(cards).length == 0) return {idf: {}, maxIDF: 0};

//only consider cards that have a body, even if we were provided a set that included others
Expand Down

0 comments on commit 4312d72

Please sign in to comment.