From f88b4fb44d2b59b54bb4241e539fe07fe3b4957d Mon Sep 17 00:00:00 2001 From: Alexander Lill Date: Sun, 5 May 2024 15:26:17 +0200 Subject: [PATCH] #4 - Add new transaction types and handle missing data This commit handles new transaction types (such as depositing cryptocurrencies to Kraken) and cases where the `refid` is `Unknown`, currently happening for a few transactions in the Kraken CSV export, that are missing the refid due to a bug. --- README.md | 2 +- resources/portfolio | 2 +- src/ledger_processor.py | 70 +++++++++++++++++++++++++++++++++++------ 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 36143a7..2693de9 100644 --- a/README.md +++ b/README.md @@ -87,5 +87,5 @@ python cli.py -fc 'EUR' -o './output/' -v './input/Alle_historischen_Kurse.csv' ### transactions_special_depot.csv ![](./doc/depot_transactions_special.png) -And do not forget to check the following box to import staking rewards as "deposits": +And do not forget to check the following box to import staking rewards and crypto deposits as "deposits" instead of as buy: ![](./doc/depot_transactions_special2.png) \ No newline at end of file diff --git a/resources/portfolio b/resources/portfolio index a476976..9c0fdd5 160000 --- a/resources/portfolio +++ b/resources/portfolio @@ -1 +1 @@ -Subproject commit a47697663f741cfde7d2d2dcd3e260b9e4a56008 +Subproject commit 9c0fdd58c21eb149d68bf299173812c9fb1f6799 diff --git a/src/ledger_processor.py b/src/ledger_processor.py index 87e05c2..97dcc61 100644 --- a/src/ledger_processor.py +++ b/src/ledger_processor.py @@ -277,8 +277,9 @@ def _process_trade(self, transaction_id, transaction): tfiat_transactions = list(filter(lambda t: self.__are_same_currency(t["asset"], self._fiat_currency), raw_transactions)) tcrypto_transactions = list(filter(lambda t: not self.__are_same_currency(t["asset"], self._fiat_currency), raw_transactions)) - assert len(tfiat_transactions) == 1, "too many transactions found!" - assert len(tcrypto_transactions) == 1, "too many transactions found!" + if len(tfiat_transactions) != 1 or len(tcrypto_transactions) != 1: + self.__print_transaction_debug_info("Trade does not consist of 1 fiat and 1 crypto transaction, skipping...", transaction) + return [], [] tfiat = tfiat_transactions[0] tcrypto = tcrypto_transactions[0] @@ -375,6 +376,7 @@ def _process_trade(self, transaction_id, transaction): depot_transactions.append(dt) + # If there is a fee, we create a sell transaction, and a cost transaction with same amount. if fee_crypto != "": sellt = DepotTransaction(date, @@ -642,23 +644,30 @@ def __is_staking_transfer(self, transaction): return False def __print_transaction_debug_info(self, message, transaction): - print(message + ", detailed transaction:") - print(json.dumps(transaction)) + print(message + ", detailed transaction:", json.dumps(transaction)) def _process_transaction(self, transaction_id, transaction): parsing_info = transaction.get("meta", {}).get("parsing_info", "") transaction_types = set(transaction.get("types", [])) - if parsing_info == "dup" and transaction_types == {"deposit"}: + if parsing_info in ["dup", "nondup"] and transaction_types == {"deposit"}: return self._process_deposit(transaction_id, transaction) elif parsing_info == "dup" and transaction_types == {"withdrawal"}: return self._process_withdrawal(transaction_id, transaction) - elif parsing_info == "dup" and transaction_types == {"trade"}: + elif parsing_info in ["dup", "nondup"] and transaction_types == {"trade"}: return self._process_trade(transaction_id, transaction) elif parsing_info == "dup" and transaction_types == {"spend","receive"}: return self._process_trade(transaction_id, transaction) elif parsing_info == "dup_asset_amount_match" and transaction_types == {"deposit","staking"}: return self._process_staking(transaction_id, transaction) + elif parsing_info == "nondup" and transaction_types == {"staking"}: + return self._process_staking(transaction_id, transaction) + elif parsing_info == "nondup" and transaction_types == {"earn"}: + return self._process_staking(transaction_id, transaction) + elif parsing_info == "nondup" and transaction_types == {"transfer"}: + if self.__is_staking_transfer(transaction): + print(f"Ignoring staking transfer... ({parsing_info}, {transaction_types})") + return [], [] elif parsing_info == "dup" and transaction_types == {"transfer", "withdrawal"}: if self.__is_staking_transfer(transaction): print(f"Ignoring staking transfer... ({parsing_info}, {transaction_types})") @@ -676,8 +685,12 @@ def _process_transaction(self, transaction_id, transaction): elif parsing_info == "nondup" and transaction_types == {"withdrawal"}: return self._process_fiat_withdrawal(transaction_id, transaction) else: - self.__print_transaction_debug_info(f"Can't process unknown case [ELSE] ({parsing_info}, {transaction_types})", transaction) - return [], [] + if parsing_info == "dup_asset_amount_match": + self.__print_transaction_debug_info(f"Can't process unknown case, but could be false positive [FP] ({parsing_info}, {transaction_types})", transaction) + return [], [] + else: + self.__print_transaction_debug_info(f"Can't process unknown case [ELSE] ({parsing_info}, {transaction_types})", transaction) + return [], [] def get_transactions(self): account_transactions = self.account_transactions @@ -741,7 +754,43 @@ def _parse_transactions(self): transactions[refid]["types"].append(etype) transactions[refid]["meta"]["parsing_info"] = "dup" - nondups_to_process = list(filter(lambda item: item["refid"] not in refid_dups, raw_ledger)) + # Process transactions classified as dups which have a missing refid + unknown_nondups = [] + if "Unknown" in transactions: + unknown_transactions = transactions["Unknown"] + del transactions["Unknown"] + + # unknown_dups can be ignored, they are transactions that add and subtract same amount of same asset + # let's search for unknown nondups, which are real transactions with wrong refid due to a Kraken bug. + + unknown_df = pd.DataFrame(unknown_transactions.get("raw", [])) + unknown_df["abs_amount"] = unknown_df["amount"].apply(self.__get_abs_amount) + unknown_df["norm_asset"] = unknown_df["asset"].apply(self.__normalize_currency_abbreviation) + unknown_nondups = unknown_df.groupby(["abs_amount", "norm_asset"], as_index=False).agg({'txid':'first', 'refid':'count'}) + unknown_nondups = unknown_nondups[unknown_nondups["refid"] < 2] + unknown_nondups = list(unknown_nondups["txid"]) + + unknown_nondups_to_process = list(filter(lambda item: item["txid"] in unknown_nondups, raw_ledger)) + for entry in unknown_nondups_to_process: + entry["refid"] = entry["txid"] + refid = entry["refid"] + + if refid not in transactions: + transactions[refid] = {} + transactions[refid]["raw"] = [] + transactions[refid]["types"] = [] + transactions[refid]["meta"] = {} + + transactions[refid]["raw"].append(entry) + transactions[refid]["types"].append(etype) + transactions[refid]["meta"]["parsing_info"] = "nondup" + else: + pass # It is enough to add these once, as they are the same entry twice + + def dupfilter(item): + return item["refid"] not in refid_dups and item["refid"] != "Unknown" + + nondups_to_process = list(filter(dupfilter, raw_ledger)) for entry in nondups_to_process: refid = entry["refid"] etype = entry["type"] @@ -755,7 +804,7 @@ def _parse_transactions(self): if refid in self._refids_to_ignore: continue - found_matching_transactions = False + found_matching_transactions = refid in unknown_nondups for other_entry in nondups_to_process: other_refid = other_entry["refid"] @@ -796,6 +845,7 @@ def _parse_transactions(self): transactions[new_key]["meta"]["parsing_info"] = "dup_asset_amount_match" if not found_matching_transactions: + if refid not in transactions: transactions[refid] = {} transactions[refid]["raw"] = []