Kraken: Action Types in CSV Exports

src/book.py

@@ -19,8 +19,9 @@
 import decimal
 import logging
 import re
+from collections import defaultdict
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 import config
 import misc
@@ -32,6 +33,12 @@
 
 
 class Book:
+    # Need to track state of duplicate deposit/withdrawal entries
+    # All deposits/withdrawals are held back until they occur a second time
+    # Initialize non-existing fields with None once they're called
+    kraken_held_ops: defaultdict[str, defaultdict[str, Any]] = \
+        defaultdict(lambda: defaultdict(lambda: None))
+
     def __init__(self, price_data: PriceData) -> None:
         self.price_data = price_data
 
@@ -452,31 +459,12 @@ def _read_kraken_ledgers(self, file_path: Path) -> None:
         operation_mapping = {
             "spend": "Sell",  # Sell ordered via 'Buy Crypto' button
             "receive": "Buy",  # Buy ordered via 'Buy Crypto' button
-            "transfer": "Airdrop",
             "reward": "StakingInterest",
+            "staking": "StakingInterest",
             "deposit": "Deposit",
             "withdrawal": "Withdrawal",
         }
 
-        # Need to track state of "duplicate entries"
-        # for deposits / withdrawals;
-        # the second deposit and the first withdrawal entry
-        # need to be skipped.
-        #   dup_state["deposit"] == 0:
-        #       Deposit is broadcast to blockchain
-        #       > Taxable event (is in public trade history)
-        #   dup_state["deposit"] == 1:
-        #       Deposit is credited to Kraken account
-        #       > Skipped
-        #   dup_state["withdrawal"] == 0:
-        #       Withdrawal is requested in Kraken account
-        #       > Skipped
-        #   dup_state["withdrawal"] == 1:
-        #       Withdrawal is broadcast to blockchain
-        #       > Taxable event (is in public trade history)
-        dup_state = {"deposit": 0, "withdrawal": 0}
-        dup_skip = {"deposit": 1, "withdrawal": 0}
-
         with open(file_path, encoding="utf8") as f:
             reader = csv.reader(f)
 
@@ -515,42 +503,58 @@ def _read_kraken_ledgers(self, file_path: Path) -> None:
                         balance,
                     ) = columns
                 else:
-                    raise RuntimeError(
-                        "Unknown Kraken ledgers format: "
+                    log.error(
+                        "{file_path}: Unknown Kraken ledgers format: "
                         "Number of rows do not match known versions."
                     )
+                    raise RuntimeError
 
                 row = reader.line_num
 
-                # Skip "duplicate entries" for deposits / withdrawals
-                if _type in dup_state.keys():
-                    skip = dup_state[_type] == dup_skip[_type]
-                    dup_state[_type] = (dup_state[_type] + 1) % 2
-                    if skip:
-                        continue
-
                 # Parse data.
                 utc_time = datetime.datetime.strptime(_utc_time, "%Y-%m-%d %H:%M:%S")
                 utc_time = utc_time.replace(tzinfo=datetime.timezone.utc)
                 change = misc.force_decimal(_amount)
+                # remove the appended .S for staked assets
+                _asset = _asset.removesuffix(".S")
                 coin = kraken_asset_map.get(_asset, _asset)
                 fee = misc.force_decimal(_fee)
                 operation = operation_mapping.get(_type)
                 if operation is None:
                     if _type == "trade":
                         operation = "Sell" if change < 0 else "Buy"
-                    elif _type in ["margin trade", "rollover", "settled"]:
+                    elif _type in ["margin trade", "rollover", "settled", "margin"]:
                         log.error(
-                            f"{file_path}: {row}: Margin trading is "
-                            "currently not supported. "
-                            "Please create an Issue or PR."
+                            f"{file_path} row {row}: Margin trading is currently not "
+                            "supported. Please create an Issue or PR."
                         )
                         raise RuntimeError
+                    elif _type == "transfer":
+                        if num_columns == 9:
+                            # for backwards compatibility assume Airdrop for staking
+                            log.warning(
+                                f"{file_path} row {row}: Staking is not supported for"
+                                "old Kraken ledger formats. "
+                                "Please create an Issue or PR."
+                            )
+                            operation = "Airdrop"
+                        elif subtype == "stakingfromspot":
+                            operation = "Staking"
+                        elif subtype == "stakingtospot":
+                            operation = "StakingEnd"
+                        elif subtype in ["spottostaking", "spotfromstaking"]:
+                            # duplicate entries for staking actions
+                            continue
+                        else:
+                            log.error(
+                                f"{file_path} row {row}: Order subtype '{subtype}' is "
+                                "currently not supported. Please create an Issue or PR."
+                            )
+                            raise RuntimeError
                     else:
                         log.error(
-                            f"{file_path}: {row}: Other order type '{_type}' "
-                            "is currently not supported. "
-                            "Please create an Issue or PR."
+                            f"{file_path} row {row}: Other order type '{_type}' is "
+                            "currently not supported. Please create an Issue or PR."
                         )
                         raise RuntimeError
                 change = abs(change)
@@ -560,22 +564,109 @@ def _read_kraken_ledgers(self, file_path: Path) -> None:
                 assert coin
                 assert change
 
-                self.append_operation(
-                    operation, utc_time, platform, change, coin, row, file_path
-                )
+                # append all operations to main list per default
+                # will be overwritten for deposits / withdrawals
+                append_operation = True
+
+                # Skip duplicate entries for deposits / withdrawals and additional
+                # deposit / withdrawal lines for staking / unstaking / staking reward
+                # actions.
+                # The second deposit and the first withdrawal need to be considered,
+                # since these are the points in time where the user actually has the
+                # assets at their disposal. The first deposit and second withdrawal are
+                # in the public trade history and are skipped.
+                # For staking / unstaking / staking reward actions, deposits /
+                # withdrawals only occur once and will be ignored.
+                # The "appended" flag stores if an operation for a given refid has
+                # already been appended to the operations list:
+                # == None: Initial value, this is the first occurence
+                # == False: No operation has been appended, this is the second occurene
+                # == True: Operation has already been appended, this should not happen
+                if operation in ["Deposit", "Withdrawal"]:
+                    # If this is the first occurence, set the "appended" flag to false
+                    # and don't append the operation to the list. Instead, store the
+                    # data for verifying or appending it later.
+                    if self.kraken_held_ops[refid]["appended"] is None:
+                        append_operation = False
+                        self.kraken_held_ops[refid]["appended"] = False
+                        self.kraken_held_ops[refid]["operation"] = operation
+                        self.kraken_held_ops[refid]["utc_time"] = utc_time
+                        self.kraken_held_ops[refid]["platform"] = platform
+                        self.kraken_held_ops[refid]["change"] = change
+                        self.kraken_held_ops[refid]["coin"] = coin
+                        self.kraken_held_ops[refid]["row"] = row
+                        self.kraken_held_ops[refid]["file_path"] = file_path
+                        self.kraken_held_ops[refid]["fee"] = fee
+                    # If this is the second occurence, append a new operation, set the
+                    # "appended" flag to True and assert that the data of this operation
+                    # agrees with the data of the first occurence.
+                    elif self.kraken_held_ops[refid]["appended"] is False:
+                        append_operation = True
+                        self.kraken_held_ops[refid]["appended"] = True
+                        try:
+                            assert (
+                                operation == self.kraken_held_ops[refid]["operation"]
+                            ), "operation"
+                            assert (
+                                change == self.kraken_held_ops[refid]["change"]
+                            ), "change"
+                            assert coin == self.kraken_held_ops[refid]["coin"], "coin"
+                        except AssertionError as e:
+                            log.error(
+                                f"{file_path} row {row}: Parameters for refid {refid} "
+                                f"({operation}) do not agree: {e}. "
+                                "Please create an Issue or PR."
+                            )
+                            raise RuntimeError
+                    # If an operation with the same refid has been already appended,
+                    # this is the third occurence. Throw an error if this happens.
+                    elif self.kraken_held_ops[refid]["appended"] is True:
+                        log.error(
+                            f"{file_path} row {row}: More than two entries with refid "
+                            f"{refid} should not exist ({operation}). "
+                            "Please create an Issue or PR."
+                        )
+                        raise RuntimeError
+                    # This should never happen
+                    else:
+                        log.error(
+                            f"{file_path} row {row}: Unknown value for appended "
+                            f"operation flag {self.kraken_held_ops[refid]['appended']}."
+                            "Please create an Issue or PR."
+                        )
+                        raise RuntimeError
 
-                if fee != 0:
+                    # For deposits, this is all we need to do.
+                    # For withdrawals, we need to append the first withdrawal as soon as
+                    # the second withdrawal occurs. Therefore, overwrite the variables
+                    # with the data of the first withdrawal and append it.
+                    if append_operation and operation == "Withdrawal":
+                        # required for type annotation: convert Optional[str] to str
+                        operation = str(self.kraken_held_ops[refid]["operation"])
+                        utc_time = self.kraken_held_ops[refid]["utc_time"]
+                        platform = self.kraken_held_ops[refid]["platform"]
+                        change = self.kraken_held_ops[refid]["change"]
+                        coin = self.kraken_held_ops[refid]["coin"]
+                        row = self.kraken_held_ops[refid]["row"]
+                        file_path = self.kraken_held_ops[refid]["file_path"]
+                        fee = self.kraken_held_ops[refid]["fee"]
+
+                if append_operation:
                     self.append_operation(
-                        "Fee", utc_time, platform, fee, coin, row, file_path
+                        operation, utc_time, platform, change, coin, row, file_path
                     )
-
-        assert dup_state["deposit"] == 0, (
-            "Orphaned deposit. (Must always come in pairs). " "Is your file corrupted?"
-        )
-        assert dup_state["withdrawal"] == 0, (
-            "Orphaned withdrawal. (Must always come in pairs). "
-            "Is your file corrupted?"
-        )
+                    if fee != 0:
+                        self.append_operation(
+                            "Fee", utc_time, platform, fee, coin, row, file_path
+                        )
+                    if operation == "StakingInterest":
+                        # For Kraken, the rewarded coins are added to the staked
+                        # portfolio. TODO (for MULTI_DEPOT only): Directly add the
+                        # rewarded coins to the staking depot (not like here with the
+                        # detour of adding it to spot and then staking the same amount)
+                        self.append_operation(
+                            "Staking", utc_time, platform, change, coin, row, file_path
+                        )
 
     def _read_kraken_ledgers_old(self, file_path: Path) -> None:
 

src/core.py

@@ -202,32 +202,41 @@ class Fiat(Enum):
 #         Converts clean fiat / clean crypto pairs to "dirty" API asset pairs
 #         (e.g. ETHEUR -> XETHZEUR)
 # Analyzed using asset pairs API data:
-# https://api.kraken.com/0/public/AssetPairs (retrieved at 2021-02-21)
+# https://api.kraken.com/0/public/AssetPairs (retrieved at 2022-01-02)
 kraken_asset_map = {
     # Fiat:
-    "ZUSD": "USD",
-    "ZEUR": "EUR",
-    "ZCAD": "CAD",
-    "ZJPY": "JPY",
-    "ZGBP": "GBP",
     "CHF": "CHF",
     "ZAUD": "AUD",
+    "ZCAD": "CAD",
+    "ZEUR": "EUR",
+    "ZGBP": "GBP",
+    "ZJPY": "JPY",
+    "ZUSD": "USD",
     # Crypto:
-    "XXBT": "XBT",
+    "XETC": "ETC",
     "XETH": "ETH",
     "XLTC": "LTC",
+    "XMLN": "MLN",
+    "XREP": "REP",
+    "XXBT": "XBT",
     "XXDG": "XDG",
+    "XXLM": "XLM",
+    "XXMR": "XMR",
     "XXRP": "XRP",
+    "XZEC": "ZEC",
 }
 
 # Only these asset pairs violate the rule
 # "clean name" + "clean name" = "asset pair"
 kraken_pair_map = {
     "USDTUSD": "USDTZUSD",
-    "XETCETH": "XETCXETH",
-    "XETCXBT": "XETCXXBT",
-    "XETCEUR": "XETCZEUR",
-    "XETCUSD": "XETCZUSD",
+    "ETCETH": "XETCXETH",
+    "ETCXBT": "XETCXXBT",
+    "ETCEUR": "XETCZEUR",
+    "ETCUSD": "XETCZUSD",
+    "ETH2ETH": "ETH2.SETH",
+    "ETH2EUR": "XETHZEUR",
+    "ETH2USD": "XETHZUSD",
     "ETHXBT": "XETHXXBT",
     "ETHCAD": "XETHZCAD",
     "ETHEUR": "XETHZEUR",
@@ -238,36 +247,36 @@ class Fiat(Enum):
     "LTCEUR": "XLTCZEUR",
     "LTCJPY": "XLTCZJPY",
     "LTCUSD": "XLTCZUSD",
-    "XMLNETH": "XMLNXETH",
-    "XMLNXBT": "XMLNXXBT",
-    "XMLNEUR": "XMLNZEUR",
-    "XMLNUSD": "XMLNZUSD",
-    "XREPETH": "XREPXETH",
-    "XREPXBT": "XREPXXBT",
-    "XREPEUR": "XREPZEUR",
-    "XREPUSD": "XREPZUSD",
+    "MLNETH": "XMLNXETH",
+    "MLNXBT": "XMLNXXBT",
+    "MLNEUR": "XMLNZEUR",
+    "MLNUSD": "XMLNZUSD",
+    "REPETH": "XREPXETH",
+    "REPXBT": "XREPXXBT",
+    "REPEUR": "XREPZEUR",
+    "REPUSD": "XREPZUSD",
     "XBTCAD": "XXBTZCAD",
     "XBTEUR": "XXBTZEUR",
     "XBTGBP": "XXBTZGBP",
     "XBTJPY": "XXBTZJPY",
     "XBTUSD": "XXBTZUSD",
     "XDGXBT": "XXDGXXBT",
-    "XXLMXBT": "XXLMXXBT",
-    "XXLMAUD": "XXLMZAUD",
-    "XXLMEUR": "XXLMZEUR",
-    "XXLMGBP": "XXLMZGBP",
-    "XXLMUSD": "XXLMZUSD",
-    "XXMRXBT": "XXMRXXBT",
-    "XXMREUR": "XXMRZEUR",
-    "XXMRUSD": "XXMRZUSD",
+    "XLMXBT": "XXLMXXBT",
+    "XLMAUD": "XXLMZAUD",
+    "XLMEUR": "XXLMZEUR",
+    "XLMGBP": "XXLMZGBP",
+    "XLMUSD": "XXLMZUSD",
+    "XMRXBT": "XXMRXXBT",
+    "XMREUR": "XXMRZEUR",
+    "XMRUSD": "XXMRZUSD",
     "XRPXBT": "XXRPXXBT",
     "XRPCAD": "XXRPZCAD",
     "XRPEUR": "XXRPZEUR",
     "XRPJPY": "XXRPZJPY",
     "XRPUSD": "XXRPZUSD",
-    "XZECXBT": "XZECXXBT",
-    "XZECEUR": "XZECZEUR",
-    "XZECUSD": "XZECZUSD",
+    "ZECXBT": "XZECXXBT",
+    "ZECEUR": "XZECZEUR",
+    "ZECUSD": "XZECZUSD",
     "EURUSD": "ZEURZUSD",
     "GBPUSD": "ZGBPZUSD",
     "USDCAD": "ZUSDZCAD",