Kraken: Action Types in CSV Exports

src/book.py

@@ -452,30 +452,16 @@ def _read_kraken_ledgers(self, file_path: Path) -> None:
         operation_mapping = {
             "spend": "Sell",  # Sell ordered via 'Buy Crypto' button
             "receive": "Buy",  # Buy ordered via 'Buy Crypto' button
-            "transfer": "Airdrop",
             "reward": "StakingInterest",
+            "staking": "StakingInterest",
             "deposit": "Deposit",
             "withdrawal": "Withdrawal",
         }
 
-        # Need to track state of "duplicate entries"
-        # for deposits / withdrawals;
-        # the second deposit and the first withdrawal entry
-        # need to be skipped.
-        #   dup_state["deposit"] == 0:
-        #       Deposit is broadcast to blockchain
-        #       > Taxable event (is in public trade history)
-        #   dup_state["deposit"] == 1:
-        #       Deposit is credited to Kraken account
-        #       > Skipped
-        #   dup_state["withdrawal"] == 0:
-        #       Withdrawal is requested in Kraken account
-        #       > Skipped
-        #   dup_state["withdrawal"] == 1:
-        #       Withdrawal is broadcast to blockchain
-        #       > Taxable event (is in public trade history)
-        dup_state = {"deposit": 0, "withdrawal": 0}
-        dup_skip = {"deposit": 1, "withdrawal": 0}
+        # Need to track state of duplicate entries
+        # for deposits / withdrawals based on refid
+        refids = []
+        margin_warnings = 0
 
         with open(file_path, encoding="utf8") as f:
             reader = csv.reader(f)
@@ -522,35 +508,54 @@ def _read_kraken_ledgers(self, file_path: Path) -> None:
 
                 row = reader.line_num
 
-                # Skip "duplicate entries" for deposits / withdrawals
-                if _type in dup_state.keys():
-                    skip = dup_state[_type] == dup_skip[_type]
-                    dup_state[_type] = (dup_state[_type] + 1) % 2
-                    if skip:
+                # Skip duplicate entries for deposits / withdrawals and
+                # additional deposit / withdrawals lines for
+                # staking / unstaking / staking reward actions
+                if _type in ["deposit", "withdrawal"]:
+                    if refid not in refids:
+                        refids.append(refid)
                         continue
 
                 # Parse data.
                 utc_time = datetime.datetime.strptime(_utc_time, "%Y-%m-%d %H:%M:%S")
                 utc_time = utc_time.replace(tzinfo=datetime.timezone.utc)
                 change = misc.force_decimal(_amount)
+                # remove the appended .S for staked assets
+                _asset = _asset.replace(".S", "")
                 coin = kraken_asset_map.get(_asset, _asset)
                 fee = misc.force_decimal(_fee)
                 operation = operation_mapping.get(_type)
                 if operation is None:
                     if _type == "trade":
                         operation = "Sell" if change < 0 else "Buy"
-                    elif _type in ["margin trade", "rollover", "settled"]:
-                        log.error(
-                            f"{file_path}: {row}: Margin trading is "
-                            "currently not supported. "
-                            "Please create an Issue or PR."
-                        )
-                        raise RuntimeError
+                    elif _type in ["margin trade", "rollover", "settled", "margin"]:
+                        margin_warnings += 1
+                        continue
+                    elif _type == "transfer":
+                        if num_columns == 9:
+                            # for backwards compatibility assume Airdrop for staking
+                            log.warning(
+                                f"{file_path}: {row}: Staking is not supported for old"
+                                " Kraken ledger formats. Please create an Issue or PR."
+                            )
+                            operation = "Airdrop"
+                        elif subtype == "stakingfromspot":
+                            operation = "Staking"
+                        elif subtype == "stakingtospot":
+                            operation = "StakingEnd"
+                        elif subtype in ["spottostaking", "spotfromstaking"]:
+                            # duplicate entries for staking actions
+                            continue
+                        else:
+                            log.error(
+                                f"{file_path}: {row}: Order subtype '{subtype}' is "
+                                "currently not supported. Please create an Issue or PR."
+                            )
+                            raise RuntimeError
                     else:
                         log.error(
-                            f"{file_path}: {row}: Other order type '{_type}' "
-                            "is currently not supported. "
-                            "Please create an Issue or PR."
+                            f"{file_path}: {row}: Other order type '{_type}' is "
+                            "currently not supported. Please create an Issue or PR."
                         )
                         raise RuntimeError
                 change = abs(change)
@@ -569,13 +574,11 @@ def _read_kraken_ledgers(self, file_path: Path) -> None:
                         "Fee", utc_time, platform, fee, coin, row, file_path
                     )
 
-        assert dup_state["deposit"] == 0, (
-            "Orphaned deposit. (Must always come in pairs). " "Is your file corrupted?"
-        )
-        assert dup_state["withdrawal"] == 0, (
-            "Orphaned withdrawal. (Must always come in pairs). "
-            "Is your file corrupted?"
-        )
+        if margin_warnings:
+            log.warning(
+                f"{file_path}: {margin_warnings} margin entries. Margin trading is "
+                "currently not supported. Please create an Issue or PR."
+            )
 
     def _read_kraken_ledgers_old(self, file_path: Path) -> None:
 

src/core.py

@@ -202,32 +202,41 @@ class Fiat(Enum):
 #         Converts clean fiat / clean crypto pairs to "dirty" API asset pairs
 #         (e.g. ETHEUR -> XETHZEUR)
 # Analyzed using asset pairs API data:
-# https://api.kraken.com/0/public/AssetPairs (retrieved at 2021-02-21)
+# https://api.kraken.com/0/public/AssetPairs (retrieved at 2022-01-02)
 kraken_asset_map = {
     # Fiat:
-    "ZUSD": "USD",
-    "ZEUR": "EUR",
-    "ZCAD": "CAD",
-    "ZJPY": "JPY",
-    "ZGBP": "GBP",
     "CHF": "CHF",
     "ZAUD": "AUD",
+    "ZCAD": "CAD",
+    "ZEUR": "EUR",
+    "ZGBP": "GBP",
+    "ZJPY": "JPY",
+    "ZUSD": "USD",
     # Crypto:
-    "XXBT": "XBT",
+    "XETC": "ETC",
     "XETH": "ETH",
     "XLTC": "LTC",
+    "XMLN": "MLN",
+    "XREP": "REP",
+    "XXBT": "XBT",
     "XXDG": "XDG",
+    "XXLM": "XLM",
+    "XXMR": "XMR",
     "XXRP": "XRP",
+    "XZEC": "ZEC",
 }
 
 # Only these asset pairs violate the rule
 # "clean name" + "clean name" = "asset pair"
 kraken_pair_map = {
     "USDTUSD": "USDTZUSD",
-    "XETCETH": "XETCXETH",
-    "XETCXBT": "XETCXXBT",
-    "XETCEUR": "XETCZEUR",
-    "XETCUSD": "XETCZUSD",
+    "ETCETH": "XETCXETH",
+    "ETCXBT": "XETCXXBT",
+    "ETCEUR": "XETCZEUR",
+    "ETCUSD": "XETCZUSD",
+    "ETH2ETH": "ETH2.SETH",
+    "ETH2EUR": "XETHZEUR",
+    "ETH2USD": "XETHZUSD",
     "ETHXBT": "XETHXXBT",
     "ETHCAD": "XETHZCAD",
     "ETHEUR": "XETHZEUR",
@@ -238,36 +247,36 @@ class Fiat(Enum):
     "LTCEUR": "XLTCZEUR",
     "LTCJPY": "XLTCZJPY",
     "LTCUSD": "XLTCZUSD",
-    "XMLNETH": "XMLNXETH",
-    "XMLNXBT": "XMLNXXBT",
-    "XMLNEUR": "XMLNZEUR",
-    "XMLNUSD": "XMLNZUSD",
-    "XREPETH": "XREPXETH",
-    "XREPXBT": "XREPXXBT",
-    "XREPEUR": "XREPZEUR",
-    "XREPUSD": "XREPZUSD",
+    "MLNETH": "XMLNXETH",
+    "MLNXBT": "XMLNXXBT",
+    "MLNEUR": "XMLNZEUR",
+    "MLNUSD": "XMLNZUSD",
+    "REPETH": "XREPXETH",
+    "REPXBT": "XREPXXBT",
+    "REPEUR": "XREPZEUR",
+    "REPUSD": "XREPZUSD",
     "XBTCAD": "XXBTZCAD",
     "XBTEUR": "XXBTZEUR",
     "XBTGBP": "XXBTZGBP",
     "XBTJPY": "XXBTZJPY",
     "XBTUSD": "XXBTZUSD",
     "XDGXBT": "XXDGXXBT",
-    "XXLMXBT": "XXLMXXBT",
-    "XXLMAUD": "XXLMZAUD",
-    "XXLMEUR": "XXLMZEUR",
-    "XXLMGBP": "XXLMZGBP",
-    "XXLMUSD": "XXLMZUSD",
-    "XXMRXBT": "XXMRXXBT",
-    "XXMREUR": "XXMRZEUR",
-    "XXMRUSD": "XXMRZUSD",
+    "XLMXBT": "XXLMXXBT",
+    "XLMAUD": "XXLMZAUD",
+    "XLMEUR": "XXLMZEUR",
+    "XLMGBP": "XXLMZGBP",
+    "XLMUSD": "XXLMZUSD",
+    "XMRXBT": "XXMRXXBT",
+    "XMREUR": "XXMRZEUR",
+    "XMRUSD": "XXMRZUSD",
     "XRPXBT": "XXRPXXBT",
     "XRPCAD": "XXRPZCAD",
     "XRPEUR": "XXRPZEUR",
     "XRPJPY": "XXRPZJPY",
     "XRPUSD": "XXRPZUSD",
-    "XZECXBT": "XZECXXBT",
-    "XZECEUR": "XZECZEUR",
-    "XZECUSD": "XZECZUSD",
+    "ZECXBT": "XZECXXBT",
+    "ZECEUR": "XZECZEUR",
+    "ZECUSD": "XZECZUSD",
     "EURUSD": "ZEURZUSD",
     "GBPUSD": "ZGBPZUSD",
     "USDCAD": "ZUSDZCAD",

src/price_data.py

@@ -336,8 +336,7 @@ def _get_price_kraken(
         """
         target_timestamp = misc.to_ms_timestamp(utc_time)
         root_url = "https://api.kraken.com/0/public/Trades"
-        pair = base_asset + quote_asset
-        pair = kraken_pair_map.get(pair, pair)
+        inverse = False
 
         minutes_offset = 0
         while minutes_offset < 120:
@@ -346,10 +345,13 @@ def _get_price_kraken(
             since = misc.to_ns_timestamp(
                 utc_time - datetime.timedelta(minutes=minutes_offset)
             )
-            url = f"{root_url}?{pair=:}&{since=:}"
 
             num_retries = 10
             while num_retries:
+                pair = base_asset + quote_asset
+                pair = kraken_pair_map.get(pair, pair)
+                url = f"{root_url}?{pair=:}&{since=:}"
+
                 log.debug(
                     f"Querying trades for {pair} at {utc_time} "
                     f"(offset={minutes_offset}m): Calling %s",
@@ -361,6 +363,18 @@ def _get_price_kraken(
 
                 if not data["error"]:
                     break
+                elif data["error"] == ['EGeneral:Invalid arguments']:
+                    # cancel if inverse pair has been tried
+                    if inverse:
+                        num_retries = 0
+                        continue
+                    # try inverse pair
+                    else:
+                        log.debug("Invalid arguments error, trying inverse coin pair")
+                        inverse = not inverse
+                        temp_asset = quote_asset
+                        quote_asset = base_asset
+                        base_asset = temp_asset
                 else:
                     num_retries -= 1
                     sleep_duration = 2 ** (10 - num_retries)
@@ -391,8 +405,12 @@ def _get_price_kraken(
             if closest_match_index == -1:
                 continue
 
-            # The desired timestamp is in the future
-            if closest_match_index == len(data_timestamps_ms) - 1:
+            now_timestamp = misc.to_ms_timestamp(datetime.datetime.now().astimezone())
+            # The desired timestamp is in the future. Ignore if the target timestamp
+            # is very close to the current timestamp (for virtual sells)
+            if closest_match_index == len(data_timestamps_ms) - 1 and \
+                (now_timestamp > target_timestamp + 3600 * 1000
+                    or not config.CALCULATE_VIRTUAL_SELL):
 
                 if minutes_step == 1:
                     # Cannot remove interval any further; give up
@@ -408,6 +426,8 @@ def _get_price_kraken(
                     )
 
             price = misc.force_decimal(data[closest_match_index][0])
+            if inverse:
+                price = misc.reciprocal(price)
             return price
 
         log.warning(