diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e7e16a73..a896b1f2ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Development] +## [0.46.1 - 2025-12-10] + +### Added +- **GFQL**: Added auto_graph_renderer_switching config option for Kepler maps. + ### Fixed - **GFQL:** `Chain` now validates on construction (matching docs) and rejects invalid hops immediately; pass `validate=False` to defer validation when assembling advanced flows (fixes #860). - **GFQL / eq:** `eq()` now accepts strings in addition to numeric/temporal values (use `isna()`/`notna()` for nulls); added coverage across validator, schema validation, JSON, and GFQL runtime (fixes #862). diff --git a/demos/demos_by_use_case/fraud/icij_fincen_viz.ipynb b/demos/demos_by_use_case/fraud/icij_fincen_viz.ipynb new file mode 100644 index 0000000000..87a77ee1fe --- /dev/null +++ b/demos/demos_by_use_case/fraud/icij_fincen_viz.ipynb @@ -0,0 +1,2901 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ICIJ FinCEN Files Visualization\n", + "\n", + "This notebook demonstrates GFQL (Graph Query Language) using the ICIJ FinCEN Files dataset.\n", + "\n", + "This document will serve as an end-to-end demonstration of using Graphistry's GFQL to extract information out of a complex real-world dataset. For this purpose, we will use a dataset made available by the International Consortium of Investigative Journalists (ICIJ) as part of their reporting of leaked documents from the U.S. Department of Treasury’s Financial Crimes Enforcement Network, which include suspicious activity reports filed by U.S. banks acting as beneficiary or orginating banks in domestic transactions or as correspondent or intermediary banks in international transactions. More information about the dataset can be found at the ICIJ. In this tutorial, we will show how graph queries can be used to extract insights from large datasets and answer questions.\n", + "\n", + "Based on: https://hub.graphistry.com/docs/GFQL/gfql/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup and Registration\n", + "\n", + "First, import Graphistry and register with your credentials." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import graphistry\n", + "\n", + "# graphistry.register(api=3, protocol=\"https\", server=\"hub.graphistry.com\",\n", + "# username=\"...\", password=\"...\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download and Load Data\n", + "\n", + "Download the ICIJ FinCEN Files dataset and create a graph." + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import zipfile\n", + "import pandas as pd\n", + "\n", + "# download data\n", + "resp = requests.get(\"https://media.icij.org/uploads/2020/09/download_data_fincen_files.zip\")\n", + "with open(\"download_data_fincen_files.zip\", \"wb\") as f:\n", + " f.write(resp.content)\n", + "with zipfile.ZipFile(\"download_data_fincen_files.zip\",\"r\") as zip_ref:\n", + " zip_ref.extract(\"download_transactions_map.csv\")\n", + "\n", + "# read csv into pandas dataframe and change type of time columns in data to datetime\n", + "df_e = pd.read_csv(\"download_transactions_map.csv\")\n", + "df_e[\"begin_date\"] = pd.to_datetime(df_e[\"begin_date\"])\n", + "df_e[\"end_date\"] = pd.to_datetime(df_e[\"end_date\"])\n", + "\n", + "# create graph\n", + "g = graphistry.edges(df_e, \"originator_bank_id\", \"beneficiary_bank_id\").materialize_nodes()\n", + "\n", + "# rename id col in nodes to nodeId\n", + "df_n = g._nodes.rename(columns={'id': 'nodeId'})\n", + "g = g.nodes(df_n, 'nodeId')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Node-list sample\n", + "\n", + "View a sample of the nodes in the graph. Each node represents a financial institution." + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "nodeId", + "rawType": "object", + "type": "string" + } + ], + "ref": "cfa51504-9fe6-47d0-98d5-a73e6f549641", + "rows": [ + [ + "0", + "cimb-bank-berhad" + ], + [ + "1", + "barclays-bank-plc-ho-uk" + ], + [ + "2", + "natwest-offshore" + ], + [ + "3", + "evrofinance-mosnarbank" + ], + [ + "4", + "latvian-trade-commercial-bank" + ] + ], + "shape": { + "columns": 1, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodeId
0cimb-bank-berhad
1barclays-bank-plc-ho-uk
2natwest-offshore
3evrofinance-mosnarbank
4latvian-trade-commercial-bank
\n", + "
" + ], + "text/plain": [ + " nodeId\n", + "0 cimb-bank-berhad\n", + "1 barclays-bank-plc-ho-uk\n", + "2 natwest-offshore\n", + "3 evrofinance-mosnarbank\n", + "4 latvian-trade-commercial-bank" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g._nodes.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Edge-list sample\n", + "\n", + "View a sample of the edges in the graph. Each edge represents an aggregegation of financial transactions between two financial institutions within a specific time period." + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "icij_sar_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "filer_org_name_id", + "rawType": "object", + "type": "string" + }, + { + "name": "filer_org_name", + "rawType": "object", + "type": "string" + }, + { + "name": "begin_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "end_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "originator_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "number_transactions", + "rawType": "float64", + "type": "float" + }, + { + "name": "amount_transactions", + "rawType": "float64", + "type": "float" + } + ], + "ref": "ad822317-09b0-47f3-859e-31f29a27845c", + "rows": [ + [ + "0", + "223254", + "3297", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2015-03-25 00:00:00", + "2015-09-25 00:00:00", + "cimb-bank-berhad", + "CIMB Bank Berhad", + "Singapore", + "SGP", + "barclays-bank-plc-london-england-gbr", + "Barclays Bank Plc", + "United Kingdom", + "GBR", + "68.0", + "56898523.47" + ], + [ + "1", + "223255", + "3297", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2015-03-30 00:00:00", + "2015-09-25 00:00:00", + "cimb-bank-berhad", + "CIMB Bank Berhad", + "Singapore", + "SGP", + "barclays-bank-plc-london-england-gbr", + "Barclays Bank Plc", + "United Kingdom", + "GBR", + "118.0", + "116238361.25" + ], + [ + "2", + "223258", + "2924", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2012-07-05 00:00:00", + "2012-07-05 00:00:00", + "barclays-bank-plc-ho-uk", + "Barclays Bank Plc Ho UK", + "United Kingdom", + "GBR", + "skandinaviska-enskilda-banken-stockholm-sweden-swe", + "Skandinaviska Enskilda Banken", + "Sweden", + "SWE", + null, + "5000.0" + ], + [ + "3", + "223259", + "2924", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2012-06-20 00:00:00", + "2012-06-20 00:00:00", + "barclays-bank-plc-ho-uk", + "Barclays Bank Plc Ho UK", + "United Kingdom", + "GBR", + "skandinaviska-enskilda-banken-stockholm-sweden-swe", + "Skandinaviska Enskilda Banken", + "Sweden", + "SWE", + null, + "9990.0" + ], + [ + "4", + "223260", + "2924", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2012-05-31 00:00:00", + "2012-05-31 00:00:00", + "barclays-bank-plc-ho-uk", + "Barclays Bank Plc Ho UK", + "United Kingdom", + "GBR", + "skandinaviska-enskilda-banken-stockholm-sweden-swe", + "Skandinaviska Enskilda Banken", + "Sweden", + "SWE", + null, + "12000.0" + ] + ], + "shape": { + "columns": 16, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idicij_sar_idfiler_org_name_idfiler_org_namebegin_dateend_dateoriginator_bank_idoriginator_bankoriginator_bank_countryoriginator_isobeneficiary_bank_idbeneficiary_bankbeneficiary_bank_countrybeneficiary_isonumber_transactionsamount_transactions
02232543297the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2015-03-252015-09-25cimb-bank-berhadCIMB Bank BerhadSingaporeSGPbarclays-bank-plc-london-england-gbrBarclays Bank PlcUnited KingdomGBR68.05.689852e+07
12232553297the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2015-03-302015-09-25cimb-bank-berhadCIMB Bank BerhadSingaporeSGPbarclays-bank-plc-london-england-gbrBarclays Bank PlcUnited KingdomGBR118.01.162384e+08
22232582924the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2012-07-052012-07-05barclays-bank-plc-ho-ukBarclays Bank Plc Ho UKUnited KingdomGBRskandinaviska-enskilda-banken-stockholm-sweden...Skandinaviska Enskilda BankenSwedenSWENaN5.000000e+03
32232592924the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2012-06-202012-06-20barclays-bank-plc-ho-ukBarclays Bank Plc Ho UKUnited KingdomGBRskandinaviska-enskilda-banken-stockholm-sweden...Skandinaviska Enskilda BankenSwedenSWENaN9.990000e+03
42232602924the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2012-05-312012-05-31barclays-bank-plc-ho-ukBarclays Bank Plc Ho UKUnited KingdomGBRskandinaviska-enskilda-banken-stockholm-sweden...Skandinaviska Enskilda BankenSwedenSWENaN1.200000e+04
\n", + "
" + ], + "text/plain": [ + " id icij_sar_id filer_org_name_id \\\n", + "0 223254 3297 the-bank-of-new-york-mellon-corp \n", + "1 223255 3297 the-bank-of-new-york-mellon-corp \n", + "2 223258 2924 the-bank-of-new-york-mellon-corp \n", + "3 223259 2924 the-bank-of-new-york-mellon-corp \n", + "4 223260 2924 the-bank-of-new-york-mellon-corp \n", + "\n", + " filer_org_name begin_date end_date \\\n", + "0 The Bank of New York Mellon Corp. 2015-03-25 2015-09-25 \n", + "1 The Bank of New York Mellon Corp. 2015-03-30 2015-09-25 \n", + "2 The Bank of New York Mellon Corp. 2012-07-05 2012-07-05 \n", + "3 The Bank of New York Mellon Corp. 2012-06-20 2012-06-20 \n", + "4 The Bank of New York Mellon Corp. 2012-05-31 2012-05-31 \n", + "\n", + " originator_bank_id originator_bank originator_bank_country \\\n", + "0 cimb-bank-berhad CIMB Bank Berhad Singapore \n", + "1 cimb-bank-berhad CIMB Bank Berhad Singapore \n", + "2 barclays-bank-plc-ho-uk Barclays Bank Plc Ho UK United Kingdom \n", + "3 barclays-bank-plc-ho-uk Barclays Bank Plc Ho UK United Kingdom \n", + "4 barclays-bank-plc-ho-uk Barclays Bank Plc Ho UK United Kingdom \n", + "\n", + " originator_iso beneficiary_bank_id \\\n", + "0 SGP barclays-bank-plc-london-england-gbr \n", + "1 SGP barclays-bank-plc-london-england-gbr \n", + "2 GBR skandinaviska-enskilda-banken-stockholm-sweden... \n", + "3 GBR skandinaviska-enskilda-banken-stockholm-sweden... \n", + "4 GBR skandinaviska-enskilda-banken-stockholm-sweden... \n", + "\n", + " beneficiary_bank beneficiary_bank_country beneficiary_iso \\\n", + "0 Barclays Bank Plc United Kingdom GBR \n", + "1 Barclays Bank Plc United Kingdom GBR \n", + "2 Skandinaviska Enskilda Banken Sweden SWE \n", + "3 Skandinaviska Enskilda Banken Sweden SWE \n", + "4 Skandinaviska Enskilda Banken Sweden SWE \n", + "\n", + " number_transactions amount_transactions \n", + "0 68.0 5.689852e+07 \n", + "1 118.0 1.162384e+08 \n", + "2 NaN 5.000000e+03 \n", + "3 NaN 9.990000e+03 \n", + "4 NaN 1.200000e+04 " + ] + }, + "execution_count": 109, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g._edges.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Visualization Setup\n", + "\n", + "Create a color palette for visualizing the graph. This is not necessary to look at the graph structure but will make the resulting graph more interpretable." + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "metadata": {}, + "outputs": [], + "source": [ + "def nonlinear_palette_generator(v, linear_palette):\n", + " out = []\n", + " num_palette = len(linear_palette)\n", + " num_palette_repetitions = [round(n**v) + 1 for n in range(num_palette)]\n", + " if v < 1:\n", + " num_palette_repetitions.reverse()\n", + " for i, color in enumerate(linear_palette):\n", + " out = out + [color for _ in range(num_palette_repetitions[i])]\n", + " return out\n", + "\n", + "palette = [\"#46327e\", \"#365c8d\", \"#277f8e\", \"#1fa187\", \"#4ac16d\", \"#a0da39\", \"#fde724\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Visualize full graph with encodings." + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# render graph of entire dataset\n", + "g_out = g.bind(edge_label=\"amount_transactions\")\n", + "g_out = g_out.encode_edge_color(\"amount_transactions\",\n", + " nonlinear_palette_generator(1.2, palette),\n", + " as_continuous=True)\n", + "g_out = g_out.settings(\n", + " height=800,\n", + " url_params={\n", + " \"pointOpacity\": 0.6 if len(g_out._nodes) > 1500 else 0.9,\n", + " \"edgeOpacity\": 0.3 if len(g_out._edges) > 1500 else 0.9,\n", + " \"strongGravity\": True,\n", + " \"play\": 2000})\n", + "g_out.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Caribbean havens subgraph\n", + "\n", + "The ICIJ mentions the importance of the world's top offshore financial havens in the data. Lets find transactions involving Caribbean tax havens using GFQL chain operations." + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "metadata": {}, + "outputs": [], + "source": [ + "from graphistry.compute.predicates.is_in import is_in\n", + "from graphistry.compute.ast import n, e_forward\n", + "\n", + "carib_havens = [\n", + " \"Cayman Islands\",\n", + " \"Bermuda\",\n", + " \"Virgin Islands British\",\n", + " \"British Virgin Islands\",\n", + " \"Bahamas\",\n", + " \"Panama\",\n", + " \"Barbados\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Outgoing transactions from Caribbean havens" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "metadata": {}, + "outputs": [], + "source": [ + "chain_operations = [\n", + " n(name=\"is_carib_bank\"),\n", + " e_forward(hops=1, edge_match={\"originator_bank_country\": is_in(options=carib_havens)}),\n", + "]\n", + "g_carib_out = g.gfql(chain_operations)" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of outgoing transaction groups: 92\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "icij_sar_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "filer_org_name_id", + "rawType": "object", + "type": "string" + }, + { + "name": "filer_org_name", + "rawType": "object", + "type": "string" + }, + { + "name": "begin_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "end_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "originator_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "number_transactions", + "rawType": "float64", + "type": "float" + }, + { + "name": "amount_transactions", + "rawType": "float64", + "type": "float" + } + ], + "ref": "323b075d-0dcb-430e-98d6-01d1a502716d", + "rows": [ + [ + "0", + "225260", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-07-18 00:00:00", + "2013-07-23 00:00:00", + "caledonian-bank-limited", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "merrill-lynch-new-york-ny-usa", + "Merrill Lynch", + "United States", + "USA", + "2.0", + "985000.0" + ], + [ + "1", + "225261", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-08-16 00:00:00", + "2013-08-16 00:00:00", + "caledonian-bank-limited", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "t-bank-julius-baer-and-co-ag-zurich-switzerland-che", + "T Bank Julius Baer And Co. AG", + "Switzerland", + "CHE", + "1.0", + "500000.0" + ], + [ + "2", + "225262", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-08-28 00:00:00", + "2013-09-05 00:00:00", + "caledonian-bank-limited", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "barclays-capital-inc-new-york-usa-usa", + "Barclays Capital Inc", + "United States", + "USA", + "2.0", + "450000.0" + ], + [ + "3", + "225264", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-03-21 00:00:00", + "2013-07-31 00:00:00", + "gonet-bank-and-trust-limited", + "Gonet Bank And Trust Limited", + "Bahamas", + "BHS", + "caledonian-bank-limited-georgetown-cayman-islands-cym", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "2.0", + "400000.0" + ], + [ + "4", + "225883", + "4076", + "the-northern-trust-company", + "The Northern Trust Company", + "2015-02-04 00:00:00", + "2015-02-04 00:00:00", + "dms-bank-trust-ltd", + "DMS Bank & Trust Ltd", + "Cayman Islands", + "CYM", + "hsbc-hong-kong-hkg", + "HSBC", + "Hong Kong", + "HKG", + "1.0", + "101000.0" + ] + ], + "shape": { + "columns": 16, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idicij_sar_idfiler_org_name_idfiler_org_namebegin_dateend_dateoriginator_bank_idoriginator_bankoriginator_bank_countryoriginator_isobeneficiary_bank_idbeneficiary_bankbeneficiary_bank_countrybeneficiary_isonumber_transactionsamount_transactions
02252603213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-07-182013-07-23caledonian-bank-limitedCaledonian Bank LimitedCayman IslandsCYMmerrill-lynch-new-york-ny-usaMerrill LynchUnited StatesUSA2.0985000.0
12252613213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-08-162013-08-16caledonian-bank-limitedCaledonian Bank LimitedCayman IslandsCYMt-bank-julius-baer-and-co-ag-zurich-switzerlan...T Bank Julius Baer And Co. AGSwitzerlandCHE1.0500000.0
22252623213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-08-282013-09-05caledonian-bank-limitedCaledonian Bank LimitedCayman IslandsCYMbarclays-capital-inc-new-york-usa-usaBarclays Capital IncUnited StatesUSA2.0450000.0
32252643213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-03-212013-07-31gonet-bank-and-trust-limitedGonet Bank And Trust LimitedBahamasBHScaledonian-bank-limited-georgetown-cayman-isla...Caledonian Bank LimitedCayman IslandsCYM2.0400000.0
42258834076the-northern-trust-companyThe Northern Trust Company2015-02-042015-02-04dms-bank-trust-ltdDMS Bank & Trust LtdCayman IslandsCYMhsbc-hong-kong-hkgHSBCHong KongHKG1.0101000.0
\n", + "
" + ], + "text/plain": [ + " id icij_sar_id filer_org_name_id \\\n", + "0 225260 3213 the-bank-of-new-york-mellon-corp \n", + "1 225261 3213 the-bank-of-new-york-mellon-corp \n", + "2 225262 3213 the-bank-of-new-york-mellon-corp \n", + "3 225264 3213 the-bank-of-new-york-mellon-corp \n", + "4 225883 4076 the-northern-trust-company \n", + "\n", + " filer_org_name begin_date end_date \\\n", + "0 The Bank of New York Mellon Corp. 2013-07-18 2013-07-23 \n", + "1 The Bank of New York Mellon Corp. 2013-08-16 2013-08-16 \n", + "2 The Bank of New York Mellon Corp. 2013-08-28 2013-09-05 \n", + "3 The Bank of New York Mellon Corp. 2013-03-21 2013-07-31 \n", + "4 The Northern Trust Company 2015-02-04 2015-02-04 \n", + "\n", + " originator_bank_id originator_bank \\\n", + "0 caledonian-bank-limited Caledonian Bank Limited \n", + "1 caledonian-bank-limited Caledonian Bank Limited \n", + "2 caledonian-bank-limited Caledonian Bank Limited \n", + "3 gonet-bank-and-trust-limited Gonet Bank And Trust Limited \n", + "4 dms-bank-trust-ltd DMS Bank & Trust Ltd \n", + "\n", + " originator_bank_country originator_iso \\\n", + "0 Cayman Islands CYM \n", + "1 Cayman Islands CYM \n", + "2 Cayman Islands CYM \n", + "3 Bahamas BHS \n", + "4 Cayman Islands CYM \n", + "\n", + " beneficiary_bank_id \\\n", + "0 merrill-lynch-new-york-ny-usa \n", + "1 t-bank-julius-baer-and-co-ag-zurich-switzerlan... \n", + "2 barclays-capital-inc-new-york-usa-usa \n", + "3 caledonian-bank-limited-georgetown-cayman-isla... \n", + "4 hsbc-hong-kong-hkg \n", + "\n", + " beneficiary_bank beneficiary_bank_country beneficiary_iso \\\n", + "0 Merrill Lynch United States USA \n", + "1 T Bank Julius Baer And Co. AG Switzerland CHE \n", + "2 Barclays Capital Inc United States USA \n", + "3 Caledonian Bank Limited Cayman Islands CYM \n", + "4 HSBC Hong Kong HKG \n", + "\n", + " number_transactions amount_transactions \n", + "0 2.0 985000.0 \n", + "1 1.0 500000.0 \n", + "2 2.0 450000.0 \n", + "3 2.0 400000.0 \n", + "4 1.0 101000.0 " + ] + }, + "execution_count": 115, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Number of outgoing transaction groups:\", len(g_carib_out._edges))\n", + "g_carib_out._edges.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "nodeId", + "rawType": "object", + "type": "string" + }, + { + "name": "is_carib_bank", + "rawType": "bool", + "type": "boolean" + } + ], + "ref": "06682bbf-1f4c-46c2-bc58-1b075a11f6c2", + "rows": [ + [ + "0", + "hsbc-bank", + "True" + ], + [ + "1", + "hsbc", + "True" + ], + [ + "2", + "caledonian-bank-limited", + "True" + ], + [ + "3", + "gonet-bank-and-trust-limited", + "True" + ], + [ + "4", + "dms-bank-trust-ltd", + "True" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodeIdis_carib_bank
0hsbc-bankTrue
1hsbcTrue
2caledonian-bank-limitedTrue
3gonet-bank-and-trust-limitedTrue
4dms-bank-trust-ltdTrue
\n", + "
" + ], + "text/plain": [ + " nodeId is_carib_bank\n", + "0 hsbc-bank True\n", + "1 hsbc True\n", + "2 caledonian-bank-limited True\n", + "3 gonet-bank-and-trust-limited True\n", + "4 dms-bank-trust-ltd True" + ] + }, + "execution_count": 116, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_carib_out._nodes.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Incoming transactions to Caribbean havens" + ] + }, + { + "cell_type": "code", + "execution_count": 117, + "metadata": {}, + "outputs": [], + "source": [ + "chain_operations = [\n", + " e_forward(hops=1, edge_match={\"beneficiary_bank_country\": is_in(options=carib_havens)}),\n", + " n(name=\"is_carib_bank\")\n", + "]\n", + "g_carib_in = g.gfql(chain_operations)" + ] + }, + { + "cell_type": "code", + "execution_count": 118, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of incoming transaction groups: 98\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "icij_sar_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "filer_org_name_id", + "rawType": "object", + "type": "string" + }, + { + "name": "filer_org_name", + "rawType": "object", + "type": "string" + }, + { + "name": "begin_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "end_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "originator_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "number_transactions", + "rawType": "float64", + "type": "float" + }, + { + "name": "amount_transactions", + "rawType": "float64", + "type": "float" + } + ], + "ref": "564a8112-a18c-4102-b227-430909a639ad", + "rows": [ + [ + "0", + "224386", + "4242", + "china-investment-corporation", + "China Investment Corporation", + "2015-08-03 00:00:00", + "2015-12-24 00:00:00", + "vp-bank-ag", + "VP Bank AG", + "Liechtenstein", + "LIE", + "mmg-bank-corporation-panama-city-panama-pan", + "Mmg Bank Corporation", + "Panama", + "PAN", + "6.0", + "444091.0" + ], + [ + "1", + "225255", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-08-27 00:00:00", + "2013-08-27 00:00:00", + "pictet-and-cie", + "Pictet And Cie", + "Switzerland", + "CHE", + "caledonian-bank-limited-georgetown-cayman-islands-cym", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "1.0", + "300015.0" + ], + [ + "2", + "225256", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-08-16 00:00:00", + "2013-08-16 00:00:00", + "banco-de-santander-sa", + "Banco De Santander S.A.", + "Uruguay", + "URY", + "caledonian-bank-limited-georgetown-cayman-islands-cym", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "1.0", + "200000.0" + ], + [ + "3", + "225257", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-08-01 00:00:00", + "2013-09-06 00:00:00", + "jpmorgan-chase-bank-na", + "JPMorgan Chase Bank Na", + "United States", + "USA", + "caledonian-bank-limited-georgetown-cayman-islands-cym", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "3.0", + "388000.0" + ], + [ + "4", + "225263", + "3213", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-07-24 00:00:00", + "2013-07-25 00:00:00", + "emirates-nbd-bank-pjsc", + "Emirates Nbd Bank PJSC", + "United Arab Emirates", + "ARE", + "caledonian-bank-limited-georgetown-cayman-islands-cym", + "Caledonian Bank Limited", + "Cayman Islands", + "CYM", + "2.0", + "200000.0" + ] + ], + "shape": { + "columns": 16, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idicij_sar_idfiler_org_name_idfiler_org_namebegin_dateend_dateoriginator_bank_idoriginator_bankoriginator_bank_countryoriginator_isobeneficiary_bank_idbeneficiary_bankbeneficiary_bank_countrybeneficiary_isonumber_transactionsamount_transactions
02243864242china-investment-corporationChina Investment Corporation2015-08-032015-12-24vp-bank-agVP Bank AGLiechtensteinLIEmmg-bank-corporation-panama-city-panama-panMmg Bank CorporationPanamaPAN6.0444091.0
12252553213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-08-272013-08-27pictet-and-ciePictet And CieSwitzerlandCHEcaledonian-bank-limited-georgetown-cayman-isla...Caledonian Bank LimitedCayman IslandsCYM1.0300015.0
22252563213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-08-162013-08-16banco-de-santander-saBanco De Santander S.A.UruguayURYcaledonian-bank-limited-georgetown-cayman-isla...Caledonian Bank LimitedCayman IslandsCYM1.0200000.0
32252573213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-08-012013-09-06jpmorgan-chase-bank-naJPMorgan Chase Bank NaUnited StatesUSAcaledonian-bank-limited-georgetown-cayman-isla...Caledonian Bank LimitedCayman IslandsCYM3.0388000.0
42252633213the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-07-242013-07-25emirates-nbd-bank-pjscEmirates Nbd Bank PJSCUnited Arab EmiratesAREcaledonian-bank-limited-georgetown-cayman-isla...Caledonian Bank LimitedCayman IslandsCYM2.0200000.0
\n", + "
" + ], + "text/plain": [ + " id icij_sar_id filer_org_name_id \\\n", + "0 224386 4242 china-investment-corporation \n", + "1 225255 3213 the-bank-of-new-york-mellon-corp \n", + "2 225256 3213 the-bank-of-new-york-mellon-corp \n", + "3 225257 3213 the-bank-of-new-york-mellon-corp \n", + "4 225263 3213 the-bank-of-new-york-mellon-corp \n", + "\n", + " filer_org_name begin_date end_date \\\n", + "0 China Investment Corporation 2015-08-03 2015-12-24 \n", + "1 The Bank of New York Mellon Corp. 2013-08-27 2013-08-27 \n", + "2 The Bank of New York Mellon Corp. 2013-08-16 2013-08-16 \n", + "3 The Bank of New York Mellon Corp. 2013-08-01 2013-09-06 \n", + "4 The Bank of New York Mellon Corp. 2013-07-24 2013-07-25 \n", + "\n", + " originator_bank_id originator_bank originator_bank_country \\\n", + "0 vp-bank-ag VP Bank AG Liechtenstein \n", + "1 pictet-and-cie Pictet And Cie Switzerland \n", + "2 banco-de-santander-sa Banco De Santander S.A. Uruguay \n", + "3 jpmorgan-chase-bank-na JPMorgan Chase Bank Na United States \n", + "4 emirates-nbd-bank-pjsc Emirates Nbd Bank PJSC United Arab Emirates \n", + "\n", + " originator_iso beneficiary_bank_id \\\n", + "0 LIE mmg-bank-corporation-panama-city-panama-pan \n", + "1 CHE caledonian-bank-limited-georgetown-cayman-isla... \n", + "2 URY caledonian-bank-limited-georgetown-cayman-isla... \n", + "3 USA caledonian-bank-limited-georgetown-cayman-isla... \n", + "4 ARE caledonian-bank-limited-georgetown-cayman-isla... \n", + "\n", + " beneficiary_bank beneficiary_bank_country beneficiary_iso \\\n", + "0 Mmg Bank Corporation Panama PAN \n", + "1 Caledonian Bank Limited Cayman Islands CYM \n", + "2 Caledonian Bank Limited Cayman Islands CYM \n", + "3 Caledonian Bank Limited Cayman Islands CYM \n", + "4 Caledonian Bank Limited Cayman Islands CYM \n", + "\n", + " number_transactions amount_transactions \n", + "0 6.0 444091.0 \n", + "1 1.0 300015.0 \n", + "2 1.0 200000.0 \n", + "3 3.0 388000.0 \n", + "4 2.0 200000.0 " + ] + }, + "execution_count": 118, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Number of incoming transaction groups:\", len(g_carib_in._edges))\n", + "g_carib_in._edges.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "nodeId", + "rawType": "object", + "type": "string" + }, + { + "name": "is_carib_bank", + "rawType": "bool", + "type": "boolean" + } + ], + "ref": "da9991cc-04a3-469f-afcd-b2df1bee9b7c", + "rows": [ + [ + "0", + "hsbc-hong-kong-hkg", + "False" + ], + [ + "1", + "credit-suisse-ag", + "False" + ], + [ + "2", + "bsi-sa", + "False" + ], + [ + "3", + "abn-amro-bank-nv", + "False" + ], + [ + "4", + "deutsche-bank-ag", + "False" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodeIdis_carib_bank
0hsbc-hong-kong-hkgFalse
1credit-suisse-agFalse
2bsi-saFalse
3abn-amro-bank-nvFalse
4deutsche-bank-agFalse
\n", + "
" + ], + "text/plain": [ + " nodeId is_carib_bank\n", + "0 hsbc-hong-kong-hkg False\n", + "1 credit-suisse-ag False\n", + "2 bsi-sa False\n", + "3 abn-amro-bank-nv False\n", + "4 deutsche-bank-ag False" + ] + }, + "execution_count": 119, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_carib_in._nodes.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualize Caribbean transactions\n", + "\n", + "Encode the visualization with colors based on transaction amounts." + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 120, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_carib = graphistry \\\n", + " .edges(pd.concat([g_carib_in._edges, g_carib_out._edges], ignore_index=True), \\\n", + " \"originator_bank_id\", \"beneficiary_bank_id\") \\\n", + " .nodes(pd.concat([g_carib_in._nodes, g_carib_out._nodes], ignore_index=True).drop_duplicates())\n", + "\n", + "g_carib_styled = (\n", + " g_carib\n", + " .bind(node=\"nodeId\", point_label=\"nodeId\")\n", + " .encode_edge_color('amount_transactions',\n", + " palette=nonlinear_palette_generator(1.05, palette),\n", + " as_continuous=True)\n", + " .encode_point_color(\"is_carib_bank\",\n", + " as_categorical=True,\n", + " categorical_mapping={True: \"red\", False: \"blue\"}) \n", + " .settings(\n", + " height=800,\n", + " url_params={\n", + " \"pointOpacity\": 0.6 if len(g_carib_in._nodes) > 1500 else 0.9,\n", + " \"edgeOpacity\": 0.3 if len(g_carib_in._edges) > 1500 else 0.9,\n", + " \"strongGravity\": True,\n", + " \"play\": 2000})\n", + ")\n", + "\n", + "g_carib_styled.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Latvia-Russia subgraph\n", + "\n", + "Find all data following a specific transaction pattern: Latvia to Russia transactions in a specific amount range." + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 121, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from graphistry import contains\n", + "\n", + "chain_operations = [\n", + " e_forward(hops=1, edge_match={\"originator_bank_country\": \"Latvia\", \"beneficiary_bank_country\": \"Russia\"}),\n", + " n({\"nodeId\": contains(pat=\"\")}, name=\"is_rus_beneficiary\"),\n", + "]\n", + "g_lva_rus = g.gfql(chain_operations)\n", + "\n", + "g_lva_rus = g_lva_rus.bind(edge_label=\"amount_transactions\")\n", + "g_lva_rus = g_lva_rus.encode_edge_color(\"amount_transactions\",\n", + " nonlinear_palette_generator(1.4, palette), as_continuous=True)\n", + "g_lva_rus = g_lva_rus.encode_point_color(\"is_rus_beneficiary\",\n", + " as_categorical=True,\n", + " categorical_mapping={True: \"red\", False: \"blue\"})\n", + "\n", + "g_lva_rus = g_lva_rus.settings(\n", + " height=800,\n", + " url_params={\n", + " \"pointOpacity\": 0.6 if len(g_lva_rus._nodes) > 1500 else 0.9,\n", + " \"edgeOpacity\": 0.3 if len(g_lva_rus._edges) > 1500 else 0.9,\n", + " \"strongGravity\": True,\n", + " \"play\": 2000})\n", + "g_lva_rus.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "nodeId", + "rawType": "object", + "type": "string" + }, + { + "name": "is_rus_beneficiary", + "rawType": "bool", + "type": "boolean" + } + ], + "ref": "91b5eebd-38eb-4840-9376-a30c770e7ff0", + "rows": [ + [ + "0", + "latvian-trade-commercial-bank", + "False" + ], + [ + "1", + "ltb-bank-riga", + "False" + ], + [ + "2", + "norvik-banka-jsc", + "False" + ], + [ + "3", + "jsc-norvik-banka", + "False" + ], + [ + "4", + "rietumu-banka-jsc", + "False" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodeIdis_rus_beneficiary
0latvian-trade-commercial-bankFalse
1ltb-bank-rigaFalse
2norvik-banka-jscFalse
3jsc-norvik-bankaFalse
4rietumu-banka-jscFalse
\n", + "
" + ], + "text/plain": [ + " nodeId is_rus_beneficiary\n", + "0 latvian-trade-commercial-bank False\n", + "1 ltb-bank-riga False\n", + "2 norvik-banka-jsc False\n", + "3 jsc-norvik-banka False\n", + "4 rietumu-banka-jsc False" + ] + }, + "execution_count": 122, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_lva_rus._nodes.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "icij_sar_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "filer_org_name_id", + "rawType": "object", + "type": "string" + }, + { + "name": "filer_org_name", + "rawType": "object", + "type": "string" + }, + { + "name": "begin_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "end_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "originator_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "number_transactions", + "rawType": "float64", + "type": "float" + }, + { + "name": "amount_transactions", + "rawType": "float64", + "type": "float" + } + ], + "ref": "58886227-2ab8-48c2-b567-51fa8269bc55", + "rows": [ + [ + "0", + "223935", + "2359", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2011-02-10 00:00:00", + "2011-02-10 00:00:00", + "latvian-trade-commercial-bank", + "Latvian Trade Commercial Bank", + "Latvia", + "LVA", + "transcredit-bank-moscow-russia-rus", + "Transcredit Bank", + "Russia", + "RUS", + "1.0", + "3485.0" + ], + [ + "1", + "223936", + "2359", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2011-02-03 00:00:00", + "2011-02-03 00:00:00", + "latvian-trade-commercial-bank", + "Latvian Trade Commercial Bank", + "Latvia", + "LVA", + "vnesheconombank-moscow-russia-rus", + "Vnesheconombank", + "Russia", + "RUS", + "1.0", + "599088.0" + ], + [ + "2", + "223937", + "2359", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2011-02-02 00:00:00", + "2011-02-02 00:00:00", + "latvian-trade-commercial-bank", + "Latvian Trade Commercial Bank", + "Latvia", + "LVA", + "vnesheconombank-moscow-russia-rus", + "Vnesheconombank", + "Russia", + "RUS", + "1.0", + "1400000.0" + ], + [ + "3", + "223939", + "2359", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2011-02-28 00:00:00", + "2011-02-28 00:00:00", + "latvian-trade-commercial-bank", + "Latvian Trade Commercial Bank", + "Latvia", + "LVA", + "ojsc-nomos-bank-moscow-russia-rus", + "Ojsc 'Nomos-Bank'", + "Russia", + "RUS", + "1.0", + "205.48" + ], + [ + "4", + "223940", + "2359", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2011-02-28 00:00:00", + "2011-02-28 00:00:00", + "latvian-trade-commercial-bank", + "Latvian Trade Commercial Bank", + "Latvia", + "LVA", + "ojsc-nomos-bank-moscow-russia-rus", + "Ojsc 'Nomos-Bank'", + "Russia", + "RUS", + "1.0", + "790753.42" + ] + ], + "shape": { + "columns": 16, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idicij_sar_idfiler_org_name_idfiler_org_namebegin_dateend_dateoriginator_bank_idoriginator_bankoriginator_bank_countryoriginator_isobeneficiary_bank_idbeneficiary_bankbeneficiary_bank_countrybeneficiary_isonumber_transactionsamount_transactions
02239352359the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2011-02-102011-02-10latvian-trade-commercial-bankLatvian Trade Commercial BankLatviaLVAtranscredit-bank-moscow-russia-rusTranscredit BankRussiaRUS1.03485.00
12239362359the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2011-02-032011-02-03latvian-trade-commercial-bankLatvian Trade Commercial BankLatviaLVAvnesheconombank-moscow-russia-rusVnesheconombankRussiaRUS1.0599088.00
22239372359the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2011-02-022011-02-02latvian-trade-commercial-bankLatvian Trade Commercial BankLatviaLVAvnesheconombank-moscow-russia-rusVnesheconombankRussiaRUS1.01400000.00
32239392359the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2011-02-282011-02-28latvian-trade-commercial-bankLatvian Trade Commercial BankLatviaLVAojsc-nomos-bank-moscow-russia-rusOjsc 'Nomos-Bank'RussiaRUS1.0205.48
42239402359the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2011-02-282011-02-28latvian-trade-commercial-bankLatvian Trade Commercial BankLatviaLVAojsc-nomos-bank-moscow-russia-rusOjsc 'Nomos-Bank'RussiaRUS1.0790753.42
\n", + "
" + ], + "text/plain": [ + " id icij_sar_id filer_org_name_id \\\n", + "0 223935 2359 the-bank-of-new-york-mellon-corp \n", + "1 223936 2359 the-bank-of-new-york-mellon-corp \n", + "2 223937 2359 the-bank-of-new-york-mellon-corp \n", + "3 223939 2359 the-bank-of-new-york-mellon-corp \n", + "4 223940 2359 the-bank-of-new-york-mellon-corp \n", + "\n", + " filer_org_name begin_date end_date \\\n", + "0 The Bank of New York Mellon Corp. 2011-02-10 2011-02-10 \n", + "1 The Bank of New York Mellon Corp. 2011-02-03 2011-02-03 \n", + "2 The Bank of New York Mellon Corp. 2011-02-02 2011-02-02 \n", + "3 The Bank of New York Mellon Corp. 2011-02-28 2011-02-28 \n", + "4 The Bank of New York Mellon Corp. 2011-02-28 2011-02-28 \n", + "\n", + " originator_bank_id originator_bank \\\n", + "0 latvian-trade-commercial-bank Latvian Trade Commercial Bank \n", + "1 latvian-trade-commercial-bank Latvian Trade Commercial Bank \n", + "2 latvian-trade-commercial-bank Latvian Trade Commercial Bank \n", + "3 latvian-trade-commercial-bank Latvian Trade Commercial Bank \n", + "4 latvian-trade-commercial-bank Latvian Trade Commercial Bank \n", + "\n", + " originator_bank_country originator_iso beneficiary_bank_id \\\n", + "0 Latvia LVA transcredit-bank-moscow-russia-rus \n", + "1 Latvia LVA vnesheconombank-moscow-russia-rus \n", + "2 Latvia LVA vnesheconombank-moscow-russia-rus \n", + "3 Latvia LVA ojsc-nomos-bank-moscow-russia-rus \n", + "4 Latvia LVA ojsc-nomos-bank-moscow-russia-rus \n", + "\n", + " beneficiary_bank beneficiary_bank_country beneficiary_iso \\\n", + "0 Transcredit Bank Russia RUS \n", + "1 Vnesheconombank Russia RUS \n", + "2 Vnesheconombank Russia RUS \n", + "3 Ojsc 'Nomos-Bank' Russia RUS \n", + "4 Ojsc 'Nomos-Bank' Russia RUS \n", + "\n", + " number_transactions amount_transactions \n", + "0 1.0 3485.00 \n", + "1 1.0 599088.00 \n", + "2 1.0 1400000.00 \n", + "3 1.0 205.48 \n", + "4 1.0 790753.42 " + ] + }, + "execution_count": 123, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_lva_rus._edges.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Find Oleg Deripaska's transaction\n", + "\n", + "Use contains() and between() predicates to find a specific transaction pattern described by the ICIJ:\n", + "\n", + "1. On Aug. 15, 2013, Mallow Capital Corp., in the British Virgin Islands, sent $15.9 million from its account at Expobank in Latvia\n", + "2. The funds were sent to Deutsche Bank Trust Company Americas in New York.\n", + "3. Deutsche Bank U.S. then sent the funds to the Bank of New York Mellon, which later flagged the transaction as suspicious.\n", + "4. The funds were then credited to Mallow Capital’s account at Bank Soyuz in Russia. The transaction was labeled a “funds transfer”." + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "metadata": {}, + "outputs": [], + "source": [ + "from graphistry import between, contains\n", + "\n", + "chain_operations = [\n", + " n({\"nodeId\": contains(pat=\"expo\")}),\n", + " e_forward(hops=1, edge_match={\n", + " \"originator_bank_country\": \"Latvia\",\n", + " \"beneficiary_bank_country\": \"Russia\",\n", + " \"amount_transactions\": between(15800000, 16000000)}),\n", + " n({\"nodeId\": contains(pat=\"soyuz\")}, name=\"is_soyuz\")\n", + "]\n", + "g_od = g.gfql(chain_operations)" + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "nodeId", + "rawType": "object", + "type": "string" + }, + { + "name": "is_soyuz", + "rawType": "bool", + "type": "boolean" + } + ], + "ref": "7846a1c3-19cc-4969-bfeb-6d49c88517d2", + "rows": [ + [ + "0", + "as-expobank", + "False" + ], + [ + "1", + "bank-soyuz-moscow-russia-rus", + "True" + ] + ], + "shape": { + "columns": 2, + "rows": 2 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nodeIdis_soyuz
0as-expobankFalse
1bank-soyuz-moscow-russia-rusTrue
\n", + "
" + ], + "text/plain": [ + " nodeId is_soyuz\n", + "0 as-expobank False\n", + "1 bank-soyuz-moscow-russia-rus True" + ] + }, + "execution_count": 125, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_od._nodes.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "icij_sar_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "filer_org_name_id", + "rawType": "object", + "type": "string" + }, + { + "name": "filer_org_name", + "rawType": "object", + "type": "string" + }, + { + "name": "begin_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "end_date", + "rawType": "datetime64[ns]", + "type": "datetime" + }, + { + "name": "originator_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "originator_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_id", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_bank_country", + "rawType": "object", + "type": "string" + }, + { + "name": "beneficiary_iso", + "rawType": "object", + "type": "string" + }, + { + "name": "number_transactions", + "rawType": "float64", + "type": "float" + }, + { + "name": "amount_transactions", + "rawType": "float64", + "type": "float" + } + ], + "ref": "3a269bb6-f311-4b19-bd36-5676983283ff", + "rows": [ + [ + "0", + "239549", + "2718", + "the-bank-of-new-york-mellon-corp", + "The Bank of New York Mellon Corp.", + "2013-08-15 00:00:00", + "2013-08-15 00:00:00", + "as-expobank", + "AS Expobank", + "Latvia", + "LVA", + "bank-soyuz-moscow-russia-rus", + "Bank Soyuz", + "Russia", + "RUS", + "1.0", + "15900000.0" + ] + ], + "shape": { + "columns": 16, + "rows": 1 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idicij_sar_idfiler_org_name_idfiler_org_namebegin_dateend_dateoriginator_bank_idoriginator_bankoriginator_bank_countryoriginator_isobeneficiary_bank_idbeneficiary_bankbeneficiary_bank_countrybeneficiary_isonumber_transactionsamount_transactions
02395492718the-bank-of-new-york-mellon-corpThe Bank of New York Mellon Corp.2013-08-152013-08-15as-expobankAS ExpobankLatviaLVAbank-soyuz-moscow-russia-rusBank SoyuzRussiaRUS1.015900000.0
\n", + "
" + ], + "text/plain": [ + " id icij_sar_id filer_org_name_id \\\n", + "0 239549 2718 the-bank-of-new-york-mellon-corp \n", + "\n", + " filer_org_name begin_date end_date originator_bank_id \\\n", + "0 The Bank of New York Mellon Corp. 2013-08-15 2013-08-15 as-expobank \n", + "\n", + " originator_bank originator_bank_country originator_iso \\\n", + "0 AS Expobank Latvia LVA \n", + "\n", + " beneficiary_bank_id beneficiary_bank beneficiary_bank_country \\\n", + "0 bank-soyuz-moscow-russia-rus Bank Soyuz Russia \n", + "\n", + " beneficiary_iso number_transactions amount_transactions \n", + "0 RUS 1.0 15900000.0 " + ] + }, + "execution_count": 126, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_od._edges.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualize the specific transaction" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 128, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g_od_styled = (\n", + " g_od\n", + " .encode_edge_color('amount_transactions',\n", + " palette=nonlinear_palette_generator(1.05, palette),\n", + " as_continuous=True)\n", + " .encode_point_color(\"is_soyuz\", as_categorical=True, categorical_mapping={True: \"red\", False: \"blue\"})\n", + " .settings(\n", + " height=800,\n", + " url_params={\n", + " \"pointOpacity\": 0.6 if len(g_lva_rus._nodes) > 1500 else 0.9,\n", + " \"edgeOpacity\": 0.3 if len(g_lva_rus._edges) > 1500 else 0.9,\n", + " \"strongGravity\": True,\n", + " \"play\": 2000})\n", + ")\n", + "\n", + "g_od_styled.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## In closing\n", + "\n", + "This notebook demonstrated the key GFQL operations:\n", + "\n", + "- **Node filtering**: `n()` with attribute matching and predicates\n", + "- **Edge traversal**: `e_forward()` with hop counts and edge matching\n", + "- **Chaining operations**: `graphistry.Chain()` to combine multiple operations\n", + "- **Predicates**:\n", + " - `is_in()` for matching multiple values\n", + " - `contains()` for substring matching\n", + " - `between()` for numeric range filtering\n", + "\n", + "### Additional Resources\n", + "\n", + "- [Python GFQL API Documentation](https://pygraphistry.readthedocs.io/en/latest/gfql/overview.html)\n", + "- [GFQL REST API Documentation](https://hub.graphistry.com/docs/GFQL/gfql-api/)\n", + "- [How GFQL Chain Works](https://hub.graphistry.com/docs/GFQL/gfql-chaining/)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "rapids-24.08", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/source/notebooks/gfql.rst b/docs/source/notebooks/gfql.rst index a967eae076..99b18c033f 100644 --- a/docs/source/notebooks/gfql.rst +++ b/docs/source/notebooks/gfql.rst @@ -12,4 +12,4 @@ GFQL Graph queries GPU Benchmarking <../demos/gfql/benchmark_hops_cpu_gpu.ipynb> GFQL Remote mode <../demos/gfql/gfql_remote.ipynb> Python Remote mode <../demos/gfql/python_remote.ipynb> - + # ICIJ FinCEN Files Visualization <../demos/demos_by_user_case/icij_fincen_viz.ipynb> diff --git a/graphistry/kepler.py b/graphistry/kepler.py index 0329563962..0510d22c33 100644 --- a/graphistry/kepler.py +++ b/graphistry/kepler.py @@ -526,6 +526,8 @@ class KeplerConfig: :type overlay_blending: Optional[Literal['normal', 'additive', 'subtractive']] :param tile_style: Base map tile style configuration :type tile_style: Optional[Dict[str, Any]] + :param auto_graph_renderer_switching: Enable automatic graph renderer switching, which allows Graphistry to hide Kepler node and edge layers depending on the mode (default: True) + :type auto_graph_renderer_switching: Optional[bool] **Example: Structured parameters** :: @@ -576,7 +578,8 @@ def __init__( *, cull_unused_columns: Optional[bool] = None, overlay_blending: Optional[Literal['normal', 'additive', 'subtractive']] = None, - tile_style: Optional[Dict[str, Any]] = None + tile_style: Optional[Dict[str, Any]] = None, + auto_graph_renderer_switching: Optional[bool] = None ) -> None: ... @@ -762,7 +765,8 @@ def with_config( *, cull_unused_columns: Optional[bool] = None, overlay_blending: Optional[Literal['normal', 'additive', 'subtractive']] = None, - tile_style: Optional[Dict[str, Any]] = None + tile_style: Optional[Dict[str, Any]] = None, + auto_graph_renderer_switching: Optional[bool] = None ) -> 'KeplerEncoding': ... @@ -777,6 +781,8 @@ def with_config(self, config: Optional[KeplerConfig] = None, **kwargs) -> 'Keple :type overlay_blending: Optional[Literal['normal', 'additive', 'subtractive']] :param tile_style: Base map tile style configuration :type tile_style: Optional[Dict[str, Any]] + :param auto_graph_renderer_switching: Enable automatic graph renderer switching + :type auto_graph_renderer_switching: Optional[bool] :return: New KeplerEncoding instance with updated config :rtype: KeplerEncoding """