diff --git a/src/simmate/apps/ethereum/templates/ethereum/transactions/table.html b/src/simmate/apps/ethereum/templates/ethereum/transactions/table.html index 8c2d1f472..bb573415b 100644 --- a/src/simmate/apps/ethereum/templates/ethereum/transactions/table.html +++ b/src/simmate/apps/ethereum/templates/ethereum/transactions/table.html @@ -1,27 +1,27 @@ {% extends "data_explorer/table_entries_base.html" %} {% block table_headers %} - {% table_header "id" "Hash:Type:ID" min_width=700 %} - {% table_header "from_address" "From" min_width=400 %} - {% table_header "from_address" "To" min_width=400 %} + {% table_header "id" "Hash:Type:ID" min_width=150 %} + {% table_header "from_address" "From" min_width=150 %} + {% table_header "from_address" "To" min_width=150 %} {% table_header "chain" "Chain" min_width=150 %} {% table_header "asset" "Asset" min_width=150 %} {% table_header "amount" "Amount" min_width=150 %} {% table_header "created_at_original" "Date" min_width=250 %} {% endblock %} {% block table_rows %} - {% foreign_key_link entry %} + {% foreign_key_link entry truncate_chars=10 %} {% if entry.from_address.ens_name %} {% foreign_key_link entry.from_address "ens_name" %} {% else %} - {% foreign_key_link entry.from_address %} + {% foreign_key_link entry.from_address truncate_chars=10 %} {% endif %} {% if entry.to_address.ens_name %} {% foreign_key_link entry.to_address "ens_name" %} {% else %} - {% foreign_key_link entry.to_address %} + {% foreign_key_link entry.to_address truncate_chars=10 %} {% endif %} {{ entry.chain }} diff --git a/src/simmate/apps/ethereum/templates/ethereum/wallets/table.html b/src/simmate/apps/ethereum/templates/ethereum/wallets/table.html index 3eaa13597..050044c1a 100644 --- a/src/simmate/apps/ethereum/templates/ethereum/wallets/table.html +++ b/src/simmate/apps/ethereum/templates/ethereum/wallets/table.html @@ -1,15 +1,15 @@ {% extends "data_explorer/table_entries_base.html" %} {% block table_headers %} - {% table_header "id" "ETH Address" min_width=400 %} + {% table_header "id" "ETH Address" min_width=150 %} {% table_header "ens_name" "ENS" min_width=200 %} - {% table_header "ethereum_balance" "ETH" min_width=200 %} - {% table_header "usdc_balance" "USDC" min_width=200 %} + {% table_header "ethereum_balance" "ETH" min_width=150 %} + {% table_header "usdc_balance" "USDC" min_width=150 %} {% table_header "stablecoin_total_balance" "Total Stablecoins" min_width=200 %} - {% table_header "assets_total_value_usd" "Total Assets" min_width=200 %} + {% table_header "assets_total_value_usd" "Total Assets" min_width=150 %} {% table_header "updated_at" "Updated" min_width=250 %} {% endblock %} {% block table_rows %} - {% foreign_key_link entry %} + {% foreign_key_link entry truncate_chars=10 %} {% if entry.ens_name %} {{ entry.ens_name }} diff --git a/src/simmate/apps/price_catalog/clients/fred.py b/src/simmate/apps/price_catalog/clients/fred.py index 74bc33437..08caa5f06 100644 --- a/src/simmate/apps/price_catalog/clients/fred.py +++ b/src/simmate/apps/price_catalog/clients/fred.py @@ -18,6 +18,27 @@ class FredClient: "Electricity": "CUSR0000SEHF01", # or APU000072610 "Housing": "CSUSHPINSA", "Consumer Price Index": "CPIAUCSL", # use for inflation metrics + # + # Producer Price Index (PPI): + # sections for "Industrial Chemicals" or "Chemicals and Allied Products" + # Make sure you look at this link to see the full tree of these tickers + # and how they relate to one another (some are subcategories of others) + # https://fred.stlouisfed.org/release/tables?rid=46&eid=142872#snid=142874 + "Chemicals": "WPU06", # & Allied Products + "Industrial Chemicals": "WPU061", + # + "Inorganic Chemicals": "WPU0613", + "Organic Chemicals": "WPU0614", + "Industrial Gases": "WPU067903", + # + "Carbon": "WPU06790918", # Carbon Black + # "Carbon Dioxide": "WPU06790302", + "Nitrogen": "WPU06790303", + "Oxygen": "WPU06790304", + "Aluminum": "WPU06130209", # Aluminum containing compounds + # "Sulfuric Acid": "WPU0613020T1", + # "NaCl": "WPU06130271", # Rock Salt + # "Ethanol": "WPU06140341", } @classmethod @@ -41,7 +62,7 @@ def get_data(cls, name: str): @staticmethod def get_ticker_data(ticker: str): """ - Uses the Yahoo ticker value to grab data. Can be any ticker in the live site + Uses the FRED ticker value to grab data. Can be any ticker in the live site such as `SPY` or `GOOGL` """ url = f"https://fred.stlouisfed.org/graph/fredgraph.csv?id={ticker}" diff --git a/src/simmate/apps/price_catalog/components/__init__.py b/src/simmate/apps/price_catalog/components/__init__.py new file mode 100644 index 000000000..08a41676a --- /dev/null +++ b/src/simmate/apps/price_catalog/components/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from .priced_items_report import PricedItemsReport diff --git a/src/simmate/apps/price_catalog/components/priced_items_report.py b/src/simmate/apps/price_catalog/components/priced_items_report.py new file mode 100644 index 000000000..5f1aa3231 --- /dev/null +++ b/src/simmate/apps/price_catalog/components/priced_items_report.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +from simmate.website.htmx.components import HtmxComponent + +from ..models import PricedItem + + +class PricedItemsReport(HtmxComponent): + + template_name = "price_catalog/priced_items/report.html" + + item_names_options: list[str] = None # set in mount + # years_ago_norm_options: list[int] = PricedItem.years_ago_options + + years_ago_norm_options = [ + (1, "Δ1y"), + (5, "Δ5y"), + (10, "Δ10y"), + (25, "Δ25y"), + ("max", "Max"), + ] + + def mount(self): + + self.priced_item_id = self.request.resolver_match.kwargs["table_entry_id"] + self.priced_item = PricedItem.objects.get(id=self.priced_item_id) + + self.item_names_options = list( + PricedItem.objects.values_list("name", flat=True).order_by("name").all() + ) + self.form_data = { + "item_names": [], # ["S&P 500", "Gold"] + "use_percent": False, + "use_inflation_adj": False, + "years_ago_norm": "max", + } + self.process() # to create figure + + def post_parse(self): + # BUG: checkbox field does not show in POST data when =False. This is + # normal HTML behavior to save on bandwidth, but causes a bug here. + if "use_percent" not in self.post_data.keys(): + self.post_data["use_percent"] = False + if "use_inflation_adj" not in self.post_data.keys(): + self.post_data["use_inflation_adj"] = False + + def process(self): + + if self.form_data["item_names"]: + self.figure = PricedItem.get_report_figure( + names=[self.priced_item.name] + self.form_data["item_names"], + years_ago_norm=( + self.form_data["years_ago_norm"] + if self.form_data["years_ago_norm"] != "max" + else None + ), + percent=( + self.form_data["use_percent"] + if self.form_data["years_ago_norm"] != "max" + else None + ), + inflation_adj=self.form_data["use_inflation_adj"], + ) + + elif self.form_data["years_ago_norm"] == "max": + self.figure = self.priced_item.get_price_figure( + inflation_adj=self.form_data["use_inflation_adj"], + ) + + else: + self.figure = self.priced_item.get_delta_figure( + years_ago=self.form_data["years_ago_norm"], + percent=self.form_data["use_percent"], + inflation_adj=self.form_data["use_inflation_adj"], + ) + + # ------------------------------------------------------------------------- + + def on_change_hook__item_names(self): + # bug-fix: when only one is selected, the value isn't put in a list + if isinstance(self.form_data["item_names"], str): + self.form_data["item_names"] = [self.form_data["item_names"]] + + def on_change_hook__years_ago_norm(self): + if self.form_data["years_ago_norm"] != "max": + self.form_data["item_names"] = [] + # self.form_data["use_percent"] = True + # self.form_data["use_inflation_adj"] = True diff --git a/src/simmate/apps/price_catalog/data/README.md b/src/simmate/apps/price_catalog/data/README.md new file mode 100644 index 000000000..9bbbfc7c4 --- /dev/null +++ b/src/simmate/apps/price_catalog/data/README.md @@ -0,0 +1,5 @@ + +To quickly look up common chemicals & their prices, for "the big 3": +- Sigma-Aldrich (MilliporeSigma): https://www.sigmaaldrich.com/US/en/products/chemistry-and-biochemicals/lab-chemicals +- Fisher Scientific (Thermo Fisher Scientific): https://www.thermofisher.com/us/en/home/chemicals.html +- VWR International (Avantor): https://www.avantorsciences.com/us/en/products/chemicals diff --git a/src/simmate/apps/price_catalog/data/__init__.py b/src/simmate/apps/price_catalog/data/__init__.py new file mode 100644 index 000000000..25963639b --- /dev/null +++ b/src/simmate/apps/price_catalog/data/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from .utilities import WIKIPEDIA_PRICES_OF_ELEMENTS_DATA diff --git a/src/simmate/apps/price_catalog/data/utilities.py b/src/simmate/apps/price_catalog/data/utilities.py new file mode 100644 index 000000000..c0e9dd717 --- /dev/null +++ b/src/simmate/apps/price_catalog/data/utilities.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +from pathlib import Path + +import pandas + + +def load_wikipedia_data() -> pandas.DataFrame: + """ + Loads a snapshot dataset from the "Prices of chemical elements" page + + This dataset was pulled from: + https://en.wikipedia.org/wiki/Prices_of_chemical_elements + + Note, I used Gemini to web scrape bc I'm lazy. The following prompt was used + to generate the CSV file, and I spot checked only a little to confirm: + + ''' + Grab the table from this link and convert it to a CSV: + https://en.wikipedia.org/wiki/Prices_of_chemical_elements + + The following should be done too: + - split the "abundance" column into two columns, one for mg/kg and one for total mass + - use "e" notation instead of "x10^" so that values are floats + - standardize numerical columns (e.g. convert ranges to single value and exclude any >, <, ~, etc symbols) + - leave a cell empty instead of using text like "Not traded" + - use the columns names of... + z, symbol, name, density_kg_l, abundance_mg_kg, total_mass_kg, + price_per_kg, price_per_l, year, source, notes + ''' + + """ + datafile = Path(__file__).parent / "wikipedia_prices_of_elements.csv" + data = pandas.read_csv(datafile) + return data + + +WIKIPEDIA_PRICES_OF_ELEMENTS_DATA = load_wikipedia_data() diff --git a/src/simmate/apps/price_catalog/data/wikipedia_prices_of_elements.csv b/src/simmate/apps/price_catalog/data/wikipedia_prices_of_elements.csv new file mode 100644 index 000000000..bb0611b23 --- /dev/null +++ b/src/simmate/apps/price_catalog/data/wikipedia_prices_of_elements.csv @@ -0,0 +1,124 @@ +z,symbol,name,density_kg_l,abundance_mg_kg,total_mass_kg,price_per_kg,price_per_l,year,source,notes +1,H,Hydrogen,0.00008988,1400.0,3.878e19,1.39,0.000125,2012,DOE Hydrogen, +1,"2H (D)",Deuterium,0.0001667,1.5e-5,4.155e11,13400.0,2.23,2020,CIL, +2,He,Helium,0.0001785,0.008,2.216e14,24.0,0.00429,2018,USGS MCS, +3,Li,Lithium,0.534,20.0,5.54e17,83.5,44.55,2020,SMM, +4,Be,Beryllium,1.85,2.8,7.756e16,857.0,1590.0,2020,ISE 2020, +5,B,Boron,2.34,10.0,2.77e17,3.68,8.62,2019,CEIC Data, +6,C,Carbon,2.267,200.0,5.54e18,0.122,0.28,2018,EIA Coal, +7,N,Nitrogen,0.0012506,19.0,5.263e17,0.14,0.000175,2001,Hypertextbook, +8,O,Oxygen,0.001429,461000.0,1.277e22,0.154,0.00022,2001,Hypertextbook, +9,F,Fluorine,0.001696,585.0,1.62e19,2.0,0.00338,2017,Echemi, +10,Ne,Neon,0.0008999,0.005,1.385e14,240.0,0.21,1999,Ullmann, +11,Na,Sodium,0.971,23600.0,6.537e20,3.0,2.91,2020,SMM, +12,Mg,Magnesium,1.738,23300.0,6.454e20,2.32,4.03,2019,Preismonitor, +13,Al,Aluminium,2.698,82300.0,2.28e21,1.79,4.84,2019,Preismonitor, +14,Si,Silicon,2.3296,282000.0,7.811e21,1.7,3.97,2019,Preismonitor, +15,P,Phosphorus,1.82,1050.0,2.909e19,2.69,4.9,2019,CEIC Data, +16,S,Sulfur,2.067,350.0,9.695e18,0.0926,0.191,2019,CEIC Data, +17,Cl,Chlorine,0.003214,145.0,4.075e18,0.082,0.00026,2013,CnAgri, +18,Ar,Argon,0.0017837,3.5,9.695e16,0.931,0.00166,2019,UNLV, +19,K,Potassium,0.862,20900.0,5.789e20,12.85,11.1,2020,SMM, +20,Ca,Calcium,1.54,41500.0,1.15e21,2.28,3.52,2020,SMM, +21,Sc,Scandium,2.989,22.0,6.094e17,3460.0,10300.0,2020,ISE 2020, +22,Ti,Titanium,4.54,5650.0,1.565e20,11.4,51.8,2020,SMM, +23,V,Vanadium,6.11,120.0,3.324e18,371.0,2265.0,2020,SMM, +24,Cr,Chromium,7.15,102.0,2.825e18,9.4,67.2,2019,Preismonitor, +25,Mn,Manganese,7.44,950.0,2.632e19,1.82,13.6,2019,Preismonitor, +26,Fe,Iron,7.874,56300.0,1.565e21,0.424,3.34,2020,SMM, +27,Co,Cobalt,8.86,25.0,6.925e17,32.8,291.0,2019,Preismonitor, +28,Ni,Nickel,8.912,84.0,2.327e18,13.9,124.0,2019,Preismonitor, +29,Cu,Copper,8.96,60.0,1.662e18,6.0,53.8,2019,Preismonitor, +30,Zn,Zinc,7.134,70.0,1.939e18,2.55,18.2,2019,Preismonitor, +31,Ga,Gallium,5.907,19.0,5.263e17,148.0,872.0,2019,Preismonitor, +32,Ge,Germanium,5.323,1.5,4.155e16,962.0,5125.0,2020,SMM, +33,As,Arsenic,5.776,1.8,4.986e16,1.155,6.675,2020,SMM, +34,Se,Selenium,4.809,0.05,1.385e15,21.4,103.0,2019,Preismonitor, +35,Br,Bromine,3.122,2.4,6.648e16,4.39,13.7,2019,CEIC Data, +36,Kr,Krypton,0.003733,1.0e-4,2.77e12,290.0,1.1,1999,Ullmann, +37,Rb,Rubidium,1.532,90.0,2.493e18,15500.0,23700.0,2018,USGS MCS, +38,Sr,Strontium,2.64,370.0,1.025e19,6.605,17.4,2019,ISE 2019, +39,Y,Yttrium,4.469,33.0,9.141e17,31.0,139.0,2019,Preismonitor, +40,Zr,Zirconium,6.506,165.0,4.571e18,36.4,236.5,2020,SMM, +41,Nb,Niobium,8.57,20.0,5.54e17,73.5,630.0,2020,SMM, +42,Mo,Molybdenum,10.22,1.2,3.324e16,40.1,410.0,2019,Preismonitor, +43,Tc,Technetium,11.5,3.0e-9,8.31e7,100000.0,1200000.0,2004,CRC Handbook, +43,"99mTc","Technetium-99m",11.5,3.0e-9,8.31e7,1.9e12,2.2e13,2008,NRC, +44,Ru,Ruthenium,12.37,0.001,2.77e13,10500.0,130000.0,2020,SMM, +45,Rh,Rhodium,12.41,0.001,2.77e13,147000.0,1820000.0,2019,Preismonitor, +46,Pd,Palladium,12.02,0.015,4.155e14,49500.0,595000.0,2019,Preismonitor, +47,Ag,Silver,10.501,0.075,2.0775e15,521.0,5470.0,2019,Preismonitor, +48,Cd,Cadmium,8.69,0.159,4.4043e15,2.73,23.8,2019,Preismonitor, +49,In,Indium,7.31,0.25,6.925e15,167.0,1220.0,2019,Preismonitor, +50,Sn,Tin,7.287,2.3,6.371e16,18.7,136.0,2019,Preismonitor, +51,Sb,Antimony,6.685,0.2,5.54e15,5.79,38.7,2019,Preismonitor, +52,Te,Tellurium,6.232,0.001,2.77e13,63.5,396.0,2019,Preismonitor, +53,I,Iodine,4.93,0.45,1.2465e16,35.0,173.0,2019,Industrial Minerals, +54,Xe,Xenon,0.005887,3.0e-5,8.31e11,1800.0,11.0,1999,Ullmann, +55,Cs,Caesium,1.873,3.0,8.31e16,61800.0,116000.0,2018,USGS MCS, +56,Ba,Barium,3.594,425.0,1.177e19,0.2605,0.938,2016,USGS MYB 2016, +57,La,Lanthanum,6.145,39.0,1.08e18,4.85,29.85,2020,SMM, +58,Ce,Cerium,6.77,66.5,1.84205e18,4.64,31.4,2020,SMM, +59,Pr,Praseodymium,6.773,9.2,2.5484e17,103.0,695.0,2019,Preismonitor, +60,Nd,Neodymium,7.007,41.5,1.14955e18,57.5,403.0,2019,Preismonitor, +61,"147Pm","Promethium-147",7.26,5.0e-14,1.385e3,460000.0,3400000.0,2003,Radiochemistry Society, +62,Sm,Samarium,7.52,7.05,1.95285e17,13.9,104.0,2019,Preismonitor, +63,Eu,Europium,5.243,2.0,5.54e16,31.4,165.0,2020,ISE 2020, +64,Gd,Gadolinium,7.895,6.2,1.7174e17,28.6,226.0,2020,ISE 2020, +65,Tb,Terbium,8.229,1.2,3.324e16,658.0,5410.0,2019,Preismonitor, +66,Dy,Dysprosium,8.55,5.2,1.4404e17,307.0,2630.0,2019,Preismonitor, +67,Ho,Holmium,8.795,1.3,3.601e16,57.1,503.0,2020,ISE 2020, +68,Er,Erbium,9.066,3.5,9.695e16,26.4,240.0,2020,ISE 2020, +69,Tm,Thulium,9.321,0.52,1.4404e16,3000.0,28000.0,2003,IMAR, +70,Yb,Ytterbium,6.965,3.2,8.864e16,17.1,119.0,2020,ISE 2020, +71,Lu,Lutetium,9.84,0.8,2.216e16,643.0,6330.0,2020,ISE 2020, +72,Hf,Hafnium,13.31,3.0,8.31e16,900.0,12000.0,2017,USGS MCS, +73,Ta,Tantalum,16.654,2.0,5.54e16,305.0,5080.0,2019,ISE 2019, +74,W,Tungsten,19.25,1.3,3.601e16,35.3,679.0,2019,Preismonitor, +75,Re,Rhenium,21.02,7.0e-4,1.939e13,3580.0,75300.0,2020,SMM, +76,Os,Osmium,22.61,0.002,5.54e13,30000.0,678000.0,2025,elementsales.com, +77,Ir,Iridium,22.56,0.001,2.77e13,144000.0,3276000.0,2025,Umicore, +78,Pt,Platinum,21.46,0.005,1.385e14,27800.0,596000.0,2019,Preismonitor, +79,Au,Gold,19.282,0.004,1.108e14,75430.0,1454441.0,2024,London gold fix, +80,Hg,Mercury,13.5336,0.085,2.3545e15,30.2,409.0,2017,USGS MCS, +81,Tl,Thallium,11.85,0.85,2.3545e16,4200.0,49800.0,2017,USGS MCS, +82,Pb,Lead,11.342,14.0,3.878e17,2.0,22.6,2019,Preismonitor, +83,Bi,Bismuth,9.807,0.009,2.493e14,6.36,62.4,2019,Preismonitor, +84,"209Po","Polonium-209",9.32,3.0e-16,8.31e0,4.92e13,4.58e14,2004,CRC Handbook (ORNL), +85,At,Astatine,7.0,3.0e-20,8.31e-4,,,, +86,Rn,Radon,0.00973,4.0e-13,1.108e4,,,, +87,Fr,Francium,1.87,1.0e-18,2.77e-2,,,, +88,Ra,Radium,5.5,9.0e-7,2.493e10,,,, +89,"225Ac","Actinium-225",10.07,1.0e-11,2.77e5,2.9e13,2.9e14,2004,CRC Handbook (ORNL), +90,Th,Thorium,11.72,9.6,2.6592e17,287.0,3360.0,2010,USGS MYB 2012, +91,Pa,Protactinium,15.37,1.4e-6,3.878e10,,,, +92,U,Uranium,18.95,2.7,7.479e16,101.0,1910.0,2018,EIA Uranium Marketing, +93,Np,Neptunium,20.45,3.0e-12,8.31e4,660000.0,13500000.0,2003,Pomona, +94,"239Pu","Plutonium-239",19.84,5.0e-14,1.385e3,6490000.0,129000000.0,2019,DOE OSTI, +95,"241Am","Americium-241",13.69,,,728000.0,9970000.0,1998,NWA, +95,"243Am","Americium-243",13.69,,,750000.0,10300000.0,2004,CRC Handbook (ORNL), +96,"244Cm","Curium-244",13.51,,,185000000.0,2.5e9,2004,CRC Handbook (ORNL), +96,"248Cm","Curium-248",13.51,,,1.6e11,2.16e12,2004,CRC Handbook (ORNL), +97,"249Bk","Berkelium-249",14.79,,,1.85e11,2.74e12,2004,CRC Handbook (ORNL), +98,"249Cf","Californium-249",15.1,,,1.85e11,2.79e12,2004,CRC Handbook (ORNL), +98,"252Cf","Californium-252",15.1,,,6.0e10,9.06e11,2004,CRC Handbook (ORNL), +99,Es,Einsteinium,8.84,,,,,, +100,Fm,Fermium,9.7,,,,,, +101,Md,Mendelevium,10.3,,,,,, +102,No,Nobelium,9.9,,,,,, +103,Lr,Lawrencium,15.6,,,,,, +104,Rf,Rutherfordium,23.2,,,,,, +105,Db,Dubnium,29.3,,,,,, +106,Sg,Seaborgium,35.0,,,,,, +107,Bh,Bohrium,37.1,,,,,, +108,Hs,Hassium,40.7,,,,,, +109,Mt,Meitnerium,37.4,,,,,, +110,Ds,Darmstadtium,34.8,,,,,, +111,Rg,Roentgenium,28.7,,,,,, +112,Cn,Copernicium,14.0,,,,,, +113,Nh,Nihonium,16.0,,,,,, +114,Fl,Flerovium,9.928,,,,,, +115,Mc,Moscovium,13.5,,,,,, +116,Lv,Livermorium,12.9,,,,,, +117,Ts,Tennessine,7.2,,,,,, +118,Og,Oganesson,7.0,,,,,, \ No newline at end of file diff --git a/src/simmate/apps/price_catalog/migrations/0001_initial.py b/src/simmate/apps/price_catalog/migrations/0001_initial.py index 1ae253782..93edd6df1 100644 --- a/src/simmate/apps/price_catalog/migrations/0001_initial.py +++ b/src/simmate/apps/price_catalog/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2025-09-13 22:47 +# Generated by Django 4.2.7 on 2025-12-18 21:44 import django.db.models.deletion from django.db import migrations, models @@ -12,7 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name="MarketItem", + name="PricedItem", fields=[ ("id", models.AutoField(primary_key=True, serialize=False)), ( @@ -26,25 +26,30 @@ class Migration(migrations.Migration): ("source", models.JSONField(blank=True, null=True)), ("category", models.TextField(blank=True, null=True)), ("name", models.TextField(blank=True, null=True)), - ("ticker", models.TextField(blank=True, null=True)), - ("ticker_source", models.TextField(blank=True, null=True)), + ("preferred_source", models.TextField(blank=True, null=True)), ("comments", models.TextField(blank=True, null=True)), - ("global_abundance", models.FloatField(blank=True, null=True)), ("price", models.FloatField(blank=True, null=True)), - ("price_10y_stats", models.JSONField(blank=True, null=True)), + ("price_unit", models.TextField(blank=True, null=True)), + ("global_abundance", models.FloatField(blank=True, null=True)), + ("global_abundance_unit", models.TextField(blank=True, null=True)), + ("market_cap", models.FloatField(blank=True, null=True)), ("global_abundance_kg", models.FloatField(blank=True, null=True)), ( "global_abundance_mg_per_kg_crust", models.FloatField(blank=True, null=True), ), ("price_per_kg", models.FloatField(blank=True, null=True)), + ("price_1y_stats", models.JSONField(blank=True, null=True)), + ("price_5y_stats", models.JSONField(blank=True, null=True)), + ("price_10y_stats", models.JSONField(blank=True, null=True)), + ("price_25y_stats", models.JSONField(blank=True, null=True)), ], options={ - "db_table": "price_catalog__market_items", + "db_table": "price_catalog__priced_items", }, ), migrations.CreateModel( - name="PriceHistory", + name="PricePoint", fields=[ ("id", models.AutoField(primary_key=True, serialize=False)), ( @@ -56,31 +61,54 @@ class Migration(migrations.Migration): models.DateTimeField(auto_now=True, db_index=True, null=True), ), ("source", models.JSONField(blank=True, null=True)), + ("ticker_source", models.TextField(blank=True, null=True)), + ("ticker", models.TextField(blank=True, null=True)), ("date", models.DateTimeField(blank=True, null=True)), ("price", models.FloatField(blank=True, null=True)), - ("price_normalized", models.FloatField(blank=True, null=True)), + ("price_inflation_adj", models.FloatField(blank=True, null=True)), ("comments", models.TextField(blank=True, null=True)), + ("delta_1y", models.FloatField(blank=True, null=True)), + ("delta_1y_inflation_adj", models.FloatField(blank=True, null=True)), + ("delta_1y_percent", models.FloatField(blank=True, null=True)), + ( + "delta_1y_percent_inflation_adj", + models.FloatField(blank=True, null=True), + ), + ("delta_5y", models.FloatField(blank=True, null=True)), + ("delta_5y_inflation_adj", models.FloatField(blank=True, null=True)), + ("delta_5y_percent", models.FloatField(blank=True, null=True)), + ( + "delta_5y_percent_inflation_adj", + models.FloatField(blank=True, null=True), + ), ("delta_10y", models.FloatField(blank=True, null=True)), - ("delta_10y_normalized", models.FloatField(blank=True, null=True)), + ("delta_10y_inflation_adj", models.FloatField(blank=True, null=True)), ("delta_10y_percent", models.FloatField(blank=True, null=True)), ( - "delta_10y_percent_normalized", + "delta_10y_percent_inflation_adj", + models.FloatField(blank=True, null=True), + ), + ("delta_25y", models.FloatField(blank=True, null=True)), + ("delta_25y_inflation_adj", models.FloatField(blank=True, null=True)), + ("delta_25y_percent", models.FloatField(blank=True, null=True)), + ( + "delta_25y_percent_inflation_adj", models.FloatField(blank=True, null=True), ), ( - "market_item", + "priced_item", models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, - related_name="price_history", - to="price_catalog.marketitem", + related_name="price_points", + to="price_catalog.priceditem", ), ), ], options={ - "db_table": "price_catalog__price_history", - "unique_together": {("market_item", "date")}, + "db_table": "price_catalog__price_points", + "unique_together": {("priced_item", "ticker", "date")}, }, ), ] diff --git a/src/simmate/apps/price_catalog/models/__init__.py b/src/simmate/apps/price_catalog/models/__init__.py index 0f0e427a0..fbd6a42f7 100644 --- a/src/simmate/apps/price_catalog/models/__init__.py +++ b/src/simmate/apps/price_catalog/models/__init__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- -from .market_items import MarketItem -from .price_history import PriceHistory +from .price_points import PricePoint +from .priced_items import PricedItem diff --git a/src/simmate/apps/price_catalog/models/market_items.py b/src/simmate/apps/price_catalog/models/market_items.py deleted file mode 100644 index c1338d32a..000000000 --- a/src/simmate/apps/price_catalog/models/market_items.py +++ /dev/null @@ -1,400 +0,0 @@ -# -*- coding: utf-8 -*- - -from datetime import datetime - -import numpy -import pandas -import plotly.express as plotly_express -import plotly.graph_objects as plotly_go -from django.utils.timezone import make_aware -from rich.progress import track - -from simmate.database.base_data_types import DatabaseTable, table_column - - -class MarketItem(DatabaseTable): - - class Meta: - db_table = "price_catalog__market_items" - - category_options = [ - "Chemical Elements", - "Fuels & Energy", - "Crops & Livestock", - "Cryptocurrency", - "Market Index", - "Other", # e.g. housing - ] - category = table_column.TextField(blank=True, null=True) - - name = table_column.TextField(blank=True, null=True) - - ticker = table_column.TextField(blank=True, null=True) - - ticker_source_options = [ - "Yahoo", # Yahoo Finance - "FRED", # Federal Reserve Bank of St. Louis - "Wikipedia", # https://en.wikipedia.org/wiki/Prices_of_chemical_elements - "Vendor(s)", # Sigma Aldrich, Fischer, eMolecules, ACD, etc. - "Other", - ] - ticker_source = table_column.TextField(blank=True, null=True) - - comments = table_column.TextField(blank=True, null=True) - - # ------------------------------------------------------------------------- - - global_abundance = table_column.FloatField(blank=True, null=True) - # count - # kg - # infinite (for indexes/housing) - - price = table_column.FloatField(blank=True, null=True) - - price_10y_stats = table_column.JSONField(blank=True, null=True) - # Start - # Current - # Min - # Max - # % change - # % change normalized for inflation - - # ------------------------------------------------------------------------- - - # Normalized values - - global_abundance_kg = table_column.FloatField(blank=True, null=True) - - global_abundance_mg_per_kg_crust = table_column.FloatField(blank=True, null=True) - - price_per_kg = table_column.FloatField(blank=True, null=True) - - # ------------------------------------------------------------------------- - - @classmethod - def get_figure(cls): - - from .price_history import PriceHistory - - data = PriceHistory.objects.order_by("market_item__name", "date").to_dataframe( - [ - "market_item__name", - "date", - # "price", - # "price_normalized", - # "delta_10y", - # "delta_10y_normalized", - # "delta_10y_percent", - "delta_10y_percent_normalized", - ] - ) - fig = plotly_express.line( - data, - x="date", - y="delta_10y_percent_normalized", - color="market_item__name", - ) - fig.show(renderer="browser") - - def get_price_figure(self, normalized: bool = False): - - y_col = "price" if not normalized else "price_normalized" - columns = ["date", y_col] - data = self.price_history.order_by("date").to_dataframe(columns) - - fig = plotly_express.area( - data, - x="date", - y=y_col, - ) - - fig.update_layout(yaxis_tickprefix="$") - - fig.show(renderer="browser") - - def get_delta_10y_figure(self, normalized: bool = False): - - ten_years_ago = self.get_10y_datetime() - y_col = ( - "delta_10y_percent" if not normalized else "delta_10y_percent_normalized" - ) - columns = ["date", y_col] - data = self.price_history.filter(date__gte=ten_years_ago).to_dataframe(columns) - - # Separate positive and negative parts - rel_change = data[y_col] - pos = numpy.where(rel_change > 0, rel_change, 0) / 100 - neg = numpy.where(rel_change < 0, rel_change, 0) / 100 - - fig = plotly_go.Figure() - - fig.add_trace( - plotly_go.Scatter( - x=data.date, - y=pos, - fill="tozeroy", - name="Positive", - line=dict(color="green"), - showlegend=False, - ) - ) - - fig.add_trace( - plotly_go.Scatter( - x=data.date, - y=neg, - fill="tozeroy", - name="Negative", - line=dict(color="red"), - showlegend=False, - ) - ) - - # Black line at Zero - fig.add_trace( - plotly_go.Scatter( - x=data.date, - y=[0] * len(data), - mode="lines", - line=dict(color="black", width=3), - name="Zero", - showlegend=False, - ) - ) - - fig.update_layout(yaxis_tickformat=".0%") - - fig.show(renderer="browser") - - # ------------------------------------------------------------------------- - - @classmethod - def _load_fred_data(cls): - - from ..clients.fred import FredClient - from .price_history import PriceHistory - - all_data = FredClient.get_all_data() - - category_lookup = { - "Chemical Elements": [], - "Fuels & Energy": ["Electricity"], - "Crops & Livestock": [], - "Cryptocurrency": [], - "Market Index": ["Consumer Price Index"], - "Other": ["Housing"], - } - - for name, data in track(all_data.items()): - - category_found = False - for category, cat_names in category_lookup.items(): - if name in cat_names: - category_found = True - break - - market_item, _ = cls.objects.update_or_create( - name=name, - defaults=dict( - category=category if category_found else None, - ticker_source="FRED", - ticker=FredClient.ticker_map[name], - ), - ) - - price_objs = [ - PriceHistory( - market_item_id=market_item.id, - date=make_aware(row.observation_date), - price=row.price, # we use the day's closing price - ) - for i, row in data.iterrows() - ] - - PriceHistory.objects.bulk_create( - price_objs, - batch_size=1_000, - ignore_conflicts=True, - ) - - @classmethod - def _load_yfinance_data(cls): - from ..clients.yfinance import YahooFinanceClient - from .price_history import PriceHistory - - all_data = YahooFinanceClient.get_all_data() - - category_lookup = { - "Chemical Elements": [ - "Gold", - "Silver", - "Platinum", - "Palladium", - "Copper", - ], - "Fuels & Energy": [ - "Crude Oil", - "Natural Gas", - ], - "Crops & Livestock": [ - "Lumber", - "Soybeans", - "Corn", - "Wheat", - "Coffee", - "Sugar", - "Cocoa", - "Cotton", - "Cattle", - ], - "Cryptocurrency": [ - "Bitcoin", - "Ethereum", - "Solana", - ], - "Market Index": [ - "S&P GSCI", - "S&P 500", - "Total Market Index", - ], - "Other": [], - } - - for name, data in track(all_data.items()): - - category_found = False - for category, cat_names in category_lookup.items(): - if name in cat_names: - category_found = True - break - - market_item, _ = cls.objects.update_or_create( - name=name, - defaults=dict( - category=category if category_found else None, - ticker_source="Yahoo", - ticker=YahooFinanceClient.ticker_map[name], - ), - ) - - price_objs = [ - PriceHistory( - market_item_id=market_item.id, - date=row.Date, - price=row.Close, # we use the day's closing price - ) - for i, row in data.iterrows() - ] - - PriceHistory.objects.bulk_create( - price_objs, - batch_size=1_000, - ignore_conflicts=True, - ) - - @classmethod - def _load_wikipedia_data(cls): - pass # https://en.wikipedia.org/wiki/Prices_of_chemical_elements - - # ------------------------------------------------------------------------- - - @classmethod - def get_buying_power_series(cls): - - cpi = cls.objects.get(name="Consumer Price Index") - cpi_data = cpi.price_history.order_by("date").to_dataframe(["date", "price"]) - - # will be set to $1 and used to normalize others - todays_buying_power = ( - cpi_data.sort_values("date", ascending=False).iloc[0].price - ) - - # This plot decreases over time because of inflation. It will always have - # a value of 1 today, but for example, 10 years ago might have a value - # of 1.25 (i.e., $1 had 25% more buying power 10yrs ago relative to today). - # When this plot is multiplied by another price plot, you can see an - # inflation-adjusted plot - cpi_data["relative_buying_power"] = 1 / (cpi_data.price / todays_buying_power) - - # return only two cols to avoid confusion - return cpi_data[["date", "relative_buying_power"]] - - @classmethod - def update_price_history_calcs(cls): - - from .price_history import PriceHistory - - # grab inflation data up front. We use the Consumer Price Index - # and normalize it to buying power so that we can scale other data - buying_power_data = cls.get_buying_power_series() - - ten_years_ago = cls.get_10y_datetime() - - for entry in track(cls.objects.all()): - - entry_10y = ( - entry.price_history.filter(date__gte=ten_years_ago) - .order_by("date") - .first() - ) - price_10y = entry_10y.price - bp_factor_10y = cls.interp_w_datetime( - original_x=buying_power_data.date, - original_y=buying_power_data.relative_buying_power, - new_x=entry_10y.date, - ) - price_normalized_10y = price_10y * bp_factor_10y - - price_objs = entry.price_history.order_by("date").all() - for i in price_objs: - # slow bc I call function one new_x value at time. I can speed - # up by calling the full series, but code it uglier - bp_factor = cls.interp_w_datetime( - original_x=buying_power_data.date, - original_y=buying_power_data.relative_buying_power, - new_x=i.date, - ) - - i.price_normalized = i.price * bp_factor - i.delta_10y = i.price - price_10y - i.delta_10y_normalized = i.price_normalized - price_normalized_10y - i.delta_10y_percent = (i.delta_10y / price_10y) * 100 - i.delta_10y_percent_normalized = ( - i.delta_10y_normalized / price_normalized_10y - ) * 100 - - PriceHistory.objects.bulk_update( - objs=price_objs, - fields=[ - "price_normalized", - "delta_10y", - "delta_10y_normalized", - "delta_10y_percent", - "delta_10y_percent_normalized", - ], - ) - - @staticmethod - def interp_w_datetime(original_x, original_y, new_x): - # DEPREC: from scipy.interpolate import interp1d - # For other options look at... - # https://docs.scipy.org/doc/scipy/tutorial/interpolate/1D.html - original_x = original_x.astype("int64") // 1e9 # to seconds - new_x = pandas.Series(new_x).astype("int64") // 1e9 # to seconds - new_y = numpy.interp(new_x, original_x, original_y)[0] - return new_y - - @staticmethod - def get_10y_datetime(): - today = datetime.now() - # BUG: this might break on leap days - ten_years_ago = make_aware( - datetime( - year=today.year - 10, - month=today.month, - day=today.day, - ), - ) - return ten_years_ago - - # ------------------------------------------------------------------------- diff --git a/src/simmate/apps/price_catalog/models/price_history.py b/src/simmate/apps/price_catalog/models/price_history.py deleted file mode 100644 index 8042b29fd..000000000 --- a/src/simmate/apps/price_catalog/models/price_history.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -from simmate.database.base_data_types import DatabaseTable, table_column - -from .market_items import MarketItem - - -class PriceHistory(DatabaseTable): - - class Meta: - db_table = "price_catalog__price_history" - unique_together = (("market_item", "date"),) - - market_item = table_column.ForeignKey( - MarketItem, - on_delete=table_column.PROTECT, - blank=True, - null=True, - related_name="price_history", - ) - - date = table_column.DateTimeField(blank=True, null=True) - - price = table_column.FloatField(blank=True, null=True) - - price_normalized = table_column.FloatField(blank=True, null=True) - # for inflation - - comments = table_column.TextField(blank=True, null=True) - - # ------------------------------------------------------------------------- - - # 10 years *from the current date*, not the date of this entry - - delta_10y = table_column.FloatField(blank=True, null=True) - delta_10y_normalized = table_column.FloatField(blank=True, null=True) - - delta_10y_percent = table_column.FloatField(blank=True, null=True) - delta_10y_percent_normalized = table_column.FloatField(blank=True, null=True) - - # ------------------------------------------------------------------------- diff --git a/src/simmate/apps/price_catalog/models/price_points.py b/src/simmate/apps/price_catalog/models/price_points.py new file mode 100644 index 000000000..5bfe37d52 --- /dev/null +++ b/src/simmate/apps/price_catalog/models/price_points.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +from simmate.database.base_data_types import DatabaseTable, table_column + +from .priced_items import PricedItem + + +class PricePoint(DatabaseTable): + + class Meta: + db_table = "price_catalog__price_points" + unique_together = (("priced_item", "ticker", "date"),) + + # ------------------------------------------------------------------------- + + # This is the the "PriceSource" -- which I keep in the same table for + # simplicity, even though it results in more disk space in the db. + + priced_item = table_column.ForeignKey( + PricedItem, + on_delete=table_column.PROTECT, + blank=True, + null=True, + related_name="price_points", + ) + + ticker_source_options = [ + "Yahoo", # Yahoo Finance + "FRED", # Federal Reserve Bank of St. Louis + "Wikipedia", # https://en.wikipedia.org/wiki/Prices_of_chemical_elements + "Chemical Vendors", # Sigma Aldrich, Fischer, VWR, etc. + "Other", + ] + ticker_source = table_column.TextField(blank=True, null=True) + + ticker = table_column.TextField(blank=True, null=True) + + # ------------------------------------------------------------------------- + + date = table_column.DateTimeField(blank=True, null=True) + + price = table_column.FloatField(blank=True, null=True) + + price_inflation_adj = table_column.FloatField(blank=True, null=True) + + comments = table_column.TextField(blank=True, null=True) + + # ------------------------------------------------------------------------- + + # Cached calculations + + # N years *from today's current date*, not the date of this entry + + delta_1y = table_column.FloatField(blank=True, null=True) + delta_1y_inflation_adj = table_column.FloatField(blank=True, null=True) + delta_1y_percent = table_column.FloatField(blank=True, null=True) + delta_1y_percent_inflation_adj = table_column.FloatField(blank=True, null=True) + + delta_5y = table_column.FloatField(blank=True, null=True) + delta_5y_inflation_adj = table_column.FloatField(blank=True, null=True) + delta_5y_percent = table_column.FloatField(blank=True, null=True) + delta_5y_percent_inflation_adj = table_column.FloatField(blank=True, null=True) + + delta_10y = table_column.FloatField(blank=True, null=True) + delta_10y_inflation_adj = table_column.FloatField(blank=True, null=True) + delta_10y_percent = table_column.FloatField(blank=True, null=True) + delta_10y_percent_inflation_adj = table_column.FloatField(blank=True, null=True) + + delta_25y = table_column.FloatField(blank=True, null=True) + delta_25y_inflation_adj = table_column.FloatField(blank=True, null=True) + delta_25y_percent = table_column.FloatField(blank=True, null=True) + delta_25y_percent_inflation_adj = table_column.FloatField(blank=True, null=True) + + # ------------------------------------------------------------------------- diff --git a/src/simmate/apps/price_catalog/models/priced_items.py b/src/simmate/apps/price_catalog/models/priced_items.py new file mode 100644 index 000000000..2078fdbe7 --- /dev/null +++ b/src/simmate/apps/price_catalog/models/priced_items.py @@ -0,0 +1,577 @@ +# -*- coding: utf-8 -*- + +from datetime import datetime + +import numpy +import pandas +import plotly.express as plotly_express +import plotly.graph_objects as plotly_go +from django.utils.timezone import make_aware +from rich.progress import track + +from simmate.database.base_data_types import DatabaseTable, table_column + + +class PricedItem(DatabaseTable): + # This page is intended for evaluating long-term trends, not live trading - + # so prices are only updated & sync'd at the start of each month. + + class Meta: + db_table = "price_catalog__priced_items" + + html_display_name = "Market Data & Price Catalog" + html_description_short = ( + "Prices and economic indicators spanning common chemicals, stocks, " + "commodities, cryptocurrencies, and macroeconomic metrics." + ) + + html_entries_template = "price_catalog/priced_items/table.html" + html_entry_template = "price_catalog/priced_items/entry.html" + + # html_form_component = "priced-item-form" + # html_enabled_forms = ["search"] + + # ------------------------------------------------------------------------- + + category_options = [ + "Chemical Index", + "Chemical Elements", + # "Chemical Solvents", + # "Chemical Reagents", + "Fuels & Energy", + "Crops & Livestock", + "Cryptocurrency", + "Market Index", + "Other", # e.g. housing + ] + category = table_column.TextField(blank=True, null=True) + + name = table_column.TextField(blank=True, null=True) + + preferred_source_options = [ + "Yahoo", # Yahoo Finance + "FRED", # Federal Reserve Bank of St. Louis + "Wikipedia", # https://en.wikipedia.org/wiki/Prices_of_chemical_elements + "Chemical Vendors", # Sigma Aldrich, Fischer, VWR, etc. + "Other", + ] + preferred_source = table_column.TextField(blank=True, null=True) + + comments = table_column.TextField(blank=True, null=True) + + # ------------------------------------------------------------------------- + + price = table_column.FloatField(blank=True, null=True) + # filled using the most recent price point of the preferred source + + # per kg + # per share + # per kW/hr + price_unit = table_column.TextField(blank=True, null=True) + + global_abundance = table_column.FloatField(blank=True, null=True) + + # count + # kg + # infinite (for indexes/housing) + global_abundance_unit = table_column.TextField(blank=True, null=True) + + market_cap = table_column.FloatField(blank=True, null=True) + # price * global_abundance + + # ------------------------------------------------------------------------- + + # Standardized values + + global_abundance_kg = table_column.FloatField(blank=True, null=True) + + global_abundance_mg_per_kg_crust = table_column.FloatField(blank=True, null=True) + + price_per_kg = table_column.FloatField(blank=True, null=True) + + # ------------------------------------------------------------------------- + + # Price history stats + + # start ($) + # min ($) + # max ($) + # change (%) + # change_inflation_adj (%) for inflation + + years_ago_options = [1, 5, 10, 25] + + price_1y_stats = table_column.JSONField(blank=True, null=True) + + price_5y_stats = table_column.JSONField(blank=True, null=True) + + price_10y_stats = table_column.JSONField(blank=True, null=True) + + price_25y_stats = table_column.JSONField(blank=True, null=True) + + # ------------------------------------------------------------------------- + + enable_html_report = True + report_df_columns = ["id"] + + @classmethod + def get_report_from_df(cls, df: pandas.DataFrame): + return {"test": 123} + + # ------------------------------------------------------------------------- + + @classmethod + def get_report_figure( + cls, + names: list[str], + percent: bool = False, + inflation_adj: bool = False, + years_ago_norm: int = None, + ): + + from .price_points import PricePoint + + if years_ago_norm: + price_mode = f"delta_{years_ago_norm}y" + if percent: + price_mode += "_percent" + if inflation_adj: + price_mode += "_inflation_adj" + date_cutoff = cls.get_years_ago_datetime(years_ago_norm) + + else: + price_mode = "price" + date_cutoff = cls.get_years_ago_datetime(100) + + data = ( + PricePoint.objects.filter( + priced_item__name__in=names, + date__gte=date_cutoff, + ) + .order_by("priced_item__name", "date") + .to_dataframe(["priced_item__name", price_mode, "date"]) + ) + + figure = plotly_express.line( + data, + x="date", + y=price_mode, + color="priced_item__name", + ) + return figure + + def get_price_figure(self, inflation_adj: bool = False): + + y_col = "price" if not inflation_adj else "price_inflation_adj" + columns = ["date", y_col] + data = self.price_points.order_by("date").to_dataframe(columns) + + figure = plotly_express.area( + data, + x="date", + y=y_col, + ) + + figure.update_layout(yaxis_tickprefix="$") + return figure + + def get_delta_figure( + self, + years_ago: int, + percent: bool = False, + inflation_adj: bool = False, + ): + + if years_ago not in self.years_ago_options: + raise Exception( + f"`years_ago` must be set to one of {self.years_ago_options}" + ) + + date_cutoff = self.get_years_ago_datetime(years_ago) + + y_col = f"delta_{years_ago}y" + if percent: + y_col += "_percent" + if inflation_adj: + y_col += "_inflation_adj" + columns = ["date", y_col] + + data = ( + self.price_points.filter(date__gte=date_cutoff) + .order_by("date") + .to_dataframe(columns) + ) + + # Separate positive and negative parts + rel_change = data[y_col] + pos = numpy.where(rel_change > 0, rel_change, 0) + neg = numpy.where(rel_change < 0, rel_change, 0) + + if percent: + pos = pos / 100 + neg = neg / 100 + + figure = plotly_go.Figure() + + figure.add_trace( + plotly_go.Scatter( + x=data.date, + y=pos, + fill="tozeroy", + name="Positive", + line=dict(color="green"), + showlegend=False, + ) + ) + + figure.add_trace( + plotly_go.Scatter( + x=data.date, + y=neg, + fill="tozeroy", + name="Negative", + line=dict(color="red"), + showlegend=False, + ) + ) + + # Black line at Zero + figure.add_trace( + plotly_go.Scatter( + x=data.date, + y=[0] * len(data), + mode="lines", + line=dict(color="black", width=3), + name="Zero", + showlegend=False, + ) + ) + + if percent: + figure.update_layout(yaxis_tickformat=".0%") + else: + figure.update_layout(yaxis_tickprefix="$") + + return figure + + # ------------------------------------------------------------------------- + + @classmethod + def get_buying_power_series(cls, reference: str = "Consumer Price Index"): + + cpi = cls.objects.get(name=reference) + cpi_data = cpi.price_points.order_by("date").to_dataframe(["date", "price"]) + + # will be set to $1 and used to normalize others + todays_buying_power = ( + cpi_data.sort_values("date", ascending=False).iloc[0].price + ) + + # This plot decreases over time because of inflation. It will always have + # a value of 1 today, but for example, 10 years ago might have a value + # of 1.25 (i.e., $1 had 25% more buying power 10yrs ago relative to today). + # When this plot is multiplied by another price plot, you can see an + # inflation-adjusted plot + cpi_data["relative_buying_power"] = 1 / (cpi_data.price / todays_buying_power) + + # return only two cols to avoid confusion + return cpi_data[["date", "relative_buying_power"]] + + @classmethod + def update_price_points_calcs(cls): + + from .price_points import PricePoint + + # grab inflation data up front. We use the Consumer Price Index + # and normalize it to buying power so that we can scale other data + buying_power_data = cls.get_buying_power_series() + + for entry in track(cls.objects.all()): + + for years_ago in cls.years_ago_options: + + year_ago_dt = cls.get_years_ago_datetime(years_ago) + + entry_closest = ( + entry.price_points.filter(date__gte=year_ago_dt) + .order_by("date") + .first() + ) + + # check that the date is within at least 6months + if ( + entry_closest.date - year_ago_dt + ).total_seconds() >= 60 * 60 * 24 * 30 * 6: + continue + + price_closest = entry_closest.price + bp_factor_closest = cls.interp_w_datetime( + original_x=buying_power_data.date, + original_y=buying_power_data.relative_buying_power, + new_x=entry_closest.date, + ) + price_inflation_adj_closest = price_closest * bp_factor_closest + + price_objs = entry.price_points.order_by("date").all() + for i in price_objs: + + # This is the bottleneck + slow bc I call function one + # new_x value at time. I can speed up by calling the full + # series, but code would be uglier. I cache this data and + # run once per day, so I'm okay with the slowdown + bp_factor = cls.interp_w_datetime( + original_x=buying_power_data.date, + original_y=buying_power_data.relative_buying_power, + new_x=i.date, + ) + + price_inflation_adj = i.price * bp_factor + + delta = i.price - price_closest + + delta_inflation_adj = ( + price_inflation_adj - price_inflation_adj_closest + ) + + delta_percent = (delta / price_closest) * 100 + + delta_percent_inflation_adj = ( + delta_inflation_adj / price_inflation_adj_closest + ) * 100 + + update_map = { + "price_inflation_adj": price_inflation_adj, + f"delta_{years_ago}y": delta, + f"delta_{years_ago}y_inflation_adj": delta_inflation_adj, + f"delta_{years_ago}y_percent": delta_percent, + f"delta_{years_ago}y_percent_inflation_adj": delta_percent_inflation_adj, + } + for k, v in update_map.items(): + setattr(i, k, v) + + PricePoint.objects.bulk_update( + objs=price_objs, + fields=[ + "price_inflation_adj", + f"delta_{years_ago}y", + f"delta_{years_ago}y_inflation_adj", + f"delta_{years_ago}y_percent", + f"delta_{years_ago}y_percent_inflation_adj", + ], + ) + + # update parent object + entry.price = entry.price_points.order_by("-date").first().price + all_prices = [p.price for p in price_objs] + stats = { + "start": price_closest, + "min": min(all_prices), + "max": max(all_prices), + "change": ((entry.price - price_closest) / price_closest) * 100, + "change_inflation_adj": ( + (entry.price - price_inflation_adj_closest) / price_closest + ) + * 100, + } + setattr(entry, f"price_{years_ago}y_stats", stats) + entry.save() + + @staticmethod + def interp_w_datetime(original_x, original_y, new_x): + # DEPREC: from scipy.interpolate import interp1d + # For other options look at... + # https://docs.scipy.org/doc/scipy/tutorial/interpolate/1D.html + original_x = original_x.astype("int64") // 1e9 # to seconds + new_x = pandas.Series(new_x).astype("int64") // 1e9 # to seconds + new_y = numpy.interp(new_x, original_x, original_y)[0] + return new_y + + @staticmethod + def get_years_ago_datetime(years: int): + today = datetime.now() + ten_years_ago = make_aware( + datetime( + year=today.year - years, + month=today.month, + day=today.day, # BUG: this might break on leap days + ), + ) + return ten_years_ago + + # ------------------------------------------------------------------------- + + @classmethod + def _load_data(cls): + # cls._load_wikipedia_data() + cls._load_yfinance_data() + cls._load_fred_data() + cls.update_price_points_calcs() + + @classmethod + def _load_fred_data(cls): + + from ..clients.fred import FredClient + from .price_points import PricePoint + + all_data = FredClient.get_all_data() + + category_lookup = { + "Chemical Index": [ + "Chemicals", + "Industrial Chemicals", + "Inorganic Chemicals", + "Organic Chemicals", + "Industrial Gases", + ], + "Chemical Elements": [ + "Carbon", + "Nitrogen", + "Oxygen", + "Aluminum", + ], + "Fuels & Energy": ["Electricity"], + "Crops & Livestock": [], + "Cryptocurrency": [], + "Market Index": ["Consumer Price Index"], + "Other": ["Housing"], + } + + for name, data in track(all_data.items()): + + # sometimes a row is missing the price and/or timestamp + data.dropna(inplace=True) + + category_found = False + for category, cat_names in category_lookup.items(): + if name in cat_names: + category_found = True + break + + priced_item, _ = cls.objects.update_or_create( + name=name, + defaults=dict( + category=category if category_found else None, + ), + ) + + price_objs = [ + PricePoint( + priced_item_id=priced_item.id, + ticker_source="FRED", + ticker=FredClient.ticker_map[name], + # + date=make_aware(row.observation_date), + price=row.price, # we use the day's closing price + ) + for i, row in data.iterrows() + ] + + # TODO: ensure I don't create duplicate entries on update + PricePoint.objects.bulk_create( + price_objs, + batch_size=1_000, + ignore_conflicts=True, + ) + + @classmethod + def _load_yfinance_data(cls): + from ..clients.yfinance import YahooFinanceClient + from .price_points import PricePoint + + all_data = YahooFinanceClient.get_all_data() + + category_lookup = { + "Chemical Index": [], + "Chemical Elements": [ + "Gold", + "Silver", + "Platinum", + "Palladium", + "Copper", + ], + "Fuels & Energy": [ + "Crude Oil", + "Natural Gas", + ], + "Crops & Livestock": [ + "Lumber", + "Soybeans", + "Corn", + "Wheat", + "Coffee", + "Sugar", + "Cocoa", + "Cotton", + "Cattle", + ], + "Cryptocurrency": [ + "Bitcoin", + "Ethereum", + "Solana", + ], + "Market Index": [ + "S&P GSCI", + "S&P 500", + "Total Market Index", + ], + "Other": [], + } + + for name, data in track(all_data.items()): + + category_found = False + for category, cat_names in category_lookup.items(): + if name in cat_names: + category_found = True + break + + priced_item, _ = cls.objects.update_or_create( + name=name, + defaults=dict( + category=category if category_found else None, + ), + ) + + price_objs = [ + PricePoint( + priced_item_id=priced_item.id, + ticker_source="Yahoo", + ticker=YahooFinanceClient.ticker_map[name], + # + date=row.Date, + price=row.Close, # we use the day's closing price + ) + for i, row in data.iterrows() + ] + + PricePoint.objects.bulk_create( + price_objs, + batch_size=1_000, + ignore_conflicts=True, + ) + + @classmethod + def _load_wikipedia_data(cls): + + raise NotImplementedError() + + from ..data import WIKIPEDIA_PRICES_OF_ELEMENTS_DATA + + for row in WIKIPEDIA_PRICES_OF_ELEMENTS_DATA.itertuples(): + priced_item, _ = cls.objects.update_or_create( + name=row.name, + defaults=dict( + category="Chemical Elements", + ticker_source="Wikipedia", + ticker=row.symbol, + price=row.price_per_kg, + price_unit="per kg", + price_per_kg=row.price_per_kg, + global_abundance=row.total_mass_kg, + global_abundance_unit="kg", + market_cap=row.price_per_kg * row.total_mass_kg, + global_abundance_kg=row.total_mass_kg, + global_abundance_mg_per_kg_crust=row.abundance_mg_kg, + # TODO: store metadata of year + source + price_per_l + tec + ), + ) diff --git a/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/entry.html b/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/entry.html new file mode 100644 index 000000000..b93f4b0df --- /dev/null +++ b/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/entry.html @@ -0,0 +1,122 @@ +{% extends "data_explorer/table_entry.html" %} +{% block entrycontent %} +
+
+

{{ table_entry.name }}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Category{{ table_entry.category }}
Price + {% if table_entry.price_per_kg %} + ${{ table_entry.price_per_kg|floatformat:2 }} per kg + {% elif table_entry.price %} + ${{ table_entry.price|floatformat:2|intcomma }} {# entry.price_unit #} + {% else %} + --- + {% endif %} +
1y + {% if table_entry.price_1y_stats.change_inflation_adj > 0 %} + + {{ table_entry.price_1y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_1y_stats.change_inflation_adj < 0 %} + + {{ table_entry.price_1y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_1y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} +
5y + {% if table_entry.price_5y_stats.change_inflation_adj > 0 %} + + {{ table_entry.price_5y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_5y_stats.change_inflation_adj < 0.0 %} + + {{ table_entry.price_5y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_5y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} +
10y + {% if table_entry.price_10y_stats.change_inflation_adj > 0 %} + + {{ table_entry.price_10y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_10y_stats.change_inflation_adj < 0 %} + + {{ table_entry.price_10y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_10y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} +
25y + {% if table_entry.price_25y_stats.change_inflation_adj > 0 %} + + {{ table_entry.price_25y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_25y_stats.change_inflation_adj < 0 %} + + {{ table_entry.price_25y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif table_entry.price_25y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} +
Global Abundance + {% if table_entry.global_abundance_mg_per_kg_crust %} + ${{ table_entry.price_per_kg }} mg per kg Earth's crust + {% elif table_entry.global_abundance_kg %} + ${{ table_entry.price_per_kg }} kg + {% elif table_entry.global_abundance %} + {{ table_entry.global_abundance }} + {% else %} + --- + {% endif %} +
Market Cap---
+
+ * percents are adjusted for inflation +
+
+
{% htmx_component "priced-items-report" %}
+
+{% endblock %} diff --git a/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/report.html b/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/report.html new file mode 100644 index 000000000..68c5f7981 --- /dev/null +++ b/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/report.html @@ -0,0 +1,14 @@ +{% extends "htmx/form_base.html" %} +{% block form %} + {{ component.figure|plotly_figure }} +
+
{% htmx_button_select name="years_ago_norm" show_label=False defer=False %}
+ {% if component.form_data.years_ago_norm != "max" %} +
{% htmx_checkbox name="use_percent" show_label=False side_text="Percent (%)" defer=False %}
+ {% endif %} +
+ {% htmx_checkbox name="use_inflation_adj" show_label=False side_text="Adjust for Inflation (CPI)" defer=False %} +
+
{% htmx_selectbox name="item_names" show_label=False multiselect=True defer=False %}
+
+{% endblock %} diff --git a/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/table.html b/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/table.html new file mode 100644 index 000000000..fd7a23c29 --- /dev/null +++ b/src/simmate/apps/price_catalog/templates/price_catalog/priced_items/table.html @@ -0,0 +1,113 @@ +{% extends "data_explorer/table_entries_base.html" %} +{% block table_report %} +
+ +
+{% endblock %} +{% block table_headers %} + {% table_header "category" min_width=150 %} + {% table_header "name" min_width=150 %} + {% table_header "price" min_width=125 %} + {% table_header "price_1y_stats__change" "1y" min_width=75 %} + {% table_header "price_5y_stats__change" "5y" min_width=75 %} + {% table_header "price_10y_stats__change" "10y" min_width=75 %} + {% table_header "price_25y_stats__change" "25y" min_width=75 %} + {% table_header "global_abundance" min_width=175 %} + {% table_header "market_cap" min_width=125 %} +{% endblock %} +{% block table_rows %} + {{ entry.category }} + {% foreign_key_link entry "name" %} + + {% if entry.price_per_kg %} + ${{ entry.price_per_kg|floatformat:2 }} per kg + {% elif entry.price %} + ${{ entry.price|floatformat:2|intcomma }} {# entry.price_unit #} + {% else %} + --- + {% endif %} + + {% if entry.name == "Consumer Price Index" %} + 0 + 0 + 0 + 0 + {% else %} + + {% if entry.price_1y_stats.change_inflation_adj > 0 %} + + {{ entry.price_1y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_1y_stats.change_inflation_adj < 0 %} + + {{ entry.price_1y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_1y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} + + + {% if entry.price_5y_stats.change_inflation_adj > 0 %} + + {{ entry.price_5y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_5y_stats.change_inflation_adj < 0.0 %} + + {{ entry.price_5y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_5y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} + + + {% if entry.price_10y_stats.change_inflation_adj > 0 %} + + {{ entry.price_10y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_10y_stats.change_inflation_adj < 0 %} + + {{ entry.price_10y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_10y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} + + + {% if entry.price_25y_stats.change_inflation_adj > 0 %} + + {{ entry.price_25y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_25y_stats.change_inflation_adj < 0 %} + + {{ entry.price_25y_stats.change_inflation_adj|floatformat:1|intcomma }}% + + {% elif entry.price_25y_stats.change_inflation_adj == 0 %} + 0 + {% else %} + --- + {% endif %} + + {% endif %} + + {% if entry.global_abundance_mg_per_kg_crust %} + ${{ entry.price_per_kg }} mg per kg Earth's crust + {% elif entry.global_abundance_kg %} + ${{ entry.price_per_kg }} kg + {% elif entry.global_abundance %} + {{ entry.global_abundance }} + {% else %} + --- + {% endif %} + + --- +{% endblock %} diff --git a/src/simmate/configuration/load_settings.py b/src/simmate/configuration/load_settings.py index 16a02326d..3bb993410 100644 --- a/src/simmate/configuration/load_settings.py +++ b/src/simmate/configuration/load_settings.py @@ -263,9 +263,9 @@ def default_settings(self) -> dict: "simmate.apps.evolution.models.ChemicalSystemSearch", ], "Business and Finance": [ + "simmate.apps.price_catalog.models.PricedItem", "simmate.apps.ethereum.models.EthereumWallet", "simmate.apps.ethereum.models.EthereumTransaction", - "simmate.apps.price_catalog.models.MarketItem", ], "Other": [ "simmate.apps.eppo_gd.models.EppoCode", @@ -277,7 +277,7 @@ def default_settings(self) -> dict: "simmate.apps.chembl.models.ChemblDocument", "simmate.apps.emolecules.models.EmoleculesSupplierOffer", "simmate.apps.evolution.models.SteadystateSource", - "simmate.apps.price_catalog.models.PriceHistory", + "simmate.apps.price_catalog.models.PricePoint", ], # "Chemical Inventory & Tracking": [], # "Computational Assay Tracking": [], diff --git a/src/simmate/website/core_components/templates/core_components/apps.html b/src/simmate/website/core_components/templates/core_components/apps.html index 29667661f..e202a1ee4 100644 --- a/src/simmate/website/core_components/templates/core_components/apps.html +++ b/src/simmate/website/core_components/templates/core_components/apps.html @@ -18,7 +18,7 @@
{{ extra_app.verbose_name }}

{{ extra_app.description_short }}

Explore + href="{{ extra_app.short_name }}/">Explore diff --git a/src/simmate/website/core_components/templates/core_components/basic_elements/foreign_key_link.html b/src/simmate/website/core_components/templates/core_components/basic_elements/foreign_key_link.html index b0049c468..307b3d0bd 100644 --- a/src/simmate/website/core_components/templates/core_components/basic_elements/foreign_key_link.html +++ b/src/simmate/website/core_components/templates/core_components/basic_elements/foreign_key_link.html @@ -3,9 +3,9 @@ {% if mode == "pill" %} class="badge bg-primary rounded-pill" {% elif mode == "block" %} type="button" class="btn btn-secondary btn-sm col-10 m-0" {% endif %} {% if open_in_new %}target="_blank"{% endif %}> {% if display_column %} - {{ entry|getattribute:display_column }} + {{ entry|getattribute:display_column|truncatechars:truncate_chars }} {% else %} - {{ entry.id }} + {{ entry.id|truncatechars:truncate_chars }} {% endif %} diff --git a/src/simmate/website/core_components/templatetags/simmate_input_forms.py b/src/simmate/website/core_components/templatetags/simmate_input_forms.py index e6f5c8b9a..4870764e4 100644 --- a/src/simmate/website/core_components/templatetags/simmate_input_forms.py +++ b/src/simmate/website/core_components/templatetags/simmate_input_forms.py @@ -102,6 +102,7 @@ def foreign_key_link( entry, # db_object display_column: str = None, mode: str = "text", # other options are "pill" and "block" + truncate_chars: int = "inf", ): return locals() diff --git a/src/simmate/website/data_explorer/templates/data_explorer/table.html b/src/simmate/website/data_explorer/templates/data_explorer/table.html index 2e9d48500..abe39f635 100644 --- a/src/simmate/website/data_explorer/templates/data_explorer/table.html +++ b/src/simmate/website/data_explorer/templates/data_explorer/table.html @@ -86,27 +86,6 @@