feat: db struct and parse data

.gitignore

@@ -7,4 +7,4 @@ venv/
 hidden*
 other_data.json 
 export_result_TG_account_N4.json
-
+*.json

add_user_ids_to_postgres_v2.py

@@ -4,7 +4,7 @@
     host=hidden_2.secrets_2()["host"],
     port=hidden_2.secrets_2()["port"],
     database=hidden_2.secrets_2()["database"],
-    user=hidden_2.secrets_2()["user"]
+    user=hidden_2.secrets_2()["user"],
 )
 
 cur_1 = connection_1.cursor()
@@ -24,13 +24,9 @@
 # (w/ Python) Select all data from an existing table & print it
 # cur_1.execute("SELECT * FROM all_ip_addresses;")
 # var_2 = cur_1.fetchall()
-#print(var_2)
-#print(var_1)
+# print(var_2)
+# print(var_1)
 
 
 # (w/ Python) Connect to a DB in Postgres
 # (w/ Python) Create a new table in Postgres w/ necessary columns
-
-
-
-

config.py

@@ -0,0 +1,4 @@
+import os
+
+# DATABASE_STRING = os.getenv("DATABASE_STRING", "sqlite:///my_db.sqlite")
+DATABASE_STRING = os.getenv("DATABASE_STRING", "postgresql://postgres:postgres@localhost/postgres")

get_all_data_from_specific_chat.py

@@ -1,48 +1,53 @@
-import json 
+import json
 
-with open('export_result_TG_account_N4.json', 'r') as json_file: 
-	exported_data_nested_dict = json.load(json_file)
+with open("export_result_TG_account_N4.json", "r") as json_file:
+    exported_data_nested_dict = json.load(json_file)
 
-chat_id_input = input('Paste Telegram chat-ID (numbers only) to export all previous data from this chat: ')
+chat_id_input = input(
+    "Paste Telegram chat-ID (numbers only) to export all previous data from this chat: "
+)
 
 for k in exported_data_nested_dict["chats"]["list"]:
-	if chat_id_input == str(k["id"]):
-		chat_data_json_string = json.dumps(k)
-		print('\nFound {} chat-ID among your chats. \nHere is all raw data from this chat: '.format(chat_id_input)) 
-		print(chat_data_json_string)
-
-		# print(type(chat_data_json_string))
-		# print(k)
-		# print(type(k))
-
-
-	# if chat_id_input != str(k["id"]):
-	# 	break
-	# else:
-	# 	print('Found this chat-ID')
-
-	# try: 
-	# 	chat_id_input == str(k["id"])
-	# except: 
-	# 	print('NOT found this chat-ID') 
-	# else: 
-	# 	print('Found this chat-ID')
+    if chat_id_input == str(k["id"]):
+        chat_data_json_string = json.dumps(k)
+        print(
+            "\nFound {} chat-ID among your chats. \nHere is all raw data from this chat: ".format(
+                chat_id_input
+            )
+        )
+        print(chat_data_json_string)
+
+        # print(type(chat_data_json_string))
+        # print(k)
+        # print(type(k))
+
+    # if chat_id_input != str(k["id"]):
+    # 	break
+    # else:
+    # 	print('Found this chat-ID')
+
+    # try:
+    # 	chat_id_input == str(k["id"])
+    # except:
+    # 	print('NOT found this chat-ID')
+    # else:
+    # 	print('Found this chat-ID')
 
 
 # all_chat_ids_list = []
 
 # 	all_chat_ids_list.append(str(k["id"]))
 
 
-# if chat_id_input in all_chat_ids_list: 
+# if chat_id_input in all_chat_ids_list:
 # 	index_of_chat = ??
 # 	print('Found this chat-ID')
-# else: 
-#     print('NOT found this chat-ID')	
+# else:
+#     print('NOT found this chat-ID')
 
 
-	# chat_id = k["id"]
-	# print(chat_id)
+# chat_id = k["id"]
+# print(chat_id)
 # print(all_chat_ids_list)
 
 # number_of_chats = len(exported_data_nested_dict)
@@ -51,10 +56,10 @@
 # print(type(exported_data_nested_dict["chats"]["list"]))
 
 
-	# exported_data_string = json_file.read()
+# exported_data_string = json_file.read()
 # print(exported_data_nested_dict)
-# print(type(exported_data_nested_dict)) 
+# print(type(exported_data_nested_dict))
 
 
 # print(chat_id_input)
-# print(type(chat_id_input)) 
+# print(type(chat_id_input))

get_list_all_IPs.py

@@ -1,37 +1,38 @@
-import json, psycopg2 
-import hidden_2  # hidden_2.py file in repo with secrets about connection to the database 
+import json, psycopg2
+import hidden_2  # hidden_2.py file in repo with secrets about connection to the database
 
 conn_2 = psycopg2.connect(
-	host=hidden_2.secrets_2()['host'], 
-	port=hidden_2.secrets_2()['port'],
-	database=hidden_2.secrets_2()['database'],
-	user=hidden_2.secrets_2()['user'])
+    host=hidden_2.secrets_2()["host"],
+    port=hidden_2.secrets_2()["port"],
+    database=hidden_2.secrets_2()["database"],
+    user=hidden_2.secrets_2()["user"],
+)
 
 cur_2 = conn_2.cursor()
 # print(cur_2)
 # print(type(cur_2))
 
-var_1 = 'CREATE TABLE if not exists all_IP_addresses (id SERIAL, IP_address TEXT);'
+var_1 = "CREATE TABLE if not exists all_IP_addresses (id SERIAL, IP_address TEXT);"
 cur_2.execute(var_1)
 conn_2.commit()  # Flush to database server
 
-with open('export_result_TG_account_N4.json', 'r') as json_file:
-	nested_dict = json.load(json_file)
-list_1 = nested_dict['other_data']['ips']
+with open("export_result_TG_account_N4.json", "r") as json_file:
+    nested_dict = json.load(json_file)
+list_1 = nested_dict["other_data"]["ips"]
 # print(list_1)
 
-for dict_1 in list_1: 
-	var_2 = 'INSERT INTO all_ip_addresses (ip_address) VALUES (%s);'
-	cur_2.execute(var_2, (dict_1['ip'], ))
-conn_2.commit() 
+for dict_1 in list_1:
+    var_2 = "INSERT INTO all_ip_addresses (ip_address) VALUES (%s);"
+    cur_2.execute(var_2, (dict_1["ip"],))
+conn_2.commit()
 
-cur_2.execute('SELECT * FROM all_ip_addresses;') 
+cur_2.execute("SELECT * FROM all_ip_addresses;")
 var_3 = cur_2.fetchall()
 # print(type(var_3))
-print('Updated database: ', var_3)  # To show the results via Terminal
+print("Updated database: ", var_3)  # To show the results via Terminal
 
 
-	# print(dict_1['ip'])  # Get all IP addresses 
+# print(dict_1['ip'])  # Get all IP addresses
 # var_2 = 'INSERT INTO all_ip_addresses (ip_address) VALUES ;'
 # cur_2.execute(var_2)
 # conn_2.commit()
@@ -45,7 +46,7 @@
 # cur_1 = conn_1.cursor()
 
 # cur_1.execute('SELECT * FROM account;')
-# a_1 = cur_1.fetchall() 
+# a_1 = cur_1.fetchall()
 # print(a_1)
 # print(type(a_1))
 

models.py

@@ -0,0 +1,50 @@
+from sqlalchemy import Column
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer, BigInteger
+from sqlalchemy import String
+from sqlalchemy import DateTime
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.orm import relationship
+from sqlalchemy import create_engine
+import config
+
+Base = declarative_base()
+
+engine = create_engine(config.DATABASE_STRING, echo=True, future=True)
+
+
+class TelegramChat(Base):
+    __tablename__ = "telegram_chat"
+
+    id = Column(BigInteger, primary_key=True)
+    telegram_id = Column(BigInteger, unique=True)
+    name = Column(String(100))
+    type = Column(String(50))
+
+    messages = relationship(
+        "TelegramMessage", back_populates="message", cascade="all, delete"
+    )
+
+    def __repr__(self):
+        return f"Chat(id={self.chat_id!r}, name={self.name!r})"
+
+
+class TelegramMessage(Base):
+    __tablename__ = "telegram_message"
+
+    id = Column(BigInteger, primary_key=True)
+    telegram_id = Column(BigInteger, unique=True)
+    date = Column(DateTime)
+    unix_timestamp = Column(BigInteger)
+    from_name = Column(String(100))
+    from_id = Column(String(100))
+
+    chat_id = Column(BigInteger, ForeignKey("telegram_chat.telegram_id"), nullable=False)
+
+    chat = relationship("TelegramChat", back_populates="messages")
+
+    def __repr__(self):
+        return f"Message(id={self.id!r}, from_name={self.from_name!r})"
+
+
+Base.metadata.create_all(engine)

parse_fields.py

@@ -0,0 +1,21 @@
+import json
+
+with open("result.json") as f:
+    data = json.load(f)
+
+
+chat_fields = set()
+message_fields = set()
+text_entities_max_count = 0
+
+for chat in data["chats"]["list"]:
+    chat_fields |= set(chat.keys())
+    for message in chat["messages"]:
+        message_fields |= set(message.keys())
+        if len(message['text_entities']) > text_entities_max_count:
+            text_entities_max_count = len(message['text_entities'])
+            print(message['id'])
+
+print(chat_fields)
+print(message_fields)
+print(f'{text_entities_max_count=}')

parse_telegram_data.py

@@ -0,0 +1,47 @@
+import json
+from sqlalchemy import create_engine
+from sqlalchemy.sql import text
+
+import config
+
+engine = create_engine(config.DATABASE_STRING, future=True)
+
+with open("result.json") as f:
+    data = json.load(f)
+
+with engine.connect() as con:
+
+    for chat in data["chats"]["list"]:
+        statement = text(
+            """
+            INSERT INTO telegram_chat(telegram_id, name, type) 
+            VALUES(:id, :name, :type)
+            ON CONFLICT (telegram_id) DO NOTHING 
+            """
+        )
+        con.execute(
+            statement,
+            {"id": chat["id"], "name": chat.get("name"), "type": chat["type"]},
+        )
+
+        for message in chat["messages"]:
+            statement = text(
+                """
+                INSERT INTO telegram_message(telegram_id, date, unix_timestamp, from_name, from_id, chat_id) 
+                VALUES(:id, :date, :unix_timestamp, :from_name, :from_id, :chat_id)
+                ON CONFLICT (telegram_id) DO NOTHING 
+                """
+            )
+            con.execute(
+                statement,
+                {
+                    "id": message["id"],
+                    "date": message["date"],
+                    "unix_timestamp": message["date_unixtime"],
+                    "from_name": message.get("from"),
+                    "from_id": message.get("from_id"),
+                    "chat_id": chat["id"],
+                },
+            )
+
+        con.commit()