TelegramChannelCondenser/scraper.py at main · shawntangy/TelegramChannelCondenser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
from database import past_messages
import requests
from bs4 import BeautifulSoup
from difflib import SequenceMatcher


# message_details = {"user1":
# {"SGBIGPURCHASES": 2}
# }

# to retrieve initial source code length
def get_html(group_name, number):
    url = "https://t.me/" + group_name + "/" + str(number)
    response = requests.get(url)
    html = response.text
    return html


# to retrieve message from html
def get_msg(html):
    soup = BeautifulSoup(html, "html.parser")
    message = str(soup.select_one("meta[property=\"og:description\"]"))
    # removes first 15 char, last 29
    message = message[15:-29]
    return message


# to retrieve actual content
def scrape(message_details):
    pstmsg = []
    print("entered scrape")
    for i in message_details:
        for key, value in message_details[i].items():
            group_name = key
            THRESHOLD = len(get_html(group_name, 1))
            last_message = value
            message = poll(message_details, i, group_name, last_message, THRESHOLD)
            pstmsg.append(message)
            while (message != 0):
                message = poll(message_details, i, group_name, message_details[i][group_name], THRESHOLD)
                pstmsg.append(message)

    past_messages[i] = pstmsg


def similarity(msg_list, tmp_list, user):
    for j in range(len(msg_list)):
        for i in range(1, len(msg_list[j])):
            a = msg_list[j][i]
            if (a != 0):
                for k in past_messages[user]:
                    if (SequenceMatcher(None, str(a), str(k)).ratio() > 0.8):
                        msg_list[j][i] = 0
                        tmp_list.pop()
    return tmp_list


def get_update(message_details):
    print("entered update")
    msg_list = []
    for i in message_details:
        tmp_list = past_messages[i]
        for key, value in message_details[i].items():
            group_name = key
            THRESHOLD = len(get_html(group_name, 1))
            last_message = value
            message = poll(message_details, i, group_name, last_message, THRESHOLD)
            msg_list.append([group_name, message])
            tmp_list.append(message)
            while (message != 0):
                message = poll(message_details, i, group_name, message_details[i][group_name], THRESHOLD)
                print("message here")
                print(message)
                if (message != 0):
                    msg_list.append([group_name, message])
                    tmp_list.append(message)
            print("msg_list here1")
            print(msg_list)
            # msg_list.pop()
            tmp_list = similarity(msg_list, tmp_list, i)
            print("msg_list here2")
            print(msg_list)
            print("tmp_list below")
            print(tmp_list)
            past_messages[i] = tmp_list
        return msg_list


def poll(message_details, user, group_name, number, THRESHOLD):
    html = get_html(group_name, number)

    if (len(html) - THRESHOLD >= 2000):  # means valid message
        print(number, " is valid")
        number += 1
        print("user and group_name below")
        print(user, group_name)
        message_details[user][group_name] = number
        # format content and return
        return get_msg(html)

    else:
        # Fails means check next 5 if deleted
        print("entered else")
        for j in range(5):
            print(j)
            if (len(get_html(group_name, number + j + 1)) - THRESHOLD >= 2000):
                number = number + j + 1
                message_details[user][group_name] = number
                html = get_html(group_name, number)
                return get_msg(html)

        print("already latest")
        return 0

# THRESHOLD = len(get_html("SGBIGPURCHASES", 1))

# schedule.every(5).seconds.do(scrape, message_details)

# last_message = scrape(message_details)

# while True:
# schedule.run_pending()
# print(last_message)