From befe47710979b32d1faca9ed4455e2ffea5c3efc Mon Sep 17 00:00:00 2001 From: Francisco Alvarez Date: Tue, 1 Feb 2022 09:03:41 -0500 Subject: [PATCH 1/5] Allow filtering based on recipient. Add config variable for toggling --- policyd_rate_limit/policyd-rate-limit.yaml | 2 ++ policyd_rate_limit/policyd.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/policyd_rate_limit/policyd-rate-limit.yaml b/policyd_rate_limit/policyd-rate-limit.yaml index 696a67a..9a8d365 100644 --- a/policyd_rate_limit/policyd-rate-limit.yaml +++ b/policyd_rate_limit/policyd-rate-limit.yaml @@ -61,6 +61,8 @@ limit_by_sasl: True limit_by_sender: False # If sasl username and sender address not found or disabled, apply limits by ip addresses. limit_by_ip: False +# If enabled, filters based on recipient address +limit_by_recipient: False # A list of ip networks in cidr notation on which limits are applied. An empty list is equal # to limit_by_ip: False, put "0.0.0.0/0" and "::/0" for every ip addresses. diff --git a/policyd_rate_limit/policyd.py b/policyd_rate_limit/policyd.py index 5ad8747..341be97 100644 --- a/policyd_rate_limit/policyd.py +++ b/policyd_rate_limit/policyd.py @@ -230,6 +230,9 @@ def action(self, connection, request): # else, if activated, we filter by sender elif config.limit_by_sender and u'sender' in request: id = request[u'sender'] + # else, if activated, we filter by recipient + elif config.limit_by_recipient and u'recipient' in request: + id = request[u'recipient'] # else, if activated, we filter by ip source addresse elif ( config.limit_by_ip and From d5f4bd336cd99f4ce4e77821ec84dbec54c1efad Mon Sep 17 00:00:00 2001 From: Francisco Alvarez Date: Tue, 1 Feb 2022 09:18:30 -0500 Subject: [PATCH 2/5] Add new variable to .conf --- policyd_rate_limit/policyd-rate-limit.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/policyd_rate_limit/policyd-rate-limit.conf b/policyd_rate_limit/policyd-rate-limit.conf index f8db71d..e50e33a 100644 --- a/policyd_rate_limit/policyd-rate-limit.conf +++ b/policyd_rate_limit/policyd-rate-limit.conf @@ -45,6 +45,9 @@ limits_by_id = {} limit_by_sasl = True limit_by_ip = False +# If enabled, filters based on recipient address +limit_by_recipient = False + limited_networks = [] # actions return to postfix, see http://www.postfix.org/access.5.html for a list of actions. From c113ef2d4fa279c76ae96a9e0874f6b5d39cd24a Mon Sep 17 00:00:00 2001 From: Francisco Alvarez Date: Mon, 7 Feb 2022 16:35:31 -0500 Subject: [PATCH 3/5] Convert report to basic email-safe HTML --- policyd_rate_limit/utils.py | 111 +++++++++++++++--------------------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/policyd_rate_limit/utils.py b/policyd_rate_limit/utils.py index 91b75a5..6c229f2 100644 --- a/policyd_rate_limit/utils.py +++ b/policyd_rate_limit/utils.py @@ -366,94 +366,77 @@ def gen_report(cur): text = [] if not config.report_only_if_needed or report: if report: - text = ["Below is the table of users who hit a limit since the last cleanup:", ""] + text = ["Below is the table of users who hit a limit since the last cleanup:

", ""] # dist to groups deltas by ids report_d = collections.defaultdict(list) - max_d = {'id': 2, 'delta': 5, 'hit': 3} for (id, delta, hit) in report: report_d[id].append((delta, hit)) - max_d['id'] = max(max_d['id'], len(id)) - max_d['delta'] = max(max_d['delta'], len(str(delta)) + 1) - max_d['hit'] = max(max_d['hit'], len(str(hit))) + # sort by hits report.sort(key=lambda x: x[2]) # table header - text.append( - "|%s|%s|%s|" % ( - print_fw("id", max_d['id']), - print_fw("delta", max_d['delta']), - print_fw("hit", max_d['hit']) - ) - ) - # table header/data separation - text.append( - "|%s+%s+%s|" % ( - print_fw("", max_d['id'], filler='-'), - print_fw("", max_d['delta'], filler='-'), - print_fw("", max_d['hit'], filler='-') - ) - ) + text.append("") + text.append("") + text.append("") + text.append("") + text.append("") for (id, _, _) in report: # sort by delta report_d[id].sort() for (delta, hit) in report_d[id]: # add a table row - text.append( - "|%s|%s|%s|" % ( - print_fw(id, max_d['id'], align_left=False), - print_fw("%ss" % delta, max_d['delta'], align_left=False), - print_fw(hit, max_d['hit'], align_left=False) - ) - ) + text.append("") + text.append("") + text.append("") else: text = ["No user hit a limit since the last cleanup"] - text.extend(["", "-- ", "policyd-rate-limit"]) + text.append("
IDDeltaHit
" + str(id) + "" + str(delta) + "s" + str(hit) + "

-- policyd-rate-limit") return text def send_report(text): - # check that smtp_server is wekk formated - if isinstance(config.smtp_server, (list, tuple)): - if len(config.smtp_server) >= 2: - server = smtplib.SMTP(config.smtp_server[0], config.smtp_server[1]) - elif len(config.smtp_server) == 1: - server = smtplib.SMTP(config.smtp_server[0], 25) - else: - raise ValueError("bad smtp_server should be a tuple (server_adress, port)") + # check that smtp_server is well formatted + if isinstance(config.smtp_server, (list, tuple)): + if len(config.smtp_server) >= 2: + server = smtplib.SMTP(config.smtp_server[0], config.smtp_server[1]) + elif len(config.smtp_server) == 1: + server = smtplib.SMTP(config.smtp_server[0], 25) else: raise ValueError("bad smtp_server should be a tuple (server_adress, port)") + else: + raise ValueError("bad smtp_server should be a tuple (server_adress, port)") - try: - # should we use starttls ? - if config.smtp_starttls: - server.starttls() - # should we use credentials ? - if config.smtp_credentials: - if ( - isinstance(config.smtp_credentials, (list, tuple)) and - len(config.smtp_credentials) >= 2 - ): - server.login(config.smtp_credentials[0], config.smtp_credentials[1]) - else: - ValueError("bad smtp_credentials should be a tuple (login, password)") - - if not isinstance(config.report_to, list): - report_to = [config.report_to] + try: + # should we use starttls ? + if config.smtp_starttls: + server.starttls() + # should we use credentials ? + if config.smtp_credentials: + if ( + isinstance(config.smtp_credentials, (list, tuple)) and + len(config.smtp_credentials) >= 2 + ): + server.login(config.smtp_credentials[0], config.smtp_credentials[1]) else: - report_to = config.report_to - for rcpt in report_to: - # Start building the mail report - msg = MIMEMultipart() - msg['Subject'] = config.report_subject or "" - msg['From'] = config.report_from or "" - msg['To'] = rcpt - msg.attach(MIMEText("\n".join(text), 'plain')) - server.sendmail(config.report_from or "", rcpt, msg.as_string()) - finally: - print('report is sent') - server.quit() + ValueError("bad smtp_credentials should be a tuple (login, password)") + + if not isinstance(config.report_to, list): + report_to = [config.report_to] + else: + report_to = config.report_to + for rcpt in report_to: + # Start building the mail report + msg = MIMEMultipart() + msg['Subject'] = config.report_subject or "" + msg['From'] = config.report_from or "" + msg['To'] = rcpt + msg.attach(MIMEText("\n".join(text), 'html')) + server.sendmail(config.report_from or "", rcpt, msg.as_string()) + finally: + print('report is sent') + server.quit() def database_init(): From a87336a871e098063c78dfae57c9d65f1541d151 Mon Sep 17 00:00:00 2001 From: Francisco Alvarez Date: Mon, 7 Feb 2022 16:43:19 -0500 Subject: [PATCH 4/5] Add flag and report to display all sent emails since last cleanup --- policyd_rate_limit/config.py | 2 ++ policyd_rate_limit/policyd-rate-limit.conf | 3 ++ policyd_rate_limit/policyd-rate-limit.yaml | 3 ++ policyd_rate_limit/utils.py | 37 +++++++++++++++++++++- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/policyd_rate_limit/config.py b/policyd_rate_limit/config.py index 7cfb296..6b7b129 100644 --- a/policyd_rate_limit/config.py +++ b/policyd_rate_limit/config.py @@ -67,6 +67,8 @@ # if True, send a report to report_email about users reaching limits each time --clean is called report = False +# If True, send a report of the total emails sent per users each time --clean is called +report_totals: False # from who to send emails reports report_from = None # address to send emails reports to. It can be a single email or a list of emails diff --git a/policyd_rate_limit/policyd-rate-limit.conf b/policyd_rate_limit/policyd-rate-limit.conf index e50e33a..323df90 100644 --- a/policyd_rate_limit/policyd-rate-limit.conf +++ b/policyd_rate_limit/policyd-rate-limit.conf @@ -59,6 +59,9 @@ db_error_action = "dunno" # if True, send a report to report_email about users reaching limits each time --clean is called report = False +# If True, send a report of the total emails sent per users each time --clean is called +report_totals: False + # from who to send emails reports report_from = None # address to send emails reports to. It can be a single email or a list of emails diff --git a/policyd_rate_limit/policyd-rate-limit.yaml b/policyd_rate_limit/policyd-rate-limit.yaml index 9a8d365..2b75214 100644 --- a/policyd_rate_limit/policyd-rate-limit.yaml +++ b/policyd_rate_limit/policyd-rate-limit.yaml @@ -80,6 +80,9 @@ db_error_action: "dunno" # If True, send a report to report_to about users reaching limits each time --clean is called report: False +# If True, send a report of the total emails sent per users each time --clean is called +report_totals: False + # from who to send emails reports. Must be defined if report: True report_from: null # Address to send emails reports to. Must be defined if report: True diff --git a/policyd_rate_limit/utils.py b/policyd_rate_limit/utils.py index 6c229f2..9a09d16 100644 --- a/policyd_rate_limit/utils.py +++ b/policyd_rate_limit/utils.py @@ -328,7 +328,11 @@ def clean(): # remove old record older than 2*max_delta expired = int(time.time() - max_delta - max_delta) report_text = "" + report_totals_text = "" with cursor() as cur: + # if report_totals is True, generate report before deleting table contents. + if config.report_totals and config.report_to: + report_totals_text = gen_totals_report(cur) cur.execute("DELETE FROM mail_count WHERE date <= %s" % config.format_str, (expired,)) print("%d records deleted" % cur.rowcount) # if report is True, generate a mail report @@ -336,9 +340,11 @@ def clean(): report_text = gen_report(cur) # The mail report has been successfully send, flush limit_report cur.execute("DELETE FROM limit_report") - # send report + # send reports if len(report_text) != 0: send_report(report_text) + if len(report_totals_text) != 0: + send_report(report_totals_text) try: if config.backend == PGSQL_DB: @@ -395,6 +401,35 @@ def gen_report(cur): return text +def gen_totals_report(cur): + cur.execute("SELECT id, date FROM mail_count") + # list to sort ids by hits + report = list(cur.fetchall()) + text = [] + if report: + text = ["Total quantity of emails sent since the last cleanup:

", ""] + # dist to groups deltas by ids + report_d = collections.defaultdict() + for (id, date ) in report: + if id in report_d.keys(): + report_d[id] += 1 + else: + report_d[id] = 1 + + # table header + text.append("") + text.append("") + text.append("") + text.append("") + + for (id, count) in report_d.items(): + # add a table row + text.append( "") + text.append( "") + text.append("
User/IPCount
" + str(id) + "" + str(count) + "

-- policyd-rate-limit") + return text + + def send_report(text): # check that smtp_server is well formatted From 848e2a0d153ff0611d1813556cfceb215be6bece Mon Sep 17 00:00:00 2001 From: Francisco Alvarez Date: Fri, 1 Apr 2022 13:13:29 -0400 Subject: [PATCH 5/5] add warning email and parameter to bin file to schedule cron --- policyd-rate-limit | 7 +++ policyd_rate_limit/policyd.py | 5 ++ policyd_rate_limit/utils.py | 86 ++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/policyd-rate-limit b/policyd-rate-limit index c77a74b..cb5f094 100755 --- a/policyd-rate-limit +++ b/policyd-rate-limit @@ -22,6 +22,7 @@ from policyd_rate_limit.utils import config if __name__ == "__main__": # pragma: no branch parser = argparse.ArgumentParser() parser.add_argument("--clean", help="clean old records from the database", action="store_true") + parser.add_argument("--warn", help="warn useer useaga of email limit", action="store_true") parser.add_argument( "--get-config", help="return the value of a config parameter", @@ -76,6 +77,12 @@ if __name__ == "__main__": # pragma: no branch except ValueError as error: sys.stderr.write("%s\n" % error) sys.exit(8) + if args.warn: + try: + utils.warn() + except ValueError as error: + sys.stderr.write("%s\n" % error) + sys.exit(8) # else we gonna lauch the policyd daemon else: # we check if policyd-rate-limit is not already running by lookig at config.pidfile diff --git a/policyd_rate_limit/policyd.py b/policyd_rate_limit/policyd.py index 341be97..eaa5846 100644 --- a/policyd_rate_limit/policyd.py +++ b/policyd_rate_limit/policyd.py @@ -259,6 +259,11 @@ def action(self, connection, request): if config.debug: sys.stderr.write("%03d/%03d hit since %ss\n" % (nb, mail_nb, delta)) sys.stderr.flush() + print("nb " + str(nb)) + print(str(mail_nb)) + if nb >= mail_nb - mail_nb * .1: + utils.send_warning_report(id, nb) + raise Pass() if nb >= mail_nb: action = config.fail_action if config.report and delta in config.report_limits: diff --git a/policyd_rate_limit/utils.py b/policyd_rate_limit/utils.py index 9a09d16..a1f9b99 100644 --- a/policyd_rate_limit/utils.py +++ b/policyd_rate_limit/utils.py @@ -430,7 +430,86 @@ def gen_totals_report(cur): return text -def send_report(text): +def warn(): + with cursor() as cur: + if config.report: + report_recipients = gen_warning_report(cur) + # send reports + if report_recipients: + for (rec, data) in report_recipients.items(): + send_report(data, rec) + + try: + if config.backend == PGSQL_DB: + # setting autocommit to True disable the transations. This is needed to run VACUUM + cursor.get_db().autocommit = True + with cursor() as cur: + if config.backend == PGSQL_DB: + cur.execute("VACUUM ANALYZE") + elif config.backend == SQLITE_DB: + cur.execute("VACUUM") + elif config.backend == MYSQL_DB: + if config.report: + cur.execute("OPTIMIZE TABLE mail_count, limit_report") + else: + cur.execute("OPTIMIZE TABLE mail_count") + finally: + if config.backend == PGSQL_DB: + cursor.get_db().autocommit = False + + +def gen_warning_report(cur): + cur.execute("SELECT id, date FROM mail_count") + # list to sort ids by hits + report = list(cur.fetchall()) + emailRec = collections.defaultdict() + + if report: + # dist to groups deltas by ids + report_d = collections.defaultdict() + for (id, date) in report: + if id in report_d.keys(): + report_d[id] += 1 + else: + report_d[id] = 1 + + for (id, count) in report_d.items(): + text = [] + name = str(id) + alert_level = 0 + + for limit, time_period in config.limits_by_id.get(u'recipient', config.limits): + msg = "" + msg2 = "" + if count >= limit * .9: + msg = "
You are currently over 90% of the" + elif count >= limit * .75: + msg = "
You are currently over 75% of the" + elif count >= limit * .5: + msg = "
You are currently over 50% of the" + + if time_period >= 86400: + msg2 = " allowed email limit in a " + str(time_period / 86400) + " day period" + elif time_period >= 3600: + msg2 = " allowed email limit in a " + str(time_period / 3600) + " hour period." + elif time_period >= 60: + msg2 = " allowed email limit in a " + str(time_period / 60) + " minute period." + else: + msg2 = " allowed email limit in a " + str(time_period) + " second period." + + msg3 = "
Total emails sent: " + str(count) + "/" + str(limit) + "
" + + if msg != "" and msg2 != "" and limit > alert_level: + text.append(name) + text.append(msg+msg2) + text.append(msg3) + emailRec[name] = text + alert_level = limit + + return emailRec + + +def send_report(text, extraTo=""): # check that smtp_server is well formatted if isinstance(config.smtp_server, (list, tuple)): @@ -461,6 +540,11 @@ def send_report(text): report_to = [config.report_to] else: report_to = config.report_to + + if extraTo != "": + report_to.append(extraTo) + print("Extra to " + str(extraTo)) + for rcpt in report_to: # Start building the mail report msg = MIMEMultipart()