diff --git a/policyd-rate-limit b/policyd-rate-limit index c77a74b..cb5f094 100755 --- a/policyd-rate-limit +++ b/policyd-rate-limit @@ -22,6 +22,7 @@ from policyd_rate_limit.utils import config if __name__ == "__main__": # pragma: no branch parser = argparse.ArgumentParser() parser.add_argument("--clean", help="clean old records from the database", action="store_true") + parser.add_argument("--warn", help="warn useer useaga of email limit", action="store_true") parser.add_argument( "--get-config", help="return the value of a config parameter", @@ -76,6 +77,12 @@ if __name__ == "__main__": # pragma: no branch except ValueError as error: sys.stderr.write("%s\n" % error) sys.exit(8) + if args.warn: + try: + utils.warn() + except ValueError as error: + sys.stderr.write("%s\n" % error) + sys.exit(8) # else we gonna lauch the policyd daemon else: # we check if policyd-rate-limit is not already running by lookig at config.pidfile diff --git a/policyd_rate_limit/config.py b/policyd_rate_limit/config.py index 974b79c..105ef26 100644 --- a/policyd_rate_limit/config.py +++ b/policyd_rate_limit/config.py @@ -67,6 +67,8 @@ # if True, send a report to report_email about users reaching limits each time --clean is called report = False +# If True, send a report of the total emails sent per users each time --clean is called +report_totals: False # from who to send emails reports report_from = None # address to send emails reports to. It can be a single email or a list of emails diff --git a/policyd_rate_limit/policyd-rate-limit.conf b/policyd_rate_limit/policyd-rate-limit.conf index f8db71d..323df90 100644 --- a/policyd_rate_limit/policyd-rate-limit.conf +++ b/policyd_rate_limit/policyd-rate-limit.conf @@ -45,6 +45,9 @@ limits_by_id = {} limit_by_sasl = True limit_by_ip = False +# If enabled, filters based on recipient address +limit_by_recipient = False + limited_networks = [] # actions return to postfix, see http://www.postfix.org/access.5.html for a list of actions. @@ -56,6 +59,9 @@ db_error_action = "dunno" # if True, send a report to report_email about users reaching limits each time --clean is called report = False +# If True, send a report of the total emails sent per users each time --clean is called +report_totals: False + # from who to send emails reports report_from = None # address to send emails reports to. It can be a single email or a list of emails diff --git a/policyd_rate_limit/policyd-rate-limit.yaml b/policyd_rate_limit/policyd-rate-limit.yaml index 0d07dcc..2248466 100644 --- a/policyd_rate_limit/policyd-rate-limit.yaml +++ b/policyd_rate_limit/policyd-rate-limit.yaml @@ -68,6 +68,8 @@ limit_by_sasl: True limit_by_sender: False # If sasl username and sender address not found or disabled, apply limits by ip addresses. limit_by_ip: False +# If enabled, filters based on recipient address +limit_by_recipient: False # A list of ip networks in cidr notation on which limits are applied. An empty list is equal # to limit_by_ip: False, put "0.0.0.0/0" and "::/0" for every ip addresses. @@ -85,6 +87,9 @@ db_error_action: "dunno" # If True, send a report to report_to about users reaching limits each time --clean is called report: False +# If True, send a report of the total emails sent per users each time --clean is called +report_totals: False + # from who to send emails reports. Must be defined if report: True report_from: null # Address to send emails reports to. Must be defined if report: True diff --git a/policyd_rate_limit/policyd.py b/policyd_rate_limit/policyd.py index 997c120..4086b0a 100644 --- a/policyd_rate_limit/policyd.py +++ b/policyd_rate_limit/policyd.py @@ -271,6 +271,9 @@ def action(self, connection, request): # else, if activated, we filter by sender elif config.limit_by_sender and u'sender' in request: id = request[u'sender'] + # else, if activated, we filter by recipient + elif config.limit_by_recipient and u'recipient' in request: + id = request[u'recipient'] # else, if activated, we filter by ip source addresse elif ( config.limit_by_ip and @@ -322,6 +325,8 @@ def action(self, connection, request): ) ) sys.stderr.flush() + if nb + recipient_count >= mail_nb - mail_nb * .1: + utils.send_warning_report(id, nb) if nb + recipient_count > mail_nb: action = config.fail_action if config.report and delta in config.report_limits: diff --git a/policyd_rate_limit/utils.py b/policyd_rate_limit/utils.py index a8d1fb8..a1b29d3 100644 --- a/policyd_rate_limit/utils.py +++ b/policyd_rate_limit/utils.py @@ -328,7 +328,11 @@ def clean(): # remove old record older than 2*max_delta expired = int(time.time() - max_delta - max_delta) report_text = "" + report_totals_text = "" with cursor() as cur: + # if report_totals is True, generate report before deleting table contents. + if config.report_totals and config.report_to: + report_totals_text = gen_totals_report(cur) cur.execute("DELETE FROM mail_count WHERE date <= %s" % config.format_str, (expired,)) print("%d records deleted" % cur.rowcount) # if report is True, generate a mail report @@ -336,9 +340,11 @@ def clean(): report_text = gen_report(cur) # The mail report has been successfully send, flush limit_report cur.execute("DELETE FROM limit_report") - # send report + # send reports if len(report_text) != 0: send_report(report_text) + if len(report_totals_text) != 0: + send_report(report_totals_text) try: if config.backend == PGSQL_DB: @@ -366,54 +372,144 @@ def gen_report(cur): text = [] if not config.report_only_if_needed or report: if report: - text = ["Below is the table of users who hit a limit since the last cleanup:", ""] + text = ["Below is the table of users who hit a limit since the last cleanup:

", ""] # dist to groups deltas by ids report_d = collections.defaultdict(list) - max_d = {'id': 2, 'delta': 5, 'hit': 3} for (id, delta, hit) in report: report_d[id].append((delta, hit)) - max_d['id'] = max(max_d['id'], len(id)) - max_d['delta'] = max(max_d['delta'], len(str(delta)) + 1) - max_d['hit'] = max(max_d['hit'], len(str(hit))) + # sort by hits report.sort(key=lambda x: x[2]) # table header - text.append( - "|%s|%s|%s|" % ( - print_fw("id", max_d['id']), - print_fw("delta", max_d['delta']), - print_fw("hit", max_d['hit']) - ) - ) - # table header/data separation - text.append( - "|%s+%s+%s|" % ( - print_fw("", max_d['id'], filler='-'), - print_fw("", max_d['delta'], filler='-'), - print_fw("", max_d['hit'], filler='-') - ) - ) + text.append("") + text.append("") + text.append("") + text.append("") + text.append("") for (id, _, _) in report: # sort by delta report_d[id].sort() for (delta, hit) in report_d[id]: # add a table row - text.append( - "|%s|%s|%s|" % ( - print_fw(id, max_d['id'], align_left=False), - print_fw("%ss" % delta, max_d['delta'], align_left=False), - print_fw(hit, max_d['hit'], align_left=False) - ) - ) + text.append("") + text.append("") + text.append("") else: text = ["No user hit a limit since the last cleanup"] - text.extend(["", "-- ", "policyd-rate-limit"]) + text.append("
IDDeltaHit
" + str(id) + "" + str(delta) + "s" + str(hit) + "

-- policyd-rate-limit") + return text + +def gen_totals_report(cur): + cur.execute("SELECT id, date FROM mail_count") + # list to sort ids by hits + report = list(cur.fetchall()) + text = [] + if report: + text = ["Total quantity of emails sent since the last cleanup:

", ""] + # dist to groups deltas by ids + report_d = collections.defaultdict() + for (id, date ) in report: + if id in report_d.keys(): + report_d[id] += 1 + else: + report_d[id] = 1 + + # table header + text.append("") + text.append("") + text.append("") + text.append("") + + for (id, count) in report_d.items(): + # add a table row + text.append( "") + text.append( "") + text.append("
User/IPCount
" + str(id) + "" + str(count) + "

-- policyd-rate-limit") return text -def send_report(text): - # check that smtp_server is wekk formated +def warn(): + with cursor() as cur: + if config.report: + report_recipients = gen_warning_report(cur) + # send reports + if report_recipients: + for (rec, data) in report_recipients.items(): + send_report(data, rec) + + try: + if config.backend == PGSQL_DB: + # setting autocommit to True disable the transations. This is needed to run VACUUM + cursor.get_db().autocommit = True + with cursor() as cur: + if config.backend == PGSQL_DB: + cur.execute("VACUUM ANALYZE") + elif config.backend == SQLITE_DB: + cur.execute("VACUUM") + elif config.backend == MYSQL_DB: + if config.report: + cur.execute("OPTIMIZE TABLE mail_count, limit_report") + else: + cur.execute("OPTIMIZE TABLE mail_count") + finally: + if config.backend == PGSQL_DB: + cursor.get_db().autocommit = False + + +def gen_warning_report(cur): + cur.execute("SELECT id, date FROM mail_count") + # list to sort ids by hits + report = list(cur.fetchall()) + emailRec = collections.defaultdict() + + if report: + # dist to groups deltas by ids + report_d = collections.defaultdict() + for (id, date) in report: + if id in report_d.keys(): + report_d[id] += 1 + else: + report_d[id] = 1 + + for (id, count) in report_d.items(): + text = [] + name = str(id) + alert_level = 0 + + for limit, time_period in config.limits_by_id.get(u'recipient', config.limits): + msg = "" + msg2 = "" + if count >= limit * .9: + msg = "
You are currently over 90% of the" + elif count >= limit * .75: + msg = "
You are currently over 75% of the" + elif count >= limit * .5: + msg = "
You are currently over 50% of the" + + if time_period >= 86400: + msg2 = " allowed email limit in a " + str(time_period / 86400) + " day period" + elif time_period >= 3600: + msg2 = " allowed email limit in a " + str(time_period / 3600) + " hour period." + elif time_period >= 60: + msg2 = " allowed email limit in a " + str(time_period / 60) + " minute period." + else: + msg2 = " allowed email limit in a " + str(time_period) + " second period." + + msg3 = "
Total emails sent: " + str(count) + "/" + str(limit) + "
" + + if msg != "" and msg2 != "" and limit > alert_level: + text.append(name) + text.append(msg+msg2) + text.append(msg3) + emailRec[name] = text + alert_level = limit + + return emailRec + + +def send_report(text, extraTo=""): + # check that smtp_server is well formatted if isinstance(config.smtp_server, (list, tuple)): if len(config.smtp_server) >= 2: server = smtplib.SMTP(config.smtp_server[0], config.smtp_server[1]) @@ -442,13 +538,19 @@ def send_report(text): report_to = [config.report_to] else: report_to = config.report_to + + if extraTo != "": + report_to.append(extraTo) + print("Extra to " + str(extraTo)) + for rcpt in report_to: # Start building the mail report msg = MIMEMultipart() msg['Subject'] = config.report_subject or "" msg['From'] = config.report_from or "" msg['To'] = rcpt - msg.attach(MIMEText("\n".join(text), 'plain')) + + msg.attach(MIMEText("\n".join(text), 'html')) server.sendmail(config.report_from or "", rcpt, msg.as_string()) finally: print('report is sent')