techsneeze · jnew-gh · Jul 25, 2022 · Jul 25, 2022 · Jul 25, 2022 · Jul 25, 2022
diff --git a/README.md b/README.md
@@ -1,11 +1,12 @@
 # dmarcts-report-parser
-A Perl based tool to parse DMARC reports, based on John Levine's [rddmarc](http://www.taugh.com/rddmarc/), but extended by the following features:
-* Allow to read messages from an IMAP server and not only from the local filesystem.
-* Store much more XML values into the database (for example the missing SPF and DKIM results from the policy_evaluated section) and also the entire XML for later reference.
+A Perl based tool to parse DMARC and TLS reports, based on John Levine's [rddmarc](http://www.taugh.com/rddmarc/), but extended by the following features:
+* Reports can be read from an IMAP server as well as the local filesystem.
 * Supports MySQL and PostgreSQL.
-* Needed database tables and columns are created automatically, user only needs to provide a database. The database schema is compatible to the one used by rddmarc, but extends it by additional fields. Users can switch from rddmarc to dmarcts-report-parser without having to do any changes to the database by themselves.
-* Due to limitations in stock configurations of MySQL/MariaSQL on some distros, it may be necessary
-to add the following to your configuration (i.e. in /etc/mysql/mariadb.conf.d/50-server.cnf):
+* Store more DMARC report XML values into the database (for example the missing SPF and DKIM results from the policy_evaluated section), including the entire report XML for later reference.
+* TLS report JSON values are stored in the database, including the entire report JSON for later reference.
+* Needed database tables and columns are created automatically, user only needs to provide a database.
+* The database schema is compatible to the one used by rddmarc, but extends it by additional tables and fields. Users can switch from rddmarc to dmarcts-report-parser without having to do any changes to the database by themselves.
+* Due to limitations in stock configurations of MySQL/MariaSQL on some distros, it may be necessary to add the following to your configuration (i.e. in /etc/mysql/mariadb.conf.d/50-server.cnf):
 
 ```
 innodb_large_prefix	= on
@@ -19,33 +20,31 @@ To install dependencies...
 
 ### on Debian:
 ```
-apt-get install libfile-mimeinfo-perl libmail-imapclient-perl libmime-tools-perl libxml-simple-perl \
-libio-socket-inet6-perl libio-socket-ip-perl libperlio-gzip-perl \
+apt-get install libfile-mimeinfo-perl libmail-imapclient-perl libmime-tools-perl libxml-simple-perl libio-socket-inet6-perl libio-socket-ip-perl libperlio-gzip-perl libjson-perl
 libmail-mbox-messageparser-perl unzip
 ```
 Plus `libdbd-mysql-perl` for MySQL or `libdbd-pg-perl` for PostgreSQL.
 ### on Fedora (Fedora 23):
 ```
-sudo dnf install perl-File-MimeInfo perl-Mail-IMAPClient perl-MIME-tools perl-XML-Simple perl-DBI \
- perl-Socket6 perl-PerlIO-gzip unzip
+sudo dnf install perl-File-MimeInfo perl-Mail-IMAPClient perl-MIME-tools perl-XML-Simple perl-DBI perl-Socket6 perl-PerlIO-gzip unzip perl-JSON
 ```
 Plus `perl-DBD-MySQL` for MySQL or `perl-DBD-Pg` for PostgreSQL.
 ### on CentOS (CentOS 7):
 ```
 yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-yum install perl-File-MimeInfo perl-Mail-IMAPClient perl-MIME-tools perl-XML-Simple perl-DBI \
- perl-Socket6 perl-PerlIO-gzip unzip perl-Mail-Mbox-MessageParser
+yum install perl-File-MimeInfo perl-Mail-IMAPClient perl-MIME-tools perl-XML-Simple perl-DBI perl-Socket6 perl-PerlIO-gzip unzip perl-Mail-Mbox-MessageParser perl-JSON
  ```
 Plus `perl-DBD-MySQL` for MySQL or `perl-DBD-Pg` for PostgreSQL.
 ### on FreeBSD (FreeBSD 11.4):
 ```
-sudo pkg install p5-File-MimeInfo p5-Mail-IMAPClient p5-MIME-tools p5-XML-Simple p5-DBI p5-Socket6 p5-PerlIO-gzip p5-Mail-Mbox-MessageParser unzip
+sudo pkg install p5-File-MimeInfo p5-Mail-IMAPClient p5-MIME-tools p5-XML-Simple p5-DBI p5-Socket6 p5-PerlIO-gzip p5-Mail-Mbox-MessageParser unzip p5-JSON
 ```
 Plus `p5-DBD-MySQL` for MySQL or `p5-DBD-Pg` for PostgreSQL.
  ### on macOS (macOS 10.13):
 ```
 brew install mysql shared-mime-info
 update-mime-database /usr/local/share/mime
+perl -MCPAN -e 'install JSON'
 perl -MCPAN -e 'install Mail::IMAPClient'
 perl -MCPAN -e 'install Mail::Mbox::MessageParser'
 perl -MCPAN -e 'install File::MimeInfo'
@@ -84,8 +83,8 @@ $imapport         = '143';
 $imapssl          = '0';        # If set to 1, remember to change server port to 993 and disable imaptls.
 $imaptls          = '0';        # Enabled as the default and best-practice.
 $tlsverify        = '0';        # Enable verify server cert as the default and best-practice.
-$imapignoreerror  = '0';          # set it to 1 if you see an "ERROR: message_string() 
-                                # expected 119613 bytes but received 81873 you may 
+$imapignoreerror  = '0';          # set it to 1 if you see an "ERROR: message_string()
+                                # expected 119613 bytes but received 81873 you may
                                 # need the IgnoreSizeErrors option" because of malfunction
                                 # imap server as MS Exchange 2007, ...
 $imapreadfolder   = 'dmarc';
@@ -94,14 +93,14 @@ $imapreadfolder   = 'dmarc';
 # the --delete option!)
 $imapmovefolder = 'dmarc/processed';
 
-# maximum size of XML files to store in database, long files can cause transaction aborts
-$maxsize_xml = 50000;
+# maximum size of data files to store in database, long files can cause transaction aborts
+$raw_data_max_size = 50000;
 # store XML as base64 encopded gzip in database (save space, harder usable)
-$compress_xml = 0;
+$raw_data_compress = 0;
 
-# if there was an error during file processing (message does not contain XML or ZIP parts, 
-# or a database error) the parser reports an error and does not delete the file, even if 
-# delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed, 
+# if there was an error during file processing (message does not contain XML, JSON or ZIP parts,
+# or a database error) the parser reports an error and does not delete the file, even if
+# delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed,
 # however not for database errors.
 $delete_failed = 0;
 ```
@@ -125,15 +124,15 @@ One of the following source options must be provided:
 #        -i : Read reports from messages on IMAP server as defined in the config file.
 #        -m : Read reports from mbox file(s) provided in PATH.
 #        -e : Read reports from MIME email file(s) provided in PATH.
-#        -x : Read reports from xml file(s) provided in PATH.
-#        -z : Read reports from zip file(s) provided in PATH.
+#        -x : Read reports from xml or json file(s) provided in PATH.
+#        -z : Read reports from zip or gzip file(s) provided in PATH.
 ```
 
 The following options are always allowed:
 ```
 #        -d : Print debug info.
 #        -r : Replace existing reports rather than failing.
-#  --delete : Delete processed message files (the XML is stored in the
+#  --delete : Delete processed message files (the XML or JSON is stored in the
 #             database for later reference).
 ```
 

diff --git a/dbx_Pg.pl b/dbx_Pg.pl
@@ -56,6 +56,47 @@
 				"CREATE INDEX rptrecord_idx_serial6 ON rptrecord (serial, ip6);",
 				],
 			},
+		"tls_report" => {
+			column_definitions 		=> [
+				"serial"					, "bigint"									, "GENERATED ALWAYS AS IDENTITY",
+				"mindate"				, "timestamp without time zone"		, "NOT NULL",
+				"maxdate"				, "timestamp without time zone"		, "NULL",
+				"domain"					, "character varying(255)"				, "NULL",
+				"org"						, "character varying(255)"				, "NOT NULL",
+				"reportid"				, "character varying(255)"				, "NOT NULL",
+				"contact"				, "character varying(255)"				, "NULL",
+				"policy_type"			, "character varying(255)"				, "NULL",
+				"policy_string"		, "character varying(255)"				, "NULL",
+				"summary_failure"		, "bigint"									, "GENERATED ALWAYS AS IDENTITY",
+				"summary_successful"	, "bigint"									, "GENERATED ALWAYS AS IDENTITY",
+				"raw_json"				, "text"										, "",
+				],
+			additional_definitions 		=> "PRIMARY KEY (serial), UNIQUE KEY domain (domain,reportid)",
+			table_options			=> "ROW_FORMAT=COMPRESSED",
+			indexes				=> [],
+				"CREATE UNIQUE INDEX tls_report_uidx_domain ON tls_report (domain, reportid);"
+			},
+			"tls_rptrecord" => {
+			column_definitions 		=> [
+				"id"								, "bigint"							, "GENERATED ALWAYS AS IDENTITY",
+				"serial"							, "bigint"							, "NOT NULL",
+				"result_type"					, "character varying(255)"		, "",
+				"sending_mta_ip"				, "bigint"							, "",
+				"sending_mta_ip6"				, "bytea"							, "",
+				"receiving_mx_hostname"		, "character varying(255)"		, "",
+				"receiving_mx_helo"			, "character varying(255)"		, "",
+				"receiving_ip"					, "bigint"							, "",
+				"receiving_ip6"				, "bytea"							, "",
+				"failed_session_count"		, "bigint"							, "NOT NULL",
+				"additional_information"	, "character varying(255)"		, "",
+				"failure_reason_code"		, "character varying(255)"		, "",
+				],
+			additional_definitions 		=> "KEY serial (serial,sending_mta_ip), KEY serial6 (serial,sending_mta_ip6)",
+			table_options			=> "",
+			indexes				=> [],
+				"CREATE INDEX tls_rptrecord_idx_serial ON tls_rptrecord (serial, ip);",
+				"CREATE INDEX tls_rptrecord_idx_serial6 ON tls_rptrecord (serial, ip6);",
+			},
 		},
 
 	add_column => sub {

diff --git a/dbx_mysql.pl b/dbx_mysql.pl
@@ -51,6 +51,44 @@
 			table_options			=> "",
 			indexes				=> [],
 			},
+		"tls_report" => {
+			column_definitions 		=> [
+				"serial"					, "int"				, "unsigned NOT NULL AUTO_INCREMENT",
+				"mindate"				, "timestamp"		, "NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP",
+				"maxdate"				, "timestamp"		, "NULL",
+				"domain"					, "varchar(255)"	, "NULL",
+				"org"						, "varchar(255)"	, "NOT NULL",
+				"reportid"				, "varchar(255)"	, "NOT NULL",
+				"contact"				, "varchar(255)"	, "NULL",
+				"policy_type"			, "varchar(20)"	, "NULL",
+				"policy_string"		, "text"				, "NULL",
+				"summary_failure"		, "int"				, "unsigned NULL",
+				"summary_successful"	, "int"				, "unsigned NULL",
+				"raw_json"				, "mediumtext"		, "",
+				],
+			additional_definitions 		=> "PRIMARY KEY (serial), UNIQUE KEY domain (domain,reportid)",
+			table_options			=> "ROW_FORMAT=COMPRESSED",
+			indexes				=> [],
+			},
+			"tls_rptrecord" => {
+			column_definitions 		=> [
+				"id"								, "int"					, "unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY",
+				"serial"							, "int"					, "unsigned NOT NULL",
+				"result_type"					, "varchar(255)"		, "",
+				"sending_mta_ip"				, "int"					, "unsigned",
+				"sending_mta_ip6"				, "binary(16)"			, "",
+				"receiving_mx_hostname"		, "varchar(255)"		, "",
+				"receiving_mx_helo"			, "varchar(255)"		, "",
+				"receiving_ip"					, "int"					, "unsigned",
+				"receiving_ip6"				, "binary(16)"			, "",
+				"failed_session_count"		, "int"					, "unsigned NOT NULL",
+				"additional_information"	, "varchar(255)"		, "",
+				"failure_reason_code"		, "varchar(255)"		, "",
+				],
+			additional_definitions 		=> "KEY serial (serial,sending_mta_ip), KEY serial6 (serial,sending_mta_ip6)",
+			table_options			=> "",
+			indexes				=> [],
+			},
 		},
 
 	add_column => sub {

diff --git a/dmarcts-report-parser.conf.sample b/dmarcts-report-parser.conf.sample
@@ -23,8 +23,8 @@ $imapport         = '143';
 $imapssl          = '0';        # If set to 1, remember to change server port to 993 and disable imaptls.
 $imaptls          = '1';        # Enabled as the default and best-practice.
 $tlsverify        = '1';        # Enable verify server cert as the default and best-practice.
-$imapignoreerror  = '0';          # set it to 1 if you see an "ERROR: message_string() 
-                                # expected 119613 bytes but received 81873 you may 
+$imapignoreerror  = '0';          # set it to 1 if you see an "ERROR: message_string()
+                                # expected 119613 bytes but received 81873 you may
                                 # need the IgnoreSizeErrors option" because of malfunction
                                 # imap server as MS Exchange 2007, ...
 $imapreadfolder   = 'dmarc';
@@ -38,12 +38,12 @@ $imapmovefolder = 'dmarc/processed';
 $imapmovefoldererr = 'Inbox.notProcessed';
 
 # maximum size of XML files to store in database, long files can cause transaction aborts
-$maxsize_xml = 50000;
+$raw_data_max_size = 50000;
 # store XML as base64 encopded gzip in database (save space, harder usable)
-$compress_xml = 0;
+$raw_data_compress = 0;
 
-# if there was an error during file processing (message does not contain XML or ZIP parts, 
-# or a database error) the parser reports an error and does not delete the file, even if 
-# delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed, 
+# if there was an error during file processing (message does not contain XML or ZIP parts,
+# or a database error) the parser reports an error and does not delete the file, even if
+# delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed,
 # however not for database errors.
 $delete_failed = 0;