Skip to content

Commit

Permalink
Update for Zeek 6, and fix longstanding bug.
Browse files Browse the repository at this point in the history 
In the past, empty sets and tables were incorrectly stored as NULL in
the postgres tables. Now they are stored as empty arrays.

This bug was noticed, due to stricter behavior of new Zeek releases when
reading data. If you have old tables, you might have to update them, and
replace fields containing nulls with empty arrays.
  • Loading branch information
@0xxon
0xxon committed Jul 21, 2023
1 parent b2b7090 commit 4268bcf
Show file tree
Hide file tree
Showing 10 changed files with 62 additions and 77 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
cmake_minimum_required(VERSION 3.15 FATAL_ERROR)

project(ZeekPluginPostgres)

Expand Down
6 changes: 3 additions & 3 deletions src/PostgresWriter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ std::tuple<bool, std::string, int> PostgreSQL::CreateParams(const Value* val)
case zeek::TYPE_TABLE:
case zeek::TYPE_VECTOR:
{
bro_int_t size;
zeek_int_t size;
Value** vals;

std::string out("{");
Expand All @@ -330,7 +330,7 @@ std::tuple<bool, std::string, int> PostgreSQL::CreateParams(const Value* val)
}

if ( ! size )
return std::make_tuple(false, std::string(), 0);
return std::make_tuple(true, std::string("{}"), 2);

for ( int i = 0; i < size; ++i )
{
Expand Down Expand Up @@ -384,7 +384,7 @@ bool PostgreSQL::DoWrite(int num_fields, const Field* const* fields, Value** val
params.push_back(CreateParams(vals[i]));

std::vector<const char*> params_char; // vector in which we compile the character pointers that we
// then pass to PQexecParams. These do not have to be cleaned up because the srings will be
// then pass to PQexecParams. These do not have to be cleaned up because the strings will be
// cleaned up automatically.
std::vector<int> params_length; // vector in which we compile the lengths of the parameters that we
// then pass to PQexecParams
Expand Down
21 changes: 2 additions & 19 deletions tests/Baseline/postgres.write-basic/ssh.out
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
id|b|i|e|c|p|sn|a|d|t|iv|s|sc|ss|se|vc|ve|f|vo|so
1|t|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|XXXXXXXXXX.XXXXXX|100|hurz|{4,2,3,1}|{CC,BB,AA}||{10,20,30}||SSHTest::foo
{
if (0 < SSHTest::i)
return (Foo);
else
return (Bar);

}||
2|t|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|XXXXXXXXXX.XXXXXX|100|hurz|{4,2,3,1}|{"\"","{\"\"\\hello","a b
cd~e","{{{{{}'","","\\\"\\{}"}||{10,20,30}||SSHTest::foo
{
if (0 < SSHTest::i)
return (Foo);
else
return (Bar);

}||
(2 rows)
1 t -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 XXXXXXXXXX.XXXXXX 100 hurz {4,2,3,1} {CC,BB,AA} {} {10,20,30} {} SSHTest::foo\n{ \nif (0 < SSHTest::i) \n\treturn (Foo);\nelse\n\treturn (Bar);\n\n} \N \N
2 t -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 XXXXXXXXXX.XXXXXX 100 hurz {4,2,3,1} {"\\"","{\\"\\"\\\\hello","a\tb\nc\rd~e","{{{{{}'","","\\\\\\"\\\\{}"} {} {10,20,30} {} SSHTest::foo\n{ \nif (0 < SSHTest::i) \n\treturn (Foo);\nelse\n\treturn (Bar);\n\n} \N \N
2 changes: 1 addition & 1 deletion tests/Baseline/postgres.write-bytea/ssh.out
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
id|b|i|e|c|p|sn|a|d|t|iv|s|sc|ss|se|vc|ve|f
1|t|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|XXXXXXXXXX.XXXXXX|100|\x6875727a|{4,2,3,1}|{"\\x4343","\\x4242","\\x4141"}||{10,20,30}||\x535348546573743a3a666f6f0a7b200a6966202830203c20535348546573743a3a6929200a0972657475726e2028466f6f293b0a656c73650a0972657475726e2028426172293b0a0a7d
1|t|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|XXXXXXXXXX.XXXXXX|100|\x6875727a|{4,2,3,1}|{"\\x4343","\\x4242","\\x4141"}|{}|{10,20,30}|{}|\x535348546573743a3a666f6f0a7b200a6966202830203c20535348546573743a3a6929200a0972657475726e2028466f6f293b0a656c73650a0972657475726e2028426172293b0a0a7d
(1 row)
2 changes: 1 addition & 1 deletion tests/Baseline/postgres.write-error/ssh.out
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
id|b|i|e|c|p|sn|a|d|t|iv|s|sc|ss|se|vc|ve|f
1|t|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|XXXXXXXXXX.XXXXXX|100|h|{4,2,3,1}|{CC,BB,AA}||{10,20,30}||SSHTest::foo
1|t|-42|SSH::LOG|21|123|10.0.0.0/24|1.2.3.4|3.14|XXXXXXXXXX.XXXXXX|100|h|{4,2,3,1}|{CC,BB,AA}|{}|{10,20,30}|{}|SSHTest::foo
{
if (0 < SSHTest::i)
return (Foo);
Expand Down
69 changes: 35 additions & 34 deletions tests/Baseline/postgres.write-wikipedia/conn.out
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,37 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
id|ts|uid|id.orig_h|id.orig_p|id.resp_h|id.resp_p|proto|service|duration|orig_bytes|resp_bytes|conn_state|local_orig|local_resp|missed_bytes|history|orig_pkts|orig_ip_bytes|resp_pkts|resp_ip_bytes|tunnel_parents
1|1300475167.096535|CHhAvVGS1DHFjwGM9|141.142.220.202|5353|224.0.0.251|5353|udp|dns||||S0|||0|D|1|73|0|0|
2|1300475167.097012|ClEkJM2Vm5giqnMf4h|fe80::217:f2ff:fed7:cf65|5353|ff02::fb|5353|udp|dns||||S0|||0|D|1|199|0|0|
3|1300475167.099816|C4J4Th3PJpwUYZZ6gc|141.142.220.50|5353|224.0.0.251|5353|udp|dns||||S0|||0|D|1|179|0|0|
4|1300475168.853899|CmES5u32sYpV7JYN|141.142.220.118|43927|141.142.2.2|53|udp|dns|0.000435|38|89|SF|||0|Dd|1|66|1|117|
5|1300475168.854378|CP5puj4I8PtEU4qzYg|141.142.220.118|37676|141.142.2.2|53|udp|dns|0.00042|52|99|SF|||0|Dd|1|80|1|127|
6|1300475168.854837|C37jN32gN3y3AZzyf6|141.142.220.118|40526|141.142.2.2|53|udp|dns|0.000392|38|183|SF|||0|Dd|1|66|1|211|
7|1300475168.857956|C0LAHyvtKSQHyJxIl|141.142.220.118|32902|141.142.2.2|53|udp|dns|0.000317|38|89|SF|||0|Dd|1|66|1|117|
8|1300475168.858306|CFLRIC3zaTU1loLGxh|141.142.220.118|59816|141.142.2.2|53|udp|dns|0.000343|52|99|SF|||0|Dd|1|80|1|127|
9|1300475168.858713|C9rXSW3KSpTYvPrlI1|141.142.220.118|59714|141.142.2.2|53|udp|dns|0.000375|38|183|SF|||0|Dd|1|66|1|211|
10|1300475168.891644|C9mvWx3ezztgzcexV7|141.142.220.118|58206|141.142.2.2|53|udp|dns|0.000339|38|89|SF|||0|Dd|1|66|1|117|
11|1300475168.892037|CNnMIj2QSd84NKf7U3|141.142.220.118|38911|141.142.2.2|53|udp|dns|0.000335|52|99|SF|||0|Dd|1|80|1|127|
12|1300475168.892414|C7fIlMZDuRiqjpYbb|141.142.220.118|59746|141.142.2.2|53|udp|dns|0.000421|38|183|SF|||0|Dd|1|66|1|211|
13|1300475168.893988|CpmdRlaUoJLN3uIRa|141.142.220.118|45000|141.142.2.2|53|udp|dns|0.000384|38|89|SF|||0|Dd|1|66|1|117|
14|1300475168.894422|C1Xkzz2MaGtLrc1Tla|141.142.220.118|48479|141.142.2.2|53|udp|dns|0.000317|52|99|SF|||0|Dd|1|80|1|127|
15|1300475168.894787|CqlVyW1YwZ15RhTBc4|141.142.220.118|48128|141.142.2.2|53|udp|dns|0.000423|38|183|SF|||0|Dd|1|66|1|211|
16|1300475168.901749|CBA8792iHmnhPLksKa|141.142.220.118|56056|141.142.2.2|53|udp|dns|0.000402|36|131|SF|||0|Dd|1|64|1|159|
17|1300475168.902195|CGLPPc35OzDQij1XX8|141.142.220.118|55092|141.142.2.2|53|udp|dns|0.000374|36|198|SF|||0|Dd|1|64|1|226|
18|1300475169.899438|Cipfzj1BEnhejw8cGf|141.142.220.44|5353|224.0.0.251|5353|udp|dns||||S0|||0|D|1|85|0|0|
19|1300475170.862384|CV5WJ42jPYbNW9JNWf|141.142.220.226|137|141.142.220.255|137|udp|dns|2.613017|350|0|S0|||0|D|7|546|0|0|
20|1300475171.675372|CPhDKt12KQPUVbQz06|fe80::3074:17d5:2052:c324|65373|ff02::1:3|5355|udp|dns|0.100096|66|0|S0|||0|D|2|162|0|0|
21|1300475171.677081|CAnFrb2Cvxr5T7quOc|141.142.220.226|55131|224.0.0.252|5355|udp|dns|0.100021|66|0|S0|||0|D|2|122|0|0|
22|1300475173.116749|C8rquZ3DjgNW06JGLl|fe80::3074:17d5:2052:c324|54213|ff02::1:3|5355|udp|dns|0.099801|66|0|S0|||0|D|2|162|0|0|
23|1300475173.117362|CzrZOtXqhwwndQva3|141.142.220.226|55671|224.0.0.252|5355|udp|dns|0.099849|66|0|S0|||0|D|2|122|0|0|
24|1300475173.153679|CaGCc13FffXe6RkQl9|141.142.220.238|56641|141.142.220.255|137|udp|dns||||S0|||0|D|1|78|0|0|
25|1300475168.652003|CtPZjS20MLrsMUOJi2|141.142.220.118|35634|208.80.152.2|80|tcp||0.061329|463|350|OTH|||0|DdA|2|567|1|402|
26|1300475168.902635|CiyBAq1bBLNaTiTAc|141.142.220.118|35642|208.80.152.2|80|tcp|http|0.120041|534|412|S1|||0|ShADad|4|750|3|576|
27|1300475168.855305|C3eiCBGOLw3VtHfOj|141.142.220.118|49996|208.80.152.3|80|tcp|http|0.218501|1171|733|S1|||0|ShADad|6|1491|4|949|
28|1300475168.85533|CwjjYJ2WqgTbAqiHl6|141.142.220.118|49997|208.80.152.3|80|tcp|http|0.21972|1125|734|S1|||0|ShADad|6|1445|4|950|
29|1300475168.859163|Ck51lg1bScffFj34Ri|141.142.220.118|49998|208.80.152.3|80|tcp|http|0.215893|1130|734|S1|||0|ShADad|6|1450|4|950|
30|1300475168.892913|CykQaM33ztNt0csB9a|141.142.220.118|49999|208.80.152.3|80|tcp|http|0.220961|1137|733|S1|||0|ShADad|6|1457|4|949|
31|1300475168.892936|CtxTCR2Yer0FR1tIBg|141.142.220.118|50000|208.80.152.3|80|tcp|http|0.229603|1148|734|S1|||0|ShADad|6|1468|4|950|
32|1300475168.895267|CLNN1k2QMum1aexUK7|141.142.220.118|50001|208.80.152.3|80|tcp|http|0.227284|1178|734|S1|||0|ShADad|6|1498|4|950|
33|1300475168.724007|CUM0KZ3MLUfNB0cl11|141.142.220.118|48649|208.80.152.118|80|tcp|http|0.119905|525|232|S1|||0|ShADad|4|741|3|396|
34|1300475169.780331|CFSwNi4CNGxcuffo49|141.142.220.235|6705|173.192.163.128|80|tcp|||||OTH|||0|^h|0|0|1|48|
1|XXXXXXXXXX.XXXXXX|CHhAvVGS1DHFjwGM9|141.142.220.202|5353|224.0.0.251|5353|udp|dns||||S0|f|f|0|D|1|73|0|0|
2|XXXXXXXXXX.XXXXXX|ClEkJM2Vm5giqnMf4h|fe80::217:f2ff:fed7:cf65|5353|ff02::fb|5353|udp|dns||||S0|t|f|0|D|1|199|0|0|
3|XXXXXXXXXX.XXXXXX|C4J4Th3PJpwUYZZ6gc|141.142.220.50|5353|224.0.0.251|5353|udp|dns||||S0|f|f|0|D|1|179|0|0|
4|XXXXXXXXXX.XXXXXX|CmES5u32sYpV7JYN|141.142.220.118|43927|141.142.2.2|53|udp|dns|0.000435|38|89|SF|f|f|0|Dd|1|66|1|117|
5|XXXXXXXXXX.XXXXXX|CP5puj4I8PtEU4qzYg|141.142.220.118|37676|141.142.2.2|53|udp|dns|0.00042|52|99|SF|f|f|0|Dd|1|80|1|127|
6|XXXXXXXXXX.XXXXXX|C37jN32gN3y3AZzyf6|141.142.220.118|40526|141.142.2.2|53|udp|dns|0.000392|38|183|SF|f|f|0|Dd|1|66|1|211|
7|XXXXXXXXXX.XXXXXX|C0LAHyvtKSQHyJxIl|141.142.220.118|32902|141.142.2.2|53|udp|dns|0.000317|38|89|SF|f|f|0|Dd|1|66|1|117|
8|XXXXXXXXXX.XXXXXX|CFLRIC3zaTU1loLGxh|141.142.220.118|59816|141.142.2.2|53|udp|dns|0.000343|52|99|SF|f|f|0|Dd|1|80|1|127|
9|XXXXXXXXXX.XXXXXX|C9rXSW3KSpTYvPrlI1|141.142.220.118|59714|141.142.2.2|53|udp|dns|0.000375|38|183|SF|f|f|0|Dd|1|66|1|211|
10|XXXXXXXXXX.XXXXXX|C9mvWx3ezztgzcexV7|141.142.220.118|58206|141.142.2.2|53|udp|dns|0.000339|38|89|SF|f|f|0|Dd|1|66|1|117|
11|XXXXXXXXXX.XXXXXX|CNnMIj2QSd84NKf7U3|141.142.220.118|38911|141.142.2.2|53|udp|dns|0.000335|52|99|SF|f|f|0|Dd|1|80|1|127|
12|XXXXXXXXXX.XXXXXX|C7fIlMZDuRiqjpYbb|141.142.220.118|59746|141.142.2.2|53|udp|dns|0.000421|38|183|SF|f|f|0|Dd|1|66|1|211|
13|XXXXXXXXXX.XXXXXX|CpmdRlaUoJLN3uIRa|141.142.220.118|45000|141.142.2.2|53|udp|dns|0.000384|38|89|SF|f|f|0|Dd|1|66|1|117|
14|XXXXXXXXXX.XXXXXX|C1Xkzz2MaGtLrc1Tla|141.142.220.118|48479|141.142.2.2|53|udp|dns|0.000317|52|99|SF|f|f|0|Dd|1|80|1|127|
15|XXXXXXXXXX.XXXXXX|CqlVyW1YwZ15RhTBc4|141.142.220.118|48128|141.142.2.2|53|udp|dns|0.000423|38|183|SF|f|f|0|Dd|1|66|1|211|
16|XXXXXXXXXX.XXXXXX|CBA8792iHmnhPLksKa|141.142.220.118|56056|141.142.2.2|53|udp|dns|0.000402|36|131|SF|f|f|0|Dd|1|64|1|159|
17|XXXXXXXXXX.XXXXXX|CGLPPc35OzDQij1XX8|141.142.220.118|55092|141.142.2.2|53|udp|dns|0.000374|36|198|SF|f|f|0|Dd|1|64|1|226|
18|XXXXXXXXXX.XXXXXX|Cipfzj1BEnhejw8cGf|141.142.220.44|5353|224.0.0.251|5353|udp|dns||||S0|f|f|0|D|1|85|0|0|
19|XXXXXXXXXX.XXXXXX|CV5WJ42jPYbNW9JNWf|141.142.220.226|137|141.142.220.255|137|udp|dns|2.613017|350|0|S0|f|f|0|D|7|546|0|0|
20|XXXXXXXXXX.XXXXXX|CPhDKt12KQPUVbQz06|fe80::3074:17d5:2052:c324|65373|ff02::1:3|5355|udp|dns|0.100096|66|0|S0|t|f|0|D|2|162|0|0|
21|XXXXXXXXXX.XXXXXX|CAnFrb2Cvxr5T7quOc|141.142.220.226|55131|224.0.0.252|5355|udp|dns|0.100021|66|0|S0|f|f|0|D|2|122|0|0|
22|XXXXXXXXXX.XXXXXX|C8rquZ3DjgNW06JGLl|fe80::3074:17d5:2052:c324|54213|ff02::1:3|5355|udp|dns|0.099801|66|0|S0|t|f|0|D|2|162|0|0|
23|XXXXXXXXXX.XXXXXX|CzrZOtXqhwwndQva3|141.142.220.226|55671|224.0.0.252|5355|udp|dns|0.099849|66|0|S0|f|f|0|D|2|122|0|0|
24|XXXXXXXXXX.XXXXXX|CaGCc13FffXe6RkQl9|141.142.220.238|56641|141.142.220.255|137|udp|dns||||S0|f|f|0|D|1|78|0|0|
25|XXXXXXXXXX.XXXXXX|CtPZjS20MLrsMUOJi2|141.142.220.118|35634|208.80.152.2|80|tcp||0.061329|463|350|OTH|f|f|0|DdA|2|567|1|402|
26|XXXXXXXXXX.XXXXXX|CiyBAq1bBLNaTiTAc|141.142.220.118|35642|208.80.152.2|80|tcp|http|0.120041|534|412|S1|f|f|0|ShADad|4|750|3|576|
27|XXXXXXXXXX.XXXXXX|C3eiCBGOLw3VtHfOj|141.142.220.118|49996|208.80.152.3|80|tcp|http|0.218501|1171|733|S1|f|f|0|ShADad|6|1491|4|949|
28|XXXXXXXXXX.XXXXXX|CwjjYJ2WqgTbAqiHl6|141.142.220.118|49997|208.80.152.3|80|tcp|http|0.21972|1125|734|S1|f|f|0|ShADad|6|1445|4|950|
29|XXXXXXXXXX.XXXXXX|Ck51lg1bScffFj34Ri|141.142.220.118|49998|208.80.152.3|80|tcp|http|0.215893|1130|734|S1|f|f|0|ShADad|6|1450|4|950|
30|XXXXXXXXXX.XXXXXX|CykQaM33ztNt0csB9a|141.142.220.118|49999|208.80.152.3|80|tcp|http|0.220961|1137|733|S1|f|f|0|ShADad|6|1457|4|949|
31|XXXXXXXXXX.XXXXXX|CtxTCR2Yer0FR1tIBg|141.142.220.118|50000|208.80.152.3|80|tcp|http|0.229603|1148|734|S1|f|f|0|ShADad|6|1468|4|950|
32|XXXXXXXXXX.XXXXXX|CLNN1k2QMum1aexUK7|141.142.220.118|50001|208.80.152.3|80|tcp|http|0.227284|1178|734|S1|f|f|0|ShADad|6|1498|4|950|
33|XXXXXXXXXX.XXXXXX|CUM0KZ3MLUfNB0cl11|141.142.220.118|48649|208.80.152.118|80|tcp|http|0.119905|525|232|S1|f|f|0|ShADad|4|741|3|396|
34|XXXXXXXXXX.XXXXXX|CFSwNi4CNGxcuffo49|141.142.220.235|6705|173.192.163.128|80|tcp|||||OTH|f|f|0|^h|0|0|1|48|
(34 rows)
Loading

0 comments on commit 4268bcf

Please sign in to comment.