diff --git a/LICENSE b/LICENSE index 8dada3e..984f944 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -199,3 +199,5 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + diff --git a/M2LPlugin/README.md b/M2LPlugin/README.md new file mode 100644 index 0000000..d251be5 --- /dev/null +++ b/M2LPlugin/README.md @@ -0,0 +1,10 @@ +Testing: + +#1 +mvn clean install + +#2 +./test.sh examples/example_mspl_log_0.xml bro_json_config.json + +- This will validate the given M2L with schema/MSPL_XML_Schema.xsd and then convert the M2L into bro JSON config. + diff --git a/M2LPlugin/examples/example.json b/M2LPlugin/examples/example.json new file mode 100644 index 0000000..0f1b7ee --- /dev/null +++ b/M2LPlugin/examples/example.json @@ -0,0 +1,37 @@ +{ + "rules": [ + { "id": "rule1", + "event": "EVENT_CONNECTION", + "operation": "count.bro", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "source", + "value": { "address": "123.45.67.89" } + } + ] + }, + { "id": "rule2", + "event": "EVENT_FILE", + "operation": "detect-MHR.bro", + "parameters": [ ], + "action": "log", + "conditions": [ + { "type": "mime-type", + "value": "application/pdf" + }, + { "type": "mime-type", + "value": "application/x-dosexec" + } + ] + } + ] +} diff --git a/M2LPlugin/examples/example_mspl_log_0.base64 b/M2LPlugin/examples/example_mspl_log_0.base64 new file mode 100644 index 0000000..dd816aa --- /dev/null +++ b/M2LPlugin/examples/example_mspl_log_0.base64 @@ -0,0 +1 @@ +PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9InllcyI/Pgo8SVRSZXNvdXJjZSB4bWxucz0iaHR0cDovL21vZGVsaW9zb2Z0L3hzZGRlc2lnbmVyL2EyMmJkNjBiLWVlM2QtNDI1Yy04NjE4LWJlYjZhODU0MDUxYS9JVFJlc291cmNlLnhzZCIgSUQ9Ik1TUExfMDI1MzU2M2UtYzM3Ni00NzdiLWI2MjctYjMzNTc0ODg0NDkxIj4KICA8Y29uZmlndXJhdGlvbiB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIgogICAgICAgICAgICAgICAgIHhzaTp0eXBlPSJSdWxlU2V0Q29uZmlndXJhdGlvbiI+CiAgICA8Y2FwYWJpbGl0eT4KICAgICAgPE5hbWU+TG9nZ2luZzwvTmFtZT4KICAgIDwvY2FwYWJpbGl0eT4KICAgIDxkZWZhdWx0QWN0aW9uIHhzaTp0eXBlPSJMb2dnaW5nQWN0aW9uIj4KICAgICAgPGxvZ2dpbmdBY3Rpb25UeXBlPmxvZ19jb25uZWN0aW9uPC9sb2dnaW5nQWN0aW9uVHlwZT4KICAgIDwvZGVmYXVsdEFjdGlvbj4KICAgIDxjb25maWd1cmF0aW9uUnVsZT4KICAgICAgPGNvbmZpZ3VyYXRpb25SdWxlQWN0aW9uIHhzaTp0eXBlPSJMb2dnaW5nQWN0aW9uIj4KICAgICAgICA8bG9nZ2luZ0FjdGlvblR5cGU+bG9nX2Nvbm5lY3Rpb248L2xvZ2dpbmdBY3Rpb25UeXBlPgogICAgICA8L2NvbmZpZ3VyYXRpb25SdWxlQWN0aW9uPgogICAgICA8Y29uZmlndXJhdGlvbkNvbmRpdGlvbiB4c2k6dHlwZT0iTG9nZ2luZ0NvbmRpdGlvbiI+CiAgICAgICAgPGlzQ05GPmZhbHNlPC9pc0NORj4KICAgICAgICA8ZXZlbnRDb25kaXRpb24+CiAgICAgICAgICA8ZXZlbnRzPkVWRU5UX0NPTk5FQ1RJT048L2V2ZW50cz4KICAgICAgICAgIDxpbnRlcnZhbD4zMDwvaW50ZXJ2YWw+CiAgICAgICAgICA8dGhyZXNob2xkPjUwPC90aHJlc2hvbGQ+CiAgICAgICAgPC9ldmVudENvbmRpdGlvbj4KICAgICAgICA8cGFja2V0Q29uZGl0aW9uPgogICAgICAgICAgPFNvdXJjZUFkZHJlc3M+MTIzLjQ1LjY3Ljg5LDEyMy40NS42Ny45MCwxMjMuNDUuNjcuOTEsPC9Tb3VyY2VBZGRyZXNzPgogICAgICAgIDwvcGFja2V0Q29uZGl0aW9uPgogICAgICA8L2NvbmZpZ3VyYXRpb25Db25kaXRpb24+CiAgICAgIDxleHRlcm5hbERhdGEgeHNpOnR5cGU9IlByaW9yaXR5Ij4KICAgICAgICA8dmFsdWU+MDwvdmFsdWU+CiAgICAgIDwvZXh0ZXJuYWxEYXRhPgogICAgICA8TmFtZT5SdWxlMDwvTmFtZT4KICAgICAgPGlzQ05GPmZhbHNlPC9pc0NORj4KICAgICAgPEhTUEwgSFNQTF9pZD0iSFNQTDNfU29uX0lTUCIgSFNQTF90ZXh0PSJzb24gZW5hYmxlIGxvZ2dpbmcgY291bnRfY29ubmVjdGlvbiwgIHZ0dF9hZGRyZXNzLCAgIi8+CiAgICA8L2NvbmZpZ3VyYXRpb25SdWxlPgogICAgPHJlc29sdXRpb25TdHJhdGVneSB4c2k6dHlwZT0iRk1SIi8+CiAgICA8TmFtZT5NU1BMXzAyNTM1NjNlLWMzNzYtNDc3Yi1iNjI3LWIzMzU3NDg4NDQ5MTwvTmFtZT4KICA8L2NvbmZpZ3VyYXRpb24+CjwvSVRSZXNvdXJjZT4K \ No newline at end of file diff --git a/M2LPlugin/examples/example_mspl_log_0.json b/M2LPlugin/examples/example_mspl_log_0.json new file mode 100644 index 0000000..55cf865 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_log_0.json @@ -0,0 +1 @@ +{"rules":[{"id":"Rule0","operation":"count","event":"EVENT_CONNECTION","action":"log","parameters":[{"type":"object","value":"OBJ_CONNECTION"}],"conditions":[{"type":"interval","value":30},{"type":"threshold","value":50},{"type":"source","value":{"address":"123.45.67.89"}},{"type":"source","value":{"address":"123.45.67.90"}},{"type":"source","value":{"address":"123.45.67.91"}}]}]} \ No newline at end of file diff --git a/M2LPlugin/examples/example_mspl_log_0.xml b/M2LPlugin/examples/example_mspl_log_0.xml new file mode 100644 index 0000000..9461ef6 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_log_0.xml @@ -0,0 +1,36 @@ + + + + + Logging + + + log_connection + + + + log_connection + + + false + + EVENT_CONNECTION + 30 + 50 + + + 123.45.67.89,123.45.67.90,123.45.67.91, + + + + 0 + + Rule0 + false + + + + MSPL_0253563e-c376-477b-b627-b33574884491 + + diff --git a/M2LPlugin/examples/example_mspl_log_2.base64 b/M2LPlugin/examples/example_mspl_log_2.base64 new file mode 100644 index 0000000..7672678 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_log_2.base64 @@ -0,0 +1,30 @@ +PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9InllcyI/Pgo8 +SVRSZXNvdXJjZSB4bWxucz0iaHR0cDovL21vZGVsaW9zb2Z0L3hzZGRlc2lnbmVyL2EyMmJkNjBi +LWVlM2QtNDI1Yy04NjE4LWJlYjZhODU0MDUxYS9JVFJlc291cmNlLnhzZCIgSUQ9Ik1TUExfOTE5 +MGNiM2ItYzA2Yi00NmFkLWEzNmMtYTkzZDA5NzJjMjYzIj4KICAgIDxjb25maWd1cmF0aW9uIHht +bG5zOnhzaT0iaHR0cDovL3d3dy53My5vcmcvMjAwMS9YTUxTY2hlbWEtaW5zdGFuY2UiIHhzaTp0 +eXBlPSJSdWxlU2V0Q29uZmlndXJhdGlvbiI+CiAgICAgICAgPGNhcGFiaWxpdHk+CiAgICAgICAg +ICAgIDxOYW1lPkxvZ2dpbmc8L05hbWU+CiAgICAgICAgPC9jYXBhYmlsaXR5PgogICAgICAgIDxk +ZWZhdWx0QWN0aW9uIHhzaTp0eXBlPSJMb2dnaW5nQWN0aW9uIj4KICAgICAgICAgICAgPGxvZ2dp +bmdBY3Rpb25UeXBlPmxvZ19jb25uZWN0aW9uPC9sb2dnaW5nQWN0aW9uVHlwZT4KICAgICAgICA8 +L2RlZmF1bHRBY3Rpb24+CiAgICAgICAgPGNvbmZpZ3VyYXRpb25SdWxlPgogICAgICAgICAgICA8 +Y29uZmlndXJhdGlvblJ1bGVBY3Rpb24geHNpOnR5cGU9IkxvZ2dpbmdBY3Rpb24iPgogICAgICAg +ICAgICAgICAgPGxvZ2dpbmdBY3Rpb25UeXBlPmxvZ19jb25uZWN0aW9uPC9sb2dnaW5nQWN0aW9u +VHlwZT4KICAgICAgICAgICAgPC9jb25maWd1cmF0aW9uUnVsZUFjdGlvbj4KICAgICAgICAgICAg +PGNvbmZpZ3VyYXRpb25Db25kaXRpb24geHNpOnR5cGU9IkxvZ2dpbmdDb25kaXRpb24iPgogICAg +ICAgICAgICAgICAgPGlzQ05GPmZhbHNlPC9pc0NORj4KICAgICAgICAgICAgICAgIDxldmVudENv +bmRpdGlvbj4KICAgICAgICAgICAgICAgICAgICA8ZXZlbnRzPkVWRU5UX0NPTk5FQ1RJT048L2V2 +ZW50cz4KICAgICAgICAgICAgICAgICAgICA8aW50ZXJ2YWw+MzA8L2ludGVydmFsPgogICAgICAg +ICAgICAgICAgICAgIDx0aHJlc2hvbGQ+NTA8L3RocmVzaG9sZD4KICAgICAgICAgICAgICAgIDwv +ZXZlbnRDb25kaXRpb24+CiAgICAgICAgICAgICAgICA8cGFja2V0Q29uZGl0aW9uPgogICAgICAg +ICAgICAgICAgICAgIDxTb3VyY2VBZGRyZXNzPjEyMy40NS42Ny44OSwxMjMuNDUuNjcuOTAsMTIz +LjQ1LjY3LjkxLDwvU291cmNlQWRkcmVzcz4KICAgICAgICAgICAgICAgIDwvcGFja2V0Q29uZGl0 +aW9uPgogICAgICAgICAgICA8L2NvbmZpZ3VyYXRpb25Db25kaXRpb24+CiAgICAgICAgICAgIDxl +eHRlcm5hbERhdGEgeHNpOnR5cGU9IlByaW9yaXR5Ij4KICAgICAgICAgICAgICAgIDx2YWx1ZT4w +PC92YWx1ZT4KICAgICAgICAgICAgPC9leHRlcm5hbERhdGE+CiAgICAgICAgICAgIDxOYW1lPlJ1 +bGUwPC9OYW1lPgogICAgICAgICAgICA8aXNDTkY+ZmFsc2U8L2lzQ05GPgogICAgICAgICAgICA8 +SFNQTCBIU1BMX2lkPSJIU1BMM19Tb25fSVNQIiBIU1BMX3RleHQ9InNvbiBlbmFibGUgbG9nZ2lu +ZyBjb3VudF9jb25uZWN0aW9uLCAgdnR0X2FkZHJlc3MsICAiLz4KICAgICAgICA8L2NvbmZpZ3Vy +YXRpb25SdWxlPgogICAgICAgIDxyZXNvbHV0aW9uU3RyYXRlZ3kgeHNpOnR5cGU9IkZNUiIvPgog +ICAgICAgIDxOYW1lPk1TUExfOTE5MGNiM2ItYzA2Yi00NmFkLWEzNmMtYTkzZDA5NzJjMjYzPC9O +YW1lPgogICAgPC9jb25maWd1cmF0aW9uPgo8L0lUUmVzb3VyY2U+Cg== diff --git a/M2LPlugin/examples/example_mspl_log_2.xml b/M2LPlugin/examples/example_mspl_log_2.xml new file mode 100644 index 0000000..271e230 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_log_2.xml @@ -0,0 +1,35 @@ + + + + + Logging + + + log_connection + + + + log_connection + + + false + + EVENT_CONNECTION + 30 + 50 + + + 123.45.67.89,123.45.67.90,123.45.67.91, + + + + 0 + + Rule0 + false + + + + MSPL_9190cb3b-c06b-46ad-a36c-a93d0972c263 + + diff --git a/M2LPlugin/examples/example_mspl_log_3.xml b/M2LPlugin/examples/example_mspl_log_3.xml new file mode 100644 index 0000000..94577db --- /dev/null +++ b/M2LPlugin/examples/example_mspl_log_3.xml @@ -0,0 +1,38 @@ + + + + + Logging + + + log_connection + + + + log_connection + + + false + + EVENT_CONNECTION + 30 + 50 + + + + www.black-site.com,chat-paradise.com,chat.free.fr,chat.gratis.es, + + + + 0 + + Rule0 + false + + + + MSPL_b1a390f5-21b6-4cb2-b1ba-711e399d4833 + + \ No newline at end of file diff --git a/M2LPlugin/examples/example_mspl_mwd_0.base64 b/M2LPlugin/examples/example_mspl_mwd_0.base64 new file mode 100644 index 0000000..fefafa4 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_mwd_0.base64 @@ -0,0 +1 @@ +PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9InllcyI/Pgo8SVRSZXNvdXJjZSB4bWxucz0iaHR0cDovL21vZGVsaW9zb2Z0L3hzZGRlc2lnbmVyL2EyMmJkNjBiLWVlM2QtNDI1Yy04NjE4LWJlYjZhODU0MDUxYS9JVFJlc291cmNlLnhzZCIgSUQ9Ik1TUExfMzA5MWQxMzUtZWI2Ny00OGM3LWJmNjItMTIwMTVhNDdmMjVmIj4KICA8Y29uZmlndXJhdGlvbiB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIgogICAgICAgICAgICAgICAgIHhzaTp0eXBlPSJSdWxlU2V0Q29uZmlndXJhdGlvbiI+CiAgICA8Y2FwYWJpbGl0eT4KICAgICAgPE5hbWU+T2ZmbGluZV9tYWx3YXJlX2FuYWx5c2lzPC9OYW1lPgogICAgPC9jYXBhYmlsaXR5PgogICAgPGRlZmF1bHRBY3Rpb24geHNpOnR5cGU9IkFudGktbWFsd2FyZUFjdGlvbiI+CiAgICAgIDxhbnRpLW1hbHdhcmVBY3Rpb25UeXBlPjwvYW50aS1tYWx3YXJlQWN0aW9uVHlwZT4KICAgIDwvZGVmYXVsdEFjdGlvbj4KICAgIDxjb25maWd1cmF0aW9uUnVsZT4KICAgICAgPGNvbmZpZ3VyYXRpb25SdWxlQWN0aW9uIHhzaTp0eXBlPSJBbnRpLW1hbHdhcmVBY3Rpb24iPgogICAgICAgIDxhbnRpLW1hbHdhcmVBY3Rpb25UeXBlPjwvYW50aS1tYWx3YXJlQWN0aW9uVHlwZT4KICAgICAgPC9jb25maWd1cmF0aW9uUnVsZUFjdGlvbj4KICAgICAgPGNvbmZpZ3VyYXRpb25Db25kaXRpb24geHNpOnR5cGU9IkFudGktbWFsd2FyZUNvbmRpdGlvbiI+CiAgICAgICAgPGlzQ05GPmZhbHNlPC9pc0NORj4KICAgICAgICA8YXBwbGljYXRpb25MYXllckNvbmRpdGlvbj4KICAgICAgICAgIDxtaW1lVHlwZT5hcHBsaWNhdGlvbi94LWRvc2V4ZWMsPC9taW1lVHlwZT4KICAgICAgICA8L2FwcGxpY2F0aW9uTGF5ZXJDb25kaXRpb24+CiAgICAgIDwvY29uZmlndXJhdGlvbkNvbmRpdGlvbj4KICAgICAgPGV4dGVybmFsRGF0YSB4c2k6dHlwZT0iUHJpb3JpdHkiPgogICAgICAgIDx2YWx1ZT4wPC92YWx1ZT4KICAgICAgPC9leHRlcm5hbERhdGE+CiAgICAgIDxOYW1lPlJ1bGUwPC9OYW1lPgogICAgICA8aXNDTkY+ZmFsc2U8L2lzQ05GPgogICAgICA8SFNQTCBIU1BMX2lkPSJIU1BMNF9Tb25fSVNQIiBIU1BMX3RleHQ9InNvbiBlbmFibGUgbWFsd2FyZV9kZXRlY3Rpb24gc2Nhbl94ZG9zZXhlYywgICIvPgogICAgPC9jb25maWd1cmF0aW9uUnVsZT4KICAgIDxjb25maWd1cmF0aW9uUnVsZT4KICAgICAgPGNvbmZpZ3VyYXRpb25SdWxlQWN0aW9uIHhzaTp0eXBlPSJBbnRpLW1hbHdhcmVBY3Rpb24iPgogICAgICAgIDxhbnRpLW1hbHdhcmVBY3Rpb25UeXBlPjwvYW50aS1tYWx3YXJlQWN0aW9uVHlwZT4KICAgICAgPC9jb25maWd1cmF0aW9uUnVsZUFjdGlvbj4KICAgICAgPGNvbmZpZ3VyYXRpb25Db25kaXRpb24geHNpOnR5cGU9IkFudGktbWFsd2FyZUNvbmRpdGlvbiI+CiAgICAgICAgPGlzQ05GPmZhbHNlPC9pc0NORj4KICAgICAgICA8YXBwbGljYXRpb25MYXllckNvbmRpdGlvbj4KICAgICAgICAgIDxtaW1lVHlwZT5hcHBsaWNhdGlvbi9wZGYsPC9taW1lVHlwZT4KICAgICAgICA8L2FwcGxpY2F0aW9uTGF5ZXJDb25kaXRpb24+CiAgICAgIDwvY29uZmlndXJhdGlvbkNvbmRpdGlvbj4KICAgICAgPGV4dGVybmFsRGF0YSB4c2k6dHlwZT0iUHJpb3JpdHkiPgogICAgICAgIDx2YWx1ZT4xPC92YWx1ZT4KICAgICAgPC9leHRlcm5hbERhdGE+CiAgICAgIDxOYW1lPlJ1bGUxPC9OYW1lPgogICAgICA8aXNDTkY+ZmFsc2U8L2lzQ05GPgogICAgICA8SFNQTCBIU1BMX2lkPSJIU1BMNV9Tb25fSVNQIiBIU1BMX3RleHQ9InNvbiBlbmFibGUgbWFsd2FyZV9kZXRlY3Rpb24gc2Nhbl9wZGYsICAiLz4KICAgIDwvY29uZmlndXJhdGlvblJ1bGU+CiAgICA8cmVzb2x1dGlvblN0cmF0ZWd5IHhzaTp0eXBlPSJGTVIiLz4KICAgIDxOYW1lPk1TUExfMzA5MWQxMzUtZWI2Ny00OGM3LWJmNjItMTIwMTVhNDdmMjVmPC9OYW1lPgogIDwvY29uZmlndXJhdGlvbj4KPC9JVFJlc291cmNlPgo= \ No newline at end of file diff --git a/M2LPlugin/examples/example_mspl_mwd_0.base64.tmp b/M2LPlugin/examples/example_mspl_mwd_0.base64.tmp new file mode 100644 index 0000000..ca50205 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_mwd_0.base64.tmp @@ -0,0 +1,48 @@ + + + + + Offline_malware_analysis + + + + + + + + + + false + + application/x-dosexec, + + + + 0 + + Rule0 + false + + + + + + + + false + + application/pdf, + + + + 1 + + Rule1 + false + + + + MSPL_3091d135-eb67-48c7-bf62-12015a47f25f + + diff --git a/M2LPlugin/examples/example_mspl_mwd_0.xml b/M2LPlugin/examples/example_mspl_mwd_0.xml new file mode 100644 index 0000000..ca50205 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_mwd_0.xml @@ -0,0 +1,48 @@ + + + + + Offline_malware_analysis + + + + + + + + + + false + + application/x-dosexec, + + + + 0 + + Rule0 + false + + + + + + + + false + + application/pdf, + + + + 1 + + Rule1 + false + + + + MSPL_3091d135-eb67-48c7-bf62-12015a47f25f + + diff --git a/M2LPlugin/examples/example_mspl_mwd_2.base64 b/M2LPlugin/examples/example_mspl_mwd_2.base64 new file mode 100644 index 0000000..19ceb12 --- /dev/null +++ b/M2LPlugin/examples/example_mspl_mwd_2.base64 @@ -0,0 +1,45 @@ +PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9InllcyI/Pgo8 +SVRSZXNvdXJjZSB4bWxucz0iaHR0cDovL21vZGVsaW9zb2Z0L3hzZGRlc2lnbmVyL2EyMmJkNjBi +LWVlM2QtNDI1Yy04NjE4LWJlYjZhODU0MDUxYS9JVFJlc291cmNlLnhzZCIgSUQ9Ik1TUExfODRl +NDM4ZjktNzA3MS00MGRkLThlZWYtZTk1NWU1MTdhMmExIj4KICAgIDxjb25maWd1cmF0aW9uIHht +bG5zOnhzaT0iaHR0cDovL3d3dy53My5vcmcvMjAwMS9YTUxTY2hlbWEtaW5zdGFuY2UiIHhzaTp0 +eXBlPSJSdWxlU2V0Q29uZmlndXJhdGlvbiI+CiAgICAgICAgPGNhcGFiaWxpdHk+CiAgICAgICAg +ICAgIDxOYW1lPk9mZmxpbmVfbWFsd2FyZV9hbmFseXNpczwvTmFtZT4KICAgICAgICA8L2NhcGFi +aWxpdHk+CiAgICAgICAgPGRlZmF1bHRBY3Rpb24geHNpOnR5cGU9IkFudGktbWFsd2FyZUFjdGlv +biI+CiAgICAgICAgICAgIDxhbnRpLW1hbHdhcmVBY3Rpb25UeXBlPjwvYW50aS1tYWx3YXJlQWN0 +aW9uVHlwZT4KICAgICAgICA8L2RlZmF1bHRBY3Rpb24+CiAgICAgICAgPGNvbmZpZ3VyYXRpb25S +dWxlPgogICAgICAgICAgIDwhLS0gQWRkZWQgYXMgdGhlIGZpbGUgZGlkbid0IHZhbGlkYXRlIGFn +YWluc3QgdGhlIHNjaGVtYSAtLT4KICAgICAgICAgICA8Y29uZmlndXJhdGlvblJ1bGVBY3Rpb24g +eHNpOnR5cGU9IkFudGktbWFsd2FyZUFjdGlvbiIgPgogICAgICAgICAgICAgICA8YW50aS1tYWx3 +YXJlQWN0aW9uVHlwZT5TdHJpbmc8L2FudGktbWFsd2FyZUFjdGlvblR5cGU+IAogICAgICAgICAg +IDwvY29uZmlndXJhdGlvblJ1bGVBY3Rpb24+CiAgICAgICAgICAgPCEtLSAtLT4KICAgICAgICAg +ICAgPGNvbmZpZ3VyYXRpb25Db25kaXRpb24geHNpOnR5cGU9IkFudGktbWFsd2FyZUNvbmRpdGlv +biI+CiAgICAgICAgICAgICAgICA8aXNDTkY+ZmFsc2U8L2lzQ05GPgogICAgICAgICAgICAgICAg +PGFwcGxpY2F0aW9uTGF5ZXJDb25kaXRpb24+CiAgICAgICAgICAgICAgICAgICAgPG1pbWVUeXBl +PmFwcGxpY2F0aW9uL3gtZG9zZXhlYyw8L21pbWVUeXBlPgogICAgICAgICAgICAgICAgPC9hcHBs +aWNhdGlvbkxheWVyQ29uZGl0aW9uPgogICAgICAgICAgICA8L2NvbmZpZ3VyYXRpb25Db25kaXRp +b24+CiAgICAgICAgICAgIDxleHRlcm5hbERhdGEgeHNpOnR5cGU9IlByaW9yaXR5Ij4KICAgICAg +ICAgICAgICAgIDx2YWx1ZT4wPC92YWx1ZT4KICAgICAgICAgICAgPC9leHRlcm5hbERhdGE+CiAg +ICAgICAgICAgIDxOYW1lPlJ1bGUwPC9OYW1lPgogICAgICAgICAgICA8aXNDTkY+ZmFsc2U8L2lz +Q05GPgogICAgICAgICAgICA8SFNQTCBIU1BMX2lkPSJIU1BMNF9Tb25fSVNQIiBIU1BMX3RleHQ9 +InNvbiBlbmFibGUgbWFsd2FyZV9kZXRlY3Rpb24gbWFsd2FyZV9kZXRlY3Rpb24sICBzY2FuX3hk +b3NleGVjLCAgIi8+CiAgICAgICAgPC9jb25maWd1cmF0aW9uUnVsZT4KICAgICAgICA8Y29uZmln +dXJhdGlvblJ1bGU+CiAgICAgICAgICAgPCEtLSBBZGRlZCBhcyB0aGUgZmlsZSBkaWRuJ3QgdmFs +aWRhdGUgYWdhaW5zdCB0aGUgc2NoZW1hIC0tPgogICAgICAgICAgIDxjb25maWd1cmF0aW9uUnVs +ZUFjdGlvbiB4c2k6dHlwZT0iQW50aS1tYWx3YXJlQWN0aW9uIiA+CiAgICAgICAgICAgICAgIDxh +bnRpLW1hbHdhcmVBY3Rpb25UeXBlPlN0cmluZzwvYW50aS1tYWx3YXJlQWN0aW9uVHlwZT4gCiAg +ICAgICAgICAgPC9jb25maWd1cmF0aW9uUnVsZUFjdGlvbj4KICAgICAgICAgICA8IS0tIC0tPgog +ICAgICAgICAgICA8Y29uZmlndXJhdGlvbkNvbmRpdGlvbiB4c2k6dHlwZT0iQW50aS1tYWx3YXJl +Q29uZGl0aW9uIj4KICAgICAgICAgICAgICAgIDxpc0NORj5mYWxzZTwvaXNDTkY+CiAgICAgICAg +ICAgICAgICA8YXBwbGljYXRpb25MYXllckNvbmRpdGlvbj4KICAgICAgICAgICAgICAgICAgICA8 +bWltZVR5cGU+YXBwbGljYXRpb24vcGRmLDwvbWltZVR5cGU+CiAgICAgICAgICAgICAgICA8L2Fw +cGxpY2F0aW9uTGF5ZXJDb25kaXRpb24+CiAgICAgICAgICAgIDwvY29uZmlndXJhdGlvbkNvbmRp +dGlvbj4KICAgICAgICAgICAgPGV4dGVybmFsRGF0YSB4c2k6dHlwZT0iUHJpb3JpdHkiPgogICAg +ICAgICAgICAgICAgPHZhbHVlPjE8L3ZhbHVlPgogICAgICAgICAgICA8L2V4dGVybmFsRGF0YT4K +ICAgICAgICAgICAgPE5hbWU+UnVsZTE8L05hbWU+CiAgICAgICAgICAgIDxpc0NORj5mYWxzZTwv +aXNDTkY+CiAgICAgICAgICAgIDxIU1BMIEhTUExfaWQ9IkhTUEw1X1Nvbl9JU1AiIEhTUExfdGV4 +dD0ic29uIGVuYWJsZSBtYWx3YXJlX2RldGVjdGlvbiBtYWx3YXJlX2RldGVjdGlvbiwgIHNjYW5f +cGRmLCAgIi8+CiAgICAgICAgPC9jb25maWd1cmF0aW9uUnVsZT4KICAgICAgICA8cmVzb2x1dGlv +blN0cmF0ZWd5IHhzaTp0eXBlPSJGTVIiLz4KICAgICAgICA8TmFtZT5NU1BMXzg0ZTQzOGY5LTcw +NzEtNDBkZC04ZWVmLWU5NTVlNTE3YTJhMTwvTmFtZT4KICAgIDwvY29uZmlndXJhdGlvbj4KPC9J +VFJlc291cmNlPgo= diff --git a/M2LPlugin/examples/example_mspl_mwd_2.xml b/M2LPlugin/examples/example_mspl_mwd_2.xml new file mode 100644 index 0000000..3e3761d --- /dev/null +++ b/M2LPlugin/examples/example_mspl_mwd_2.xml @@ -0,0 +1,51 @@ + + + + + Offline_malware_analysis + + + + + + + + String + + + + false + + application/x-dosexec, + + + + 0 + + Rule0 + false + + + + + + String + + + + false + + application/pdf, + + + + 1 + + Rule1 + false + + + + MSPL_84e438f9-7071-40dd-8eef-e955e517a2a1 + + diff --git a/M2LPlugin/lib/commons-codec-1.9.jar b/M2LPlugin/lib/commons-codec-1.9.jar new file mode 100644 index 0000000..ef35f1c Binary files /dev/null and b/M2LPlugin/lib/commons-codec-1.9.jar differ diff --git a/M2LPlugin/lib/javax.json-1.0.4.jar b/M2LPlugin/lib/javax.json-1.0.4.jar new file mode 100644 index 0000000..09967d8 Binary files /dev/null and b/M2LPlugin/lib/javax.json-1.0.4.jar differ diff --git a/M2LPlugin/lib/javax.json-api-1.0.jar b/M2LPlugin/lib/javax.json-api-1.0.jar new file mode 100644 index 0000000..d276c79 Binary files /dev/null and b/M2LPlugin/lib/javax.json-api-1.0.jar differ diff --git a/M2LPlugin/lib/mspl_class.jar b/M2LPlugin/lib/mspl_class.jar new file mode 100644 index 0000000..945ae19 Binary files /dev/null and b/M2LPlugin/lib/mspl_class.jar differ diff --git a/M2LPlugin/pom.xml b/M2LPlugin/pom.xml new file mode 100644 index 0000000..6d4dced --- /dev/null +++ b/M2LPlugin/pom.xml @@ -0,0 +1,47 @@ + + 4.0.0 + eu.securedfp7.m2lservice.plugin + M2LPluginBro + 0.1 + jar + M2L Plugin for Bro PSA + + + mspl + mspl_class + system + ${project.basedir}/lib/mspl_class.jar + LATEST + + + javax.json + javax.json-api + 1.0 + + + org.glassfish + javax.json + 1.0.4 + + + commons-codec + commons-codec + 1.9 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + 1.7 + 1.7 + + + + + diff --git a/M2LPlugin/schema/MSPL_XML_Schema.xsd b/M2LPlugin/schema/MSPL_XML_Schema.xsd new file mode 100644 index 0000000..0693133 --- /dev/null +++ b/M2LPlugin/schema/MSPL_XML_Schema.xsd @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/M2LPlugin/schema/old_MSPL_XML_Schema.xsd b/M2LPlugin/schema/old_MSPL_XML_Schema.xsd new file mode 100644 index 0000000..15b74b7 --- /dev/null +++ b/M2LPlugin/schema/old_MSPL_XML_Schema.xsd @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/AddressValue.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/AddressValue.java new file mode 100644 index 0000000..da2ac04 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/AddressValue.java @@ -0,0 +1,62 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.net.URI; +import java.net.URISyntaxException; + +import javax.json.JsonObjectBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonException; + +public class AddressValue extends Value< URI > { + + public AddressValue( final String type, + final String host, + final int port ) throws URISyntaxException { + super( type, + new URI( null, null, host, port, null, null, null ) ); + } + + public AddressValue( final String type, + final String host ) throws URISyntaxException { + super( type, + new URI( null, null, host, -1, null, null, null ) ); + } + + public AddressValue( final String type, + final int port ) throws URISyntaxException { + super( type, + new URI( null, null, null, port, null, null, null ) ); + } + + public JsonObjectBuilder toJson( final JsonBuilderFactory factory ) { + + if ( !this.validate() ) { + throw new JsonException( "Invalid Value" ); + } + + final JsonObjectBuilder builder = factory.createObjectBuilder(); + builder.add( "type", this.type ); + + final JsonObjectBuilder valBuilder = factory.createObjectBuilder(); + final String host = this.value.getHost(); + if ( host != null ) { + valBuilder.add( "address", host ); + } + + final int port = this.value.getPort(); + if ( port >= 0 ) { + valBuilder.add( "port", port ); + } + + builder.add( "value", valBuilder ); + + return builder; + } + + public boolean validate() { + return ( this.type != null + || this.value != null + || ( this.value.getHost() == null + && this.value.getPort() == -1 ) ); + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/BadConfigException.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/BadConfigException.java new file mode 100644 index 0000000..7eba209 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/BadConfigException.java @@ -0,0 +1,10 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.lang.Exception; + +public class BadConfigException extends Exception { + + public BadConfigException( String message ) { + super( message ); + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/ConfigWriter.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/ConfigWriter.java new file mode 100644 index 0000000..5703382 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/ConfigWriter.java @@ -0,0 +1,47 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.util.List; +import java.util.LinkedList; +import java.lang.IllegalStateException; +import java.io.OutputStream; +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; + +import javax.json.Json; +import javax.json.JsonObjectBuilder; +import javax.json.JsonArrayBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonWriterFactory; +import javax.json.JsonWriter; +import javax.json.JsonException; + +class ConfigWriter { + + public static void write( OutputStream out, List< Rule > rules ) throws JsonException { + + try { + final JsonBuilderFactory factory = Json.createBuilderFactory( null ); + final JsonObjectBuilder builder = factory.createObjectBuilder(); + + final JsonArrayBuilder ruleBuilder = factory.createArrayBuilder(); + for ( final Rule rule : rules ) { + ruleBuilder.add( rule.toJson( factory ) ); + } + builder.add( "rules", ruleBuilder ); + final JsonObject object = builder.build(); + final JsonWriterFactory wFactory = Json.createWriterFactory( null ); + final JsonWriter writer = wFactory.createWriter( out ); + + writer.write( object ); + writer.close(); + + } catch ( JsonException e ) { + // I/O error + throw e; + } catch ( IllegalStateException e ) { + throw new JsonException( e.toString() ); + } + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/HSPLInfo.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/HSPLInfo.java new file mode 100644 index 0000000..941fdc4 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/HSPLInfo.java @@ -0,0 +1,52 @@ +package eu.securedfp7.m2lservice.plugin; + +import javax.json.JsonObjectBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonException; + +public class HSPLInfo { + + private String id; + private String text; + + public HSPLInfo( final String id, + final String text ) { + this.id = id; + this.text = text; + } + + public String getId() { + return this.id; + } + + public String getText() { + return this.text; + } + + public JsonObjectBuilder toJson( final JsonBuilderFactory factory ) { + + if ( !this.validate() ) { + throw new JsonException( "Invalid Rule" ); + } + + final JsonObjectBuilder builder = factory.createObjectBuilder(); + builder.add( "id", this.id ); + builder.add( "text", this.text ); + + return builder; + } + + public boolean validate() { + + if ( this.id == null ) { + return false; + } + + if ( this.text == null ) { + return false; + } + + return true; + } + +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/IntValue.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/IntValue.java new file mode 100644 index 0000000..7276af8 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/IntValue.java @@ -0,0 +1,33 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.lang.Integer; + +import javax.json.JsonObjectBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonException; + +public class IntValue extends Value< Integer > { + + public IntValue( final String type, + final int value ) { + super( type, new Integer( value ) ); + } + + public JsonObjectBuilder toJson( final JsonBuilderFactory factory ) { + + if ( !this.validate() ) { + throw new JsonException( "Invalid Value" ); + } + + final JsonObjectBuilder builder = factory.createObjectBuilder(); + builder.add( "type", this.type ); + builder.add( "value", this.value.intValue() ); + + return builder; + } + + public boolean validate() { + return ( this.type != null + || this.value != null ); + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/M2LPlugin.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/M2LPlugin.java new file mode 100644 index 0000000..1ad1f18 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/M2LPlugin.java @@ -0,0 +1,187 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.util.List; +import java.util.LinkedList; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.LinkedList; + +import java.lang.IllegalStateException; +import javax.json.JsonObjectBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonWriterFactory; +import javax.json.JsonWriter; +import javax.json.JsonException; + +import java.nio.file.Files; +import java.nio.file.Paths; +import org.apache.commons.codec.binary.Base64; + +/** + * Provides the Medium to Low Level (M2L) translation service for BroNSM. + * + * @author VTT Technical Research Centre of Finland Ltd + * @version 0.2 2016/03/22 + */ + +public class M2LPlugin { + + private static String securityControl = "BroNSM"; + private static String version = "0.2"; + private static String devlopedBy = "VTT Technical Research Centre of Finland Ltd"; + private static String providedBy = "SECURED project"; + + public M2LPlugin() { + } + + public String getType() { + return this.securityControl; + } + + public String getVersion() { + return this.version; + } + + public String developedBy() { + return this.devlopedBy; + } + + public String providedBy() { + return this.providedBy; + } + + /** + * Perform the translation + * + * @param MSPLFileName + * : MSPL file name + * @param securityControlFileName + * : output file name + * @return 0 if OK, + * -1 if can't read MSPLFileName IOException, + * -2 if BadConfigException. and + * -3 if JsonException occurs. + */ + + public int getConfiguration( String MSPLFileName, + String securityControlFileName) { + int result = 1; + FileInputStream in = null; + FileOutputStream out = null; + + try { + // Check if the input file is encoded as Base64 + // TODO: We simply decode into a temp file and pass that to + // MSPLParser, should refactor... + // NOTE: We do not delete the temp file. + boolean isBase64Encoded = false; + try { + final String inputString = new String(Files.readAllBytes(Paths.get(MSPLFileName))); + if(Base64.isBase64(inputString.getBytes())){ + isBase64Encoded = true; + FileOutputStream tempOut = null; + try { + MSPLFileName = MSPLFileName + ".tmp"; + tempOut = new FileOutputStream(MSPLFileName); + final byte[] decodedBytes = Base64.decodeBase64(inputString.getBytes()); + tempOut.write(decodedBytes); + } catch ( final IOException e) { + e.printStackTrace(); + } finally { + if( tempOut != null ) { + try { + tempOut.close(); + } catch ( final IOException e ) { + e.printStackTrace(); + } + } + } + } + } catch ( final IOException e1 ) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + System.out.println( "isBase64Encoded: " + isBase64Encoded ); + // TODO: fix below. + // replace quotations and \n from the input files + try { + String inputString = new String(Files.readAllBytes(Paths.get(MSPLFileName))); + inputString = inputString.replace("\\\"", "\""); + inputString = inputString.replace("\\n", ""); + FileOutputStream outCleaned = new FileOutputStream(MSPLFileName); + outCleaned.write(inputString.getBytes()); + outCleaned.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + // Do the actual M2L -> Bro JSON config conversion + in = new FileInputStream( MSPLFileName ); + out = new FileOutputStream( securityControlFileName ); + final List< Rule > rules = new LinkedList< Rule >(); + final MSPLParser parser = new MSPLParser(); + + parser.parse( in, rules ); + ConfigWriter.write( out, rules ); + + // If the input file is encoded in base64 we need to convert the output file to base64 + // Simple write the config again encoded to base64 + // TODO: Modify ConfigWriter to write base64, if needed. + if(isBase64Encoded){ + FileOutputStream outB64 = null; + try { + final String inputString = new String(Files.readAllBytes(Paths.get(securityControlFileName))); + outB64 = new FileOutputStream(securityControlFileName); + final byte[] encodedBytes = Base64.encodeBase64(inputString.getBytes()); + outB64.write(encodedBytes); + } catch ( final IOException e) { + e.printStackTrace(); + } finally { + if( outB64 != null ) { + try { + outB64.close(); + } catch ( final IOException e ) { + e.printStackTrace(); + } + } + } + } + result = 0; + + } catch ( final IOException e ) { + result = -1; + e.printStackTrace(); + } catch ( final BadConfigException e ) { + result = -2; + e.printStackTrace(); + System.out.println("Booyah! No can do..Just crash?"); + } catch ( final JsonException e ) { + result = -3; + e.printStackTrace(); + } finally { + if ( in != null ) { + try { + in.close(); + } catch ( final IOException e ) { + e.printStackTrace(); + } + } + if ( out != null ) { + try { + out.close(); + } catch ( final IOException e ) { + e.printStackTrace(); + } + } + } + + // TODO: What do we return in case of an exception? + return result; + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/MSPLParser.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/MSPLParser.java new file mode 100644 index 0000000..ef4dae7 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/MSPLParser.java @@ -0,0 +1,629 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.lang.Integer; +import java.lang.NumberFormatException; + +import java.util.List; +import java.util.LinkedList; +import java.util.regex.PatternSyntaxException; + +import java.io.InputStream; + +import java.net.URL; +import java.net.URISyntaxException; + +import java.math.BigInteger; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; + +import main.java.mspl_class.ITResource; +import main.java.mspl_class.Configuration; +import main.java.mspl_class.MaliciousFileAnalysisCapability; +import main.java.mspl_class.LoggingCapability; +import main.java.mspl_class.Capability; +import main.java.mspl_class.AntiMalwareAction; +import main.java.mspl_class.LoggingAction; +import main.java.mspl_class.ConfigurationAction; +import main.java.mspl_class.FileSystemCondition; +import main.java.mspl_class.ApplicationLayerCondition; +import main.java.mspl_class.AntiMalwareCondition; +import main.java.mspl_class.EventCondition; +import main.java.mspl_class.PacketFilterCondition; +import main.java.mspl_class.LoggingCondition; +import main.java.mspl_class.ConfigurationCondition; +import main.java.mspl_class.HSPL; +import main.java.mspl_class.ConfigurationRule; +import main.java.mspl_class.ExternalData; +import main.java.mspl_class.LSTP; +import main.java.mspl_class.FMR; +import main.java.mspl_class.ATP; +import main.java.mspl_class.ALL; +import main.java.mspl_class.MSTP; +import main.java.mspl_class.DTP; +import main.java.mspl_class.ResolutionStrategy; +import main.java.mspl_class.RuleSetConfiguration; +import main.java.mspl_class.LevelType; +import main.java.mspl_class.HTTPCondition; + +public class MSPLParser { + + private Rule.Action defaultAction = Rule.Action.INVALID; + private List< Rule > rules = new LinkedList< Rule >(); + + public void parse( final InputStream mspl, + final List< Rule > to ) throws BadConfigException { + try { + final JAXBContext ctx = JAXBContext.newInstance( ITResource.class ); + final Unmarshaller um = ctx.createUnmarshaller(); + final ITResource root = (ITResource)um.unmarshal( mspl ); + + this.visit( root ); + + if ( this.rules.isEmpty() ) { + throw new BadConfigException( "No rules found" ); + } + + to.addAll( this.rules ); + + return; + + } catch ( final JAXBException e) { + throw new BadConfigException( e.getMessage() ); + + } finally { + this.rules.clear(); + this.defaultAction = Rule.Action.INVALID; + } + } + + // Implements the visitor pattern + + private void visit( final ITResource in ) throws BadConfigException { + final String id = in.getID(); + + this.visit( in.getConfiguration() ); + } + + private void visit( final Configuration in ) throws BadConfigException { + + // Handle in subclasses: + // final List< Capability > capabilities = in.getCapability(); + + if ( in instanceof RuleSetConfiguration ) { + this.visit( (RuleSetConfiguration)in ); + } else { + throw new BadConfigException( "Unexpected Configuration type" ); + } + } + + private void visit( final RuleSetConfiguration in ) throws BadConfigException { + final String name = in.getName(); + + final List< Capability > capabilities = in.getCapability(); + for ( final Capability capability : capabilities ) { + this.visit( capability ); + } + + // Might be null + final ConfigurationAction action = in.getDefaultAction(); + if ( action != null ) { + this.visit( action, null ); + } + + // NOTE: might be empty! + final List< ConfigurationRule > rules = in.getConfigurationRule(); + if ( rules == null || rules.isEmpty() ) { + throw new BadConfigException( "At least one rule must be" + + " present" ); + } + + for ( final ConfigurationRule rule : rules ) { + this.visit( rule ); + } + + this.visit( in.getResolutionStrategy() ); + } + + private void visit( final Capability in ) throws BadConfigException { +// final String name = in.getName() + if ( in instanceof MaliciousFileAnalysisCapability ) { + this.visit( (MaliciousFileAnalysisCapability)in ); + } else if ( in instanceof LoggingCapability ) { + this.visit( (LoggingCapability)in ); + } else if ( in instanceof Capability ) { + // ? + } else { + throw new BadConfigException( "Unexpected Capability type" ); + } + } + + private void visit( final MaliciousFileAnalysisCapability in ) throws BadConfigException { +// final boolean online = in.isSupportOnlineTraficAnalysis(); +// final boolean offline = in.isSupportOfflineTraficAnalysis(); +// final String fileType = in.getFileType(); + } + + private void visit( final LoggingCapability in ) throws BadConfigException { +// final String resType = in.getResourceType(); + } + + private void visit( final ConfigurationAction in, + final Rule rule ) throws BadConfigException { + if ( in instanceof AntiMalwareAction ) { + this.visit( (AntiMalwareAction)in, rule ); + } else if ( in instanceof LoggingAction ) { + this.visit( (LoggingAction)in, rule ); + } else { + throw new BadConfigException( "Unexpected Action" ); + } + } + + private void visit( final AntiMalwareAction in, + final Rule rule ) throws BadConfigException { +// final String type = in.getAntiMalwareActionType(); + if ( rule == null ) { + this.defaultAction = Rule.Action.MALWARE_DETECTION; + } else { + rule.setAction( Rule.Action.MALWARE_DETECTION ); + } + } + + private void visit( final LoggingAction in, + final Rule rule ) throws BadConfigException { +// final String type = in.getLoggingActionType(); + if ( rule == null ) { + this.defaultAction = Rule.Action.LOG; + } else { + rule.setAction( Rule.Action.LOG ); + } + } + + private void visit( final ConfigurationRule in ) throws BadConfigException { + final String name = in.getName(); + final boolean cnf = in.isIsCNF(); + + final Rule rule = new Rule(); + rule.setId( name ); + + // Action is either specified in the rule or the default action: + + final ConfigurationAction ca = in.getConfigurationRuleAction(); + if ( ca != null ) { + this.visit( ca, rule ); + } + + if ( rule.getAction() == Rule.Action.INVALID ) { + if ( this.defaultAction == Rule.Action.INVALID ) { + throw new BadConfigException( "Undefined Action" ); + } + rule.setAction( this.defaultAction ); + } + + this.visit( in.getConfigurationCondition(), rule ); + + // Might be null + final ExternalData data = in.getExternalData(); + if ( data != null ) { + this.visit( data, rule ); + } + + // Might be empty! + List< HSPL > hspls = in.getHSPL(); + for ( final HSPL hspl : hspls ) { + this.visit( hspl, rule ); + } + + if ( !rule.validate() ) { + throw new BadConfigException( "Invalid Rule: " + name ); + } + + this.rules.add( rule ); + } + + private void visit( final ConfigurationCondition in, + final Rule rule ) throws BadConfigException { + // Handled in subclasses: + // final boolean cnf = in.isIsCNF(); + + if ( in instanceof AntiMalwareCondition ) { + this.visit( (AntiMalwareCondition)in, rule ); + + rule.setOperation( "detect-MHR" ); + rule.setEvent( Rule.Event.FILE ); + + } else if ( in instanceof LoggingCondition ) { + this.visit( (LoggingCondition)in, rule ); + + rule.setOperation( "count" ); + } else { + throw new BadConfigException( "Unexpected Condition type" ); + } + } + + private void visit( final AntiMalwareCondition in, + final Rule rule ) throws BadConfigException { + final boolean cnf = in.isIsCNF(); + + final FileSystemCondition fsc = in.getFileSystemCondition(); + if ( fsc != null ) { + throw new BadConfigException( "FileSystemCondition is not" + + " defined with" + + " AntiMalwareCondition" ); + } + + final ApplicationLayerCondition ac = in.getApplicationLayerCondition(); + if ( ac == null ) { + throw new BadConfigException( "ApplicationLayerCondition must be " + + "present in AntiMalwareCondition" ); + } + + this.visit( ac, rule ); + + final EventCondition event = in.getEventCondition(); + if ( event != null ) { + throw new BadConfigException( "EventCondition is not" + + " defined with" + + " AntiMalwareCondition" ); + } + } + + private void visit( final FileSystemCondition in, + final Rule rule ) throws BadConfigException { +// final String file = in.getFilename(); // Might be null +// final String path = in.getPath(); // Might be null + + // Might be null + final PacketFilterCondition pf = in.getPacketFilterCondition(); + if ( pf != null ) { + this.visit( in.getPacketFilterCondition(), rule ); + } + } + + private List< String > parseAddressList( final String string ) throws BadConfigException { + + final List< String > list = new LinkedList< String >(); + if ( string == null ) { + return list; + } + + final String[] parts; + try { + parts = string.split( "," ); + } catch ( final PatternSyntaxException e ) { + throw new BadConfigException( "Internal error" ); + } + + // TODO: currently expects valid IP / hostname + + for ( final String part : parts ) { + list.add( part.trim() ); // AddressValue constructor does syntax checking! + } + + return list; + } + + private List< Integer > parsePortList( final String string ) throws BadConfigException { + + final List< Integer > list = new LinkedList< Integer >(); + if ( string == null ) { + return list; + } + + final String[] parts; + try { + parts = string.split( "," ); + } catch ( final PatternSyntaxException e ) { + throw new BadConfigException( "Internal error" ); + } + + for ( final String part : parts ) { + try { + list.add( Integer.valueOf( part.trim() ) ); + } catch ( final NumberFormatException e ) { + throw new BadConfigException( "Invalid port" ); + } + } + + return list; + } + + private void visit( final PacketFilterCondition in, + final Rule rule ) throws BadConfigException { + final String src = in.getSourceAddress(); // Might be null + final String dst = in.getDestinationAddress();// Might be null + final String srcPort = in.getSourcePort(); // Might be null + final String dstPort = in.getDestinationPort(); // Might be null +// TODO: +// final String direction = in.getDirection(); // Might be null +// final String iFace = in.getInterface(); // Might be null +// final String protocol = in.getProtocolType(); // Might be null + + final List< String > srcs = this.parseAddressList( src ); + final List< String > dsts = this.parseAddressList( dst ); + final List< Integer > sPorts = this.parsePortList( srcPort ); + final List< Integer > dPorts = this.parsePortList( dstPort ); + + if ( srcs.isEmpty() ) { + for ( final Integer port : sPorts ) { + try { + rule.addCondition( new AddressValue( "source_port", port.intValue() ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + } else { + if ( sPorts.isEmpty() ) { + for ( final String host : srcs ) { + try { + rule.addCondition( new AddressValue( "source", host ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + } else { + for ( final Integer port : sPorts ) { + final int p = port.intValue(); + for ( final String host : srcs ) { + try{ + rule.addCondition( new AddressValue( "source", host, p ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + } + } + } + + if ( dsts.isEmpty() ) { + for ( final Integer port : dPorts ) { + try { + rule.addCondition( new AddressValue( "destination_port", port.intValue() ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + } else { + if ( dPorts.isEmpty() ) { + for ( final String host : dsts ) { + try { + rule.addCondition( new AddressValue( "destination", host ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + } else { + for ( final String host : dsts ) { + for ( final Integer port : dPorts ) { + try { + rule.addCondition( new AddressValue( "destination", host, port.intValue() ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + } + } + } + + // Might be empty! + final List< String > states = in.getState(); + for ( final String state : states ) { + // TODO: check valid states! + rule.addCondition( new StringValue( "state", state ) ); + } + } + + private void visit( final ApplicationLayerCondition in, + final Rule rule ) throws BadConfigException { + final String url = in.getURL(); // Might be null + final HTTPCondition http = in.getHttpCondition(); // Might be null + final String extension = in.getFileExtension(); // Might be null + final String mime = in.getMimeType(); // Might be null + final Integer maxConn = in.getMaxconn(); // Might be null + final String dstDomain = in.getDstDomain(); // Might be null + final String srcDomain = in.getSrcDomain(); // Might be null + final String urlRegEx = in.getURLRegex(); // Might be null + + if ( http != null + || extension != null + || maxConn != null + || dstDomain != null + || srcDomain != null + || urlRegEx != null + // expect exactly one condition: + || ( mime != null && url != null ) ) { + throw new BadConfigException( "Unexpected ApplicationLayerCondition" ); + } + + if ( mime != null ) { + String value = mime.trim(); + if ( value.endsWith( "," ) ) { + value = value.substring( 0, value.length() - 1 ); + } + + rule.addCondition( new StringValue( "mime-type", value ) ); + + return; + } + + if ( url != null ) { + // Let's assume its a string consisting of comma-separated names + List< String > hosts = parseAddressList( url.trim() ); + + for ( final String host : hosts ) { + try { + // TODO: let's assume all the names represent destinations, + // since there is really no way to say what it is. + rule.addCondition( new AddressValue( "destination", host ) ); + } catch ( final URISyntaxException e ) { + throw new BadConfigException( e.getMessage() ); + } + } + + return; + } + + throw new BadConfigException( "Invalid ApplicationLayerCondition" ); + } + + private void visit( final EventCondition in, + final Rule rule ) throws BadConfigException { + + final String event = in.getEvents(); + if ( event == null ) { + throw new BadConfigException( "Exactly Event must be present" + + " in EventCondition" ); + } + + if ( !event.equals( "EVENT_CONNECTION" ) ) { + throw new BadConfigException( "Unexpected Event in" + + " EventCondition" ); + } + rule.setEvent( Rule.Event.CONNECTION ); + + final BigInteger interval = in.getInterval(); + if ( interval != null ) { + rule.addCondition( new IntValue( "interval", + interval.intValue() ) ); + } + + final BigInteger threshold = in.getThreshold(); + if (threshold != null ) { + rule.addCondition( new IntValue( "threshold", + threshold.intValue() ) ); + } + } + + private void visit( final LoggingCondition in, + final Rule rule ) throws BadConfigException { + final boolean cnf = in.isIsCNF(); + + final EventCondition event = in.getEventCondition(); + if ( event == null ) { + throw new BadConfigException( "Exactly one EventCondition" + + " must be present in LoggingCondition" ); + } + + this.visit( event, rule ); + + final String object = in.getObject(); + if ( object != null ) { + if ( !object.equals( "OBJ_CONNECTION" ) ) { + throw new BadConfigException( "Unexpected Object" ); + } + + rule.addParameter( new StringValue( "object", object ) ); + + } else { + // Compensate missing value: + rule.addParameter( new StringValue( "object", "OBJ_CONNECTION" ) ); + } + + // Might be empty! + final List< PacketFilterCondition > pfs = in.getPacketCondition(); + for ( final PacketFilterCondition pf : pfs ) { + this.visit( pf, rule ); + } + + final List< ApplicationLayerCondition > als = in.getApplicationCondition(); + for ( final ApplicationLayerCondition al : als ) { + this.visit( al, rule ); + } + + // Require at least one condition: + if ( ( pfs == null || pfs.isEmpty() ) + && ( als == null || als.isEmpty() ) ) { + throw new BadConfigException( "One or more PacketFilterConditions" + + " or ApplicationLayerConditions must" + + " be present in LoggingCondition" ); + } + } + + private void visit( final ExternalData in, + final Rule rule ) throws BadConfigException { + // ? + } + + private void visit( final HSPL in, + final Rule rule ) throws BadConfigException { + final String id = in.getHSPLId(); + final String text = in.getHSPLText(); + + rule.setHSPL( new HSPLInfo( id, text ) ); + } + + private void visit( final ResolutionStrategy in ) throws BadConfigException { + + if ( in instanceof LSTP ) { + this.visit( (LSTP)in ); + } else if ( in instanceof FMR ) { + this.visit( (FMR)in ); + } else if ( in instanceof ATP ) { + this.visit( (ATP)in ); + } else if ( in instanceof ALL ) { + this.visit( (ALL)in ); + } else if ( in instanceof MSTP ) { + this.visit( (MSTP)in ); + } else if ( in instanceof DTP ) { + this.visit( (DTP)in ); + } else { + throw new BadConfigException( "Unexpected ResolutionStrategy " + + "type" ); + } + } + + private void visit( final LSTP in ) throws BadConfigException { + + // Might be empty! + final List< ExternalData > datas = in.getExternalData(); + for ( final ExternalData data : datas ) { + this.visit( data, null ); + } + } + + private void visit( final FMR in ) throws BadConfigException { + + // Might be empty! + final List< ExternalData > datas = in.getExternalData(); + for ( final ExternalData data : datas ) { + this.visit( data, null ); + } + } + + private void visit( final ATP in ) throws BadConfigException { + + // Might be empty! + final List< ExternalData > datas = in.getExternalData(); + for ( final ExternalData data : datas ) { + this.visit( data, null ); + } + } + + private void visit( final ALL in ) throws BadConfigException { + + // Might be empty! + final List< ExternalData > datas = in.getExternalData(); + for ( final ExternalData data : datas ) { + this.visit( data, null ); + } + } + + private void visit( final MSTP in ) throws BadConfigException { + + // Might be empty! + final List< ExternalData > datas = in.getExternalData(); + for ( final ExternalData data : datas ) { + this.visit( data, null ); + } + } + + private void visit( final DTP in ) throws BadConfigException { + + // Might be empty! + final List< ExternalData > datas = in.getExternalData(); + for ( final ExternalData data : datas ) { + this.visit( data, null ); + } + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Rule.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Rule.java new file mode 100644 index 0000000..0d173d2 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Rule.java @@ -0,0 +1,172 @@ +package eu.securedfp7.m2lservice.plugin; + +import java.util.List; +import java.util.LinkedList; + +import javax.json.JsonObjectBuilder; +import javax.json.JsonArrayBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonException; + +public class Rule { + + public enum Event { + INVALID, + CONNECTION, + FILE + } + + public enum Action { + INVALID, + LOG, + MALWARE_DETECTION + } + + private String id = null; + private HSPLInfo hspl = null; + private String operation = null; + private Event event = Event.INVALID; + private Action action = Action.INVALID; + private List< Value > parameters = new LinkedList< Value >(); + private List< Value > conditions = new LinkedList< Value >(); + + public Rule() { + } + + public String getId() { + return this.id; + } + + public void setId( final String id ) { + this.id = id; + } + + public HSPLInfo getHSPL() { + return this.hspl; + } + + public void setHSPL( final HSPLInfo hspl ) { + this.hspl = hspl; + } + + public String getOperation() { + return this.operation; + } + + public void setOperation( final String op ) { + this.operation = op; + } + + public Event getEvent() { + return this.event; + } + + public void setEvent( final Event ev ) { + this.event = ev; + } + + public Action getAction() { + return this.action; + } + + public void setAction( final Action a ) { + this.action = a; + } + + public void addParameter( final Value v ) { + this.parameters.add( v ); + } + + public void addCondition( final Value v ) { + this.conditions.add( v ); + } + + // TODO: this is ugly: + private String eventToString( final Event ev ) { + + switch ( ev ) { + case INVALID: return null; + case CONNECTION: return "EVENT_CONNECTION"; + case FILE: return "EVENT_FILE"; + default: return null; + } + } + + // TODO: this is ugly: + private String actionToString( final Action ac ) { + + switch( ac ) { + case INVALID: return null; + case LOG: return "log"; + case MALWARE_DETECTION: return "log"; // Currently we only support logging + default: return null; + } + } + + public JsonObjectBuilder toJson( final JsonBuilderFactory factory ) { + + if ( !this.validate() ) { + throw new JsonException( "Invalid Rule" ); + } + + final JsonObjectBuilder builder = factory.createObjectBuilder(); + builder.add( "id", this.id ); + builder.add( "hspl", this.hspl.toJson( factory ) ); + builder.add( "operation", this.operation ); + builder.add( "event", this.eventToString( this.event ) ); + builder.add( "action", this.actionToString( this.action ) ); + + final JsonArrayBuilder parmBuilder = factory.createArrayBuilder(); + for ( Value item : this.parameters ) { + parmBuilder.add( item.toJson( factory ) ); + } + + builder.add( "parameters", parmBuilder ); + + final JsonArrayBuilder condBuilder = factory.createArrayBuilder(); + for ( Value item : this.conditions ) { + condBuilder.add( item.toJson( factory ) ); + } + + builder.add( "conditions", condBuilder ); + + return builder; + } + + public boolean validate() { + + if ( this.id == null ) { + return false; + } + + if ( this.hspl == null || !this.hspl.validate() ) { + return false; + } + + if ( this.operation == null ) { + return false; + } + + if ( this.event == Event.INVALID ) { + return false; + } + + if ( this.action == Action.INVALID ) { + return false; + } + + for ( final Value item : this.parameters ) { + if ( !item.validate() ) { + return false; + } + } + + for ( final Value item : this.conditions ) { + if ( !item.validate() ) { + return false; + } + } + + return true; + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/StringValue.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/StringValue.java new file mode 100644 index 0000000..e7cf383 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/StringValue.java @@ -0,0 +1,31 @@ +package eu.securedfp7.m2lservice.plugin; + +import javax.json.JsonObjectBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonException; + +public class StringValue extends Value< String > { + + public StringValue( final String type, + final String value ) { + super ( type, value ); + } + + public JsonObjectBuilder toJson( final JsonBuilderFactory factory ) { + + if ( !this.validate() ) { + throw new JsonException( "Invalid Value" ); + } + + final JsonObjectBuilder builder = factory.createObjectBuilder(); + builder.add( "type", this.type ); + builder.add( "value", this.value ); + + return builder; + } + + public boolean validate() { + return ( this.type != null + || this.value != null ); + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Tester.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Tester.java new file mode 100644 index 0000000..7d5da40 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Tester.java @@ -0,0 +1,61 @@ +package eu.securedfp7.m2lservice.plugin; + +// For validating the XMLSchema +import javax.xml.XMLConstants; +import javax.xml.transform.Source; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.*; +import org.xml.sax.SAXException; +import java.net.*; +import java.io.*; + +public class Tester { + + // For testing + public static void main( final String[] args ) { + System.out.println( "################################"); + System.out.println( "Tester."); + String validateRes = validateSchemaReturnError(args[ 0 ]); + if(validateRes != null){ + System.out.println("##Oops! Your MSPL (" + args[ 0 ] + ") does not validate with the schema, reason: \n" + validateRes); + }else{ + System.out.println("##Great! Your MSPL (" + args[ 0 ] + " is well formed!"); + } + System.out.println( "################################"); + + System.out.println( "input: " + args[ 0 ] ); + System.out.println( "output: " + args[ 1 ] ); + final M2LPlugin plugin = new M2LPlugin(); + final int status = plugin.getConfiguration( args[ 0 ], args[ 1 ] ); + System.out.println( "status: " + status ); + } + + private static String validateSchemaReturnError(String MSPLFileName) { + String ret = null; + Source xmlFile = new StreamSource(new File(MSPLFileName)); + SchemaFactory schemaFactory = SchemaFactory + .newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Source schemaFileValidate; + Schema schema; + Validator validator = null; + try { + // NOTE: assumes you run this from M2LPluginBro folder, modify if needed. + schemaFileValidate = new StreamSource(new File("./schema/MSPL_XML_Schema.xsd")); + schema = schemaFactory.newSchema(schemaFileValidate); + validator = schema.newValidator(); + } catch (SAXException e) { + e.printStackTrace(); + } + try { + validator.validate(xmlFile); + //System.out.println("####" + xmlFile.getSystemId() + " is valid"); + } catch (SAXException e) { + //System.out.println(xmlFile.getSystemId() + " is NOT valid"); + ret = e.getLocalizedMessage(); + }catch ( final IOException e) { + e.printStackTrace(); + } + + return ret; + } +} diff --git a/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Value.java b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Value.java new file mode 100644 index 0000000..b49ad90 --- /dev/null +++ b/M2LPlugin/src/main/java/eu/securedfp7/m2lservice/plugin/Value.java @@ -0,0 +1,36 @@ +package eu.securedfp7.m2lservice.plugin; + +import javax.json.JsonObjectBuilder; +import javax.json.JsonBuilderFactory; +import javax.json.JsonException; + +public abstract class Value< T > { + + protected String type = null; + protected T value = null; + + protected Value( final String t, final T v ) { + this.type = t; + this.value = v; + } + + public String getType() { + return this.type; + } + + public void setType( final String t ) { + this.type = t; + } + + public T getValue() { + return this.value; + } + + public void setValue( final T v ) { + this.value = v; + } + + public abstract JsonObjectBuilder toJson( final JsonBuilderFactory factory ); + + public abstract boolean validate(); +} diff --git a/M2LPlugin/test.sh b/M2LPlugin/test.sh new file mode 100644 index 0000000..552c45a --- /dev/null +++ b/M2LPlugin/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +java -cp ./lib/javax.json-1.0.4.jar:./lib/mspl_class.jar:./lib/commons-codec-1.9.jar:./target/M2LPluginBro-0.1.jar eu.securedfp7.m2lservice.plugin.Tester $1 $2 diff --git a/M2LPlugin/validate.sh b/M2LPlugin/validate.sh new file mode 100644 index 0000000..8ed7968 --- /dev/null +++ b/M2LPlugin/validate.sh @@ -0,0 +1,8 @@ +#/bin/sh + +if [ ! -f "$1" ]; then + echo "usage: $0 FILE" + exit 1 +fi + +xmllint --schema ./schema/MSPL_XML_Schema.xsd --noout --nonet --dropdtd $1 diff --git a/NED_files/PSCM/userList b/NED_files/PSCM/userList new file mode 100644 index 0000000..e6c6643 --- /dev/null +++ b/NED_files/PSCM/userList @@ -0,0 +1 @@ +user1 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 user2 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 user3 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 user4 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 user5 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 user6 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 user10 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 test 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 test1 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 bro 19ceda366d4b785cee1daa69f75f0caff81d1f73e0ffd19d6556002bf9c98ad0 diff --git a/NED_files/README.md b/NED_files/README.md new file mode 100644 index 0000000..dcf2c80 --- /dev/null +++ b/NED_files/README.md @@ -0,0 +1 @@ +# Placeholder diff --git a/NED_files/TVDM/PSAManifest/BroLogging_manifest.xml b/NED_files/TVDM/PSAManifest/BroLogging_manifest.xml new file mode 100644 index 0000000..35e7a07 --- /dev/null +++ b/NED_files/TVDM/PSAManifest/BroLogging_manifest.xml @@ -0,0 +1,125 @@ + + + + + BroLogging + Bro Logging + Offers network monitoring and logging capabilities + https://www.secured-fp7.eu/ + 1.00 + VTT + VTT + Copyright 2016 VTT Technical Research Centre of Finland Ltd + + This file is part of Bro PSA + + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Freeware + + + + + BroLogging + bro + network traffic analysis + + + Logging + + + + + bro + BroLogging + This plugin converts MSPL to Bro PSA configuration + + brologging_M2L_plugin + BroLogging + http://195.235.93.146:8080/v1/PSA/M2Lplugins/PSA-brologging + none + + brologging_M2L_plugin.jar + + + + + 10 + + + 10 + + 2 + 10 + + + + + + + 1 + x86_64 + Intel + + 1 + + + 2 + + + 10 + + + 2000 + + + + Debian + 7.0 + x86_64 + + + + + cold migration + stateless + + + + + + img + + brologging_M2L_plugin + brologging_M2L_plugin.jar + java + + 1 + + + + + 100 + + + 10 + + + 10 + + + diff --git a/NED_files/TVDM/PSAManifest/BroMalware_manifest.xml b/NED_files/TVDM/PSAManifest/BroMalware_manifest.xml new file mode 100644 index 0000000..a4fc38b --- /dev/null +++ b/NED_files/TVDM/PSAManifest/BroMalware_manifest.xml @@ -0,0 +1,125 @@ + + + + + BroMalware + Bro Malware Detection + Offers malware detection capabilities + https://www.secured-fp7.eu/ + 1.00 + VTT + VTT + Copyright 2016 VTT Technical Research Centre of Finland Ltd + + This file is part of Bro PSA + + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Freeware + + + + + BroMalware + bro + anti-malware detector + + + Offline_malware_analysis + + + + + bro + BroMalware + This plugin converts MSPL to Bro PSA configuration + + bromalware_M2L_plugin + BroMalware + http://195.235.93.146:8080/v1/PSA/M2Lplugins/PSA-bromalware + none + + bromalware_M2L_plugin.jar + + + + + 10 + + + 10 + + 2 + 10 + + + + + + + 1 + x86_64 + Intel + + 1 + + + 2 + + + 10 + + + 2000 + + + + Debian + 7.0 + x86_64 + + + + + cold migration + stateless + + + + + + img + + bromalware_M2L_plugin + bromalware_M2L_plugin.jar + java + + 1 + + + + + 100 + + + 10 + + + 10 + + + diff --git a/NED_files/TVDM/PSAManifest/broPSA b/NED_files/TVDM/PSAManifest/broPSA new file mode 100644 index 0000000..8aa35b8 --- /dev/null +++ b/NED_files/TVDM/PSAManifest/broPSA @@ -0,0 +1,22 @@ +{ + "PSA_id":"BroPSA", + "disk": "veryLightPSA-bro-1G.img", + "interface": [ + { + "network":"data", + "type":"data_in" + }, + { + "network":"data", + "type":"data_out" + }, + { + "network":"control", + "type":"manage" + } + ], + "memory": "256", + "IP": true, + "os-architecture": "x86_64", + "vcpu": "1" +} diff --git a/NED_files/TVDM/psaConfigs/broPSA/psaConf b/NED_files/TVDM/psaConfigs/broPSA/psaConf new file mode 100644 index 0000000..8e53402 --- /dev/null +++ b/NED_files/TVDM/psaConfigs/broPSA/psaConf @@ -0,0 +1,92 @@ +{ + + "rules": [ + { "id": "rule1", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination", + "value": { "address": "91.197.85.151" } + } + ] + }, + { "id": "rule2", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination", + "value": { "address": "81.209.67.238" } + } + ] + }, + { "id": "rule3", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination_port", + "value": { "port": 80 } + } + ] + }, + { "id": "rule4", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_FILE", + "operation": "detect-MHR", + "parameters": [ ], + "action": "log", + "conditions": [ + { "type": "mime-type", + "value": "application/pdf" + }, + { "type": "mime-type", + "value": "application/x-dosexec" + } + ] + } + ] +} diff --git a/NED_files/TVDM/userGraph/bro b/NED_files/TVDM/userGraph/bro new file mode 100644 index 0000000..33778b9 --- /dev/null +++ b/NED_files/TVDM/userGraph/bro @@ -0,0 +1,26 @@ +{ + "name": "user_profile_type", + "user_token": "", + "profile_type": "AD", + + "PSASet": [ + + { + "id": "broPSA", + "security_controls": [ + + { + "imgName": "veryLightPSA-bro-1G.img", + "conf_id":"psaConf" + } + + ] + + } + + ], + + "ingress_flow": ["12345"], + "egress_flow": ["12345"] + +} diff --git a/PSA/BroManager.py b/PSA/BroManager.py new file mode 100644 index 0000000..c908f4a --- /dev/null +++ b/PSA/BroManager.py @@ -0,0 +1,462 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# BroManager.py +# +# An interface to Bro. +# +# Author: jounih / VTT Technical Research Centre of Finland Ltd., 2015 +# jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import threading +import subprocess +import os +import os.path +import logging + +# In the new image, the broccoli location is here +import sys +sys.path.append( '/opt/bro/lib/broctl/' ) + +from broccoli import Connection +import ConfigLoader +import ModuleLoader + +# A simple Thread-object that is used to call Broccoli's +# processInput() every now and then to keep Bro event +# handling running. + +class InputThread( threading.Thread ): + + polling = None # threading.Event instance for waiting + pollingInterval = 1 # Polling interval in seconds + connection = None # Bro connection to poll + + def __init__( self, connection ): + self.connection = connection + threading.Thread.__init__( self ) + self.polling = threading.Event() + + def run( self ): + while ( ( not self.polling.is_set() ) + and self.connection != None ): + self.connection.processInput() + self.polling.wait( self.pollingInterval ) + +# Exception class for Bro related exceptions. +class BroException( Exception ): + def __init__( self, value ): + super( BroException, self ).__init__( value ) + self.value = value + + def __str__( self ): + return repr( self.value ) + +# BroManager is *the* interface to Bro +# +# Allows starting and stopping Bro and loading new modules +# +# State management: +# BroManager controls an instance of the Bro network monitor. To keep +# state management simple there are essentially two states: 'running' +# and 'stopped'. All changes (e.g. loading modules, adding rules) should +# be made while Bro is stopped. +# + +class BroManager( object ): + # TODO: These should be configurable: + configFile = '/opt/bro/share/bro/site/secured.bro' + broctlPath = '/opt/bro/bin/broctl' + + SCRIPT_PRE_INIT = 'pre-init.bro' + SCRIPT_POST_INIT = 'post-init.bro' + + # Bro PSA installation base directory. All directiories are relative to this + baseDir = None + # Module directory: self.baseDir + '/modules' + moduleDir = None + # Current Inputhread object. None if there is not connection. + thread = None + # Bro Connectio object instance if Bro manager is currently connected to + # one, None otherwise. + connection = None + # Dictionary of currently loaded Bro modules. + modules = { } # { "name": BroModule } + # A logger to which Bro modules report their logs. + # TODO: this is a quick hack: a better aproach should be implemented. + logger = None + + def __init__( self, base=None, logger=None ): + if not os.getuid() == 0: + raise Exception( 'BroManager requires root access!' ) + + self.logger = logger + if base != None: + self.baseDir = base + else: + self.baseDir = '/home/psa/pythonScript/' + self.moduleDir = self.baseDir + '/modules' + ModuleLoader.init( self.baseDir + '/modules.json' ) + + def __del__( self ): + self.disconnect() + + def isConnected( self ): + return ( self.connection != None ) + + # NOTE: it could be a good idea to use connect / disconnect internally + # only and always use startBro / stopBro / loadConfig externally. However, + # now connect() is also use to connect (or check) if a bro instance is + # already running. + + def connect( self ): # throws IOError + """ + Connect to a running Bro instance. + + Creates a thread that starts calling Broccoli's processInput() + periodically. + """ + if self.connection != None: + self.disconnect() + + # Note: all Bro modules must be loaded *BEFORE* the Bro Connection + # is created. The reason for this is that Broccoli Python interface + # only registers event handlers for those @events that it has seen + # the moment when the Connection is created. Thus, loading new modules + # requires creating a new Connection. + + # TODO: make the address configurable + + self.connection = Connection( "127.0.0.1:47760", connect = False ) + + try: + self.connection.connect() + except IOError as e: + self.connection = None + raise e + + self.thread = InputThread( self.connection ) + self.thread.start() + + def disconnect( self ): + """ + Close the connection with the Bro instance. + + Stops the thread polling Broccoli's processInput() + """ + if self.thread != None: + if self.thread.polling: + self.thread.polling.set() + self.thread.connection = None + self.thread.join() + self.thread = None + if self.connection != None: + self.connection.connDelete() + self.connection = None + + def _loadModule( self, name ): + """ + Loads a module corresponding to 'name' if one is + found in the modules.json file. + """ + module = ModuleLoader.load( name ) + if module == None: + logging.error( 'Could not load module: ' + name) + return None + + # Create an instance of the module + instance = module( self.logger ) + self.modules[ name ] = instance + logging.info( 'Module loading succesfull: ' + name) + return instance + + def _getOrLoadModule( self, key ): + """ + Return a module corresponding to 'key'. If one is not present + try to load it based on the modules.json file. + """ + try: + return self.modules[ key ] + except KeyError: + return self._loadModule( key ) + + def loadConfig( self, filename ): + """ + Loads a configuration file. Before loading, each module is disabled. + Configuration rules are passed to corresponding modules. In case such + module has not been loaded, they are loaded according to modules.json + file. + + Note: this function should only be called when Bro is stopped! + """ + + # This function used to reset each module, but requiring Bro to + # be stopped makes more sense and keeps the state management + # easier. + if self.connection != None: + raise BroException( 'Invalid state' ) + + self._disablePreInitScript() + self._disableAllModules() + self._disablePostInitScript() + + self.modules = { } + rules = ConfigLoader.load( filename ) + + self._broctl_cmd( 'cleanup', 'all' ) + + logging.info( 'Enabling pre-init script' ) + self._enablePreInitScript() + + for rule in rules: + module = self._getOrLoadModule( rule.operation ) + if module == None: + logging.warning( 'No module for operation ' + + rule.operation + + ' (' + rule.ruleId + ')' ) + continue + + if not module.enabled: + self._enableModule( module ) + + logging.info( 'Setting rule %s for module %s' + % ( rule.ruleId, module.broScript ) ) + if not module.onRule( rule ): + logging.warning( 'Invalid rule: ' + rule.ruleId ) + + logging.info( 'Enabling post-init script' ) + self._enablePostInitScript() + + # Note: broctl install must be run when ever the local policy scripts + # are modified. This means each time a module is enabled or disabled. + # However, it is not a good idea to run them in enable/disableModule + # functions separately for each change. + + self._broctl_cmd( 'check' ) + self._broctl_cmd( 'install' ) + + # The 'broctl update' command is only needed if Bro is already running. + # However, update won't update all Bro state, so stopping and restarting + # Bro for any updates is a safer way. We don't expect this to happen + # often! + + # self._broctl_cmd( 'update' ) + + + def startBro( self ): + """ + Starts bro instance and calls each modules onStart-callback. + """ + + if self.connection != None: + raise BroException( 'Invalid state' ) + + logging.info( 'Starting Bro' ) + # Newer Bro versions have commend 'deploy', which must be + # run when ever the scripts are modified. It should be equivalent + # of 'check', 'install' and 'restart' + self._broctl_cmd( 'cleanup', '--all' ) + self._broctl_cmd( 'check' ) + self._broctl_cmd( 'install' ) + #self._broctl_cmd( 'update' ) + self._broctl_cmd( 'start' ) + self.connect() + + logging.info( 'Starting modules' ) + for key, module in self.modules.iteritems(): + if module.enabled: + logging.info( 'Module: ' + module.broScript ) + module.onStart( self.connection ) + logging.info( 'Done' ) + logging.info( 'Bro Started' ) + + def stopBro( self ): + """ + Stops running bro instance. Each module's onStop-callback is called + before bro is stopped to allow any cleanup actions necessary. + """ + + if self.connection == None: + raise BroException( 'Invalid state' ) + + logging.info( 'Stopping Bro' ) + logging.info( 'Stopping modules' ) + for key, module in self.modules.iteritems(): + if module.enabled: + logging.info( 'Module: ' + module.broScript ) + module.onStop() + logging.info( 'Done' ) + + self.disconnect() + self._broctl_cmd( 'stop' ) + logging.info( 'Bro Stopped' ) + + def restartBro( self ): + self.stopBro() + self.startBro() + + def _broctl_cmd( self, cmd, *args ): + """ + Execute a command using broctl + """ + cArgs = [ self.broctlPath, cmd ] + for arg in args: + cArgs.append( arg ) + + logging.info( 'Calling broctl: ' + str( cArgs ) ) + # will wait for completion of cmd + rv = subprocess.call( cArgs ) + if rv == 1: + raise Exception( 'Error: broctl ' + cmd + ' failed!' ) + + def _enableModule( self, module ): + """ + Enables a specific Bro module. + + If the module is not listed in the Bro configuration file, it is added + there. If the module is listed in the file, but commented out, the + comment character is removed. + + Note: does not call module's onStart callback! + Note: Bro must be restarted in order of these changes to take effect. + """ + # See if the module name already exists in the configuration file: + path = self.moduleDir + '/' + module.broScript + rv = subprocess.call( [ 'grep', + '--quiet', + '@load ' + path, + self.configFile ] ) + if rv != 0: # No match found: add a new line + with open( self.configFile, 'a' ) as f: + f.write( '\n@load ' + path + '\n' ) + rv = 0 + else: # Remove comment chracater before the load directive + pattern = 's|^#*@load ' + path + '|@load ' + path + '|g' + rv = subprocess.call( [ 'sed', + '-i.bak', + '--silent', + pattern, + self.configFile ] ) + + if rv == 0: + module.enabled = True + return rv + + def _disableModule( self, module ): + """ + Disable a specific Bro module. + + Essentially comments out the module from Bro configuration file. + + Note: does not call module's onStop callback! + Note: Bro must be restarted in order of these changes to take effect. + """ + # Comment the load directive out + pattern = 's|^@load ' + self.moduleDir + '/' + module.broScript + pattern += '|#@load ' + self.moduleDir + '/' + module.broScript + '|g' + rv = subprocess.call( [ 'sed', + '-i.bak', + '--silent', + pattern, + self.configFile ] ) + + if rv == 0: + module.enabled = False + return rv + + def _disableAllModules( self ): + """ + Disables all Bro modules. + + Essentially comments out all modules in the module directory from the + Bro configuration file. This includes modules that are not listed in + the current module configuration. The main purpose of this function is + to ensure clean restart of Bro. + + Does not affect the currently loaded modules in any way. + + Note: Bro must be restarted in order of these changes to take effect. + """ + + # Comment the load directive out in order to disable the module + pattern = 's|^@load ' + self.moduleDir + '/' + pattern += '|#@load ' + self.moduleDir + '/|g' + rv = subprocess.call( [ 'sed', + '-i.bak', + '--silent', + pattern, + self.configFile ] ) + return ( rv == 0 ) + + def _enablePreInitScript( self ): + script = self.moduleDir + '/' + self.SCRIPT_PRE_INIT + line = '@load ' + script + # Remove the all instances of the line first to make sure that + # the line is included only once and that its the first line! + # This might usually not be needed, but let's make it anyways + # to be sure that we don't have any unexpected side effects! + _fileRemoveLines( self.configFile, line ) + _fileRemoveEmptyLines( self.configFile ) + # Only add the line if the pre-init script actually exists + if _fileExists( script ): + _filePrependLine( self.configFile, line ) + _fileRemoveEmptyLines( self.configFile ) + + def _enablePostInitScript( self ): + script = self.moduleDir + '/' + self.SCRIPT_POST_INIT + line = '@load ' + script + # Remove the all instances of the line first to make sure that + # the line is included only once and that its the last line! + # This might usually not be needed, but let's make it anyways + # to be sure that we don't have any unexpected side effects! + _fileRemoveLines( self.configFile, line ) + _fileRemoveEmptyLines( self.configFile ) + # Only add the line if the post-init script actually exists + if _fileExists( script ): + _fileAppendLine( self.configFile, line ) + _fileRemoveEmptyLines( self.configFile ) + + def _disablePreInitScript( self ): + script = self.moduleDir + '/' + self.SCRIPT_PRE_INIT + line = '@load ' + script + _fileRemoveLines( self.configFile, line ) + _fileRemoveEmptyLines( self.configFile ) + + def _disablePostInitScript( self ): + script = self.moduleDir + '/' + self.SCRIPT_POST_INIT + line = '@load ' + script + _fileRemoveLines( self.configFile, line ) + _fileRemoveEmptyLines( self.configFile ) + +# The file handling scripts below use mostly sed magic to do their things. +# This might not be the best or most pythonianic way to do the operations, +# but as most of the file-related functions above also make it this way +# let's continue the habbit... + +def _fileExists( f ): + return os.path.isfile( f ) + +def _fileContainsLine( f, line ): + return ( subprocess.call( [ 'grep', '--quiet', 'line', f ] ) != 0 ) + +def _fileRemoveLines( f, line ): + pattern = 's|^' + line + '||g' + rv = subprocess.call( [ 'sed', '-i.bak', pattern, f ] ) + return rv + +def _fileRemoveEmptyLines( f ): + rv = subprocess.call( [ 'sed', '-i.bak', '/^\s*$/d', f ] ) + return rv + +def _filePrependLine( f, line ): + # Sed magic doesn't work for empty files + # However, in that case we only need to append + if os.stat( f ).st_size == 0: + _fileAppendLine( f, line ) + else: + pattern = '1s|^|' + line + '\\n|g' + subprocess.call( [ 'sed', '-i.bak', '--silent', pattern, f ] ) + +def _fileAppendLine( f, line ): + with open( f, 'a') as fi: + fi.write( '\n' + line + '\n' ) diff --git a/PSA/Config.py b/PSA/Config.py new file mode 100644 index 0000000..9e1e5b1 --- /dev/null +++ b/PSA/Config.py @@ -0,0 +1,167 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# Config.py +# +# PSA configuration file parsing +# +# Author: anon, +# jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import ConfigParser +import os +#import copy +import logging + +def resolve_psa_home(): + try: + home = os.environ[ 'PSA_HOME' ] + if not os.path.isdir( home ): + error = 'Environment variable $PSA_HOME is not a valid directory' + raise RuntimeError( error ) + if not os.path.isabs( home ): + error ='Environment variable $PSA_HOME path is not absolute' + raise RuntimeError( error ) + return home + except KeyError: + logging.warning( 'Environment variable $PSA_HOME not set' ) + logging.info( 'Using current working directory as $PSA_HOME' ) + return os.getcwd() + +def normalize_path( base, path ): + return os.path.join( base, path) + +def getboolean_default( config, section, option, default ): + try: + return config.getboolean( section, option ) + except ConfigParser.NoOptionError as e: + return default + +def get_default( config, section, option, default ): + try: + return config.get( section, option ) + except ConfigParser.NoOptionError as e: + return default + +class Configuration( object ): + _instance = None # Singleton + + def __new__( cls, *args, **kwargs ): + if not cls._instance: + cls._instance = super( Configuration, cls ).__new__( cls, *args, **kwargs ) + return cls._instance + + def __init__( self ): + config = ConfigParser.RawConfigParser() + #config.read( 'psa.conf' ) + config.read( 'psaEE.conf' ) + + # Hard-coded options + self._PSA_HOME = resolve_psa_home() + self._LOG_FILE = 'PSA.log' + + + + # Optional + self._VERBOSE = getboolean_default( config, 'configuration', + 'verbose', False ) + self._DEBUG = getboolean_default( config, 'configuration', + 'debug', False ) + self._TEST_MODE = getboolean_default( config, 'configuration', + 'test_mode', False ) + + self._TEST_MODE_IP = get_default( config, 'configuration', + 'test_mode_ip', None ) + self._TEST_MODE_DNS = get_default( config, 'configuration', + 'test_mode_dns', None ) + self._TEST_MODE_NETMASK = get_default( config, 'configuration', + 'test_mode_netmask', None ) + self._TEST_MODE_GATEWAY = get_default( config, 'configuration', + 'test_mode_gateway', None ) + + # Required options: + self._PSC_ADDRESS = config.get( 'configuration', 'psc_address' ) + self._PSA_CONFIG_PATH = config.get( 'configuration', 'psa_config_path' ) + self._PSA_ID = config.get( 'configuration', 'psa_id' ) + self._PSA_SCRIPTS_PATH = config.get( 'configuration', 'scripts_path' ) + self._PSA_API_VERSION = config.get( 'configuration', 'psa_api_version' ) + self._PSA_VERSION = config.get( 'configuration', 'psa_version' ) + self._PSA_NAME = config.get( 'configuration', 'psa_name' ) + self._PSA_LOG_LOCATION = config.get( 'configuration', 'psa_log_location' ) + + # Make all relative paths absolute based on $PSA_HOME + base = self._PSA_HOME + self._LOG_FILE = normalize_path( base, self._LOG_FILE ) + self._PSA_CONFIG_PATH = normalize_path( base, self._PSA_CONFIG_PATH ) + self._PSA_SCRIPTS_PATH = normalize_path( base, self._PSA_SCRIPTS_PATH ) + self._PSA_LOG_LOCATION = normalize_path( base, self._PSA_LOG_LOCATION ) + + self._CONF_ID = config.get( 'configuration', 'conf_id' ) + + @property + def PSA_HOME( self ): + return self._PSA_HOME + + @property + def TEST_MODE( self ): + return self._TEST_MODE + + @property + def TEST_MODE_IP( self ): + return self._TEST_MODE_IP + + @property + def TEST_MODE_DNS( self ): + return self._TEST_MODE_DNS + + @property + def TEST_MODE_NETMASK( self ): + return self._TEST_MODE_NETMASK + + @property + def TEST_MODE_GATEWAY( self ): + return self._TEST_MODE_GATEWAY + + @property + def LOG_FILE( self ): + return self._LOG_FILE + + @property + def VERBOSE( self ): + return self._VERBOSE + + @property + def PSC_ADDRESS( self ): + return self._PSC_ADDRESS + + @property + def PSA_CONFIG_PATH( self ): + return self._PSA_CONFIG_PATH + + @property + def PSA_SCRIPTS_PATH( self ): + return self._PSA_SCRIPTS_PATH + + @property + def PSA_ID( self ): + return self._PSA_ID + + @property + def PSA_NAME( self ): + return self._PSA_NAME + + @property + def PSA_API_VERSION( self ): + return self._PSA_API_VERSION + + @property + def PSA_VERSION( self ): + return self._PSA_VERSION + + @property + def PSA_LOG_LOCATION( self ): + return self._PSA_LOG_LOCATION + + # @property + # def CONF_ID(self): + # return self._CONF_ID diff --git a/PSA/ConfigLoader.py b/PSA/ConfigLoader.py new file mode 100644 index 0000000..d87a9f1 --- /dev/null +++ b/PSA/ConfigLoader.py @@ -0,0 +1,90 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# ConfigLoader.py +# +# Loads a JSON configuration file and performs some sanity checks. +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import json + +class ParseError( Exception ): + def __init__( self, value ): + super( ParseError, self ).__init__( value ) + self.value = value + + def __str__ ( self ): + return repr( self.value ) + +class ObjectEnum( object ): + Connection, Port, Address = range( 3 ) + +def parseObjectEnum( value ): + if value == 'OBJ_CONNECTION': + return ObjectEnum.Connection + if value == 'OBJ_PORT': + return ObjectEnum.Port + if value == 'OBJ_ADDRESS': + return ObjectEnum.Address + raise ParseError( 'Invalid ObjectEnum: ' + value ) + +class EventEnum( object ): + File, Connection = range( 2 ) + +def parseEventEnum( value ): + if value == 'EVENT_FILE': + return EventEnum.File + if value == 'EVENT_CONNECTION': + return EventEnum.Connection + raise ParseError( 'Invalid EventEnum: ' + value ) + +class ActionEnum( object ): + Log = range( 1 ) + +def parseActionEnum( value ): + if value == 'log': + return ActionEnum.Log + raise ParseError( 'Invalid ActionEnum: ' + value ) + +def parseMultiValueDictionary( data ): + to = {} + for item in data: + key = item[ 'type' ] + value = item[ 'value' ] + to.setdefault( key, [] ) + to[ key ].append( value ) + return to + +def parseHSPL( data ): + to = {} + to[ 'id' ] = data[ 'id' ] + to[ 'text' ] = data[ 'text' ] + return to + +class Rule( object ): + + ruleId = None # Rule ID string + event = None # Event Enum + operation = None # Operation name (bro module name) + action = None # Action Enum + parameters = {} # Dictionary of parameters: type as a key, list of values + conditions = {} # Dictionary of conditions: type as a key, list of values + + def __init__( self, data ): + self.ruleId = data[ 'id' ] + self.hspl = parseHSPL( data[ 'hspl' ] ) + self.event = parseEventEnum( data[ 'event' ] ) + self.action = parseActionEnum( data[ 'action' ] ) + self.operation = data[ 'operation' ] + self.parameters = parseMultiValueDictionary( data[ 'parameters' ] ) + self.conditions = parseMultiValueDictionary( data[ 'conditions' ] ) + +def load( filename ): + out = [] + with open( filename, 'r' ) as data_file: + data = json.load( data_file ) + rules = data[ 'rules' ] + for rule in rules: + out.append( Rule( rule ) ) + return out diff --git a/PSA/DEBUG.md b/PSA/DEBUG.md new file mode 100644 index 0000000..5a6c6cf --- /dev/null +++ b/PSA/DEBUG.md @@ -0,0 +1,81 @@ +# Adding extra Bro scripts + +This document describes how to load user defined Bro NSM scripts on Bro PSA. +Since BroPSA loads scripts dynamically, normal Bro configuration files +cannot be used (easily) for debugging, e.g., for redefining variables. This +document describes two approaches of adding such Bro scripts. + +Examples in this document consider adding a local repository of file hashes for +the 'detect-MHR' module. This feature can be used, e.g., for testing. Normally, +these file hashes can be added simply by redefining the bro variable +MHR::local_hashes as shown in the example below. However, because of BroPSA's +dynamic module loading, this is not possible. + +## Option 1: Using pre- and post-init scripts + +BroPSA allows users to define Bro scripts that are loaded before or after the +actual BroPSA module Bro scripts are loaded. Pre- and post-init scripts must be +placed on files called *modules/pre-init.bro* or *modules/post-init.bro*, +respectively. If either of these files exist when BroPSA's configuration is set, +then it will be automatically added into the BroPSA's Bro configuration. + +**Example**: + +Create file *modules/post-init.bro* with the following content and then start +BroPSA normally: + +``` +redef ignore_checksums = T; +redef tcp_max_initial_window = 0; +redef tcp_max_above_hole_without_any_acks = 0; +redef tcp_excessive_data_without_further_acks = 0; + +redef MHR::local_hashes += { [ "afba7d3f3addd136afb4b13a49703e979fb4f590" ] + = [ $kind="sha1", $description="detected T170.pdf" ], + [ "f2e5efd7b47d1fb5b68d355191cfed1a66b82c79" ] + = [ $kind="sha1", $description="detected 7z1514.exe" ] }; +``` + +## Option 2: Using BroLoader-module + +BroLoader-module is a dummy BroPSA module that does not do anything else, but +triggers a Bro script file called modules/config.bro to be loaded. This script +file can be used to load certain Bro scripts dynamically. Compared to using pre- +and post-init scripts BroLoader-module offers extra flexibility: it can be used +to load Bro scripts between BroPSA modules, not just before or after all the +modules are loaded. Since BroLoader is a normal BroPSA module, it is loaded +according to the load order defined by the BroPSA's configuration file. + + +**Example**: +Create file *modules/config.bro* with the following content: + +``` +redef ignore_checksums = T; +redef tcp_max_initial_window = 0; +redef tcp_max_above_hole_without_any_acks = 0; +redef tcp_excessive_data_without_further_acks = 0; + +redef MHR::local_hashes += { [ "afba7d3f3addd136afb4b13a49703e979fb4f590" ] + = [ $kind="sha1", $description="detected T170.pdf" ], + [ "f2e5efd7b47d1fb5b68d355191cfed1a66b82c79" ] + = [ $kind="sha1", $description="detected 7z1514.exe" ] }; +``` + +Add a new rule to *psaConfig/psaconf* after any rules related to the +'detect-MHR' module (e.g. as the last rule). This rule will cause +*modules/config.bro* file to be loaded. Start BroPSA normally. + +``` + { "id": "load-config", + "hspl": { + "id": "-", + "text": "-" + }, + "event": "EVENT_FILE", + "operation": "load-config", + "parameters": [ ], + "action": "log", + "conditions": [] + } +``` diff --git a/PSA/ModuleLoader.py b/PSA/ModuleLoader.py new file mode 100644 index 0000000..5bba03a --- /dev/null +++ b/PSA/ModuleLoader.py @@ -0,0 +1,69 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# ModuleLoader.py +# +# Loads python modules. +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import os +import json +import logging + +# Modules filename +moduleFile = None + +def init( filename ): + global moduleFile + moduleFile = filename + +def _loadModule( path ): + """ + First converts 'path' from file path representation to + Python module name, i.e., removes file extension, converts + slashes to dots, and removes . or .. from the start of the + path if any (thus, paths will be relative to this directory). + """ + path = path.strip() + path = os.path.normpath( path ) + if path.endswith( '.py' ): + path = path[:-3] + changed = True + while changed: + changed = False + while path.startswith( '.' ): + path = path[1:] + changed = True + while path.startswith( '/' ): + path = path[1:] + changed = True + name = path.replace( '/', '.' ) + logging.info( 'Loading: ' + name ) + module = __import__( name, fromlist=[ '' ] ) + return getattr( module, 'module' ) + + +def load( name ): + """ + Loads a module by name 'name' if one is listed in the + modules file. Returns content of the variable called 'module' + which should contain the module class declaration. If anything + goes wrong, None is returned. + """ + logging.info( 'Searching module: ' + moduleFile ) + try: + with open( moduleFile, 'r' ) as config: + data = json.load( config ) + modules = data[ 'modules' ] + for module in modules: + moduleName = module[ 'name' ] + logging.info( 'Scanning: ' + moduleName ) + if moduleName == name: + logging.info( 'Found module: ' + name + + ' (' + module[ 'module' ] + ')' ) + return _loadModule( module[ 'module' ] ) + except Exception as e: + logging.warning( 'Module loading failed: ' + str( e ) ) + + return None diff --git a/PSA/README.md b/PSA/README.md new file mode 100644 index 0000000..077e397 --- /dev/null +++ b/PSA/README.md @@ -0,0 +1 @@ +# Bro PSA diff --git a/PSA/boot_psa.sh b/PSA/boot_psa.sh new file mode 100644 index 0000000..6560168 --- /dev/null +++ b/PSA/boot_psa.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ -z "$PSA_HOME" ]; then + echo "error: 'PSA_HOME' is not set." >&2 + exit 1 +fi + +if [ ! -d "$PSA_HOME" ]; then + echo "error: 'PSA_HOME' is not a valid directory." >&2 + exit 1 +fi + +ip=$(ifconfig eth0 | grep "inet addr" | awk '{print $2}' | cut -d: -f2) +gunicorn -k gevent -b $ip:8080 --log-file $PSA_HOME/GUNICORN.log --log-level debug psaEE:app & diff --git a/PSA/boot_script_psa b/PSA/boot_script_psa new file mode 100644 index 0000000..963536f --- /dev/null +++ b/PSA/boot_script_psa @@ -0,0 +1,23 @@ +#!/bin/bash + +# Place this in /etc/network/if-up.d/ + +PSA_HOME="/home/psa" + +if [ -z "$PSA_HOME" ]; then + echo "error: 'PSA_HOME' is not set." >&2 + exit 0 +fi + +if [ ! -d "$PSA_HOME" ]; then + echo "error: 'PSA_HOME' is not a valid directory." >&2 + exit 0 +fi + +[ "$IFACE" = 'eth2' ] || exit 0 + +ifconfig eth2 mtu 1496 +dhclient -1 eth2 +cd $PSA_HOME/pythonScript +ip=$(ifconfig eth2 | grep "inet addr" | awk '{print $2}' | cut -d: -f2) +gunicorn -k gevent -b $ip:8080 --log-file $PSA_HOME/GUNICORNz.log --log-level debug psaEE:app & diff --git a/PSA/dumpLogFile.py b/PSA/dumpLogFile.py new file mode 100644 index 0000000..c07a199 --- /dev/null +++ b/PSA/dumpLogFile.py @@ -0,0 +1,30 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# + +''' + File: dumpLogFile.py + Description: + REST resource to dump content of the log file from the PSC + For development purpose only! Disable this in production (TBD) + +''' + +import falcon +#import json +import logging +import sys + +class dumpLogFile(): + def __init__(self): + pass + + def on_get(self, req, resp): + try: + in_file = open("PSA.log","r") + log = in_file.read() + in_file.close() + resp.status = falcon.HTTP_200 + resp.body = log + except Exception as e: + logging.exception(sys.exc_info()[0]) + resp.status = falcon.HTTP_501 diff --git a/PSA/execInterface.py b/PSA/execInterface.py new file mode 100644 index 0000000..a6300c3 --- /dev/null +++ b/PSA/execInterface.py @@ -0,0 +1,248 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# File: execInterface.py +# Created: 27/08/2014 +# Author: BSC, VTT +# Modified: 2016 +# Author: VTT, jju, jk +# +# Description: +# Web service running on the PSA receiving the +# configuration for the PSA from the PSC +# +# + +import falcon +import requests +import logging +import json +import sys +import subprocess +import datetime + +from BroManager import BroManager + +# Bro instance: +bro = None + +class execInterface(): + + def __init__ ( self, home, configsPath, scriptsPath, psaLogLocation, psaID, pscAddr, psaAPIVersion ): + self.psaHome = home + self.confsPath = configsPath + self.scripts_path = scriptsPath + self.log_location = psaLogLocation + self.psaID = psaID + self.pscAddr = pscAddr + self.psaAPI = psaAPIVersion + + def on_post( self, request, response, command ): + print "onPost" + try: + res = {} + res[ "command" ] = command + if command == "init": + # receive the configuration, or init package + script_file = self.confsPath + "/psaconf" + fp=open(script_file, 'wb') + while True: + chunk = request.stream.read(4096) + fp.write(chunk) + if not chunk: + break + fp.close() + + # Make script executable for current user + # hazardous.. we're root + #st = os.stat(script_file) + #os.chmod(script_file, st.st_mode | stat.S_IEXEC) + + # Run the init.sh and return it's return value + res["ret_code"] = str(self.callInitScript()) + logging.info("PSA "+self.psaID+" configuration registered") + elif command == "start": + res["ret_code"] = str(self.callStartScript()) + elif command == "stop": + res["ret_code"] = str(self.callStopScript()) + else: + logging.info("POST: unknown command: " + command) + response.status = falcon.HTTP_404 + return + + response.body = json.dumps(res) + response.status = falcon.HTTP_200 + response.set_header("Content-Type", "application/json") + + except Exception as e: + logging.exception( sys.exc_info()[0] ) + response.status = falcon.HTTP_501 + + def on_get(self, request, response, command): + try: + res = {} + res["command"] = command + if command == "status": + res["ret_code"] = self.callStatusScript().replace("\n", "") + elif command == "configuration": + res["ret_code"] = self.callGetConfigurationScript() + elif command == "internet": + res["ret_code"] = self.callGetInternetScript() + elif command == "log": + # Return PSA log or 501 + log = self.callGetLogScript() + if log != None: + response.body = log + response.status = falcon.HTTP_200 + response.set_header("Content-Type", "text/plain; charset=UTF-8") + else: + response.status = falcon.HTTP_501 + return + elif command == 'brolog': + log = self.callGetBroLogScript() + if log != None: + response.body = log + response.status = falcon.HTTP_200 + response.set_header("Content-Type", "text/plain; charset=UTF-8") + else: + response.status = falcon.HTTP_501 + return + else: + logging.info("GET: unknown command: " + command) + response.status = falcon.HTTP_404 + return + + response.body = json.dumps(res) + response.status = falcon.HTTP_200 + response.set_header("Content-Type", "application/json") + except Exception as e: + logging.exception(sys.exc_info()[0]) + response.status = falcon.HTTP_501 + + def callInitScript( self ): + global bro + logging.info ("callInitScript()" ) + + if bro != None: + bro.stopBro() + del bro + + bro = BroManager( self.psaHome, self ) + bro.loadConfig( self.confsPath + "/psaconf" ) + + #ret = subprocess.call([ self.scripts_path + 'init.sh']) + #return ret + + logging.info( 'BroManager initialized: %r' % ( bro != None ) ) + + return 0 + + def callStartScript( self ): + logging.info( "callStartScript()" ) + + if bro == None: + logging.critical( 'BroManager instance not found.' ) + self.callInitScript() + + try: +# bro.start() + try: + bro.connect() + logging.info( 'Bro is already running.' ) + return 0 + except IOError as e: + logging.info( 'No running instances of Bro found.' ) + bro.startBro() + logging.info( 'Bro is running.' ) + return 0 + except Exception as e: + logging.critical( 'Fatal error while connecting to Bro' ) + logging.critical( e ) + +# ret = subprocess.call([ self.scripts_path + 'start.sh']) +# return ret + return 1 + + def callStopScript( self ): + logging.info( "callStopScript()" ) + + if bro != None: + bro.stopBro() + logging.info( 'Bro stopped.' ) + else: + logging.info( 'Bro is not running.' ) + +# ret = subprocess.call([ self.scripts_path + 'stop.sh']) +# return ret + + return 0 + + def callStatusScript( self ): + proc = subprocess.Popen( [ self.scripts_path + 'status.sh' ], + stdout = subprocess.PIPE, + shell = True ) + ( out, err ) = proc.communicate() + return out + + def callGetConfigurationScript( self ): + logging.info( "callGetConfigurationScript()" ) + proc = subprocess.Popen( [ self.scripts_path + 'current_config.sh' ], + stdout = subprocess.PIPE, + shell = True ) + ( out, err ) = proc.communicate() + return out + + def callGetInternetScript (self ): + logging.info( "callGetInternetScript()" ) + proc = subprocess.Popen( [ self.scripts_path + 'ping.sh' ], + stdout = subprocess.PIPE, + shell = True ) + ( out, err ) = proc.communicate() + return out + + def callGetLogScript( self ): + logging.info( "callGetLogScript()" ) + try: + filename = self.confsPath + "/bro.log" + #filename = self.log_location + with open( filename, "r" ) as f: + return f.read() + except Exception as e: + logging.exception( sys.exc_info()[0] ) + return None + + def get_client_address( self, environ ): + try: + return environ[ 'HTTP_X_FORWARDED_FOR' ].split( ',' )[ -1 ].strip() + except KeyError: + return environ[ 'REMOTE_ADDR' ] + + def callGetBroLogScript( self ): + logging.info( "callGetBroLogScript()" ) + try: + filename = self.confsPath + "/bro.log" + with open( filename, "r") as f: + return f.read() + except Exception as e: + logging.exception( sys.exc_info()[ 0 ] ) + return None + + def onEvent( self, logEntry ): + filename = self.confsPath + "/bro.log" + line = str( datetime.datetime.utcnow() ) + ': ' + logEntry + with open( filename, "a" ) as logFile: + logFile.write( line ) + + def onNotifyEvent( self, policy, title, info): + self.sendPsaEvent(policy, title, info) + + def sendPsaEvent(self, policy, title, info): + logging.info( "sendPsaEvent()" ) + header = {"Content-Type": "application/json"} + ev = {"psa_id": self.psaID, "event_title": title, "event_body": info, "extra_info": policy, "hspl_id": "", "mspl_id": ""} + url = self.pscAddr + "/" + self.psaAPI + "/psaEvent/" + self.psaID + + try: + requests.post(url, data=json.dumps(ev), headers=header) + except Exception as e: + logging.exception( sys.exc_info()[ 0 ] ) + diff --git a/PSA/getConfiguration.py b/PSA/getConfiguration.py new file mode 100644 index 0000000..b1c647c --- /dev/null +++ b/PSA/getConfiguration.py @@ -0,0 +1,124 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# File: getConfiguration.py +# Created: 05/09/2014 +# Author: BSC + +# Modified: 29/10/2015 +# Author: VTT +# +# Description: +# Web service running on the PSA interacting with the PSC +# +# + +import json +import requests +import logging +from psaExceptions import psaExceptions +import subprocess +import base64 + +class getConfiguration(): + + #def __init__(self, pscAddr, configsPath, confID, psaID): + def __init__(self, pscAddr, configsPath, scriptsPath, psaID, psaAPIVersion): + self.pscAddr = pscAddr + self.configsPath = configsPath + self.scripts_path = scriptsPath + #self.confID = confID + self.psaID = psaID + self.psaAPI = psaAPIVersion + + def send_start_event(self): + logging.info("PSA: send_start_event") + logging.info("PSA: "+self.psaID+" calling PSC") + resp = requests.get(self.pscAddr + "/" + self.psaAPI + "/psa_up/" + self.psaID) + logging.info("PSA: "+self.psaID+" calling PSC done") + return resp.content + + def pullPSAconf( self, execIf ): + + header = {'Content-Type':'application/octet-stream'} + + #resp = requests.get(self.pscAddr+"/getConf/"+self.psaID+"/"+self.confID, headers=header) + resp = requests.get(self.pscAddr + "/" +self.psaAPI + "/getConf/"+self.psaID, headers=header) + + # NOTE: pylint will complain about 'requests.codes.ok' since it has no + # way of knowning statically that it exists as request.codes is + # contructed dynamically. + + if resp.status_code == requests.codes.ok: # pylint: disable=E1101 + #fp=open(self.configsPath+"/"+self.confID,'wb') + #fp=open(self.configsPath+"/"+self.psaID,'wb') + # We don't have multiple security controls inside one PSA image at the moment. + json_config = False + try: + conf = json.loads(resp.content) + logging.info("PSA JSON conf received:") + logging.info(conf) + # Handle different config formats + if conf["conf_type"] == "base64": + decoded_conf = base64.b64decode(conf["conf"]) + elif conf["conf_type"] == "text": + decoded_conf = conf["conf"] + else: + # Use default format, presume text. + decoded_conf = conf["conf"] + json_config = True + except Exception as e: + logging.info("Could not load JSON config, reverting to old text format") + decoded_conf = resp.content + + fp=open(self.configsPath+"/psaconf", 'wb') + fp.write(decoded_conf) + fp.close() + +# self.callInitScript() + execIf.callInitScript() + if json_config: + self.enforceConfiguration(conf) + + logging.info("PSA "+self.psaID+" configuration registered") + return resp.content + else: + logging.error("Bad configuration request for PSA "+self.psaID) + raise psaExceptions.confRetrievalFailed() + + + # header = {'Accept':'application/octet-stream', 'Content-Type':'application/octet-stream'} + # resp = requests.get(self.pscAddr+"/getConfiguration/"+self.confURI, data={}, headers=header) + # if (resp.status_code != 200): + # msg = "PSC is not able to provide the conf for: [PSAid] " + self.psaID + ", [confURI] " + self.confURI + # raise psaExceptions.confRetrievalFailed(msg) + + # TODO check script validity + #return resp.text + + #TODO: this should be the same as in execInterface.py!!! +# def callInitScript(self): +# logging.info("callInitScript()") +# ret = subprocess.call(['.' + self.scripts_path + 'init.sh']) +# return ret + + def enforceConfiguration(self, jsonConf): + req_keys = ("IP", "dns", "netmask", "gateway") + has_req = False + if all (key in jsonConf for key in req_keys): + has_req = True + + if has_req: + logging.info("PSA requires IP, configuring...") + ip = jsonConf["IP"] + dns = jsonConf["dns"] + netmask = jsonConf["netmask"] + gateway = jsonConf["gateway"] + logging.info("ip: " + str(ip)) + logging.info("gateway: " + str(gateway)) + logging.info("dns: " + str(dns)) + logging.info("netmask: " + str(netmask)) + ret = subprocess.call( [ self.scripts_path + 'ip_conf.sh', ip, gateway, dns, netmask ] ) + #ret = subprocess.call(['.' + self.scripts_path + 'ip_conf.sh', ip, gateway, dns, netmask]) + logging.info("Result of setting config: " + str(ret)) + else: + logging.info("PSA doesn't require IP, skipping configuration.") diff --git a/PSA/interfaces b/PSA/interfaces new file mode 100644 index 0000000..ccf3246 --- /dev/null +++ b/PSA/interfaces @@ -0,0 +1,32 @@ +# PSA interface file +# Place this in /etc/network in your PSA image template + +# This file describes the network interfaces available on your system +# and how to activate them. For more information, see interfaces(5). + +# The loopback network interface +auto lo br0 eth2 +iface lo inet loopback + +# The primary network interface + +iface eth0 inet manual +iface eth1 inet manual +#iface eth2 inet dhcp +iface eth2 inet manual + +iface br0 inet manual + pre-up ip link set eth0 down + pre-up ip link set eth1 down + pre-up brctl addbr br0 + pre-up brctl addif br0 eth0 eth1 + pre-up ip addr flush dev eth0 + pre-up ip addr flush dev eth1 + pre-up ip link set eth0 up + pre-up ip link set eth1 up + pre-up ip link set br0 up + post-down ip link set eth0 down + post-down ip link set eth1 down + post down ip link set br0 down + post-down brctl delif br0 eth0 eth1 + post-down brctl delbr br0 diff --git a/PSA/json/psaStartup.json b/PSA/json/psaStartup.json new file mode 100644 index 0000000..846d571 --- /dev/null +++ b/PSA/json/psaStartup.json @@ -0,0 +1,9 @@ +{ + + "name": "psa_startup_file", + "user_token": "token1", + "psaID": "12345", + "pscAddr": "http://127.0.0.1:4321", + "confURI": "12345" + +} diff --git a/PSA/modules.json b/PSA/modules.json new file mode 100644 index 0000000..a31322f --- /dev/null +++ b/PSA/modules.json @@ -0,0 +1,16 @@ +{ + "modules": [ + { + "name": "count", + "module": "modules/Count.py" + }, + { + "name": "detect-MHR", + "module": "modules/MHR.py" + }, + { + "name": "load-config", + "module": "modules/BroLoader.py" + } + ] +} diff --git a/PSA/modules/BroEventDispatcher.py b/PSA/modules/BroEventDispatcher.py new file mode 100644 index 0000000..5bc0cb9 --- /dev/null +++ b/PSA/modules/BroEventDispatcher.py @@ -0,0 +1,41 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# BroEventDispatcher.py +# +# A simple event dispatcher. +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import logging + +callbacks = { } + +def init(): + pass + +def register( key, obj ): + """ + Register a callback for key 'key' + """ + global callbacks + callbacks[ key ] = obj + +def unregister( key ): + """ + Unregisters callback for key 'key' + """ + global callbacks + del callbacks[ key ] + +def dispatch( key, data ): + """ + Dispatch event 'data' to the callback registered for key 'key' + """ + global callbacks + try: + cb = callbacks[ key ] + if cb != None: + cb.onEvent( data ) + except Exception as e: + logging.warning( 'No dispatcher for key: ' + key + ': ' + str( e ) ) diff --git a/PSA/modules/BroLoader.py b/PSA/modules/BroLoader.py new file mode 100644 index 0000000..f97aeb7 --- /dev/null +++ b/PSA/modules/BroLoader.py @@ -0,0 +1,33 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# BroLoader.py +# +# A dummy module that loads config.bro file +# +# The rule for this module should be the las one in the list! +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +from modules.BroModule import BroModule + +class BroLoaderModule( BroModule ): + + rules = { } + + def __init__( self, logger ): + super( BroLoaderModule, self ).__init__( 'config.bro', logger ) + + def onStart( self, connection ): + super( BroLoaderModule, self ).onStart( connection ) + + def onStop( self ): + super( BroLoaderModule, self ).onStop() + + def onRule( self, rule ): + return True + + def onEvent( self, data ): + pass + +module = BroLoaderModule diff --git a/PSA/modules/BroModule.py b/PSA/modules/BroModule.py new file mode 100644 index 0000000..1a7e0b0 --- /dev/null +++ b/PSA/modules/BroModule.py @@ -0,0 +1,73 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# BroModule.py +# +# A parent interface for all Bro modules. +# +# Each Bro module should define the functions declared in the +# BroModule class. Futhremore, each module must define module +# variable 'module' that contains the BroModule class defined +# in the module. The module variable is used by the BroManager +# to instantiate the module object. +# +# Any Bro event handlers (@event) should be registered to +# BroEventDispacther. This dispatcher is used to circument the fact +# that Broccoli Python interface expects the event handler to be +# a module function (not a class memeber). +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + + +# NOTE: any communication with Bro should only happen if Bro +# is running, i.e., the module is in state 'Started'! Otherwise +# Gunicorn worker will boot unexpectably. + +class BroModule( object ): + + class State( object ): + Started, Stopped = range( 2 ) + + broScript = None # Bro scrip's filename + enabled = False # If the module is enabled currently + connection = None # Bro connection for sending events + state = State.Stopped # Modules current state + logger = None # Logger to send log events to + + def __init__( self, filename, logger ): + self.broScript = filename + self.logger = logger + + def onRule( self, rule ): + """ + Add a single configuration rule to module + """ + return False + + def onStart( self, connection ): + """ + Called when Bro is started. + + Bro is already running when this callback is called. The callback + should be used to pass rule information to modules .bro script. + """ + self.connection = connection + self.state = self.State.Started + + def onStop( self ): + """ + Called when Bro is being stopped. + + This callback should be used to perform any cleanup actions necessary. + """ + + self.state = self.State.Stopped + + def onEvent( self, event ): + """ + Called if a Bro event is dispatched to this module. + """ + pass + +# Example module variable definition: +#module = BroModule diff --git a/PSA/modules/CertValidation.bro b/PSA/modules/CertValidation.bro new file mode 100644 index 0000000..b8063eb --- /dev/null +++ b/PSA/modules/CertValidation.bro @@ -0,0 +1,114 @@ +# -*- Mode:Bro;indent-tabs-mode:nil;-*- +# +# CertValidation.bro +# +# Certificate Validation module +# +# Heavily based on validate-certs.bro script +# +# Author: sl / VTT / 2016 +# + +@load ./psa-utils +@load protocols/ssl/validate-certs + +module CVModule; + +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; # Timestamp + op: string &log; # Type of event + id: string &log; # Name of the rule + }; + +event on_cv_config( req: CVConfigRecord ) { + Log::write( CVModule::LOG, + [ $ts = network_time(), + $id = req$op, + $msg = ( req?$mime ? req$mime : "-" ) ] ); + + # Possibly setting up some root certs to trust + + switch ( req$op ) { + case "add": # Add a root cert + break; + default: # Invalid operation + return; + } +} + + +# this event occurs whenever a SSL connection is established +event ssl_established( c: connection ) &priority=3 +{ + logging.info("SSL established!"); + + local cert = c$ssl$cert_chain[0]$x509$certificate; + + local id = ""; + local hashes = ""; + for ( i in c$ssl$cert_chain ) + { + if ( i > 0 ) + hashes += " "; + hashes += c$ssl$cert_chain[i]$sha1; + + } + local name = c$ssl$cert_chain[0]$x509$certificate$subject; + local message = c$ssl$validation_status; + logging.info( id + " \"" + name + "\" " + hashes + " \"" + message + "\""; + + send_log_event( id, name, hashes, msg ); +} + +# A log event for cert validations. + +type CVLogRecord: record { + id: string; # Operation ID + ts: string; # time + hashes: string; # Cert hashes of the whole chain + name: string; # Cert subject + msg: string; # Trusted, expired or some other reason. +}; + +# Event handler: + +global cv_log: event( data: CVLogRecord ); + +# Auxilliary function to formatting and sending CVLogRecords: + +function send_log_event( id: string, name: string, hashes: string, msg: string ) +{ + local source = ""; + + local rec: CVLogRecord; + rec$ts = network_time(); + rec$id = id; + rec$hashes = hashes; + rec$name = name; + rec$msg = msg; + + event cv_log( rec ); +} + + +event bro_init() &priority=9 +{ + if ( !Log::create_stream( LOG, [ $columns=Info ] ) ) + { + print "CertValidation.bro: Log creation failed!"; + } + + Log::write( CVModule::LOG, + [ $ts = network_time(), + $id = "Init", + $msg = "" ] ); + + PSA::subscribe_events( /on_cv_config/ ); + PSA::subscribe_events( /ssl_established/ ); +} + +} diff --git a/PSA/modules/CertValidation.py b/PSA/modules/CertValidation.py new file mode 100644 index 0000000..9511802 --- /dev/null +++ b/PSA/modules/CertValidation.py @@ -0,0 +1,65 @@ + +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# CertValidation.py +# +# Implements a certificate chain verification module that utilises +# validate-certs.bro +# + +import logging + +from broccoli import event, record_type, record +from modules.BroModule import BroModule + +# Log event: +CVLogRecord = record_type( 'id', # Operation ID + 'ts', # When the cert was detected + 'hashes', # Cert hashes + 'name', # Cert subject + 'msg' ) # Message (Trusted/expired/etc) + +# Key for receiving Bro events. +CVModuleKey = 'CVModuleEvent' + + +class CVModule( BroModule ): + + def __init__( self, logger ): + super( CVModule, self ).__init__( 'CertValidation.bro', logger ) + logging.info( 'CVModule init' ); + + def onStart( self, connection ): + super( CVModule, self ).onStart( connection ) + + def onStop( self ): + super( CVModule, self ).onStop() + + def onRule( self, rule ): + logging.info( 'Rule received' ); + + def _sendRule( ): + logging.info( 'Passing rule to bro' ); + + def _log_event( self, data ): + + try: + fmt = "[%s] %s (%s: %s): %s\n" + line = fmt % ( data.ts, + data.id, + data.name, + data.hashes, + data.msg ) + + self.logger.onEvent( line ) + except Exception as e: + logging.error( e ) + +# Dispatching events: +@event(CVLogRecord) +def cv_log( data ): + logging.info( "Event: Certificate validated" ) + BroEventDispatcher.dispatch( CVModuleKey, data ) + + +module = CVModule diff --git a/PSA/modules/Count.py b/PSA/modules/Count.py new file mode 100644 index 0000000..c306f7f --- /dev/null +++ b/PSA/modules/Count.py @@ -0,0 +1,463 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# Count.py +# +# Implements a count module that communicates with ccount.bro +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import logging +from collections import deque +import calendar +import time +import uuid + +from broccoli import event, record_type, record, addr, port, count +from modules.BroModule import BroModule +import modules.BroEventDispatcher as BroEventDispatcher + +# Record used to pass configuration rules to bro +CountConfigRecord = record_type( 'op', # Operation: src_addr, src_port, + # dst_addr, or dst_port + 'id', # Rule ID + 'address', # Address to match: IP or + # hostname, or an empty if only + # port is matched) + 'service' ) # Port to match (0/tcp if only + # address is matched) +# Bro's response record +CountReportRecord = record_type( 'rule', # Rule ID of matching rule + 'ts', # Start time of the period + 'num_occurences', # Number of events occured + 'first_occurence', # Timestamp of first occurence + 'last_occurence', # Timestamp of last occurence + 'period' ) # Duration of the period (in seconds) + +# Bro's record for indicating end of a measurement period (all records sent) +CountPeriodRecord = record_type( 'ts', # Start time of the period + 'period' ) # Duration of the period in seconds + +CountModuleKey = 'CountModuleEvent' + +class BroRule( object ): + + ruleId = None + rule = None + op = None + address = None + service = None + counter = None + + def __init__( self ): + pass + + def record( self ): + rec = record( CountConfigRecord ) + rec.op = str( self.op ) + rec.id = str( self.ruleId ) + rec.address = str( self.address ) + rec.service = port( str( self.service ) + '/tcp' ) + return rec + +# Bro summaries a produced in measurement period of one minute. +# An object of this counter class is used to combine measurements +# of several adjacent perioids with each other. + +# NOTE: because of how Bro SumStats works, reports are sent immediately +# when the reporting threshold is exceed: this may mean, that the reported +# count does not sum all the counts of that period. In addition, if the +# reporting period is longer than one minute, the same events may cause +# multiple log entries to be reported, as the counter simply sums counts +# of all periods fitting into the interval. +# +# Fixing these issues would require: +# a) an event to be generated even if there is no events during the period. +# This event should preferably come from bro (SumStats) as otherwise +# (e.g. using a timer in Python) there is no way to know if there was no +# event, or we are just experiencing a delay. +# b) another option to fix this could be to always record log entries when the +# interval has ended: at counter tick (or sometimes using a watchdog if +# there have not been any events for a certain (long) period), before +# discarding old periods, we could check if there is a full interval +# (i.e., collected periods before the new timestamp form an interval) +# and trigger a log event for that and possible the new interval. + +class Counter( object ): + + DEFAULT_INTERVAL = 60 + DEFAULT_THRESHOLD = 1 + + interval = DEFAULT_INTERVAL + threshold = DEFAULT_THRESHOLD + occurences = 0 + queue = None + + def __init__( self, iv, th ): + self.interval = iv + self.threshold = th + self.queue = deque() + + def tick( self, ts, period, count ): + if self.queue: + if ts == self.queue[ -1 ][ 0 ]: + # Already handled (i.e. end-of-period report, when + # there was also a count report this period) + if self.queue[ -1 ][ 2 ] != count and count != 0: + logging.error( 'Invalid state: may have missed a count report' ) + return None + + self.occurences += count + self.queue.append( ( ts, period, count ) ) + + events = [] + + # Is it possible, that we have an event? + if self.occurences > 0 and self.occurences >= self.threshold: + # Yes, lets iterate over the queue to find all events: + m = len( self.queue ) + for i in range( 0, m ): + item = self.queue[ i ] + start = item[ 0 ] + end = start + self.interval + # Is there a full interval? + if end <= ts + period: + # Yes, lets count it's occurences + c = 0 + for j in range( i, m ): + n = self.queue[ j ] + if n[ 0 ] + n[ 1 ] <= end: + c += n[ 2 ] + else: + break + + # and make an event, if they exceed the threshold: + if c >= self.threshold: + events.append( ( c, self.interval, start ) ) + else: + break + + # Let's remove all periods that are reported or cannot macth + # any new intervals: + self.trim( ts + period - self.interval ) + + return events + + def trim( self, ts ): + while self.queue: + item = self.queue[ 0 ] + if item[ 0 ] <= ts: + self.occurences -= item[ 2 ] + self.queue.popleft() + else: + break + + def reset( self ): + self.occurences = 0 + self.queue.clear() + +class CountModule( BroModule ): + + rules = { } # BroRule (not Rule) objects! + + def __init__( self, logger ): + super( CountModule, self ).__init__( 'ccount.bro', logger ) + BroEventDispatcher.register( CountModuleKey, self ) + + def onStart( self, connection ): + super( CountModule, self ).onStart( connection ) + self.reset( False ) + self._sendAllRules() + + def onStop( self ): + super( CountModule, self ).onStop() + + def _sendRule( self, rule ): + """ + Send a single rule to bro module + """ + try: + rec = rule.record() + logging.info( 'Passing rule to Bro: ' + rec.id ) + self.connection.send( 'on_count_config', rec ) + except Exception as e: + logging.warning( 'Config exception for rule: ' + rule.ruleId + + ' (' + rule.rule.ruleId + ')' ) + logging.exception( e ) + + def _sendAllRules( self ): + for key, rule in self.rules.iteritems(): + self._sendRule( rule ) + + def _addRule( self, rule, broRule ): + + broRule.ruleId = str( uuid.uuid4().hex ) + logging.info( 'Generated ID for BroRule: ' + broRule.ruleId + + ' (' + rule.ruleId + ')' ) + self.rules[ broRule.ruleId ] = broRule + broRule.rule = rule + + # Only send rules if connected to Bro + if self.state == BroModule.State.Started: + self._sendRule( broRule ) + #return True + + def onRule( self, rule ): + """ + Parses rules to Bro module's format. + """ + + # TODO: currently only supports one condition per rule + # => otherwise rule will be split to many conditions! + + # TODO: currently only supports tcp-ports. Broccoli does not + # support 'port/unkown'. This is compensated in the bro + # module by converting all ports 0/tcp to 0/unknown. + iv = Counter.DEFAULT_INTERVAL + if 'interval' in rule.conditions: + items = rule.conditions[ 'interval' ] + if len ( items ) > 1 : + logging.error( "Rule may only have at most one 'interval' condition." ) + return False + iv = int( items[ 0 ] ) + + if iv % 60 != 0: + logging.warning( "Only intervals multiple of one minute are supported!" ) + new = iv + 60 - iv % 60 + logging.info( "Using the next multiple (" + str( new ) + " seconds)" + + " instead of " + str( iv ) + " seconds" ) + iv = new + + th = Counter.DEFAULT_THRESHOLD + if 'threshold' in rule.conditions: + items = rule.conditions[ 'threshold' ] + if len (items ) > 1 : + logging.error( "Rule may only have at most one 'threshold' condition." ) + return False + + th = int( items[ 0 ] ) + if th < 1 : + logging.error( "Invalid threshold: " + th ) + return False + + rv = False + + if 'source' in rule.conditions: + items = rule.conditions[ 'source' ] + for item in items: + service = 0 + if 'port' in item: + service = item[ 'port' ] + + b = BroRule() + b.counter = Counter( iv, th ) + b.op = 'src_addr' + b.address = item[ 'address' ] + b.service = service + self._addRule( rule, b ) + rv = True + #return self._addRule( rule, b ) + + if 'destination' in rule.conditions: + items = rule.conditions[ 'destination' ] + for item in items: + service = 0 + if 'port' in item: + service = item[ 'port' ] + + b = BroRule() + b.counter = Counter( iv, th ) + b.op = 'dst_addr' + b.address = item[ 'address' ] + b.service = service + self._addRule( rule, b ) + rv = True + #return self._addRule( rule, b ) + + if 'source_port' in rule.conditions: + items = rule.conditions[ 'source_port' ] + for item in items: + b = BroRule() + b.counter = Counter( iv, th ) + b.op = 'src_port' + b.address = '' + b.service = item[ 'port' ] + self._addRule( rule, b ) + rv = True + #return self._addRule( rule, b ) + + if 'destination_port' in rule.conditions: + items = rule.conditions[ 'destination_port' ] + for item in items: + b = BroRule() + b.counter = Counter( iv, th ) + b.op = 'dst_port' + b.address = '' + b.service = item[ 'port' ] + self._addRule( rule, b ) + rv = True + #return self._addRule( rule, b ) + + return rv + + + def reset( self, resetRules = True ): + # Only send rules if connected to Bro + if self.state == BroModule.State.Started: + b = BroRule() + b.rule = None + b.ruleId = 'reset' + b.op = 'reset' + b.address = '' + b.service = 0 + self._sendRule( b ) + if resetRules: + self.rules = { } + + def _formatLogEvent( self, broRule, status ): + rule = broRule.rule + ts = status[ 2 ] # end time + occurences = status[ 0 ] + period = broRule.counter.interval + + # There should be at most one matching condition for a BroRule! + + if broRule.op == 'src_addr': + if 'source' in rule.conditions: + items = rule.conditions[ 'source' ] + for item in items: + service = 'any' + if 'port' in item: + service = str( item[ 'port' ] ) + return ( ts, + rule.ruleId, + rule.hspl[ 'id' ], + occurences, + period, + 'source', + broRule.address, + broRule.service ) + #item[ 'address' ], + #service ) + else: + return None + + if broRule.op == 'dst_addr': + if 'destination' in rule.conditions: + items = rule.conditions[ 'destination' ] + for item in items: + service = 'any' + if 'port' in item: + service = str( item[ 'port' ] ) + return ( ts, + rule.ruleId, + rule.hspl[ 'id' ], + occurences, + period, + 'destination', + broRule.address, + broRule.service ) + #item[ 'address' ], + #service ) + else: + return None + + if broRule.op == 'src_port': + if 'source_port' in rule.conditions: + items = rule.conditions[ 'source_port' ] + for item in items: + return ( ts, + rule.ruleId, + rule.hspl[ 'id' ], + occurences, + period, + 'source_port', + 'any', + str( broRule.service ) ) + #str( item[ 'port' ] ) ) + else: + return None + + if broRule.op == 'dst_port': + if 'destination_port' in rule.conditions: + items = rule.conditions[ 'destination_port' ] + for item in items: + return ( ts, + rule.ruleId, + rule.hspl[ 'id' ], + occurences, + period, + 'destination_port', + 'any', + str( broRule.service ) ) + #str( item[ 'port' ] ) ) + else: + return None + + return None + + def onEvent( self, data ): + logging.info( "ts: " + str( int( data.ts ) ) ) + + if hasattr( data, 'rule' ): + self.onCountEvent( data ) + else: + self.onEndOfPeriod( data ) + + def onRuleFired( self, rule, status ): + logging.info( 'onEvent: ' + rule.ruleId + ' (' + rule.rule.ruleId + ')' ) + try: + ev = self._formatLogEvent( rule, status ) + if ev == None: + return + + fmt = "[%s] Rule '%s' (HSPL: %s) fired %d times within %d seconds: " \ + "on condition '%s' with address '%s' and port '%s'\n" + self.logger.onEvent( fmt % ev ) + except Exception as e: + logging.error( e ) + + def onCountEvent( self, data ): + try: + rule = self.rules[ data.rule ] + logging.info( "rule: " + rule.ruleId + " " + str( data.num_occurences ) ) + ts = int( data.ts ) + status = rule.counter.tick( ts, data.period, data.num_occurences ) + if status: + logging.info( "log events: " + str( len( status ) ) ) + # Rule fired! + for entry in status: + self.onRuleFired( rule, entry ) + else: + logging.debug( 'event not fired: status: ' + str( status ) ) + except Exception as e: + logging.error( e ) + + def onEndOfPeriod( self, data ): + try: + ts = int( data.ts ) + for key, rule in self.rules.iteritems(): + logging.info( "rule: " + rule.ruleId + " " + str( 0 ) ) + status = rule.counter.tick( ts, data.period, 0 ) + if status: + logging.info( "log events: " + str( len( status ) ) ) + # Rule fired! + for entry in status: + self.onRuleFired( rule, entry ) + else: + logging.debug( 'event not fired: status: ' + str( status ) ) + except Exception as e: + logging.error( e ) + +@event( CountReportRecord ) +def report_count( data ): + logging.info( 'Event: CountReportRecord' ) + BroEventDispatcher.dispatch( CountModuleKey, data ) + + +@event( CountPeriodRecord ) +def report_period( data ): + logging.info( 'Event: CountPeriodRecord' ) + BroEventDispatcher.dispatch( CountModuleKey, data ) + +module = CountModule diff --git a/PSA/modules/MHR.bro b/PSA/modules/MHR.bro new file mode 100644 index 0000000..f974e74 --- /dev/null +++ b/PSA/modules/MHR.bro @@ -0,0 +1,492 @@ +# -*- Mode:Bro;indent-tabs-mode:nil;-*- +# +# MHR.bro +# +# Detect file downloads that have hash values matching files in Team +# Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/). +# +# Acknowledgement: this script is based on the Bro Cymru's Malware Hash Registry +# example script provided by the Bro Project. +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +@load base/utils/files +@load base/utils/time +@load base/files/hash + +@load base/protocols/conn +@load base/protocols/dhcp +@load base/protocols/dnp3 +@load base/protocols/dns +@load base/protocols/ftp +@load base/protocols/http +@load base/protocols/irc +@load base/protocols/modbus +@load base/protocols/pop3 +@load base/protocols/radius +@load base/protocols/snmp +@load base/protocols/smtp +@load base/protocols/socks +@load base/protocols/ssh +@load base/protocols/ssl +@load base/protocols/syslog +@load base/protocols/tunnels + +@load base/frameworks/communication +@load base/frameworks/files +@load frameworks/files/hash-all-files + +@load ./psa-utils + +module MHR; + +export { + +# # File types to attempt matching against the Malware Hash Registry. +# const match_file_types = /application\/x-dosexec/ +# | /application\/vnd.ms-cab-compressed/ +# | /application\/pdf/ +# | /application\/x-shockwave-flash/ +# | /application\/x-java-applet/ +# | /application\/jar/ +# | /video\/mp4/ &redef; + + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; # Timestamp + id: string &log; + msg: string &log; + }; + + #global log_malware: event( rec: Info ); + + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + + ## The malware hash registry runs each malware sample through several + ## A/V engines. Team Cymru returns a percentage to indicate how + ## many A/V engines flagged the sample as malicious. This threshold + ## allows you to require a minimum detection rate. + const notice_threshold = 10 &redef; + + + # Objects of this type describe file hashes that are registered locally for + # detection. For now, the file hash type should always be 'sha1'. The + # description field should contain a human readable description of this + # file. This description is added to related log messages. + # The actual file hash is used as a key in the 'local_hashes' table and + # is not present in this record. + + type LocalHash: record { + kind: string; # Type of hash 'sha1' + description: string; # Description of the hash + }; + + # Global table of locally registered file hashes, e.g., hashes that + # are reported as malware even if they are not registered to the + # malware registry. This is useful for testing, but also allows admins + # to add monitoring for files not registered by Cymru. + # + # NOTE: use redef in a configuration file to add hashes: do not add them + # into this file! + + const local_hashes: table [ string ] of LocalHash = {} &redef; + + # + # redef MHR::local_hashes += { [ "hash-value-1" ] = [ $kind="sha1", $description="" ], + # [ "hash-value-2" ] = [ $kind="sha1", $description="" ] }; + # +} + +# An enumeration that describe all the possible file states (for detection): + +type FileState: enum { New, Hashed, Gapped }; + +# Objects of this type are used to keep track of currently +# transfered files. + +type CurrentFile: record { + id: string; + status: FileState; +}; + +global current_files: table[ string ] of CurrentFile = { }; + +# Definition of configuration event +type MHRConfigRecord: record { + op: string; # Operation: add/reset + mime: string &optional; +}; + +global match_mimes : set [ string ] = { } &redef; + +event on_mhr_config( req: MHRConfigRecord ) { + + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = req$op, + $msg = ( req?$mime ? req$mime : "-" ) ] ); + + # TODO: + # There seem to be no way of creating patterns dynamically (after bro_init)! + # Possible solution: redef 'match_file_types' in a bro-file and restart bro. + + switch ( req$op ) { + case "add": # Add a new rule + if ( req$mime !in match_mimes ) + { + add match_mimes[ req$mime ]; + } + break; + case "reset": + # Remove all rules: + match_mimes = set( ); + break; + default: # Invalid operation + return; + } +} + +# A log event for detected malware + +type MHRRecord: record { + id: string; # Operation ID + ts: time; # File detection time + hash: string; # Sha1 hash + fid: string; # Bro's file ID + name: string; # Filename, if available + service: string; # Service (e.g., HTTP) using which the file was loaded + source: string; # List of space separated addresses + mime: string; # Mime type of the file + detected: time; # First time the malware was detected + rate: count; # Times the malware has been detected + url: string; # VirusTotal URL for the malware + msg: string; # Optional message (not used in 'macth' unless its a local match) +}; + +# Event handler: + +global mhr_alert: event( data: MHRRecord ); + +# A log event for hashed file, errors, etc. + +type MHRLogRecord: record { + id: string; # Operation ID + ts: time; # File detection time + hash: string; # Sha1 hash + fid: string; # Bro's file ID + name: string; # Filename, if available + service: string; # Service (e.g., HTTP) using which the file was loaded + source: string; # List of space separated addresses + mime: string; # Mime type of the file + msg: string; # Optional message (not used in 'macth' unless its a local match) +}; + +# Event handler: + +global mhr_log: event( data: MHRLogRecord ); + +# Auxilliary function to formatting and sending MHRLogRecords: + +function send_log_event( f: fa_file, id: string, hash: string, msg: string ) +{ + local source = ""; + + for ( i in f$info$tx_hosts ) + { + source = cat( source, " ", i ); + } + + local rec: MHRLogRecord; + rec$id = id; + rec$ts = f$info$ts; + rec$hash = hash; + rec$fid = f$id; + rec$name = ( f$info?$filename ? f$info$filename : "" ); + rec$service = f$source; + rec$source = source; + rec$mime = ( f$info?$mime_type ? f$info$mime_type : "" ); + rec$msg = msg; + + event mhr_log( rec ); +} + +function send_alert_event( f: fa_file, + hash: string, + url: string, + detected: time, + rate: count, + msg: string ) +{ + local source = ""; + + for ( i in f$info$tx_hosts ) + { + source = cat( source, " ", i ); + } + + local rec: MHRRecord; + rec$id = "match"; + rec$ts = f$info$ts; + rec$hash = hash; + rec$fid = f$id; + rec$name = ( f$info?$filename ? f$info$filename : "" ); + rec$service = f$source; + rec$source = source; + rec$mime = f$info$mime_type; + rec$rate = rate; + rec$detected = detected; + rec$url = url; + rec$msg = msg; + + event mhr_alert( rec ); +} + +# Actual registry lookup: + +function do_mhr_lookup( hash: string, f: fa_file ) +{ + # Uncomment for testing: a known malware hash brbbot.exe + #hash="2c9e509de4b3ec03589b5c95baba06a9387195e6"; + + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Performing lookup", + $msg = f$id ] ); + + # Log all hashed files at this point + send_log_event( f, "log", hash, "file hashed" ); + + local hash_domain = fmt( "%s.malware.hash.cymru.com", hash ); + when ( local MHR_result = lookup_hostname_txt( hash_domain ) ) + { + # Data is returned as " " + local MHR_answer = split_string1( MHR_result, / / ); + + if ( |MHR_answer| == 2 ) + { + local mhr_detect_rate = to_count( MHR_answer[ 2 ] ); + if ( mhr_detect_rate >= notice_threshold ) + { + local mhr_first_detected = double_to_time( to_double( MHR_answer[ 1 ] ) ); + #local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected); + #local message = fmt( "Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected ); + local virustotal_url = fmt( match_sub_url, hash ); + # We don't have the full fa_file record here in order to + # avoid the "when" statement cloning it (expensive!). + + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Macth", + $msg = hash ] ); + + send_alert_event( f, hash, virustotal_url, mhr_first_detected, + mhr_detect_rate, "" ); + } + else + { + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "No match", + $msg = f$id ] ); + + } + } + else # Do a local lookup + { + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Performing local lookup", + $msg = f$id ] ); + + if ( hash in local_hashes ) + { + local data = local_hashes[ hash ]; + + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Macth", + $msg = hash ] ); + + send_alert_event( f, hash, "", current_time(), + 0, data$description ); + } + else + { + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "No match", + $msg = f$id ] ); + + } + } + } +} + +function check_mime( mime_type : string ) : bool +{ + # Check for direct match: + if ( mime_type in match_mimes ) + { + return T; + } + else # Check for patrial matches + { + # A clumsy way of doing this, but we cannot generate pattern + # dynamically :'( + + for ( mime in match_mimes ) + { + # If file's mime-type string contains 'mime': + if ( strstr( mime_type, mime ) != 0 ) + { + return T; + } + } + } + + return F; +} + +event file_hash( f: fa_file, kind: string, hash: string ) +{ + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "File hashed", + $msg = hash ] ); + + # Only handle sha1 hashes + if ( kind == "sha1" ) + { + # Mark file as hashed + if ( f$id !in current_files ) + { + current_files[ f$id ] = CurrentFile( $id = f$id, + $status = Hashed ); + } + else + { + current_files[ f$id ]$status = Hashed; + } + + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Checking mime", + $msg = f$info?$mime_type ] ); + + if ( f$info?$mime_type ) + { + if ( check_mime( f$info$mime_type ) ) + { + do_mhr_lookup( hash, f ); + } + } + else # mime-type not available + { + send_log_event( f, "log", hash, "mime-type missing" ); + } + } +} + +# Make note of every detected file in order to follow their state: +event file_new( f: fa_file ) +{ + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "File detected", + $msg = f$id ] ); + + current_files[ f$id ] = CurrentFile( $id = f$id, + $status = New ); +} + +# Make note that not all file parts could be detected (there will be no hash) +event file_gap( f: fa_file, offset: count, len: count ) +{ + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Gap detected", + $msg = f$id ] ); + + if ( f$id !in current_files ) + { + current_files[ f$id ] = CurrentFile( $id = f$id, + $status = Gapped ); + } + else + { + current_files[ f$id ]$status = Gapped; + } +} + +# Remove file state and send an event in case of +# any errors were detected. +# NOTE: this function might be called before the +# hash lookup returns: nothing during the +# lookup or after it should depend on the +# stored file information (which is removed +# in this function)! + +event file_state_remove( f: fa_file ) +{ + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "File ended", + $msg = f$id ] ); + + if ( f$id !in current_files ) + { + # We are fucked up! + return; + } + + local entry = current_files[ f$id ]; + + switch ( entry$status ) { + case New: fallthrough; + case Gapped: + if ( f$info?$mime_type ) + { + if ( check_mime( f$info$mime_type ) ) + { + send_log_event( f, "log", "", "file not hashed" ); + } + } + else + { + send_log_event( f, "log", "", "mime-type missing" ); + } + + break; + case Hashed: + # Nothing to do: event is sent if the hash matched + break; + default: + # TODO: Log: Invalid status! + break; + } + + delete current_files[ f$id ]; +} + + + +event bro_init() &priority=9 +{ + #Log::create_stream( LOG, [ $columns=Info, $ev=log_malware ] ); # return True if ok + if ( !Log::create_stream( LOG, [ $columns=Info ] ) ) + { + print "MHR.bro: Log creation failed!"; + } + + Log::write( MHR::LOG, + [ $ts = network_time(), + $id = "Init", + $msg = "" ] ); + + PSA::subscribe_events( /on_mhr_config/ ); +} \ No newline at end of file diff --git a/PSA/modules/MHR.py b/PSA/modules/MHR.py new file mode 100644 index 0000000..cfa7d33 --- /dev/null +++ b/PSA/modules/MHR.py @@ -0,0 +1,194 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# MHR.py +# +# Implements a malware detection module that communicates with MHR.bro +# +# Author: jju, jk / VTT Technical Research Centre of Finland Ltd., 2016 +# + +import logging + +from broccoli import event, record_type, record +from modules.BroModule import BroModule +import modules.BroEventDispatcher as BroEventDispatcher + +MHRConfigRecord = record_type( 'op', # Operation type (add|reset) + 'mime' ) # Mime to add + +# Bro's response records: + +# Alert event: +MHRRecord = record_type( 'id', # Operation ID (match) + 'ts', # When the file was detected + 'hash', # Matchin sha1 hash + 'fid', # Bro's file ID + 'name', # Filename, if available + 'service', # Service (e.g., HTTP) using which the file + # was loaded + 'source', # List of space separated addresses + 'mime', # Mime type of the file + 'detected', # First time the malware was detected + 'rate', # Times the malware has been detected + 'url', # VirusTotal URL for the malware + 'msg' ) # Message (not included in 'match') + +# Log event: +MHRLogRecord = record_type( 'id', # Operation ID (match) + 'ts', # When the file was detected + 'hash', # Matchin sha1 hash + 'fid', # Bro's file ID + 'name', # Filename, if available + 'service', # Service (e.g., HTTP) using which the + # file was loaded + 'source', # List of space separated addresses + 'mime', # Mime type of the file + 'msg' ) # Message (not included in 'match') + +# Key for receiving Bro events. +MHRModuleKey = 'MHRModuleEvent' + +class MHRModule( BroModule ): + + rules = { } + + def __init__( self, logger ): + super( MHRModule, self ).__init__( 'MHR.bro', logger ) + BroEventDispatcher.register( MHRModuleKey, self ) + + def onRule( self, rule ): + + # Current only checks uses mime-type condition: + + if 'mime-type' in rule.conditions: + self.rules[ rule.ruleId ] = rule + + if self.state == BroModule.State.Started: + self._sendRule( rule ) + + return True + + return False + + def onStart( self, connection ): + super( MHRModule, self ).onStart( connection ) + self.reset( False ) + self._sendAllRules() + + def onStop( self ): + super( MHRModule, self ).onStop() + + def _sendRule( self, rule ): + """ + Send a single rule to bro module + """ + mimes = rule.conditions[ 'mime-type' ] + for mime in mimes: + try: + rec = record( MHRConfigRecord ) + rec.op = 'add' + rec.mime = str( mime ) + logging.info( 'Passing rule to Bro: ' + rule.ruleId + + ' (' + mime + ')' ) + self.connection.send( 'on_mhr_config', rec ) + except Exception: + logging.warning( 'Config exception for rule: ' + rule.ruleId ) + + def _sendAllRules( self ): + for key, rule in self.rules.iteritems(): + self._sendRule( rule ) + + def reset( self, resetRules = True ): + # Only send rules if connected to Bro + if self.state == BroModule.State.Started: + try: + rec = record( MHRConfigRecord ) + rec.op = 'reset' + rec.mime = 'reset' + logging.info( 'Passing rule to Bro: reset' ) + self.connection.send( 'on_mhr_config', rec ) + except Exception: + logging.warning( 'Config exception for rule: reset' ) + + if resetRules: + self.rules = { } + + def _log_alert( self, rule, data ): + + try: + fmt = "[%s] Rule '%s'(HSPL: %s) fired on file %s (%s, %s) from %s (%s): %s\n" + + text = '' + if not data.msg or data.msg == None or data.msg == '' : + text = data.url + else: + text = data.msg + ' (local hash)' + + line = fmt % ( data.ts, + rule.ruleId, + rule.hspl[ 'id' ], + data.fid, + data.mime, + data.hash, + data.source, + data.service, + text ) + # Log and alert + self.logger.onEvent( line ) + + fmt2 = "File (%s, %s) from %s (%s): %s" + info = fmt2 % ( data.mime, + data.hash, + data.source, + data.service, + text ) + self.logger.onNotifyEvent( rule.hspl['text'], 'Detected malicious file!', info ) + except Exception as e: + logging.error( e ) + + def _log_event( self, data ): + + try: + fmt = "[%s] Info: file %s (%s, %s) from %s (%s): %s\n" + line = fmt % ( data.ts, + data.fid, + data.mime, + data.hash, + data.source, + data.service, + data.msg ) + + self.logger.onEvent( line ) + except Exception as e: + logging.error( e ) + + + def onEvent( self, data ): + logging.error( 'Event ' + data.id ) + + if data.id == 'match': + count = 0 + for key, rule in self.rules.iteritems(): + if data.mime in rule.conditions[ 'mime-type' ]: + count += 1 + self._log_alert( rule, data ) + # Make sure that a log entry is generated even if + # mime matching in Bro and Python aren't equivalent: + if count == 0: + self._log_alert( '?', data ) + + elif data.id == 'log': + self._log_event( data ) + +# Dispatching events: +@event(MHRRecord) +def mhr_alert( data ): + BroEventDispatcher.dispatch( MHRModuleKey, data ) + +# Dispatching events: +@event(MHRLogRecord) +def mhr_log( data ): + BroEventDispatcher.dispatch( MHRModuleKey, data ) + +# Required for module loading: +module = MHRModule diff --git a/PSA/modules/__init__.py b/PSA/modules/__init__.py new file mode 100644 index 0000000..0da6d90 --- /dev/null +++ b/PSA/modules/__init__.py @@ -0,0 +1 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- diff --git a/PSA/modules/ccount.bro b/PSA/modules/ccount.bro new file mode 100644 index 0000000..4a4883a --- /dev/null +++ b/PSA/modules/ccount.bro @@ -0,0 +1,346 @@ +# -*- Mode:Bro;indent-tabs-mode:nil;-*- +# +# count.bro +# +# Bro script that can be configured dynamically to count established connections +# that fulfill certain conditions, such as, source or destination addresses or +# ports. +# +# Acknowledgement: this script is originally based on the Bro SumStats example +# script provided by the Bro Project. +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +@load base/protocols/conn +@load base/protocols/dhcp +@load base/protocols/dnp3 +@load base/protocols/dns +@load base/protocols/ftp +@load base/protocols/http +@load base/protocols/irc +@load base/protocols/modbus +@load base/protocols/pop3 +@load base/protocols/radius +@load base/protocols/snmp +@load base/protocols/smtp +@load base/protocols/socks +@load base/protocols/ssh +@load base/protocols/ssl +@load base/protocols/syslog +@load base/protocols/tunnels + +@load base/files/hash +@load base/files/extract +@load base/files/unified2 +@load base/files/x509 + +@load base/frameworks/communication +@load base/frameworks/sumstats + +@load ./psa-utils + +module CCount; + +# Definitions for logging framework: +export { + + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; # Timestamp + op: string &log; # Type of event + id: string &log; # Name of the rule + address: addr &log; # IP address + service: port &log; + }; + + #global log_cc: event( rec: Info ); +} + +# Definition of configuration event +type CountConfigRecord: record { + op: string; # Operation + id: string; # Name of the rule + address: string; # Related address, an IP address or a hostname + service: port; # Related port +}; + +# Definition of configuration event +type Rule : record { + id: string; # Name of the rule + op: string; # Operation + address: set[addr]; # A set of related IP address + service: port; # Port +}; + +# Table of currently active rules: +global rules : table[ string ] of Rule = {}; + +# Add a new rule to rules table: +function add_rule( cc: CountConfigRecord, addresses: set[ addr ] ) { + + # Log 'any IP' + if ( |addresses| == 0 ) { + Log::write( CCount::LOG, [ $ts = network_time(), + $op = cc$op, + $id = cc$id, + $address = 0.0.0.0, + $service = cc$service ]); + } + + # Log one or more IPs + for ( a in addresses ) { + Log::write( CCount::LOG, [ $ts = network_time(), + $op = cc$op, + $id = cc$id, + $address = a, + $service = cc$service ]); + } + + rules[ cc$id ] = Rule( $id = cc$id, + $op = cc$op, + $address = addresses, + $service = cc$service ); +} + +# Event handler for configuration events: +event on_count_config( cc: CountConfigRecord ) { + + # To fix missing protocol 'unknown' in broccoli python bindings + if ( cc$service == 0/tcp ) + { + cc$service = 0/unknown; + } + + switch ( cc$op ) { + case "src_addr": fallthrough; + case "src_port": fallthrough; + case "dst_addr": fallthrough; + case "dst_port": + if ( cc$id in rules ) { + delete rules[ cc$id ]; + } + + # The address is either an IP address, a hostname, or empty. + # A hostname may resolve to several IP addresses, so we deal + # with a set of addresses instead of a single address. + + local addresses: set[ addr ]; + + # For some reason, |string|>0 doesn't fire: is this because of + # something made by broccoli-python string conversion? + + if ( cc$address != "" ) { + + # If we have a single ip, convert it to 'addr': + if ( is_valid_ip( cc$address ) ) { + add addresses[ to_addr( cc$address ) ]; + } + else # otherwise, do a lookup for IP addresses: + { + # This will block, so let's do it async: + when ( local h = lookup_hostname( cc$address ) ) { + add_rule( cc, h ); + } + return; + } + } else { + # Empty set of addresses (check port only) + } + + add_rule( cc, addresses ); + break; + case "reset": # Delete a rule + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = cc$op, + $id = cc$id, + $address = 0.0.0.0, + $service = cc$service ]); + + for ( key in rules ) + { + delete rules[ key ]; + } + break; + default: # Invalid operation + return; + } + +} + +# Attaches a observer to each connection for each rule that it fulfills. +#event connection_established( c: connection ) { +event new_connection( c: connection ) { + + # TODO: a faster way to find correct rules should be implemented. + + for ( key in rules ) + { + local rule = rules[ key ]; + + switch ( rule$op ) + { + case "src_addr": + if ( c$id$orig_h in rule$address + && ( rule$service == 0/unknown || rule$service == c$id$orig_p ) ) + { + SumStats::observe( "conn established", + SumStats::Key( $str = rule$id ), + SumStats::Observation( $num = 1 ) ); + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "add_observer", + $id = rule$op, + $address = c$id$orig_h, + $service = c$id$orig_p ]); + } + break; + case "dst_addr": + if ( c$id$resp_h in rule$address + && ( rule$service == 0/unknown || rule$service == c$id$resp_p ) ) + { + SumStats::observe( "conn established", + SumStats::Key( $str = rule$id ), + SumStats::Observation( $num = 1 ) ); + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "add_observer", + $id = rule$op, + $address = c$id$resp_h, + $service = c$id$resp_p ]); + } + break; + case "src_port": + if ( rule$service == c$id$orig_p ) + { + SumStats::observe( "conn established", + SumStats::Key( $str = rule$id ), + SumStats::Observation( $num = 1 ) ); + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "add_observer", + $id = rule$op, + $address = c$id$orig_h, + $service = c$id$orig_p ]); + } + break; + case "dst_port": + if ( rule$service == c$id$resp_p ) + { + SumStats::observe( "conn established", + SumStats::Key( $str = rule$id ), + SumStats::Observation( $num = 1 ) ); + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "add_observer", + $id = rule$op, + $address = c$id$resp_h, + $service = c$id$resp_p ]); + } + break; + default: # Invalid operation + return; + } + } +} + +# Events to send to Count.py + +# Measurement report: +type CountReportRecord: record { + rule: string; # Rule (ID) of this measurement + ts: time; # Timestamp for this measurement (start time) + num_occurences: double; # Total number of occurences within measurement period + first_occurence: time; # Timestamp of first occurence + last_occurence: time; # Timestamp of last occurence + period: count; # Measurement perioid in seconds +}; + +# End of measurement perioid notification: + +type CountPeriodRecord: record { + ts: time; # Timestamp for this measurement (start time) + period: count; # Measurement perioid in seconds +}; + +global report_count: event( data: CountReportRecord ); +global report_period: event( data: CountPeriodRecord ); + +event bro_init() &priority=9 +{ + #Log::create_stream( CCount::LOG, [ $columns = CCount::Info, + # $ev = log_cc ] ); + + if ( !Log::create_stream( CCount::LOG, [ $columns = CCount::Info ] ) ) + { + print "ccount.bro: Log creation failed!"; + } + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "init", + $id = "", + $address = 0.0.0.0, + $service = 0/unknown ] ); + + PSA::subscribe_events( /on_count_config/ ); + + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + # There will be one result for each connection responder (c$id$resp_h) + + local r1 = SumStats::Reducer( $stream = "conn established", + $apply = set( SumStats::SUM ) ); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create( [ $name = "counting connections", + $epoch = 1min, + $reducers = set( r1 ), + $epoch_result( ts: time, + key: SumStats::Key, + result: SumStats::Result ) = { + + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "log", + $id = key$str, + $address = 0.0.0.0, + $service = 0/tcp ]); + + local stats = result[ "conn established" ]; + local data: CountReportRecord; + data$rule = key$str; + data$ts = ts; + data$num_occurences = stats$sum; + data$first_occurence = stats$begin; + data$last_occurence = stats$end; + data$period = 60; + + # send event for our broccoli + event report_count( data ); + }, + $epoch_finished( ts: time ) = { + + Log::write( CCount::LOG, [ $ts = network_time(), + $op = "end", + $id = "end", + $address = 0.0.0.0, + $service = 0/tcp ]); + + local data: CountPeriodRecord; + data$ts = ts; + data$period = 60; + + # send event for our broccoli + event report_period( data ); + } ] ); +} diff --git a/PSA/modules/psa-utils.bro b/PSA/modules/psa-utils.bro new file mode 100644 index 0000000..5b30e47 --- /dev/null +++ b/PSA/modules/psa-utils.bro @@ -0,0 +1,40 @@ +# -*- Mode:Bro;indent-tabs-mode:nil;-*- +# +# psa-utils.bro +# +# Generic utilities for all Bro PSA modules. +# +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 + +@load base/frameworks/communication + +module PSA; + +export { + global subscribe_events: function( events: pattern ); +} + +function subscribe_events( events : pattern ) +{ + + if ( "PSA" in Communication::nodes ) + { + local node = Communication::nodes[ "PSA" ]; + if ( node?$events ) + { + local evs = merge_pattern( node$events, events ); + node$events = evs; + } + else + { + node$events = events; + } + } + else + { + Communication::nodes[ "PSA" ] = [ $host = 127.0.0.1, + $events = events, + $connect = F, + $ssl = F ]; + } +} \ No newline at end of file diff --git a/PSA/psaConfigs/README.md b/PSA/psaConfigs/README.md new file mode 100644 index 0000000..e9ea7bb --- /dev/null +++ b/PSA/psaConfigs/README.md @@ -0,0 +1 @@ +Runtime PSA security control configs are stored in this folder. diff --git a/PSA/psaConfigs/example.conf b/PSA/psaConfigs/example.conf new file mode 100644 index 0000000..8e53402 --- /dev/null +++ b/PSA/psaConfigs/example.conf @@ -0,0 +1,92 @@ +{ + + "rules": [ + { "id": "rule1", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination", + "value": { "address": "91.197.85.151" } + } + ] + }, + { "id": "rule2", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination", + "value": { "address": "81.209.67.238" } + } + ] + }, + { "id": "rule3", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination_port", + "value": { "port": 80 } + } + ] + }, + { "id": "rule4", + "hspl": { + "id": "hspl0", + "text": "abcd" + }, + "event": "EVENT_FILE", + "operation": "detect-MHR", + "parameters": [ ], + "action": "log", + "conditions": [ + { "type": "mime-type", + "value": "application/pdf" + }, + { "type": "mime-type", + "value": "application/x-dosexec" + } + ] + } + ] +} diff --git a/PSA/psaEE.conf b/PSA/psaEE.conf new file mode 100644 index 0000000..05482c6 --- /dev/null +++ b/PSA/psaEE.conf @@ -0,0 +1,17 @@ +[configuration] +psc_address=http://192.168.2.1:8080 +psa_config_path=psaConfigs/ +scripts_path=scripts/ +psa_id=BroMalware +psa_name=Bro PSA +psa_version=0.1.0 +psa_api_version=v0.5 +psa_log_location=psaConfigs/psa.log +conf_id= +verbose=false +debug=false +test_mode=false +#test_mode_ip=10.2.4.1 +#test_mode_dns=8.8.8.8 +#test_mode_netmask=255.0.0.0 +#test_mode_gateway=10.2.2.252 diff --git a/PSA/psaEE.py b/PSA/psaEE.py new file mode 100644 index 0000000..7d90378 --- /dev/null +++ b/PSA/psaEE.py @@ -0,0 +1,147 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# File: psaEE.py +# Created: 27/08/2014 +# Author: BSC +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# +# Description: +# Web service running on the PSA interacting with the PSC +# +# + +import falcon +#import json +import Config +import logging +import subprocess +from execInterface import execInterface +from getConfiguration import getConfiguration +from psaExceptions import psaExceptions +from dumpLogFile import dumpLogFile +import os.path + +conf = Config.Configuration() +date_format = "%m/%d/%Y %H:%M:%S" +log_format = "[%(asctime)s.%(msecs)d] [%(module)s] %(message)s" + +logging.basicConfig( filename = conf.LOG_FILE, + level = logging.DEBUG, + format = log_format, + datefmt = date_format ) + +# Enforce logging level even if handlers had already +# been added into the root logger: +logger = logging.getLogger() +logger.setLevel( logging.DEBUG ) + +#pscAddr = conf.PSC_ADDRESS +#configsPath = conf.PSA_CONFIG_PATH +#psaID = conf.PSA_ID +#confID = conf.CONF_ID + +if conf.TEST_MODE: + logging.info( 'Test Mode enabled' ) + +logging.info( "--------" ) +logging.info( "PSA EE init." ) +logging.info( "PSA ID: " + str( conf.PSA_ID ) ) +logging.info( "PSA NAME: " + str( conf.PSA_NAME ) ) +logging.info( "PSA VERSION: " + str( conf.PSA_VERSION ) ) +logging.info( "PSA-PSC API version: " + str( conf.PSA_API_VERSION ) ) +logging.info( "PSA log location: " + str( conf.PSA_LOG_LOCATION ) ) +logging.info( "--------" ) + +# instantiate class object to manage REST interface to the PSC +execIntf = execInterface( conf.PSA_HOME, + conf.PSA_CONFIG_PATH, + conf.PSA_SCRIPTS_PATH, + conf.PSA_LOG_LOCATION, + conf.PSA_ID, + conf.PSC_ADDRESS, + str(conf.PSA_API_VERSION)) +#confHand = getConfiguration(pscAddr, configsPath, confID, psaID) +confHand = None +if not conf.TEST_MODE: + confHand = getConfiguration( conf.PSC_ADDRESS, + conf.PSA_CONFIG_PATH, + conf.PSA_SCRIPTS_PATH, + conf.PSA_ID, + str(conf.PSA_API_VERSION) ) + +# start the HTTP falcon proxy and adds reachable resources as routes +app = falcon.API() +base = '/' + str( conf.PSA_API_VERSION ) + '/execInterface/' +app.add_route( base + '{command}', execIntf ) + +dumpLog = dumpLogFile() +#FOR DEBUGGING ONLY, REMOVE IN PRODUCTION +app.add_route( base + 'dump-log-ctrl', dumpLog ) + +logging.info("execInterface routes added.") + +# Inform our PSC that we are up +#TODO +''' +try: + start_res = confHand.send_start_event() + # We don't need to enable anything + #proc = subprocess.Popen(confScript, stdout=subprocess.PIPE, shell=True) + #(out, err) = proc.communicate() +except psaExceptions as exc: + pass +''' +# Pull configuration and start the PSA. +try: + if not conf.TEST_MODE: + confScript = confHand.pullPSAconf( execIntf ) + + else: # Do local test setup + + # Check that some psaconf file exists + if not os.path.isfile( conf.PSA_CONFIG_PATH + '/psaconf' ): + raise psaExceptions.confRetrievalFailed() + + execIntf.callInitScript() + + if conf.TEST_MODE_IP != None: + + # Only run ip_conf.sh if all the parameters are present + if ( conf.TEST_MODE_DNS == None + or conf.TEST_MODE_NETMASK == None + or conf.TEST_MODE_GATEWAY == None ): + raise psaExceptions.confRetrievalFailed() + + logging.info( 'PSA requires IP, configuring...' ) + ip = conf.TEST_MODE_IP + dns = conf.TEST_MODE_DNS + netmask = conf.TEST_MODE_NETMASK + gateway = conf.TEST_MODE_GATEWAY + logging.info( 'ip: ' + str( ip ) ) + logging.info( 'gateway: ' + str( gateway ) ) + logging.info( 'dns: ' + str( dns ) ) + logging.info( 'netmask: ' + str( netmask ) ) + + ret = subprocess.call( [ conf.PSA_SCRIPTS_PATH + 'ip_conf.sh', + ip, gateway, dns, netmask ] ) + logging.info( 'Result of setting config: ' + str( ret ) ) + else: + logging.info( "PSA doesn't require IP, skipping configuration." ) + logging.info('PSA '+ conf.PSA_ID + ' configuration registered' ) + + execIntf.callStartScript() + +except psaExceptions.confRetrievalFailed as e: + print e + +logging.info( "PSA start done." ) + +# http request to ask for the configuration and start the script +''' +try: + confScript = confHand.pullPSAconf() + proc = subprocess.Popen(confScript, stdout=subprocess.PIPE, shell=True) + (out, err) = proc.communicate() +except psaExceptions as exc: + pass +''' diff --git a/PSA/psaExceptions.py b/PSA/psaExceptions.py new file mode 100644 index 0000000..5b9bf84 --- /dev/null +++ b/PSA/psaExceptions.py @@ -0,0 +1,14 @@ +# -*- Mode:Python;indent-tabs-mode:nil; -*- +# +# File: psaExceptions.py +# Created: 05/09/2014 +# Author: BSC +# +# Description: +# Custom execption class to manage error in the PSC +# + +class psaExceptions( object ): + + class confRetrievalFailed( Exception ): + pass diff --git a/PSA/pylintrc b/PSA/pylintrc new file mode 100644 index 0000000..f0a6913 --- /dev/null +++ b/PSA/pylintrc @@ -0,0 +1,280 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=C0326 + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct attribute names in class +# bodies +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/PSA/scripts/current_config.sh b/PSA/scripts/current_config.sh new file mode 100644 index 0000000..715d728 --- /dev/null +++ b/PSA/scripts/current_config.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# status.sh +# Description: +# This script return the current configuration. +# +# This script is called by the PSA API when the PSA's current runtime configuration is requested. +# +# Return value: +# Current configuration +# + +PSA_HOME=/home/psa/pythonScript + +if [ -z "$PSA_HOME" ]; then + echo "error: 'PSA_HOME' is not set." >&2 + exit 1 +fi + +if [ ! -d "$PSA_HOME" ]; then + echo "error: 'PSA_HOME' is not a valid directory." >&2 + exit 1 +fi + +#PSA_HOME=/home/admini/SECURED/ +#PSA_HOME=/home/psa/pythonScript + +COMMAND_OUTPUT="$(cat $PSA_HOME/psaConfigs/psaconf)" +printf '%s\n' "${COMMAND_OUTPUT[@]}" +exit 1; diff --git a/PSA/scripts/init.sh b/PSA/scripts/init.sh new file mode 100644 index 0000000..25e869b --- /dev/null +++ b/PSA/scripts/init.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +#if [ -z "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not set." >&2 +# exit 1 +#fi + +#if [ ! -d "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not a valid directory." >&2 +# exit 1 +#fi + +exit 0; + diff --git a/PSA/scripts/ip_conf.sh b/PSA/scripts/ip_conf.sh new file mode 100644 index 0000000..e9d0eff --- /dev/null +++ b/PSA/scripts/ip_conf.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# ip_conf.sh +# +# This script is called by the PSA API when the PSA should be configured with IP +# address. +# +# NOTE: This script is called right after init.sh script at the start-up of a +# PSA. +# +# !!! +# This should have the base setup for IP. init.sh should not change these +# values, since it will overwrite these values at the moment. +# !!! +# +# --> (We can change the logic to call this after init.sh always?) +# + +# Just a place-holder as Bro PSA does not use this script.. +#if [ -z "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not set." >&2 +# exit 1 +#fi + +#if [ ! -d "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not a valid directory." >&2 +# exit 1 +#fi + +# Please, define the interface this PSA requires the IP for. +CLIENT_IFACE=br0 +if [ "$#" -ne 4 ] +then + echo "Illegal number of params. Should be 4 (IP, gateway, dns, netmask)" + exit 1; +fi + +echo "-------------" +echo "IP:" + $1 +echo "gateway:" + $2 +echo "dns:" + $3 +echo "netmask:" + $4 + +# Note that now we just replace any existing conf, since this should be the only +# DNS for the PSA. +SEARCH='nameserver '$3 +if grep -Fxq "$SEARCH" /etc/resolv.conf +then + echo "Had dns already" +else + echo "Didn't have dns, setting" + echo -e "$SEARCH" > /etc/resolv.conf +fi + +/sbin/ifconfig $CLIENT_IFACE $1 netmask $4 +ip route delete default +/sbin/route add default gw $2 $CLIENT_IFACE diff --git a/PSA/scripts/ping.sh b/PSA/scripts/ping.sh new file mode 100644 index 0000000..66d0b56 --- /dev/null +++ b/PSA/scripts/ping.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# ping.sh +# +# This script is called by the PSA API when the PSA is requested to ping. +# +# Return value: +# ping result +# + +COMMAND_OUTPUT="$(ping -c 3 www.google.com)" +echo ${COMMAND_OUTPUT} +exit 1; diff --git a/PSA/scripts/start.sh b/PSA/scripts/start.sh new file mode 100644 index 0000000..e233888 --- /dev/null +++ b/PSA/scripts/start.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# start.sh +# Created: 1/02/2016 +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +# Just a place-holder as Bro PSA does not use this script.. +#if [ -z "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not set." >&2 +# exit 1 +#fi + +#if [ ! -d "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not a valid directory." >&2 +# exit 1 +#fi + +echo "ERROR: this script should not be called" +exit 0 diff --git a/PSA/scripts/status.sh b/PSA/scripts/status.sh new file mode 100644 index 0000000..b2398ff --- /dev/null +++ b/PSA/scripts/status.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# status.sh +# Created: 1/02/2016 +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# +# Description: +# Script that returns the current status of the Bro PSA. +# +# This script is called by the PSA API when the PSA's runtime status is +# requested. +# +# Return value: +# 1: alive +# 2: not alive +# + +#if [ -z "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not set." >&2 +# exit 1 +#fi + +#if [ ! -d "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not a valid directory." >&2 +# exit 1 +#fi + +BROCTL=/opt/bro/bin/broctl +LINE=`$BROCTL status 2>&1 | grep "running"` + +if [ "$?" -eq 0 ] ; then + echo 1 + exit 1 +fi + +echo 0 +exit 0 diff --git a/PSA/scripts/stop.sh b/PSA/scripts/stop.sh new file mode 100644 index 0000000..b8c4da8 --- /dev/null +++ b/PSA/scripts/stop.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# stop.sh +# Created: 1/02/2016 +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# + +# Just a place-holder as Bro PSA does not use this script.. + +#if [ -z "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not set." >&2 +# exit 1 +#fi + +#if [ ! -d "$PSA_HOME" ]; then +# echo "error: 'PSA_HOME' is not a valid directory." >&2 +# exit 1 +#fi + +echo "ERROR: this script should not be called" +exit 0 diff --git a/PSA/secured.bro b/PSA/secured.bro new file mode 100644 index 0000000..8a7b403 --- /dev/null +++ b/PSA/secured.bro @@ -0,0 +1,111 @@ +##! Local site policy. It will be dynamically updated when policy is enabled/disabled +##! +##! This file will not be overwritten when upgrading or reinstalling! +# Note: Bro supports writing to files, but not reading from them. + + +# Disable checksum validation +redef ignore_checksums = T; +redef tcp_max_initial_window = 0; +redef tcp_max_above_hole_without_any_acks = 0; +redef tcp_excessive_data_without_further_acks = 0; + +# Implemented policies + +# Enables pinging a bro node +#@load /home/admini/SECURED/policies/broping-record.bro + +# Count connections +#@load /home/admini/SECURED/policies/count_conns.bro + +# Weak keys notice +#@load /home/admini/SECURED/policies/weak-keys.bro + +# Hash calculation +#@load /home/admini/SECURED/policies/hash-files.bro + +# Interesting scripts + +# Log some information about web applications being used by users +# on your network. +#@load misc/app-stats + +# Scripts that do asset tracking. +#@load protocols/conn/known-hosts +#@load protocols/conn/known-services +#@load protocols/ssl/known-certs + +# This script enables SSL/TLS certificate validation. +#@load protocols/ssl/validate-certs +# This script prevents the logging of SSL CA certificates in x509.log +#@load protocols/ssl/log-hostcerts-only +# Uncomment the following line to check each SSL certificate hash against the ICSI +# certificate notary service; see http://notary.icsi.berkeley.edu . +#@load protocols/ssl/notary + +# Enable MD5 and SHA1 hashing for all files. +#@load frameworks/files/hash-all-files + +# Detect SHA1 sums in Team Cymru's Malware Hash Registry. +#@load frameworks/files/detect-MHR + + + + +# Some general scripts and some other scripts that might be interesting + +# This script logs which scripts were loaded during each run. +@load misc/loaded-scripts + +# Apply the default tuning scripts for common tuning settings. +@load tuning/defaults + +# Load the scan detection script. +#@load misc/scan + +# Detect traceroute being run on the network. +#@load misc/detect-traceroute + +# Generate notices when vulnerable versions of software are discovered. +# The default is to only monitor software found in the address space defined +# as "local". Refer to the software framework's documentation for more +# information. +#@load frameworks/software/vulnerable + +# Detect software changing (e.g. attacker installing hacked SSHD). +#@load frameworks/software/version-changes + +# This adds signatures to detect cleartext forward and reverse windows shells. +#@load-sigs frameworks/signatures/detect-windows-shells + +# Load all of the scripts that detect software in various protocols. +#@load protocols/ftp/software +#@load protocols/smtp/software +#@load protocols/ssh/software +#@load protocols/http/software +# The detect-webapps script could possibly cause performance trouble when +# running on live traffic. Enable it cautiously. +#@load protocols/http/detect-webapps + +# This script detects DNS results pointing toward your Site::local_nets +# where the name is not part of your local DNS zone and is being hosted +# externally. Requires that the Site::local_zones variable is defined. +#@load protocols/dns/detect-external-names + +# Script to detect various activity in FTP sessions. +#@load protocols/ftp/detect + +# If you have libGeoIP support built in, do some geographic detections and +# logging for SSH traffic. +#@load protocols/ssh/geo-data +# Detect hosts doing SSH bruteforce attacks. +#@load protocols/ssh/detect-bruteforcing +# Detect logins using "interesting" hostnames. +#@load protocols/ssh/interesting-hostnames + +# Detect SQL injection attacks. +#@load protocols/http/detect-sqli + +# Uncomment the following line to enable detection of the heartbleed attack. Enabling +# this might impact performance a bit. +#@load policy/protocols/ssl/heartbleed diff --git a/PSA/test/configs/post-init.bro b/PSA/test/configs/post-init.bro new file mode 100644 index 0000000..9a54276 --- /dev/null +++ b/PSA/test/configs/post-init.bro @@ -0,0 +1,19 @@ +# This file can be used to make Bro PSA Detect-MHR module to consider the files +# fecthed by the following scripts to be considered as malware: +# +# download_pdf.sh +# download_exe.sh +# +# Usage: copy this files under Bro PSA's modules directory +# (PSA/modules/post-init.bro) before booting Bro PSA. +# + +redef ignore_checksums = T; +redef tcp_max_initial_window = 0; +redef tcp_max_above_hole_without_any_acks = 0; +redef tcp_excessive_data_without_further_acks = 0; + +redef MHR::local_hashes += { [ "afba7d3f3addd136afb4b13a49703e979fb4f590" ] + = [ $kind="sha1", $description="detected T170.pdf" ], + [ "f2e5efd7b47d1fb5b68d355191cfed1a66b82c79" ] + = [ $kind="sha1", $description="detected 7z1514.exe" ] }; diff --git a/PSA/test/download_exe.sh b/PSA/test/download_exe.sh new file mode 100644 index 0000000..12a7dd0 --- /dev/null +++ b/PSA/test/download_exe.sh @@ -0,0 +1,2 @@ +#!/bin/sh +wget --no-proxy http://www.7-zip.org/a/7z1514.exe diff --git a/PSA/test/download_http.sh b/PSA/test/download_http.sh new file mode 100644 index 0000000..8786a6d --- /dev/null +++ b/PSA/test/download_http.sh @@ -0,0 +1,2 @@ +#!/bin/sh +wget www.vtt.fi diff --git a/PSA/test/download_http_google_49.sh b/PSA/test/download_http_google_49.sh new file mode 100644 index 0000000..b9a2d97 --- /dev/null +++ b/PSA/test/download_http_google_49.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for ((n=0;n<49;n++)); +do + wget www.google.com +done + +echo "done." diff --git a/PSA/test/download_http_google_50.sh b/PSA/test/download_http_google_50.sh new file mode 100644 index 0000000..d8c124a --- /dev/null +++ b/PSA/test/download_http_google_50.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for ((n=0;n<50;n++)); +do + wget www.google.com +done + +echo "done." diff --git a/PSA/test/download_http_vtt_49.sh b/PSA/test/download_http_vtt_49.sh new file mode 100644 index 0000000..8a55a89 --- /dev/null +++ b/PSA/test/download_http_vtt_49.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for ((n=0;n<49;n++)); +do + wget www.vtt.fi +done + +echo "done." diff --git a/PSA/test/download_http_vtt_50.sh b/PSA/test/download_http_vtt_50.sh new file mode 100644 index 0000000..dd2be16 --- /dev/null +++ b/PSA/test/download_http_vtt_50.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for ((n=0;n<50;n++)); +do + wget www.vtt.fi +done + +echo "done." diff --git a/PSA/test/download_pdf.sh b/PSA/test/download_pdf.sh new file mode 100644 index 0000000..c72daa8 --- /dev/null +++ b/PSA/test/download_pdf.sh @@ -0,0 +1,2 @@ +#!/bin/sh +wget www.vtt.fi/inf/pdf/technology/2014/T170.pdf diff --git a/PSA/test/gunicorn_brolog.sh b/PSA/test/gunicorn_brolog.sh new file mode 100644 index 0000000..85e18c1 --- /dev/null +++ b/PSA/test/gunicorn_brolog.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl -X GET http://10.0.2.15:8080/v0.5/execInterface/brolog diff --git a/PSA/test/gunicorn_start.sh b/PSA/test/gunicorn_start.sh new file mode 100644 index 0000000..1ab5ab0 --- /dev/null +++ b/PSA/test/gunicorn_start.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl -X POST http://10.0.2.15:8080/v0.5/execInterface/start diff --git a/PSA/test/gunicorn_status.sh b/PSA/test/gunicorn_status.sh new file mode 100644 index 0000000..76c75d2 --- /dev/null +++ b/PSA/test/gunicorn_status.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl http://10.0.2.15:8080/v0.5/execInterface/status diff --git a/PSA/test/gunicorn_stop.sh b/PSA/test/gunicorn_stop.sh new file mode 100644 index 0000000..93a429b --- /dev/null +++ b/PSA/test/gunicorn_stop.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +curl -X POST http://10.0.2.15:8080/v0.5/execInterface/stop diff --git a/PSA/util/cleanup.sh b/PSA/util/cleanup.sh new file mode 100644 index 0000000..2523c00 --- /dev/null +++ b/PSA/util/cleanup.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# File: cleanup.sh +# Created: 28/01/2016 +# Author: jju / VTT Technical Research Centre of Finland Ltd., 2016 +# +# Description: +# +# A simple script to cleanup the development directory +# + +# All paths should be relative! + +# subdirectiories to clean. All directories listed are cleaned +# from generic temporary files, such as .pyc and *~ +subdirs="modules, json, psaConfig, test, scripts" + +# Specific temporary files that should be removed, e.g. log +# files. +tmpfiles="GUNICORN.log, PSA.log, psaConfigs/bro.log pylint.out" + +echo "rm -f ./*.pyc ./*~" +rm -f ./*.pyc ./*~ + +dirs=(${subdirs//,/ }) +for dir in "${dirs[@]}" +do + if [ -n "$dir" -a -d "$dir" ]; then + echo "rm -f ./$dir/*.pyc ./$dir/*~" + rm -f ./$dir/*.pyc ./$dir/*~ + fi +done + +files=(${tmpfiles//,/ }) +for file in "${files[@]}" +do + echo "rm -f ./$file" + rm -f ./$file +done + +exit 0 diff --git a/PSA/util/kill.sh b/PSA/util/kill.sh new file mode 100644 index 0000000..539a6b6 --- /dev/null +++ b/PSA/util/kill.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +PIDS=`ps aux | grep guni | grep python | sed 's/^[^ \t]*[ \t]*\([0-9]*\).*/\1/g' | tr '\n' ' '` + +kill -s 9 $PIDS diff --git a/PSA/util/pylint.sh b/PSA/util/pylint.sh new file mode 100644 index 0000000..31ae01e --- /dev/null +++ b/PSA/util/pylint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +pylint *.py modules > pylint.out + +echo "done: check pylint.out" +echo "" +exit 0 diff --git a/README.md b/README.md new file mode 100644 index 0000000..893b0ef --- /dev/null +++ b/README.md @@ -0,0 +1,267 @@ +# 1. End-user + +## 1.1 Description / general readme + +Bro PSA provides network monitoring capabilities to end users. It can be used to +monitor user's network traffic according to specified policies and log security +related events, such as suspicious connections and suspected malicious files. + +## 1.2 Features / Capabilities + +The list of capabilities supported by this PSA includes: + +* Logging of established connections during certain time interval filtered based + on source and/or destination addresses and ports. Log events can be configured + to trigger only if the amount of specified events exceeds certain threshold + within a time interval. This capability may be used, e.g., log network usage + or detect abnormal network traffic. + +* Scanning downloaded files for known malware. This PSA's malware detection + capabilities are based on detecting certain types of files, e.g. PDFs, in the + network traffic to or from the user's device. The PSA computes hashes of such + files and compares these against a repository of known hashes of malicious + files. Type of files that are to be scanned can be configured. + +## 1.3 Security policy examples + +Examples of the policies that can be enables using the SECURED GGUI includes: + +```I; Enable; Logging; {propose, count_connection }; {traffic_target, address}``` + +- This policy enables logging of all connections to or from a specific address. + +```I; Enable; Malware_detection; {type_Content, scan_xdosexec } ``` + +- This policy enables scanning of all Windows native executable files detected + in the network traffic. + +## 1.4 Support, troubleshooting / known issues + +None + +# 2. Developer / admin + +## 2.1 Description / general readme + +The Bro PSA provides network monitoring capabilities to end users. It can be +used to monitor user's network traffic according to specified policies and log +security related events, such as suspicious connections and suspected malicious +files. + +The Bro PSA is implemented using the [Bro Network Security Monitoring platform](https://www.bro.org). +Bro is a passive network traffic analyzer primarily used for security monitoring +of suspicious activities in network traffic flows. It is an open-source software +with BSD-style license. More detailed description of The Bro NSM can be found in +[Bro NSM documentation](https://www.bro.org/documentation/index.html). + +## 2.2 Components and Requirements + +Software components used by the PSA include: + +* [Bro Network Security Monitor](https://www.bro.org/) +* [BroControl](https://github.com/bro/broctl): An interactive interface for + managing a Bro installation. Allows, e.g., starting and stopping the Bro NSM + and updating its configuration. In the Bro PSA, this component is used for + controlling the Bro NSM. +* [Broccoli](https://github.com/bro/broccoli): The Bro Client Communications + Library. This component enables third-party applications to communicate + directly with the Bro NSM by sending and receiving events using Bro's + communication protocol. In the Bro PSA it is used to connect the PSA interface + with the Bro policy scripts written as Bro plugin extensions. +* [Broccoli-python](https://github.com/bro/broccoli-python): Python bindings for + Broccoli. Required, as the PSA interface is implemented using Python. + +## 2.3 Detailed architecture + +The Bro PSA is implemented using the Bro NSM. + +Bro's architecture is illustrated below. Bro taps to network traffic in order to +captures network packets. These packets are sent to Event Engine that reduces +captured low-level packets to high-level network events, such as TCP +connections, SSL sessions events, or HTTP requests. High-level events are then +dispatched to Policy Script Interpreter that executes a set of event handlers to +the events. Policy scripts define the actual actions taken on each event. Each +policy script implements a set of event handlers. Events fired at policy scripts +may, for instance, produce logs or notifications about the network traffic, or +new events for other policy scripts. + +![Bro architecture](docs/bro-architecture.png) + +Bro provides fully customizable and extensible platform for network traffic +analysis. It utilizes its own event-based, syntactically C-like, scripting +language with large set of pre-built functionalities. External C-library, +Broccoli, allows interfacing with third-party programs. + +The Bro PSA is implemented as a separate Python application that interfaces with +the Bro NSM using Broccoli. + +The architecture of the PSA is illustrated below. Each capability supported by +the PSA is implemented as its own Bro policy script (Bro plug-in). These policy +scripts determine what kind of information the PSA listens to and subscribe for +related network events in Bro's event stream. Plugins filter low-level network +events, e.g. establishment of connection to certain address and port, to higher +level log event for the PSA, e.g. *"there have been 100 connection attempts from +certain address to local port 80 within one minute"*. + +![Bro PSA architecture](docs/bro-psa-architecture.png) + +The PSA uses two interfaces to communicate with the Bro NSM: +* BroControl is used to control the Bro NSM. For example, it is used to start + and stop monitoring activities, as well as, to install new or remove old + policy scripts (Bro plugins). +* Broccoli is used to pass configuration information to policy scripts and to + receive log events from them. + +Currently, two Bro plug-ins have been implemented: + +* **Logging plug-in**: allows logging of established connections during certain + time interval filtered based on source and/or destination addresses and ports. + Log events can be configured to trigger only if the amount of specified events + exceeds certain threshold within a time interval. This capability may be used, + e.g., log network usage or detect abnormal network traffic. + +* **Malware Detection plug-in**: allows scanning of downloaded files for known + malware. This PSA's malware detection capabilities are based on detecting + certain types of files, e.g. PDFs, in the network traffic to or from the + user's device. The PSA computes hashes of such files and compares these + against a repository of known hashes of malicious files. Type of files that + are to be scanned can be configured. + +## 2.4 Virtual machine image creation + +See [PSA Developer guide](https://github.com/SECURED-FP7/secured-psa-develop-test) +for creating a virtual machine base image. + +After obtaining the base image, install the following software components into +the image: + +* Bro Network Security Monitor ([installation instructions](https://github.com/bro/bro/blob/master/doc/install/install.rst)). +* BroControl ([installation instructions](https://github.com/bro/broctl/blob/master/doc/broctl.rst)) +* Broccoli ([installation instructions](https://github.com/bro/broccoli/blob/master/README)) +* Broccoli-python ([installation instructions](https://github.com/bro/broccoli-python/blob/master/README)) + +**NOTE**: Bro PSA has been developed and tested on Bro NSM version 2.4+, it is required to use Bro NSM +version 2.4+, as the Bro policy scripts (plug-ins) might not work correctly in +other versions. + +Copy [Bro PSA](PSA) files +into the following folder in the base image: + +``` +$HOME/pythonScript/ +``` + +Configure Bro and BroCtl (default: /opt/bro/etc/broctl.cfg). At least the +following options should be set: + +``` +BroArgs = -b +``` + +* This option sets the Bro NSM to operate on 'bare mode' in an attempt to + minimize its resource usage. This causes many of Bro's default scripts and + policies not to be loaded when the Bro NSM starts. + +``` +SitePolicyStandalone = <$HOME>/pythonScript/secured.bro +``` + +* This option defines local Bro policies to be taken from SECURED specific + policy file. This policy file is provided with the PSA + [secured.bro](PSA/secured.bro) and the file path should be modified to match + the Bro PSA installation. + +Change passwords for users 'root' and 'psa' for security reasons. + +## 2.5 Support, troubleshooting / known issues + +Known issues: + +* Connection counting / logging capability currently can only handle detection + intervals on granularity of one minute. + +* Bro NSM might not be able to handle all network traffic packets, e.g., due to + congestion or because it does not have enough resources allocated. In such + case it starts to drop packets, which may cause monitoring module to miss part + of the network traffic. This might cause Malware Detection capability to miss + malicious files as the currently used detection technique relies on file + hashes which cannot be calculated to partial files. + +## 2.6 Files required + +The following files are needed to run Bro PSA correctly: + +### PSA application image + +The procedure to create a valid PSA image from scratch starts with the prerequisite instructions defined in the PSA Developer guide. + +The PSA VM image (KVM) is available at [Bro](https://vm-images.secured-fp7.eu/images/) ([checksums](https://vm-images.secured-fp7.eu/images/)). + +***NOTE**: You have to manually change the PSA ID to *BroLogging/BroMalware* in PSA conf (PSA/psaEE.conf), change the VM image name in copy_psa_sw_to_vm.sh to the name required by SPM/UPR (and possibly legacy NED's JSON PSA Manifest) if tested in a fully integrated environment and and run the copy script. For a local test setup you can use the default image name, just replace the configuration file from NED files as desired. This is due to the capability differentiation in the policy framework. + + +### Manifest + +The PSA Manifests are available at [BroLogging](NED_files/TVDM/PSAManifest/brologging_manifest.xml) and [BroMalware](NED_files/TVDM/PSAManifest/bromalware_manifest.xml) (and [legacy JSON format for old NED and manual testing](NED_files/TVDM/PSAManifest/broPSA)). + +### HSPL + +Examples of HSPL: + +```I; Enable; Logging; {propose, count_connection }; {traffic_target, address}``` + +- This policy enables logging of all connections to or from a specific address. + +```I; Enable; Malware_detection; {type_Content, scan_xdosexec } ``` + +- This policy enables scanning of all Windows native executable files detected + in the network traffic. + +### MSPL + +Example MSPLs can be found below: + + * [Logging](M2LPlugin/examples/example_mspl_log_2.xml) + * [Malware Detection](M2LPlugin/examples/example_mspl_mwd_2.xml) + +### M2L Plug-in + +The M2L plug-in is available at [M2LPlugin](M2LPlugin). Notice that it contains separate plug-ins for BroLogging and BroMalware. + +## 2.7 Features/Capabilities + +Bro PSA provides the following capabilities + +The list of capabilities are: +* Logging +* Offline_malware_analysis + +## 2.8 Testing + +Test scripts are available in the test [directory](tests) or in PSA/test. + +# 3. License + +Please refer to project [LICENSE](LICENSE) file. + +This software requires several open source software components with their own license: + +* Bro Network Security Monitor, available at [license](https://raw.githubusercontent.com/bro/bro/master/COPYING) +* BroControl, available at [license](https://raw.githubusercontent.com/bro/broctl/master/COPYING) +* Broccoli, available at [license](https://raw.githubusercontent.com/bro/broccoli/master/COPYING) +* Broccoli-python, available at [license](https://raw.githubusercontent.com/bro/broccoli-python/master/COPYING) + +# 4. Additional Information + +## 4.1 Partners involved + +* Application: VTT +* MSPL: POLITO, VTT +* M2L Plugin: VTT + +# 5. Status (OK/No/Partial) - OK + +# 6. TODO-list + +* Instructions for adding new Bro plug-ins to Bro PSA + diff --git a/copy_psa_sw_to_vm.sh b/copy_psa_sw_to_vm.sh new file mode 100644 index 0000000..561a59f --- /dev/null +++ b/copy_psa_sw_to_vm.sh @@ -0,0 +1,32 @@ +# Uses libguestfs - Installation guide: http://www.libguestfs.org/ +# +# Run this as sudo and run this file from the folder that contains PSC folder. +# +# Make sure that: +# 1) SW_PATH directory exists in the target IMG. +# 2) Make sure that intefaces and boot_script_psa have executable permission (+x). +# +# WARNING: Using this on live virtual machines can be dangerous, potentially causing disk corruption! The virtual machine must be shut down before using this script! + +# NOTE: Change img name accordingly BroMalware/BroLogging +# Remember to update PSA/psaEE.conf too: +# psa_id=BroMalware/BroLogging +# psa_name=Bro PSA + +IMG="/var/lib/libvirt/images/BroMalware.img" +SW_PATH="/home/psa/pythonScript/" + +# Copy python files +echo -n "copy PSA SW... " +virt-copy-in -a $IMG PSA/* $SW_PATH +echo "done." + +# Copy interfaces file +echo -n "copy interfaces... " +virt-copy-in -a $IMG PSA/interfaces /etc/network/ +echo "done." + +# Copy boot script that is executed when interfaces are up +echo -n "copy boot_script_psa... " +virt-copy-in -a $IMG PSA/boot_script_psa /etc/network/if-up.d/ +echo "done." diff --git a/copy_psa_to_ned.sh b/copy_psa_to_ned.sh new file mode 100644 index 0000000..64f0c79 --- /dev/null +++ b/copy_psa_to_ned.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# This script copies the PSA codes into the NED v0.6 implementation folders, namely: +# 1) PSCM/userList +# 2) TVDM/psaConfigs/[psaID] folder +# 3) TVDM/PSAManifest/[psaID] file +# 4) TVDM/userGraph/[psaID] file +# +# One parameter is required - the full path to your destination NED v0.5.1 dir, e.g., /home/ned/NED/. +# Note: This will overwrite existing configurations for the PSA_ID and the USER inside your NED! + +if [ $# -ne 1 ] ; then + echo "Usage: $0 [Full path to NED directory where the PSA files are to be copied (e.g., /home/ned/NED/)]" + exit 1 +fi + +NED_VERSION=v0.5.1 + +# Note: If you use this for other PSAs, please rename the PSA_ID as such (the config folders have to match in NED_files/TVDM/)! +PSA_ID="broPSA" +USER="bro" +PW=" secuser" +PSCM_PATH=$1PSCM/ +NED_PATH=$1 +USER_LIST=userList + +TEMPLATES=NED_files_template/TVDM/ + +if [ ! -f $PSCM_PATH$USER_LIST ]; then +echo "$PSCM_PATH$USER_LIST file does not exist." +echo "Usage: $0 [full path to NED directory where the PSA files are to be copied (e.g., /home/ned/NED/)]" + exit 1 +fi + +# 1 +################################################################################# +echo "Checking if PSA user exists in PSCM/userList" +user_pw=" secuser" +user_cred=$PSA_ID$user_pw + +if grep -q "$USER" $PSCM_PATH"$USER_LIST"; then + echo "User existed in PSCM/userList, skipping creation of new user." +else + echo "User not in PSCM/userList file, creating new user." + echo $USER$PW >> $PSCM_PATH$USER_LIST +fi + +# 2 +################################################################################# +echo "Copying PSA files into NED $NED_VERSION folders" +cp -avr NED_files/TVDM $NED_PATH diff --git a/docs/HowToAddNewBroPSAModules.md b/docs/HowToAddNewBroPSAModules.md new file mode 100644 index 0000000..144e1f5 --- /dev/null +++ b/docs/HowToAddNewBroPSAModules.md @@ -0,0 +1,372 @@ +# Adding new modules to BroPSA + +## 1 Introduction + +Each BroPSA module consist of two parts: + +* **A Python class**: this part is used as PSA's interface to the module and + to the Bro script. It's main tasks are 1) parsing configuration rules to a + format suitable for the corresponding Bro script, 2) formatting Bro script's + outputs for the BroPSA logs etc., and 3) communication with the Bro script. + However, Python part of the module may, as an example, also make more complex + computations related to the actual monitoring task as Python allows more + flexibility compared to Bro NSM scripting language. +* **A Bro script**: this part implements the actual monitoring code. + +### 1.1 Creating a new Python module + +Modules are placed in the *modules* directory of the PSA source code tree. Each +module's Python part must contain a class that inherits from the +[BroModule](../PSA/modules/BroModule.py) class. Each class must define a global +variable called *module* that must be initialized with the class constructor. +The constructor must take a single argument, a logger object, that must be +passed to parent class' constructor. It can be later accessed using a member +variable of the same name. The first argument given to the parent's constructor +is the name of the Bro scripts file corresponding to this module. + +``` +from modules.BroModule import BroModule + +... + +class MYModule( BroModule ): + + def __init__( self, logger ): + super( MyModule, self ).__init__( 'my-module.bro', logger ) + ... +... + +module = MyModule +``` + +A BroModule implements four function that can be overridden in the child +classes. + +Function *onStart* is called after Bro NSM is started and the Bro script +part of the module becomes available. The main purpose of this function is +to pass configuration options to the Bro script. Take function takes a single +parameter that is a connection object connected to Bro. It **must** be passed +to parent class' onStart function. It can be later accessed using a member +variable of the same name. + +``` + def onStart( self, connection ): + super( MyModule, self ).onStart( connection ) + ... +``` + +Function *onStop* is called before Bro NSM is stopped and the Bro script +part of the module becomes unavailable. The main purpose of this function is +to perform any tasks related to shutting down the module. The function must call +parent class' onStop function. + +``` + def onStop( self ): + super( MyModule, self ).onStop() + ... +``` + +Function *onRule* is called by BroPSA once for each configuration rule in the +PSA configuration related to the module. This function takes a single argument, +the rule object, and must return True if it was able to process the rule, or +False otherwise. + +``` + def onRule( self, rule ): + ... +``` + +Function *onEvent* is used to pass events from the Bro script to the Python part +of the module. It takes a single argument that is the event sent by the Bro +script. This function is explained in more detail in Section 2.2. + +``` + def onEvent( self, data ): + ... + +``` + +**NOTE**: Easiest way to to create a new Python module is to copy and modify +some of the existing modules. + +### 1.2 Creating a Bro NSM script + +Refer to [Bro NSM documentation](https://www.bro.org/sphinx/scripting/) + +**NOTE**: All Bro modules used with BroPSA should work in Bro NSM bare mode! + +# 2 Communication between Bro and Python + +The Python part and the Bro script part of a module communicate using Broccoli +Python bindings. + +## 2.1 From Python to Bro + +### 2.1.1 In Python code + +Import required functions from Broccoli: + +``` +from broccoli import event, record_type, record, addr, port, count +``` + +Define a record type for the message. Record contents should be a list of field +names that correspond to those defined in the Bro script (see Section 2.2.1). + +``` +MyInRecord = record_type( ... ) +``` + +**Example:** +``` +MyInRecord = record_type( 'op' ) +``` + +To send a new record, a record must first be created using the *record* +function. After this the record is filled with the actual data. Each of the +defined fields should contain *some* value. All the fields of the record must be +initialized using suitable function (e.g., str(), addr(), port(), count()) that +matches the corresponding field's type in the Bro script to ensure they are +encoded correctly. + +Records are sent using the Broccoli Connection stored in the Module object's +member variable *connection*. The first argument given to the function is the +event name, which can be freely chosen. The connection object should only be +used between the calls to modules *onStart* and *onStop* methods. Otherwise +message will not be passed to the Bro script. + +**Example:** +``` + try: + rec = record( MyInRecord ) + rec.op = str( 'MyString' ) + self.connection.send( 'on_my_event', rec ) + except Exception: + ... +``` + +**Note**: Broccoli and it's Python Bindings provide a very simple interface to +Bro script and not all the Bro script features are supported. Thus, the message +format should be relatively simple. Complex types, such as containers, should +not be used in the message content. + +### 2.1.2 In Bro code + +Module's *init* callback should subscribe for all Bro events the module wants to +receive. This is performed using utilities in [psa-utils.bro](PSA/modules/psa-utils.bro). +Thus, this script file must first be loaded into the module script: + +``` +@load ./psa-utils +``` + +Subscription is made using the function: + +``` +function subscribe_events( events : pattern ) +``` + +The pattern argument should have a Bro pattern that captures the event name +used in *send* function in the Python code. + +The Bro script must define a record type matching the record type defined in the +Python code. In addition, an event handler must be defined to capture the +corresponding event. This event handler's name must match the given event name +and take one parameter of the defined record type. + +**Example:** +``` +type MyInRecord: record +{ + op: string; +}; + +event on_my_event( rec: MyInRecord ) +{ + ... +} + +event bro_init() &priority=9 +{ + PSA::subscribe_events( /on_my_event/ ); +} +``` + +## 2.2 From Bro to Python + +### 2.2.1 In Bro code + +An ouput record must be defined similarly to defining the input record in +Section 2.1.2. In addition, an event handler must be declared for that record. + +``` +type MyOutRecord : record +{ + op: string; +}; + +global my_event: event( data: MyOutRecord ); +``` + +A new event is sent simply by creating and filling the event record and then +calling the event handler: + +``` + local data: MyOutRecord; + # fill the record + data$op = "MyData"; + event report_count( data ); + +``` + +### 2.2.2 In Python code + +In the Python code an event handler must be defined for the Bro event. +Event handlers are always global functions not related to any specific +object instaces. BroEventDispatcher is used to pass the event to the +actual BroModule object. + +In order to receive events, the module must import BroEventDispatcher +and register itself with the dispatcher using the function *register*. +The function takes a key (any string) and an object as arguments. The +given object must implement function called *onEvent* as it is defined +in the BroModule. See Section 1.1 for more details. + +**Example**: +``` +import modules.BroEventDispatcher as BroEventDispatcher + +... + +MyModuleKey = 'MyModuleEvent' + +... + +class MYModule( BroModule ): + +... + + def __init__( self, logger ): + ... + BroEventDispatcher.register( MyModuleKey, self ) + ... +``` + +A record type must be defined similarly to the input record in Section +2.1.1. + +**Example**; +``` +MyOutRecord = record_type( 'op' ) +``` + +Event handlers a defined using @event decorators. The decorator statement should +take the defined record as its argument. The actual event handler function takes +a single argument that is the Bro record. In order to pass this record to the +module object, the event handler must call BroEventDispatcher's *dispatch* +function with the key registered for the module object and the received record. + +``` +@event( MyOutRecord ) +def report_count( data ): + BroEventDispatcher.dispatch( MyModuleKey, data ) +``` + +The record will be eventually passed to module's *onEvent* function for further +processing. + +``` + def onEvent( self, data ): + ... +``` + +## 3. Loading a module at runtime + +BroPSA loads modules dynamically based on the PSA configuration file and +module description file [*modules.json*](../PSA/modules.json). The latter is +used to map *operations* in rule definitions of the former file to correct +module implementations. Each module available at runtime should have an entry +in the modules file: + +``` +{ + "modules": [ +... + { + "name": "MyOperation", + "module": "modules/MyModule.py" + } +... + ] +} +``` + +where: + +* The value of the *name* attribute must match the value of the *operation* + attribute of any rules related to this module in the BroPSA configuration + file. +* The value of the *module* attribute must contain a (relative) path to the + Python file containing the global *module* variable initialized to module's + BroModule class' constructor function. + +In order to be loaded a module must match to at least one rule in the PSA +configuration file. The rule is needed **even** if the module does not actually +use any configuration options. Thus, the *PSA/psaConfigs/psaconf* file should +contain a *rule* entry of the following format: + +``` +{ + + "rules": [ +... + { "id": "MyRule", + "hspl": { + "id": "MyRule", + "text": "MyHSPL" + }, + "event": "EVENT_CONNECTION", + "operation": "MyOperation", + "parameters": [ + { "type": "MyParameter", + "value": "MyValue" + } + ], + "action": "log", + "conditions": [ + { "type": "MyCondition", + "value": "MyValue" + } +... + ] + } +} +``` + +All of the following attributes must exist: + +* The *id* attribute must be a unique rule ID (in file scope) +* The *hspl* attribute should specify the HSPL rule related to this rule. The + *id* and the *text* attribute should come directly from the MSPL definition. + In case the configuration is written by hand, these attributes may contain + any string values, but they should still be present. +* The *operation* attribute must contain a module specific identifier that can + be chosen freely. This identifier must match to some module entry in the + *modules.json* file +* The *conditions* and *parameters* attributes may be empty lists or they may + contain module specific key-value pairs. Each of these pairs must have two + attributes: *type* and *value*. The value of the *type* attribute must be a + string and should describe the parameter. Value of the *value* attribute + can be any JSON object. Although, if the attribute has a complex values, + e.g., an object, changes might be needed into the configuration parsing + code. +* The *action* attribute must contain value *log*, as it is currently the only + supported action. +* The *event* attribute must contain one of the values *EVENT_CONNECTION* or + *EVENT_FILE*. However, this attribute is currently not used. + + + +## 4. Adding new configuration options + +You are on your own, bro... diff --git a/docs/bro-architecture.dia b/docs/bro-architecture.dia new file mode 100644 index 0000000..f947e08 Binary files /dev/null and b/docs/bro-architecture.dia differ diff --git a/docs/bro-architecture.png b/docs/bro-architecture.png new file mode 100644 index 0000000..ef6b317 Binary files /dev/null and b/docs/bro-architecture.png differ diff --git a/docs/bro-psa-architecture.dia b/docs/bro-psa-architecture.dia new file mode 100644 index 0000000..f828bca Binary files /dev/null and b/docs/bro-psa-architecture.dia differ diff --git a/docs/bro-psa-architecture.png b/docs/bro-psa-architecture.png new file mode 100644 index 0000000..6687809 Binary files /dev/null and b/docs/bro-psa-architecture.png differ diff --git a/examples/psa-conf-example.json b/examples/psa-conf-example.json new file mode 100644 index 0000000..0d4bf15 --- /dev/null +++ b/examples/psa-conf-example.json @@ -0,0 +1,76 @@ +{ + + "rules": [ + { "id": "rule1", + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination", + "value": { "address": "91.197.85.151" } + } + ] + }, + { "id": "rule2", + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination", + "value": { "address": "81.209.67.238" } + } + ] + }, + { "id": "rule3", + "event": "EVENT_CONNECTION", + "operation": "count", + "parameters": [ + { "type": "object", + "value": "OBJ_CONNECTION" + } + ], + "action": "log", + "conditions": [ + { "type": "interval", + "value": 30 }, + { "type": "threshold", + "value": 50 }, + { "type": "destination_port", + "value": { "port": 80 } + } + ] + }, + { "id": "rule4", + "event": "EVENT_FILE", + "operation": "detect-MHR", + "parameters": [ ], + "action": "log", + "conditions": [ + { "type": "mime-type", + "value": "application/pdf" + }, + { "type": "mime-type", + "value": "application/x-dosexec" + } + ] + } + ] +} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..dcf2c80 --- /dev/null +++ b/tests/README.md @@ -0,0 +1 @@ +# Placeholder