From 4633b8e5cd972fe650185bfae7822b696551c5e9 Mon Sep 17 00:00:00 2001 From: Brendan Smith <3453402+bsmithio@users.noreply.github.com> Date: Mon, 9 Oct 2023 15:52:40 -0500 Subject: [PATCH] Update content pack with RFC5424 Extractors --- config/OPNsense-pack.json | 758 +++++++++++++++++++------------------- 1 file changed, 387 insertions(+), 371 deletions(-) diff --git a/config/OPNsense-pack.json b/config/OPNsense-pack.json index 71ec61a..e2ebc6d 100644 --- a/config/OPNsense-pack.json +++ b/config/OPNsense-pack.json @@ -1,22 +1,116 @@ { "v": 1, - "id": "2a49dff2-b925-4708-8fb4-9afba67640a9", + "id": "f68cf7f1-f238-4c1a-a1b5-ceb7e0ffbdd1", "rev": 1, "name": "OPNsense Dashboard", - "summary": "This pack includes everything needed to setup Graylog for the dashboard.", + "summary": "OPNsense Dashboard", "description": "", - "vendor": "BSmithIO", - "url": "https://github.com/BSmithIO/OPNsense-Dashboard", + "vendor": "bsmithio", + "url": "https://github.com/bsmithio/OPNsense-Dashboard/", "parameters": [], "entities": [ + { + "v": "1", + "type": { + "name": "lookup_table", + "version": "1" + }, + "id": "f3295aa2-d219-4db3-8f01-e23e852da4e5", + "data": { + "default_single_value_type": { + "@type": "string", + "@value": "NULL" + }, + "cache_name": { + "@type": "string", + "@value": "6f974ae8-5d78-4552-afa9-0ec512ee5273" + }, + "name": { + "@type": "string", + "@value": "geoip" + }, + "default_multi_value_type": { + "@type": "string", + "@value": "NULL" + }, + "default_multi_value": { + "@type": "string", + "@value": "" + }, + "data_adapter_name": { + "@type": "string", + "@value": "0be5d392-f0bf-49af-b6c5-680ab356b9a0" + }, + "_scope": { + "@type": "string", + "@value": "DEFAULT" + }, + "title": { + "@type": "string", + "@value": "GeoIP" + }, + "default_single_value": { + "@type": "string", + "@value": "" + }, + "description": { + "@type": "string", + "@value": "Geo IP Lookup" + } + }, + "constraints": [ + { + "type": "server-version", + "version": ">=5.0.2+59d96f8" + } + ] + }, + { + "v": "1", + "type": { + "name": "pipeline", + "version": "1" + }, + "id": "bde4c008-a0c9-4049-ad0c-60881fa545d8", + "data": { + "title": { + "@type": "string", + "@value": "GeoIP" + }, + "description": { + "@type": "string", + "@value": "GeoIP" + }, + "source": { + "@type": "string", + "@value": "pipeline \"GeoIP\"\nstage 0 match either\nrule \"GeoIP lookup: src_ip\"\nend" + }, + "connected_streams": [ + { + "@type": "string", + "@value": "df7d5f2d-ff6a-4a4b-9044-cbd6238da087" + } + ] + }, + "constraints": [ + { + "type": "server-version", + "version": ">=5.0.2+59d96f8" + } + ] + }, { "v": "1", "type": { "name": "lookup_cache", "version": "1" }, - "id": "79d432a2-a390-4450-b7ca-7ba16eebffe6", + "id": "6f974ae8-5d78-4552-afa9-0ec512ee5273", "data": { + "_scope": { + "@type": "string", + "@value": "DEFAULT" + }, "name": { "@type": "string", "@value": "geoip" @@ -55,7 +149,7 @@ "constraints": [ { "type": "server-version", - "version": ">=4.2.1+5442e44" + "version": ">=5.0.2+59d96f8" } ] }, @@ -65,7 +159,7 @@ "name": "stream", "version": "1" }, - "id": "d060729f-292b-4894-af6a-ed2f1c258e08", + "id": "df7d5f2d-ff6a-4a4b-9044-cbd6238da087", "data": { "alarm_callbacks": [], "outputs": [], @@ -122,41 +216,7 @@ "constraints": [ { "type": "server-version", - "version": ">=4.2.1+5442e44" - } - ] - }, - { - "v": "1", - "type": { - "name": "pipeline", - "version": "1" - }, - "id": "fda2128d-4140-47b4-915a-889349953b12", - "data": { - "title": { - "@type": "string", - "@value": "GeoIP" - }, - "description": { - "@type": "string", - "@value": "GeoIP" - }, - "source": { - "@type": "string", - "@value": "pipeline \"GeoIP\"\nstage 0 match either\nrule \"GeoIP lookup: src_ip\"\nend" - }, - "connected_streams": [ - { - "@type": "string", - "@value": "d060729f-292b-4894-af6a-ed2f1c258e08" - } - ] - }, - "constraints": [ - { - "type": "server-version", - "version": ">=4.2.1+5442e44" + "version": ">=5.0.2+59d96f8" } ] }, @@ -166,8 +226,12 @@ "name": "lookup_adapter", "version": "1" }, - "id": "db9a5df6-9e1a-4d37-ad73-16a8dd08b5fa", + "id": "0be5d392-f0bf-49af-b6c5-680ab356b9a0", "data": { + "_scope": { + "@type": "string", + "@value": "DEFAULT" + }, "name": { "@type": "string", "@value": "geoip" @@ -206,87 +270,7 @@ "constraints": [ { "type": "server-version", - "version": ">=4.2.1+5442e44" - } - ] - }, - { - "v": "1", - "type": { - "name": "pipeline_rule", - "version": "1" - }, - "id": "6bbba4f4-256f-4478-abf4-8034001c5237", - "data": { - "title": { - "@type": "string", - "@value": "GeoIP lookup: src_ip" - }, - "description": { - "@type": "string", - "@value": "" - }, - "source": { - "@type": "string", - "@value": "rule \"GeoIP lookup: src_ip\"\nwhen\nhas_field(\"src_ip\")\nthen\nlet geo = lookup(\"geoip\", to_string($message.\"src_ip\"));\nset_field(\"src_ip_geo_location\", geo[\"coordinates\"]);\nset_field(\"src_ip_geo_country\", geo[\"country\"].iso_code);\nset_field(\"src_ip_geo_city\", geo[\"city\"].names.en);\nend" - } - }, - "constraints": [ - { - "type": "server-version", - "version": ">=4.2.1+5442e44" - } - ] - }, - { - "v": "1", - "type": { - "name": "lookup_table", - "version": "1" - }, - "id": "dc892e81-3ada-4ee9-8c47-d0dcda4c7d65", - "data": { - "default_single_value_type": { - "@type": "string", - "@value": "NULL" - }, - "cache_name": { - "@type": "string", - "@value": "79d432a2-a390-4450-b7ca-7ba16eebffe6" - }, - "name": { - "@type": "string", - "@value": "geoip" - }, - "default_multi_value_type": { - "@type": "string", - "@value": "NULL" - }, - "default_multi_value": { - "@type": "string", - "@value": "" - }, - "data_adapter_name": { - "@type": "string", - "@value": "db9a5df6-9e1a-4d37-ad73-16a8dd08b5fa" - }, - "title": { - "@type": "string", - "@value": "GeoIP" - }, - "default_single_value": { - "@type": "string", - "@value": "" - }, - "description": { - "@type": "string", - "@value": "Geo IP Lookup" - } - }, - "constraints": [ - { - "type": "server-version", - "version": ">=4.2.1+5442e44" + "version": ">=5.0.2+59d96f8" } ] }, @@ -296,28 +280,20 @@ "name": "input", "version": "1" }, - "id": "ec0618e2-70b3-49d8-898c-3b67ad39f4c6", + "id": "1f88a480-2960-4ee9-9ea4-87e38b22e92c", "data": { "title": { "@type": "string", "@value": "Syslog UDP" }, "configuration": { - "expand_structured_data": { - "@type": "boolean", - "@value": false - }, - "recv_buffer_size": { - "@type": "integer", - "@value": 262144 - }, "port": { "@type": "integer", "@value": 1514 }, - "number_worker_threads": { + "recv_buffer_size": { "@type": "integer", - "@value": 6 + "@value": 262144 }, "force_rdns": { "@type": "boolean", @@ -331,9 +307,21 @@ "@type": "string", "@value": "0.0.0.0" }, - "store_full_message": { + "expand_structured_data": { "@type": "boolean", "@value": false + }, + "store_full_message": { + "@type": "boolean", + "@value": true + }, + "charset_name": { + "@type": "string", + "@value": "UTF-8" + }, + "number_worker_threads": { + "@type": "integer", + "@value": 6 } }, "static_fields": {}, @@ -346,172 +334,6 @@ "@value": false }, "extractors": [ - { - "target_field": { - "@type": "string", - "@value": "filterlog_ipv4_tcp" - }, - "condition_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*,(in|out),4,.*,tcp,.*)$" - }, - "order": { - "@type": "integer", - "@value": 0 - }, - "converters": [ - { - "type": { - "@type": "string", - "@value": "CSV" - }, - "configuration": { - "column_header": { - "@type": "string", - "@value": "rule_number,sub_rule_number,anchor,tracker,interface,reason,action,direction,ip_version,tos,ecn,ttl,id,offset,ip_flags,protocol_id,protocol_name,length,src_ip,dst_ip,src_port,dst_port,datalen,flags,sequence,ack,window,urg,options" - }, - "trim_leading_whitespace": { - "@type": "boolean", - "@value": true - } - } - } - ], - "configuration": { - "regex_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*)$" - } - }, - "source_field": { - "@type": "string", - "@value": "message" - }, - "title": { - "@type": "string", - "@value": "OPNsense: IPv4 TCP" - }, - "type": { - "@type": "string", - "@value": "REGEX" - }, - "cursor_strategy": { - "@type": "string", - "@value": "COPY" - }, - "condition_type": { - "@type": "string", - "@value": "REGEX" - } - }, - { - "target_field": { - "@type": "string", - "@value": "filterlog_ipv4_icmp" - }, - "condition_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*,(in|out),4,.*,icmp,.*)$" - }, - "order": { - "@type": "integer", - "@value": 4 - }, - "converters": [ - { - "type": { - "@type": "string", - "@value": "CSV" - }, - "configuration": { - "column_header": { - "@type": "string", - "@value": "rule_number,sub_rule_number,anchor,tracker,interface,reason,action,direction,ip_version,tos,ecn,ttl,id,offset,flags,protocol_id,protocol_name,length,src_ip,dst_ip,datalen" - } - } - } - ], - "configuration": { - "regex_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*)$" - } - }, - "source_field": { - "@type": "string", - "@value": "message" - }, - "title": { - "@type": "string", - "@value": "OPNsense: IPv4 ICMP" - }, - "type": { - "@type": "string", - "@value": "REGEX" - }, - "cursor_strategy": { - "@type": "string", - "@value": "COPY" - }, - "condition_type": { - "@type": "string", - "@value": "REGEX" - } - }, - { - "target_field": { - "@type": "string", - "@value": "filterlog_ipv6_udp" - }, - "condition_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*,(in|out),6,.*,udp,.*)$" - }, - "order": { - "@type": "integer", - "@value": 3 - }, - "converters": [ - { - "type": { - "@type": "string", - "@value": "CSV" - }, - "configuration": { - "column_header": { - "@type": "string", - "@value": "rule_number,sub_rule_number,anchor,tracker,interface,reason,action,direction,ip_version,class,flowlabel,hoplimit,protocol_name,protocol_id,length,src_ip,dst_ip,src_port,dst_port,datalength" - } - } - } - ], - "configuration": { - "regex_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*)$" - } - }, - "source_field": { - "@type": "string", - "@value": "message" - }, - "title": { - "@type": "string", - "@value": "OPNsense: IPv6 UDP" - }, - "type": { - "@type": "string", - "@value": "REGEX" - }, - "cursor_strategy": { - "@type": "string", - "@value": "COPY" - }, - "condition_type": { - "@type": "string", - "@value": "REGEX" - } - }, { "target_field": { "@type": "string", @@ -519,7 +341,7 @@ }, "condition_value": { "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*,(in|out),6,.*,ipv6-icmp,.*)$" + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*,(in|out),6,.*,icmp,.*)$" }, "order": { "@type": "integer", @@ -534,7 +356,7 @@ "configuration": { "column_header": { "@type": "string", - "@value": "rule_number,sub_rule_number,anchor,tracker,interface,reason,action,direction,ip_version,class,flow,hoplimit,protocol_name,protocol_id,length,src_ip,dst_ip,datalen" + "@value": "rule-number,sub-rule-number,anchor,tracker,interface,reason,action,direction,ip-version,class,flowlabel,hoplimit,protocol-name,protocol-id,length,src-ip,dst-ip,datalength" } } } @@ -542,74 +364,16 @@ "configuration": { "regex_value": { "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*)$" + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*)$" } }, "source_field": { "@type": "string", - "@value": "message" + "@value": "full_message" }, "title": { "@type": "string", - "@value": "OPNsense: IPv6 ICMP" - }, - "type": { - "@type": "string", - "@value": "REGEX" - }, - "cursor_strategy": { - "@type": "string", - "@value": "COPY" - }, - "condition_type": { - "@type": "string", - "@value": "REGEX" - } - }, - { - "target_field": { - "@type": "string", - "@value": "filterlog_ipv4_udp" - }, - "condition_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*,(in|out),4,.*,udp,.*)$" - }, - "order": { - "@type": "integer", - "@value": 2 - }, - "converters": [ - { - "type": { - "@type": "string", - "@value": "CSV" - }, - "configuration": { - "column_header": { - "@type": "string", - "@value": "rule_number,sub_rule_number,anchor,tracker,interface,reason,action,direction,ip_version,tos,ecn,ttl,id,offset,flags,protocol_id,protocol_name,length,src_ip,dst_ip,src_port,dst_port,datalen" - }, - "trim_leading_whitespace": { - "@type": "boolean", - "@value": true - } - } - } - ], - "configuration": { - "regex_value": { - "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*)$" - } - }, - "source_field": { - "@type": "string", - "@value": "message" - }, - "title": { - "@type": "string", - "@value": "OPNsense: IPv4 UDP" + "@value": "OPNsense: RFC5424 IPv6 ICMP" }, "type": { "@type": "string", @@ -631,7 +395,7 @@ }, "condition_value": { "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*,(in|out),6,.*,tcp,.*)$" + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*,(in|out),6,.*,tcp,.*)$" }, "order": { "@type": "integer", @@ -646,7 +410,7 @@ "configuration": { "column_header": { "@type": "string", - "@value": "rule_number,sub_rule_number,anchor,tracker,interface,reason,action,direction,ipversion,class,flowlabel,hoplimit,protocol_name,protocol_id,length,src_ip,dst_ip,src_port,dst_port,datalength,flags,sequence,ack,window,urg,options" + "@value": "rule-number,sub-rule-number,anchor,tracker,interface,reason,action,direction,ipversion,class,flowlabel,hoplimit,protocol-name,protocol-id,length,src-ip,dst-ip,src-port,dst-port,datalength,tcp-flags,sequence,ack,window,urg,options,opnsense-rid" }, "trim_leading_whitespace": { "@type": "boolean", @@ -658,16 +422,240 @@ "configuration": { "regex_value": { "@type": "string", - "@value": "^(?i).*\\sfilterlog\\[[0-9]+\\]:\\s(.*)$" + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*)$" } }, "source_field": { "@type": "string", - "@value": "message" + "@value": "full_message" }, "title": { "@type": "string", - "@value": "OPNsense: IPv6 TCP" + "@value": "OPNsense: RFC5424 IPv6 TCP" + }, + "type": { + "@type": "string", + "@value": "REGEX" + }, + "cursor_strategy": { + "@type": "string", + "@value": "COPY" + }, + "condition_type": { + "@type": "string", + "@value": "REGEX" + } + }, + { + "target_field": { + "@type": "string", + "@value": "filterlog_ipv4_tcp" + }, + "condition_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*,(in|out),4,.*,tcp,.*)$" + }, + "order": { + "@type": "integer", + "@value": 0 + }, + "converters": [ + { + "type": { + "@type": "string", + "@value": "CSV" + }, + "configuration": { + "column_header": { + "@type": "string", + "@value": "rule-number,sub-rule-number,anchor,tracker,interface,reason,action,direction,ip-version,tos,ecn,ttl,id,offset,ip-flags,protocol-id,protocol-name,length,src-ip,dst-ip,src-port,dst-port,datalength,tcp-flags,sequence,f1,f2,tcp-options,opnsense-rid" + }, + "trim_leading_whitespace": { + "@type": "boolean", + "@value": true + } + } + } + ], + "configuration": { + "regex_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*)$" + } + }, + "source_field": { + "@type": "string", + "@value": "full_message" + }, + "title": { + "@type": "string", + "@value": "OPNsense: RFC5424 IPv4 TCP" + }, + "type": { + "@type": "string", + "@value": "REGEX" + }, + "cursor_strategy": { + "@type": "string", + "@value": "COPY" + }, + "condition_type": { + "@type": "string", + "@value": "REGEX" + } + }, + { + "target_field": { + "@type": "string", + "@value": "filterlog_ipv4_udp" + }, + "condition_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*,(in|out),4,.*,udp,.*)$" + }, + "order": { + "@type": "integer", + "@value": 2 + }, + "converters": [ + { + "type": { + "@type": "string", + "@value": "CSV" + }, + "configuration": { + "column_header": { + "@type": "string", + "@value": "rule-number,sub-rule-number,anchor,tracker,interface,reason,action,direction,ip-version,tos,ecn,ttl,id,offset,flags,protocol-id,protocol-name,length,src-ip,dst-ip,src-port,dst-port,opnsense-rid" + }, + "trim_leading_whitespace": { + "@type": "boolean", + "@value": true + } + } + } + ], + "configuration": { + "regex_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*)$" + } + }, + "source_field": { + "@type": "string", + "@value": "full_message" + }, + "title": { + "@type": "string", + "@value": "OPNsense: RFC5424 IPv4 UDP" + }, + "type": { + "@type": "string", + "@value": "REGEX" + }, + "cursor_strategy": { + "@type": "string", + "@value": "COPY" + }, + "condition_type": { + "@type": "string", + "@value": "REGEX" + } + }, + { + "target_field": { + "@type": "string", + "@value": "filterlog_ipv4_icmp" + }, + "condition_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*,(in|out),4,.*,icmp,.*)$" + }, + "order": { + "@type": "integer", + "@value": 4 + }, + "converters": [ + { + "type": { + "@type": "string", + "@value": "CSV" + }, + "configuration": { + "column_header": { + "@type": "string", + "@value": "rule-number,sub-rule-number,anchor,tracker,interface,reason,action,direction,ip-version,tos,ecn,ttl,id,offset,flags,protocol-id,protocol-name,length,src-ip,dst-ip,datalength" + } + } + } + ], + "configuration": { + "regex_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*)$" + } + }, + "source_field": { + "@type": "string", + "@value": "full_message" + }, + "title": { + "@type": "string", + "@value": "OPNsense: RFC5424 IPv4 ICMP" + }, + "type": { + "@type": "string", + "@value": "REGEX" + }, + "cursor_strategy": { + "@type": "string", + "@value": "COPY" + }, + "condition_type": { + "@type": "string", + "@value": "REGEX" + } + }, + { + "target_field": { + "@type": "string", + "@value": "filterlog_ipv6_udp" + }, + "condition_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*,(in|out),6,.*,udp,.*)$" + }, + "order": { + "@type": "integer", + "@value": 3 + }, + "converters": [ + { + "type": { + "@type": "string", + "@value": "CSV" + }, + "configuration": { + "column_header": { + "@type": "string", + "@value": "rule-number,sub-rule-number,anchor,tracker,interface,reason,action,direction,ip-version,class,flowlabel,hoplimit,protocol-name,protocol-id,length,src-ip,dst-ip,src-port,dst-port,opnsense-rid" + } + } + } + ], + "configuration": { + "regex_value": { + "@type": "string", + "@value": "^(?i).*\\sfilterlog.+\\[.+\\]\\s(.*)$" + } + }, + "source_field": { + "@type": "string", + "@value": "full_message" + }, + "title": { + "@type": "string", + "@value": "OPNsense: RFC5424 IPv6 UDP" }, "type": { "@type": "string", @@ -687,7 +675,35 @@ "constraints": [ { "type": "server-version", - "version": ">=4.2.1+5442e44" + "version": ">=5.0.2+59d96f8" + } + ] + }, + { + "v": "1", + "type": { + "name": "pipeline_rule", + "version": "1" + }, + "id": "ae3c665d-5f80-4040-bbe6-0261e890d1dc", + "data": { + "title": { + "@type": "string", + "@value": "GeoIP lookup: src_ip" + }, + "description": { + "@type": "string", + "@value": "" + }, + "source": { + "@type": "string", + "@value": "rule \"GeoIP lookup: src_ip\"\nwhen\nhas_field(\"src_ip\")\nthen\nlet geo = lookup(\"geoip\", to_string($message.\"src_ip\"));\nset_field(\"src_ip_geo_location\", geo[\"coordinates\"]);\nset_field(\"src_ip_geo_country\", geo[\"country\"].iso_code);\nset_field(\"src_ip_geo_city\", geo[\"city\"].names.en);\nend" + } + }, + "constraints": [ + { + "type": "server-version", + "version": ">=5.0.2+59d96f8" } ] }