================================================
So you've got all these RADIUS logs, but how do you analyse them? What is the
-easiest way to query the logs, find out when a client connected or
-disconnected, or view the top ten clients logging into the system over the last
-six hours?
+easiest way to query the logs, find out when a client connected or disconnected,
+or view the top ten clients logging into the system over the last six hours?
The logstash/elasticsearch/kibana stack is designed and built to do just that.
elasticsearch is a search engine; logstash is commonly used to feed data in,
logs in.
This directory contains the following files as a starting point for feeding
-RADIUS logs into elasticsearch via logstash.
+RADIUS logs into elasticsearch via logstash, then sample dashboards for Kibana
+to explore the data.
Files
-----
Please note that all files should be reviewed before use to determine if they
-are suitable for your configuration/system.
+are suitable for your configuration/system, especially if you are integrating
+this into an existing logstash/elasticsearch setup.
radius-mapping.sh
This shell script (which just runs curl) pushes a template mapping into the
elasticsearch cluster.
-
-radius.conf
+logstash-radius.conf
A sample configuration file for logstash that parses RADIUS 'detail' files.
It processes these by joining each record onto one line, then splitting the
- tab-delimited key-value pairs out.
+ tab-delimited key-value pairs out. Some additional data is then extracted
+ from certain key attributes.
The file will need to be edited at least to set the input method: for
experimentation the given input (stdin) may be used. If logstash is running on
input such as log-courier or logstash-forwarder may be better to get the data
over the network to the logstash server.
+ It would be best to use an input method that can join the multiple lines of
+ the detail file together and feed them to logstash as a single entry, rather
+ than using the logstash multiline filter.
+
+log-courier.conf
+
+ An example configuration for the log-courier feeder.
+
+kibana4-dashboard.json
+
+ Basic RADIUS dashboard for Kibana4.
+
+ To import the dashboard first create a new index called "radius-*" in
+ Settings/Indices. Then go to Kibana's Settings page, "Objects" and "Import".
+ Once imported open the "RADIUS detail" dashboard.
+
+kibana3-dashboard.json
+
+ Basic RADIUS dashboard for Kibana3. To import the dashboard go to Load,
+ Advanced and "Choose File".
+
Example usage
-------------
$ ./radius-mapping.sh
Feed a detail file in:
-$ /path/to/logstash -f radius.conf < acct-detail
+$ /path/to/logstash -f logstash-radius.conf < acct-detail
See also
elasticsearch web site: http://www.elastic.co/
-
Matthew Newton
-April 2015
+January 2016
--- /dev/null
+{
+ "title": "RADIUS detail",
+ "services": {
+ "query": {
+ "list": {
+ "0": {
+ "query": "*",
+ "alias": "",
+ "color": "#584477",
+ "id": 0,
+ "pin": false,
+ "type": "lucene",
+ "enable": true
+ },
+ "1": {
+ "id": 1,
+ "type": "lucene",
+ "query": "Acct-Status-Type:Start",
+ "alias": "Accounting Start",
+ "color": "#629E51",
+ "pin": false,
+ "enable": true
+ },
+ "2": {
+ "id": 2,
+ "color": "#6ED0E0",
+ "alias": "",
+ "pin": false,
+ "type": "lucene",
+ "enable": true,
+ "query": "Acct-Status-Type:Interim-Update"
+ },
+ "3": {
+ "id": 3,
+ "color": "#BF1B00",
+ "alias": "",
+ "pin": false,
+ "type": "lucene",
+ "enable": true,
+ "query": "Acct-Status-Type:Stop"
+ }
+ },
+ "ids": [
+ 0,
+ 1,
+ 2,
+ 3
+ ]
+ },
+ "filter": {
+ "list": {
+ "0": {
+ "type": "time",
+ "field": "@timestamp",
+ "from": "now-7d",
+ "to": "now",
+ "mandate": "must",
+ "active": true,
+ "alias": "",
+ "id": 0
+ }
+ },
+ "ids": [
+ 0
+ ]
+ }
+ },
+ "rows": [
+ {
+ "title": "Time series",
+ "height": "200px",
+ "editable": true,
+ "collapse": false,
+ "collapsable": true,
+ "panels": [
+ {
+ "span": 9,
+ "editable": true,
+ "type": "histogram",
+ "loadingEditor": false,
+ "mode": "count",
+ "time_field": "@timestamp",
+ "value_field": null,
+ "x-axis": true,
+ "y-axis": true,
+ "scale": 1,
+ "y_format": "none",
+ "grid": {
+ "max": null,
+ "min": 0
+ },
+ "queries": {
+ "mode": "selected",
+ "ids": [
+ 1,
+ 2,
+ 3
+ ]
+ },
+ "annotate": {
+ "enable": false,
+ "query": "*",
+ "size": 20,
+ "field": "_type",
+ "sort": [
+ "_score",
+ "desc"
+ ]
+ },
+ "auto_int": true,
+ "resolution": 100,
+ "interval": "1h",
+ "intervals": [
+ "auto",
+ "1s",
+ "1m",
+ "5m",
+ "10m",
+ "30m",
+ "1h",
+ "3h",
+ "12h",
+ "1d",
+ "1w",
+ "1y"
+ ],
+ "lines": false,
+ "fill": 0,
+ "linewidth": 3,
+ "points": false,
+ "pointradius": 5,
+ "bars": true,
+ "stack": true,
+ "spyable": true,
+ "zoomlinks": true,
+ "options": true,
+ "legend": true,
+ "show_query": true,
+ "interactive": true,
+ "legend_counts": true,
+ "timezone": "browser",
+ "percentage": false,
+ "zerofill": true,
+ "derivative": false,
+ "tooltip": {
+ "value_type": "cumulative",
+ "query_as_alias": true
+ },
+ "title": "RADIUS Accounting data"
+ },
+ {
+ "error": false,
+ "span": 3,
+ "editable": true,
+ "type": "terms",
+ "loadingEditor": false,
+ "field": "NAS-Identifier",
+ "exclude": [],
+ "missing": false,
+ "other": false,
+ "size": 20,
+ "order": "count",
+ "style": {
+ "font-size": "10pt"
+ },
+ "donut": false,
+ "tilt": false,
+ "labels": true,
+ "arrangement": "horizontal",
+ "chart": "pie",
+ "counter_pos": "above",
+ "spyable": true,
+ "queries": {
+ "mode": "selected",
+ "ids": [
+ 1
+ ]
+ },
+ "tmode": "terms",
+ "tstat": "total",
+ "valuefield": "",
+ "title": "Sessions by NAS"
+ }
+ ],
+ "notice": false
+ },
+ {
+ "title": "Graphs",
+ "height": "200px",
+ "editable": true,
+ "collapse": false,
+ "collapsable": true,
+ "panels": [
+ {
+ "error": false,
+ "span": 3,
+ "editable": true,
+ "type": "terms",
+ "loadingEditor": false,
+ "field": "Calling-Station-Id",
+ "exclude": [],
+ "missing": false,
+ "other": false,
+ "size": 10,
+ "order": "count",
+ "style": {
+ "font-size": "10pt"
+ },
+ "donut": false,
+ "tilt": false,
+ "labels": true,
+ "arrangement": "horizontal",
+ "chart": "table",
+ "counter_pos": "above",
+ "spyable": true,
+ "queries": {
+ "mode": "selected",
+ "ids": [
+ 1
+ ]
+ },
+ "tmode": "terms",
+ "tstat": "total",
+ "valuefield": "",
+ "title": "Top Calling-Station-Id"
+ },
+ {
+ "error": false,
+ "span": 3,
+ "editable": true,
+ "type": "terms",
+ "loadingEditor": false,
+ "field": "Called-Station-Id",
+ "exclude": [],
+ "missing": false,
+ "other": false,
+ "size": 10,
+ "order": "count",
+ "style": {
+ "font-size": "10pt"
+ },
+ "donut": false,
+ "tilt": false,
+ "labels": true,
+ "arrangement": "horizontal",
+ "chart": "table",
+ "counter_pos": "above",
+ "spyable": true,
+ "queries": {
+ "mode": "selected",
+ "ids": [
+ 1
+ ]
+ },
+ "tmode": "terms",
+ "tstat": "total",
+ "valuefield": "",
+ "title": "TopN Called-Station-Id"
+ },
+ {
+ "error": false,
+ "span": 3,
+ "editable": true,
+ "type": "terms",
+ "loadingEditor": false,
+ "field": "User-Name",
+ "exclude": [],
+ "missing": false,
+ "other": false,
+ "size": 10,
+ "order": "max",
+ "style": {
+ "font-size": "10pt"
+ },
+ "donut": false,
+ "tilt": false,
+ "labels": true,
+ "arrangement": "horizontal",
+ "chart": "table",
+ "counter_pos": "above",
+ "spyable": true,
+ "queries": {
+ "mode": "all",
+ "ids": [
+ 0,
+ 1,
+ 2,
+ 3
+ ]
+ },
+ "tmode": "terms_stats",
+ "tstat": "max",
+ "valuefield": "Acct-Output-Octets.long",
+ "title": "TopN data Output"
+ },
+ {
+ "error": false,
+ "span": 3,
+ "editable": true,
+ "type": "terms",
+ "loadingEditor": false,
+ "field": "User-Name",
+ "exclude": [],
+ "missing": false,
+ "other": false,
+ "size": 10,
+ "order": "max",
+ "style": {
+ "font-size": "10pt"
+ },
+ "donut": false,
+ "tilt": false,
+ "labels": true,
+ "arrangement": "horizontal",
+ "chart": "table",
+ "counter_pos": "above",
+ "spyable": true,
+ "queries": {
+ "mode": "all",
+ "ids": [
+ 0,
+ 1,
+ 2,
+ 3
+ ]
+ },
+ "tmode": "terms_stats",
+ "tstat": "max",
+ "valuefield": "Acct-Input-Octets.long",
+ "title": "TopN Data Input"
+ }
+ ],
+ "notice": false
+ },
+ {
+ "title": "Table",
+ "height": "150px",
+ "editable": true,
+ "collapse": false,
+ "collapsable": true,
+ "panels": [
+ {
+ "error": false,
+ "span": 12,
+ "editable": true,
+ "type": "table",
+ "loadingEditor": false,
+ "size": 100,
+ "pages": 5,
+ "offset": 0,
+ "sort": [
+ "@timestamp",
+ "asc"
+ ],
+ "overflow": "min-height",
+ "fields": [
+ "timestamp",
+ "User-Name",
+ "Calling-Station-Id",
+ "Called-Station-Id",
+ "Framed-IP-Address",
+ "NAS-Identifier"
+ ],
+ "highlight": [],
+ "sortable": true,
+ "header": true,
+ "paging": true,
+ "field_list": false,
+ "all_fields": false,
+ "trimFactor": 500,
+ "localTime": false,
+ "timeField": "@timestamp",
+ "spyable": true,
+ "queries": {
+ "mode": "all",
+ "ids": [
+ 0,
+ 1,
+ 2,
+ 3
+ ]
+ },
+ "style": {
+ "font-size": "9pt"
+ },
+ "normTimes": true,
+ "title": "RADIUS data"
+ }
+ ],
+ "notice": false
+ }
+ ],
+ "editable": true,
+ "failover": false,
+ "index": {
+ "interval": "day",
+ "pattern": "[radius-]YYYY.MM.DD",
+ "default": "[radius-]YYYY.MM.DD",
+ "warm_fields": false
+ },
+ "style": "dark",
+ "panel_hints": true,
+ "pulldowns": [
+ {
+ "type": "query",
+ "collapse": true,
+ "notice": false,
+ "enable": true,
+ "query": "*",
+ "pinned": true,
+ "history": [
+ "Acct-Status-Type:Stop",
+ "Acct-Status-Type:Interim-Update",
+ "Acct-Status-Type:Start",
+ "*"
+ ],
+ "remember": 10
+ },
+ {
+ "type": "filtering",
+ "collapse": true,
+ "notice": true,
+ "enable": true
+ }
+ ],
+ "nav": [
+ {
+ "type": "timepicker",
+ "collapse": false,
+ "notice": false,
+ "enable": true,
+ "status": "Stable",
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ],
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "timefield": "@timestamp",
+ "now": true,
+ "filter_id": 0
+ }
+ ],
+ "loader": {
+ "save_gist": false,
+ "save_elasticsearch": true,
+ "save_local": true,
+ "save_default": true,
+ "save_temp": true,
+ "save_temp_ttl_enable": true,
+ "save_temp_ttl": "30d",
+ "load_gist": false,
+ "load_elasticsearch": true,
+ "load_elasticsearch_size": 20,
+ "load_local": false,
+ "hide": false
+ },
+ "refresh": false
+}
\ No newline at end of file
--- /dev/null
+[
+ {
+ "_id": "RADIUS-detail",
+ "_type": "dashboard",
+ "_source": {
+ "title": "RADIUS detail",
+ "hits": 0,
+ "description": "",
+ "panelsJSON": "[{\"col\":5,\"id\":\"RADIUS-unique-User-Name-by-day\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"User-Name\",\"Calling-Station-Id\",\"Called-Station-Id\",\"Framed-IP-Address\",\"NAS-Identifier\"],\"id\":\"RADIUS-data\",\"row\":5,\"size_x\":8,\"size_y\":4,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"RADIUS-accounting-packets-histogram\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"RADIUS-table-topN-data-transferred-by-User-Name\",\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"RADIUS-Sessions-per-NAS\",\"type\":\"visualization\",\"size_x\":4,\"size_y\":4,\"col\":9,\"row\":5}]",
+ "version": 1,
+ "timeRestore": true,
+ "timeTo": "now",
+ "timeFrom": "now-7d",
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
+ }
+ }
+ },
+ {
+ "_id": "RADIUS-Accounting-Start-data",
+ "_type": "search",
+ "_source": {
+ "title": "RADIUS Accounting-Start data",
+ "description": "",
+ "hits": 0,
+ "columns": [
+ "User-Name",
+ "Calling-Station-Id",
+ "Called-Station-Id",
+ "Framed-IP-Address",
+ "NAS-Identifier"
+ ],
+ "sort": [
+ "@timestamp",
+ "desc"
+ ],
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"radius-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[{\"meta\":{\"negate\":false,\"index\":\"radius-*\",\"key\":\"Acct-Status-Type\",\"value\":\"Start\",\"disabled\":false},\"query\":{\"match\":{\"Acct-Status-Type\":{\"query\":\"Start\",\"type\":\"phrase\"}}}}]}"
+ }
+ }
+ },
+ {
+ "_id": "RADIUS-data",
+ "_type": "search",
+ "_source": {
+ "title": "RADIUS data",
+ "description": "",
+ "hits": 0,
+ "columns": [
+ "User-Name",
+ "Calling-Station-Id",
+ "Called-Station-Id",
+ "Framed-IP-Address",
+ "NAS-Identifier"
+ ],
+ "sort": [
+ "@timestamp",
+ "desc"
+ ],
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"index\":\"radius-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[]}"
+ }
+ }
+ },
+ {
+ "_id": "RADIUS-unique-User-Name-by-day",
+ "_type": "visualization",
+ "_source": {
+ "title": "RADIUS unique User-Name by day",
+ "visState": "{\n \"type\": \"histogram\",\n \"params\": {\n \"shareYAxis\": true,\n \"addTooltip\": true,\n \"addLegend\": true,\n \"scale\": \"linear\",\n \"mode\": \"stacked\",\n \"times\": [],\n \"addTimeMarker\": false,\n \"defaultYExtents\": false,\n \"setYExtents\": false,\n \"yAxis\": {}\n },\n \"aggs\": [\n {\n \"id\": \"1\",\n \"type\": \"cardinality\",\n \"schema\": \"metric\",\n \"params\": {\n \"field\": \"User-Name\"\n }\n },\n {\n \"id\": \"2\",\n \"type\": \"date_histogram\",\n \"schema\": \"segment\",\n \"params\": {\n \"field\": \"@timestamp\",\n \"interval\": \"d\",\n \"customInterval\": \"2h\",\n \"min_doc_count\": 1,\n \"extended_bounds\": {}\n }\n },\n {\n \"id\": \"3\",\n \"type\": \"terms\",\n \"schema\": \"group\",\n \"params\": {\n \"field\": \"User-Name\",\n \"size\": 50,\n \"order\": \"desc\",\n \"orderBy\": \"1\"\n }\n }\n ],\n \"listeners\": {}\n}",
+ "description": "",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\n \"filter\": []\n}"
+ },
+ "savedSearchId": "RADIUS-data"
+ }
+ },
+ {
+ "_id": "RADIUS-accounting-packets-histogram",
+ "_type": "visualization",
+ "_source": {
+ "title": "RADIUS accounting packets histogram",
+ "visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"Acct-Status-Type\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "description": "",
+ "savedSearchId": "RADIUS-data",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[]}"
+ }
+ }
+ },
+ {
+ "_id": "RADIUS-table-topN-data-transferred-by-User-Name",
+ "_type": "visualization",
+ "_source": {
+ "title": "RADIUS table topN data transferred by User-Name",
+ "visState": "{\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Acct-Output-Octets.long\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"User-Name\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"Acct-Input-Octets.long\"}}],\"listeners\":{}}",
+ "description": "",
+ "savedSearchId": "RADIUS-data",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[]}"
+ }
+ }
+ },
+ {
+ "_id": "RADIUS-Sessions-per-NAS",
+ "_type": "visualization",
+ "_source": {
+ "title": "RADIUS Sessions per NAS",
+ "visState": "{\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"NAS-Identifier\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+ "description": "",
+ "savedSearchId": "RADIUS-Accounting-Start-data",
+ "version": 1,
+ "kibanaSavedObjectMeta": {
+ "searchSourceJSON": "{\"filter\":[]}"
+ }
+ }
+ }
+]
\ No newline at end of file
--- /dev/null
+# Example log-courier configuration file for RADIUS detail files.
+#
+{
+ "general": {
+ "persist directory": "/var/lib/log-courier",
+ "log syslog": true,
+ "log stdout": false,
+ "admin listen address": "unix:/var/run/log-courier/admin.socket"
+ },
+
+ "network": {
+ "transport": "tcp",
+ "reconnect": 10,
+
+ # Servers to connect to.
+ #
+ "servers": [
+ "logstash1.example:5140",
+ "logstash2.example:5140"
+ ]
+ },
+
+ "files": [
+ {
+ # Match RADIUS detail files, but not anything that has
+ # been gzipped.
+ #
+ "paths": [ "/var/log/radius/radacct/*/detail-????????" ],
+
+ # Add a type:"radiusdetail" field to the data so that
+ # logstash can tell what type of data this is (in case
+ # log-courier is being used for other data as well).
+ #
+ "fields": {
+ "type": "radiusdetail"
+ },
+
+ # Stop watching a file if nothing has been written in 12h.
+ #
+ "dead time": "12h",
+
+ # Process multilines. If this is being used then the
+ # "multiline" section should be commented out from the
+ # logstash configuration. Logstash can then also be run
+ # with multiple workers (using -w).
+ #
+ "codec": {
+ "name": "multiline",
+ "pattern": "^[A-Z\t]",
+ "negate": false,
+ "what": "next"
+ }
+ }
+ ]
+}
+
--- /dev/null
+# logstash configuration to process RADIUS detail files
+#
+# Matthew Newton
+# January 2016
+#
+# RADIUS "detail" files are textual representations of the RADIUS
+# packets, and are written to disk by e.g. FreeRADIUS. They look
+# something like the following, with the timestamp on the first
+# line then all attributes/values tab-indented.
+#
+# Tue Mar 10 15:32:24 2015
+# Packet-Type = Access-Request
+# User-Name = "test@example.com"
+# Calling-Station-Id = "01-02-03-04-05-06"
+# Called-Station-Id = "aa-bb-cc-dd-ee-ff:myssid"
+# NAS-Port = 10
+# NAS-IP-Address = 10.9.0.4
+# NAS-Identifier = "Wireless-Controller-1"
+# Service-Type = Framed-User
+# NAS-Port-Type = Wireless-802.11
+#
+
+
+
+# Example input - read data from a file. This can be useful for
+# testing, but usually not so much for live service. For example,
+# to read in a detail file with this input you could use:
+#
+# /opt/logstash/bin/logstash -v -f logstash-radius.conf < detailfile
+
+input {
+ stdin {
+ type => radiusdetail
+ }
+}
+
+# Moving into production will likely need something more reliable.
+# There are many input methods, an example here using log-courier
+# (which supports client-site multiline processing and does not
+# lose log events if logstash is restarted).
+
+# input {
+# courier {
+# port => 5140
+# transport => "tcp"
+# }
+# }
+
+
+
+# Filter stage. Here we take the raw logs and process them into
+# something structured ready to index. Each attribute is stored as
+# a separate field in the output document.
+
+filter {
+
+ if [type] == "radiusdetail" {
+
+ # If you are using a log feeder that can join
+ # multiple lines together then that is preferrable
+ # to using multiline here, because this can not be
+ # used with threaded logstash (i.e. -w<n> at
+ # startup).
+
+ # In that case you should comment out the following
+ # section. For example, see the log-courier
+ # configuration configuration in this directory.
+
+ multiline {
+ pattern => "^[A-Z\t]"
+ negate => false
+ what => "next"
+ }
+
+ # Pull off the timestamp at the start of the
+ # detail record. Note there may be additional data
+ # after it that has been added by the local admin,
+ # so stop at a newline OR a tab.
+
+ grok {
+ match => [ "message", "^(?<timestamp>[^\n\t]+)[\n\t]" ]
+ }
+
+ # Create the @timestamp field.
+
+ date {
+ match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy",
+ "EEE MMM d HH:mm:ss yyyy" ]
+ }
+
+ # Split the attributes and values into fields.
+ # This is the bulk of processing that adds all of
+ # the RADIUS attributes as elasticsearch fields.
+
+ kv {
+ field_split => "\n"
+ source => "message"
+ trim => "\" "
+ trimkey => "\t "
+ }
+
+ # Now we try and add some useful additional
+ # information. If certain fields can be broken
+ # down into components then do that here and add
+ # the data as sub-fields. For example,
+ # Called-Station-Id might be able to be broken
+ # down to Called-Station-Id.mac and Called-Station-Id.ssid
+ # on some wireless systems, or to .ip and .port
+ # with a VPN.
+
+ # Multiple calls to grok otherwise it can stop
+ # processing once it has matched one field, but
+ # e.g. you want to pull both IP and port out of
+ # the same field in two different regex's.
+
+ # Pull out some IP addresses as field.ip:
+
+ grok {
+ break_on_match => false
+ tag_on_failure => []
+ match => [
+ "Framed-IP-Address", "^(?<Framed-IP-Address.ip>\d+\.\d+\.\d+\.\d+$)",
+ "NAS-IP-Address", "^(?<NAS-IP-Address.ip>\d+\.\d+\.\d+\.\d+$)",
+ "Calling-Station-Id", "^(?<Calling-Station-Id.ip>\d+\.\d+\.\d+\.\d+)",
+ "Called-Station-Id", "^(?<Called-Station-Id.ip>\d+\.\d+\.\d+\.\d+)"
+ ]
+ }
+
+ # Split User-Name, Operator-Name, and pull out
+ # some IP ports if they are there:
+
+ grok {
+ break_on_match => false
+ tag_on_failure => []
+ match => [
+ "User-Name", "^(?<User-Name.username>[^@]+)?(?:@(?<User-Name.realm>[^@]+))$",
+ "Operator-Name", "^(?<Operator-Name.id>.)(?<Operator-Name.value>.+)$",
+
+ "Calling-Station-Id", "\[(?<Calling-Station-Id.port>\d+)\]$",
+ "Called-Station-Id", "\[(?<Called-Station-Id.port>\d+)\]$"
+ ]
+ }
+
+ # Extract MAC addresses (and SSIDs if there).
+ # MAC address matching here is lazy, but should be
+ # good enough.
+
+ grok {
+ break_on_match => false
+ tag_on_failure => []
+ match => [
+ "Calling-Station-Id", "^(?<Calling-Station-Id.mac>[a-fA-F0-9:-]{17})$",
+ "Calling-Station-Id", "^(?<Calling-Station-Id.mac>[a-fA-F0-9\.]{14})$",
+ "Calling-Station-Id", "^(?<Calling-Station-Id.mac>[a-fA-F0-9]{12})$",
+
+ "Called-Station-Id", "^(?<Called-Station-Id.mac>[a-fA-F0-9:-]{17})(?::(?<Called-Station-Id.ssid>.*))?$",
+ "Called-Station-Id", "^(?<Called-Station-Id.mac>[a-fA-F0-9\.]{14})(?::(?<Called-Station-Id.ssid>.*))?$",
+ "Called-Station-Id", "^(?<Called-Station-Id.mac>[a-fA-F0-9]{12})(?::(?<Called-Station-Id.ssid>.*))?$"
+ ]
+ }
+
+ # With the optional sanitize_mac plugin, it's
+ # possible to make sure all MAC addresses look the
+ # same, which has obvious benefits.
+ #
+ # https://github.com/mcnewton/elk/blob/master/logstash-filters/sanitize_mac.rb
+
+ # sanitize_mac {
+ # match => {
+ # "Called-Station-Id.mac" => "Called-Station-Id.mac"
+ # "Calling-Station-Id.mac" => "Calling-Station-Id.mac"
+ # }
+ # separator => ":"
+ # fixcase => "lower"
+ # }
+
+
+ # Gigawords presents an issue because the 64-bit
+ # value is split across two attributes. Combine
+ # them both back into a single attribute so that
+ # the full value is available to use.
+
+ if ([Acct-Input-Octets]) {
+ ruby {
+ code => "event['Acct-Input-Octets.long'] =
+ event['Acct-Input-Octets'].to_i + ( event['Acct-Input-Gigawords'] ? (event['Acct-Input-Gigawords'].to_i * (2**32)) : 0)"
+ }
+ }
+
+ if ([Acct-Output-Octets]) {
+ ruby {
+ code => "event['Acct-Output-Octets.long'] =
+ event['Acct-Output-Octets'].to_i + ( event['Acct-Output-Gigawords'] ? (event['Acct-Output-Gigawords'].to_i * (2**32)) : 0)"
+ }
+ }
+
+ }
+}
+
+
+
+# Output data to the local elasticsearch cluster (called
+# "elasticsearch") using type "detail" in index "radius-DATE".
+
+output {
+ if [type] == "radiusdetail" {
+ elasticsearch {
+ host => localhost
+ protocol => http
+ cluster => elasticsearch
+ index_type => "detail"
+ index => "radius-%{+YYYY.MM.dd}"
+ flush_size => 1000
+ }
+ }
+}
+
# Matthew Newton
# April 2015
-# This should be run on an elasticsearch node. Alternatively, adjust
-# the curl URI below.
+# This should be run on an elasticsearch node. Alternatively,
+# adjust the curl URI below.
+
+# The template will be called "radius", and will apply to all
+# indices prefixed with "radius-" that contain data type "detail".
+# As not all RADIUS attributes are known to begin with it has the
+# following starting point that can be modified to suit the local
+# configuration:
+#
+# Acct-Input- or Acct-Output- attributes are numbers;
+# Acct-Session-Time is a number;
+# Everything else is a string.
+
+# Additionally, the supplied logstash config will try and extract
+# MAC addresses, IP addresses and ports from the data. These are
+# stored as sub-fields under the respective attribute. For
+# example, an attribute
+#
+# Called-Station-Id := "10.0.4.6[4500]"
+#
+# will be broken down into the following fields in elasticsearch:
+#
+# Called-Station-Id = "10.0.4.6[4500]"
+# Called-Station-Id.ip = "10.0.4.6"
+# Called-Station-Id.port = "4500"
+#
+# This mapping ensures that these have an appropriate data type.
+
curl -XPUT '127.0.0.1:9200/_template/radius' -d '
{
"order":0,
"mappings":{
"detail":{
- "dynamic_templates":[
- { "keep_message":{
- "match":"message",
- "mapping":{
- "type":"string",
- "index":"analyzed"
+
+ "properties": {
+ "@timestamp": { "format": "dateOptionalTime", "type": "date" },
+ "@version": { "type" : "string" },
+ "message": { "type" : "string" },
+ "Acct-Session-Time": { "type" : "long", "doc_values": true },
+ "offset": { "type" : "long", "doc_values": true }
+ },
+
+ "dynamic_templates": [
+
+ { "acct_io_numbers": {
+ "match_pattern": "regex",
+ "match": "^Acct-(Input|Output)-.*$",
+ "mapping": {
+ "type": "long",
+ "doc_values": true
}
}
},
- { "no_analyze_strings":{
- "match":"*",
- "match_mapping_type":"string",
- "mapping":{
- "type":"string",
- "index":"not_analyzed"
+
+ { "ipv4_address": {
+ "path_match": "*.ip",
+ "mapping": {
+ "type": "ip",
+ "doc_values": true
+ }
+ }
+ },
+
+ { "network_port": {
+ "path_match": "*.port",
+ "mapping": {
+ "type": "integer",
+ "doc_values": true
+ }
+ }
+ },
+
+ { "no_analyze_strings": {
+ "match": "*",
+ "mapping": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
}
}
}
+
]
}
}
}'
-
+++ /dev/null
-# logstash configuration to process RADIUS detail files
-#
-# Matthew Newton
-# February 2014
-#
-# RADIUS "detail" files are textual representations of the RADIUS
-# packets, and are written to disk by e.g. FreeRADIUS. They look
-# something like the following, with the timestamp on the first
-# line then all attributes/values tab-indented.
-#
-# Tue Mar 10 15:32:24 2015
-# Packet-Type = Access-Request
-# User-Name = "test@example.com"
-# Calling-Station-Id = "01-02-03-04-05-06"
-# Called-Station-Id = "aa-bb-cc-dd-ee-ff:myssid"
-# NAS-Port = 10
-# NAS-IP-Address = 10.9.0.4
-# NAS-Identifier = "Wireless-Controller-1"
-# Service-Type = Framed-User
-# NAS-Port-Type = Wireless-802.11
-#
-# This filter processes the detail file such that each attribute
-# is stored as a separate field in the output document.
-
-
-input {
- stdin {
- type => radiusdetail
- }
-}
-
-
-filter {
-
- if [type] == "radiusdetail" {
-
- # join all lines of a record together
- multiline {
- pattern => "^[^\t]"
- negate => true
- what => "previous"
- }
-
- # pull off the timestamp
- grok {
- match => [ "message", "^(?<timestamp>[^\n\t]+)[\n\t]" ]
- }
-
- # create the timestamp field
- date {
- match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy",
- "EEE MMM d HH:mm:ss yyyy" ]
- }
-
- # split the attributes and values into fields
- kv {
- field_split => "\n"
- source => "message"
- trim => "\" "
- trimkey => "\t "
- }
- }
-}
-
-output {
- if [type] == "radiusdetail" {
- elasticsearch {
- host => localhost
- protocol => http
- cluster => elasticsearch
- index_type => "detail"
- index => "radius-%{+YYYY.MM.dd}"
- flush_size => 1000
- }
- }
-}
-
projects / freeradius.git / commitdiff