# logstash configuration to process RADIUS detail files
#
# Matthew Newton
-# January 2016
+# January 2017
+#
+# This config has been tested with logstash version 5.1.2.
#
# RADIUS "detail" files are textual representations of the RADIUS
# packets, and are written to disk by e.g. FreeRADIUS. They look
-# Example input - read data from a file. This can be useful for
-# testing, but usually not so much for live service. For example,
-# to read in a detail file with this input you could use:
+# Example input - read data from a file. For example, to read in a
+# detail file with this input you could use:
+#
+# # /usr/share/logstash/bin/logstash --path.settings=/etc/logstash -f logstash-radius.conf
#
-# /opt/logstash/bin/logstash -v -f logstash-radius.conf < detailfile
input {
- stdin {
+ file {
+ path => "/path/to/radius/detail/file"
+
+ # Note when testing that logstash will remember where
+ # it got to and continue from there.
+ start_position => "beginning"
+
+ # Set the type, for below.
type => radiusdetail
+
+ # It is preferable to use a log feeder that can join
+ # multiple lines together, rather than using multiline
+ # here. For an example, see the log-courier
+ # configuration in this directory.
+
+ # If you didn't read the above, go back and read it again.
+
+ # If that is not possible you may be able to use the
+ # following section. Note that if you are using the
+ # "stdin" input, the file is chunked into 16k blobs,
+ # so every 16k a detail record is likely to be chopped
+ # in half. If you are using the "file" input (as in this
+ # example), the blank links between records are not
+ # passed through so the regex here has to be aware of
+ # that. Basically, do multiline as early as possible
+ # in your log feeder client not here and you'll avoid
+ # most issues that are likely to come up.
+
+ codec => multiline {
+ pattern => "^\t"
+ negate => false
+ what => "previous"
+ }
+
+ # If you really want to use the "stdin" input, this
+ # will work better, but be aware of the comments
+ # above.
+
+ #codec => multiline {
+ # pattern => "^[A-Z\t]"
+ # negate => false
+ # what => "next"
+ #}
}
}
# Moving into production will likely need something more reliable.
# There are many input methods, an example here using log-courier
# (which supports client-site multiline processing and does not
-# lose log events if logstash is restarted).
+# lose log events if logstash is restarted). You could also
+# investigate e.g. filebeat from Elastic.
# input {
# courier {
# port => 5140
# transport => "tcp"
+#
+# # Don't set the type here, as it's set in the
+# # log-courier config instead.
+# #type => radiusdetail
# }
# }
if [type] == "radiusdetail" {
- # If you are using a log feeder that can join
- # multiple lines together then that is preferrable
- # to using multiline here, because this can not be
- # used with threaded logstash (i.e. -w<n> at
- # startup).
-
- # In that case you should comment out the following
- # section. For example, see the log-courier
- # configuration configuration in this directory.
-
- multiline {
- pattern => "^[A-Z\t]"
- negate => false
- what => "next"
- }
-
# Pull off the timestamp at the start of the
# detail record. Note there may be additional data
# after it that has been added by the local admin,
# This is the bulk of processing that adds all of
# the RADIUS attributes as elasticsearch fields.
+ # Note issue https://github.com/logstash-plugins/logstash-filter-kv/issues/10
+ # currently means that all spaces will be stripped
+ # from all fields. If this is a problem, adjust the
+ # trim setting.
+
kv {
field_split => "\n"
source => "message"
# possible to make sure all MAC addresses look the
# same, which has obvious benefits.
#
- # https://github.com/mcnewton/elk/blob/master/logstash-filters/sanitize_mac.rb
+ # https://github.com/mcnewton/logstash-filter-sanitize_mac
# sanitize_mac {
# match => {
# "Called-Station-Id_mac" => "Called-Station-Id_mac"
# "Calling-Station-Id_mac" => "Calling-Station-Id_mac"
# }
- # separator => ":"
+ # separator => "-"
# fixcase => "lower"
# }
if ([Acct-Input-Octets]) {
ruby {
- code => "event['Acct-Input-Octets_long'] =
- event['Acct-Input-Octets'].to_i + ( event['Acct-Input-Gigawords'] ? (event['Acct-Input-Gigawords'].to_i * (2**32)) : 0)"
+ code => "event.set('Acct-Input-Octets_long', event.get('Acct-Input-Octets').to_i +
+ (event.get('Acct-Input-Gigawords') ? (event.get('Acct-Input-Gigawords').to_i * (2**32)) : 0))"
}
}
if ([Acct-Output-Octets]) {
ruby {
- code => "event['Acct-Output-Octets_long'] =
- event['Acct-Output-Octets'].to_i + ( event['Acct-Output-Gigawords'] ? (event['Acct-Output-Gigawords'].to_i * (2**32)) : 0)"
+ code => "event.set('Acct-Output-Octets_long', event.get('Acct-Output-Octets').to_i +
+ (event.get('Acct-Output-Gigawords') ? (event.get('Acct-Output-Gigawords').to_i * (2**32)) : 0))"
}
}
-# Output data to the local elasticsearch cluster (called
-# "elasticsearch") using type "detail" in index "radius-DATE".
+# Output data to the local elasticsearch cluster
+# using type "detail" in index "radius-DATE".
output {
if [type] == "radiusdetail" {
elasticsearch {
- host => localhost
- protocol => http
- cluster => elasticsearch
- index_type => "detail"
+ document_type => "detail"
index => "radius-%{+YYYY.MM.dd}"
flush_size => 1000
}
projects / freeradius.git / blobdiff