nf2csv

Revision 2137, 18.6 kB (checked in by mbr, 1 year ago)
version 2.1.1
Property svn:executable set to ``* Property svn:keywords set to `Revision`

Line
1	#!/usr/bin/perl -w
2	#
3	###########################################################################
4	#
5	# File: nf2csv
6	#
7	# Purpose: nf2csv parses iptables log messages and prints them on stdout
8	# in comma separated value format. This is most useful for
9	# generating data for the AfterGlow project
10	# (http://afterglow.sourceforge.net) to visualize iptables log
11	# data.
12	#
13	# nf2csv is part of the psad project (http://www.cipherdyne.org/psad)
14	#
15	# Author: Michael Rash (mbr@cipherdyne.org)
16	#
17	# Credits: (see the CREDITS file)
18	#
19	# Copyright (C) 2006 Michael Rash (mbr@cipherdyne.org)
20	#
21	# License (GNU Public License):
22	#
23	# This program is distributed in the hope that it will be useful,
24	# but WITHOUT ANY WARRANTY; without even the implied warranty of
25	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26	# GNU General Public License for more details.
27	#
28	# You should have received a copy of the GNU General Public License
29	# along with this program; if not, write to the Free Software
30	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
31	# USA
32	#
33	###########################################################################
34	#
35	# $Id: nf2csv 1800 2006-12-10 04:51:40Z mbr $
36	#
37
38	use Getopt::Long 'GetOptions';
39	use strict;
40
41	my $version = '2.1.1';
42	my $revision_svn = '$Revision$';
43	my $rev_num = '1';
44	($rev_num) = $revision_svn =~ m\|\$Rev.*:\s+(\S+)\|;
45
46	### regex to match an ip address
47	my $ip_re = qr\|(?:[0-2]?\d{1,2}\.){3}[0-2]?\d{1,2}\|;
48
49	### main packet data structure
50	my %pkt_NF_init = (
51
52	### data link layer
53	'src_mac' => '',
54	'dst_mac' => '',
55	'intf' => '', ### FIXME in and out interfaces?
56
57	### network layer
58	'src' => '',
59	'dst' => '',
60	'proto' => '',
61	'ip_id' => -1,
62	'ttl' => -1,
63	'tos' => '',
64	'ip_len' => -1,
65	'itype' => -1,
66	'icode' => -1,
67	'ip_opts' => '',
68	'icmp_seq' => -1,
69	'icmp_id' => -1,
70	'frag_bit' => 0,
71
72	### transport layer
73	'sp' => -1,
74	'dp' => -1,
75	'win' => -1,
76	'flags' => -1,
77	'tcp_seq' => -1,
78	'tcp_ack' => -1,
79	'tcp_opts' => '',
80	'udp_len' => -1,
81
82	### extra fields for internals (fwsnort sid matching,
83	### Netfilter logging prefixes and chains, etc.)
84	'fwsnort_sid' => 0,
85	'chain' => '',
86	'log_prefix' => '',
87	'syslog_host' => '',
88	'timestamp' => ''
89	);
90
91	my $csv_fields = '';
92	my $csv_print_uniq = 0;
93	my $csv_line_limit = 0;
94	my $csv_start_line = 0;
95	my $csv_end_line = 0;
96	my $csv_regex = 0;
97	my $csv_neg_regex = 0;
98	my $nf_log_file = '';
99	my $print_ver = 0;
100	my $debug = 0;
101	my $help = 0;
102
103	### make Getopts case sensitive
104	Getopt::Long::Configure('no_ignore_case');
105
106	&usage(1) unless (GetOptions(
107	'Messages-file=s' => \$nf_log_file, # Specify the path to file containing
108	'fields=s' => \$csv_fields, # Specify list of CSV fields.
109	'uniq-lines' => \$csv_print_uniq, # Only print unique lines in CSV
110	# output.
111	'max-lines=i' => \$csv_line_limit, # Limit the number of CSV output
112	# lines.
113	'start-line=i' => \$csv_start_line, # Starting line in CSV file.
114	'end-line=i' => \$csv_end_line, # Ending line in CSV file.
115	'regex=s' => \$csv_regex, # Require additional regex match.
116	'neg-regex=s' => \$csv_neg_regex, # Require additional negative regex
117	'debug' => \$debug, # Run in debug mode.
118	'Version' => \$print_ver, # Print the nf2csv version and exit.
119	'help' => \$help, # Display help.
120	));
121	&usage(0) if $help;
122
123	### Print the version number and exit if -V given on the command line.
124	if ($print_ver) {
125	print "[+] psad v$version (file revision: $rev_num)\n",
126	" by Michael Rash <mbr\@cipherdyne.org>\n";
127	exit 0;
128	}
129
130	### see what we should be searching for
131	my ($tokens_ar, $match_criteria_ar) = &csv_tokens();
132
133	$csv_regex = qr/$csv_regex/ if $csv_regex;
134	$csv_neg_regex = qr/$csv_neg_regex/ if $csv_neg_regex;
135
136	my %csv_uniq_lines = ();
137
138	if ($csv_start_line) {
139	die *"[] Cannot have start line > end line."**
140	if $csv_start_line > $csv_end_line;
141	}
142	my $ctr = 0;
143	my $line_ctr = 0;
144
145	my $fh = *STDIN;
146	if ($nf_log_file) {
147	open MSGS, "< $nf_log_file" or die *"[] Could not open "**,
148	"$nf_log_file: $!";
149	$fh = *MSGS;
150	}
151
152	MSG: while (<$fh>) {
153	my $pkt_str = $_;
154	$line_ctr++;
155	if ($csv_start_line) {
156	next MSG unless $line_ctr >= $csv_start_line;
157	}
158	if ($csv_end_line) {
159	last MSG if $line_ctr == $csv_end_line;
160	}
161	next MSG unless $pkt_str =~ /IN.*OUT/;
162
163	### init pkt hash
164	my %pkt = %pkt_NF_init;
165
166	my $rv = &parse_NF_pkt_str(\%pkt, $pkt_str);
167	next MSG unless $rv;
168
169	if ($csv_regex) {
170	next MSG unless $pkt{'raw'} =~ m\|$csv_regex\|;
171	}
172	if ($csv_neg_regex) {
173	next MSG unless $pkt{'raw'} !~ m\|$csv_neg_regex\|;
174	}
175	$pkt{'log_prefix'} =~ s/\W//g;
176	$pkt{'log_prefix'} =~ s/\s//g;
177	my @matched_fields = ();
178	for (my $i=0; $i <= $#$tokens_ar; $i++) {
179	my $tok = $tokens_ar->[$i];
180	if ($match_criteria_ar) {
181	my $match_hr = $match_criteria_ar->[$i];
182	if (defined $match_hr->{'num'}) {
183	unless ($pkt{$tok} =~ m\|^\d+$\|
184	and $pkt{$tok} == $match_hr->{'num'}) {
185	next MSG;
186	}
187	} elsif (defined $match_hr->{'gt'}) {
188	unless ($pkt{$tok} =~ m\|^\d+$\|
189	and $pkt{$tok} > $match_hr->{'gt'}) {
190	next MSG;
191	}
192	} elsif (defined $match_hr->{'lt'}) {
193	unless ($pkt{$tok} =~ m\|^\d+$\|
194	and $pkt{$tok} < $match_hr->{'lt'}) {
195	next MSG;
196	}
197	} elsif (defined $match_hr->{'str'}) {
198	unless ($pkt{$tok} eq $match_hr->{'str'}) {
199	next MSG;
200	}
201	} elsif (defined $match_hr->{'re'}) {
202	unless ($pkt{$tok} =~ m\|$match_hr->{'re'}m\|) {
203	next MSG;
204	}
205	} elsif (defined $match_hr->{'net'}) {
206	if ($pkt{$tok} =~ m\|$ip_re\|) {
207	unless (ipv4_in_network($match_hr->{'net'}, $pkt{$tok})) {
208	next MSG;
209	}
210	} else {
211	next MSG;
212	}
213	} elsif (defined $match_hr->{'ip'}) {
214	unless ($pkt{$tok} eq $match_hr->{'ip'}) {
215	next MSG;
216	}
217	}
218	push @matched_fields, $pkt{$tok};
219	} else {
220	push @matched_fields, $pkt{$tok};
221	}
222	}
223	next MSG unless @matched_fields;
224	my $str = '';
225	if ($csv_fields) {
226	$str .= "$_, " for @matched_fields;
227	} else {
228	$str .= "$_ " for @matched_fields;
229	}
230	$str =~ s/,\s*$//;
231	$str =~ s/\s*$//;
232	$ctr++;
233	if ($csv_print_uniq) {
234	$csv_uniq_lines{$str} = '';
235	} else {
236	print $str, "\n";
237	}
238	if ($csv_line_limit > 0) {
239	last if $ctr >= $csv_line_limit;
240	}
241	}
242	close $fh;
243	if ($csv_print_uniq) {
244	print "$_\n" for keys %csv_uniq_lines;
245	}
246
247	exit 0;
248	#============================= end main ===============================
249
250	sub csv_tokens() {
251
252	my @tokens = ();
253	my @match_criteria = ();
254
255	if ($csv_fields) {
256	my @tok_tmp = split /\s+/, $csv_fields;
257	for my $tok_str (@tok_tmp) {
258	my $token = $tok_str;
259	my $search = '';
260	if ($tok_str =~ m\|(\w+):(\S+)\|) {
261	$token = $1;
262	$search = $2;
263	}
264	$token = 'src' if $token eq 'SRC';
265	$token = 'dst' if $token eq 'DST';
266	$token = 'sp' if $token eq 'SPT';
267	$token = 'dp' if $token eq 'DPT';
268	$token = 'tos' if $token eq 'TOS';
269	$token = 'win' if $token eq 'WIN';
270	$token = 'itype' if $token eq 'TYPE';
271	$token = 'icode' if $token eq 'CODE';
272	$token = 'ttl' if $token eq 'TTL';
273	$token = 'ip_id' if $token eq 'ID';
274	$token = 'icmp_seq' if $token eq 'SEQ';
275	$token = 'proto' if $token eq 'PROTO';
276	$token = 'ip_len' if $token eq 'LEN';
277	$token = 'intf' if $token eq 'IN' or $token eq 'OUT';
278	unless (defined $pkt_NF_init{$token}) {
279	print *"[] $token is not a valid packet field; valid "**,
280	"fields are:\n";
281	for my $key (sort keys %pkt_NF_init) {
282	print " $key\n";
283	}
284	die;
285	}
286	push @tokens, $token;
287
288	if ($search) {
289	my %search_hsh = ();
290	if ($search =~ m\|^\d+$\|) {
291	$search_hsh{'num'} = $search;
292	} elsif ($search =~ m\|^>(\d+)$\|) {
293	$search_hsh{'gt'} = $1;
294	die *"[] $token value must be >= 0"**
295	unless $1 >= 0;
296	} elsif ($search =~ m\|^<(\d+)$\|) {
297	$search_hsh{'lt'} = $1;
298	die *"[] $token value must be >= 0"**
299	unless $1 >= 0;
300	} elsif ($search =~ m\|^/(.*?)/$\|) {
301	$search_hsh{'re'} = qr\|$1\|;
302	} elsif ($search =~ m\|^\"(.*?)\"$\|) {
303	$search_hsh{'str'} = $1;
304	} elsif ($search =~ m\|^$ip_re/$ip_re$\|) {
305	$search_hsh{'net'} = $search;
306	} elsif ($search =~ m\|^$ip_re/\d+$\|) {
307	$search_hsh{'net'} = $search;
308	} elsif ($search =~ m\|^$ip_re$\|) {
309	$search_hsh{'ip'} = $search;
310	} else {
311	die *"[] Unrecognized value for $token"**;
312	}
313	push @match_criteria, \%search_hsh;
314	} else {
315	push @match_criteria, {};
316	}
317	}
318	} else {
319	@tokens = qw(
320	timestamp
321	src
322	dst
323	sp
324	dp
325	proto
326	flags
327	ip_len
328	intf
329	chain
330	log_prefix
331	);
332	}
333	return \@tokens, \@match_criteria;
334	}
335
336	sub parse_NF_pkt_str() {
337	my ($pkt_hr, $pkt_str) = @_;
338
339	print STDERR "\n", $pkt_str if $debug;
340
341	$pkt_hr->{'raw'} = $pkt_str;
342
343	if ($pkt_str =~ /.kernel:\s+(.?)\s*IN=/) {
344	$pkt_hr->{'log_prefix'} = $1;
345	}
346
347	### get the in/out interface and iptables chain (the code below
348	### allows the iptables log message to contain the PHYSDEV stuff):
349	### Feb 25 12:13:27 bridge kernel: INBOUND TCP: IN=br0 PHYSIN=eth0 OUT=br0
350	### PHYSOUT=eth1 SRC=63.147.183.21 DST=11.11.79.100 LEN=48 TOS=0x00
351	### PREC=0x00 TTL=113 ID=19664 DF PROTO=TCP SPT=4918 DPT=135 WINDOW=64240
352	### RES=0x00 SYN URGP=0
353	if ($pkt_str =~ /\sIN=(\S+).*\sOUT=\s/) {
354	$pkt_hr->{'intf'} = $1;
355	$pkt_hr->{'chain'} = 'INPUT';
356	} elsif ($pkt_str =~ /\sIN=(\S+).*\sOUT=\S/) {
357	$pkt_hr->{'intf'} = $1;
358	$pkt_hr->{'chain'} = 'FORWARD';
359	} elsif ($pkt_str =~ /\sIN=\s+.*\sOUT=(\S+)/) {
360	$pkt_hr->{'intf'} = $1;
361	$pkt_hr->{'chain'} = 'OUTPUT';
362	}
363
364	if ($pkt_str =~ /\sMAC=(\S+)/) {
365	my $mac_str = $1;
366	if ($mac_str =~ /^((?:\w{2}\:){6})((?:\w{2}\:){6})/) {
367	$pkt_hr->{'dst_mac'} = $1;
368	$pkt_hr->{'src_mac'} = $2;
369	}
370	}
371	if ($pkt_hr->{'src_mac'}) {
372	$pkt_hr->{'src_mac'} =~ s/:$//;
373	print STDERR "[+] src mac addr: $pkt_hr->{'src_mac'}\n" if $debug;
374	}
375	if ($pkt_hr->{'dst_mac'}) {
376	$pkt_hr->{'dst_mac'} =~ s/:$//;
377	print STDERR "[+] dst mac addr: $pkt_hr->{'dst_mac'}\n" if $debug;
378	}
379
380	unless ($pkt_hr->{'intf'} and $pkt_hr->{'chain'}) {
381	print STDERR "[-] err packet: could not determine ",
382	"interface and chain.\n" if $debug;
383	return 0;
384	}
385
386	### get the syslog logging host for this packet
387	if ($pkt_str =~ /^\s*((?:\S+\s+){2}\S+)\s+(\S+)\s+kernel:/) {
388	$pkt_hr->{'timestamp'} = $1;
389	$pkt_hr->{'syslog_host'} = $2;
390	} else {
391	$pkt_hr->{'timestamp'} = localtime();
392	$pkt_hr->{'syslog_host'} = 'unknown';
393	}
394
395	### try to extract a snort sid (generated by fwsnort) from
396	### the packet
397	if ($pkt_str =~ /SID(\d+)/) {
398	$pkt_hr->{'fwsnort_sid'} = $1;
399	}
400
401	### get IP options if --log-ip-options is used
402	### (they appear before the PROTO= field).
403	if ($pkt_str =~ /OPT\s+$(\S+)$\s+PROTO=/) {
404	$pkt_hr->{'ip_opts'} = $1;
405	}
406
407	### May 18 22:21:26 orthanc kernel: DROP IN=eth2 OUT=
408	### MAC=00:60:1d:23:d0:01:00:60:1d:23:d3:0e:08:00 SRC=192.168.20.25
409	### DST=192.168.20.1 LEN=60 TOS=0x10 PREC=0x00 TTL=64 ID=47300 DF
410	### PROTO=TCP SPT=34111 DPT=6345 WINDOW=5840 RES=0x00 SYN URGP=0
411	if ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+)\s+TOS=(\S+)
412	\s.\s+TTL=(\d+)\s+ID=(\d+)\s.\s+PROTO=TCP\s+
413	SPT=(\d+)\s+DPT=(\d+)\s.\sWINDOW=(\d+)\s+
414	(.*)\s+URGP=/x) {
415
416	($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
417	$pkt_hr->{'tos'}, $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'},
418	$pkt_hr->{'sp'}, $pkt_hr->{'dp'}, $pkt_hr->{'win'},
419	$pkt_hr->{'flags'})
420	= ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10);
421
422	### the reserve bits are not reported by ulogd, but normal
423	### iptables syslog messages contain them.
424	$pkt_hr->{'flags'} =~ s/\sRES=\S+\s//;
425
426	$pkt_hr->{'proto'} = 'tcp';
427
428	### default to NULL
429	$pkt_hr->{'flags'} = 'NULL' unless $pkt_hr->{'flags'};
430
431	unless ($pkt_hr->{'flags'} !~ /WIN/ &&
432	$pkt_hr->{'flags'} =~ /ACK/ \|\|
433	$pkt_hr->{'flags'} =~ /SYN/ \|\|
434	$pkt_hr->{'flags'} =~ /RST/ \|\|
435	$pkt_hr->{'flags'} =~ /URG/ \|\|
436	$pkt_hr->{'flags'} =~ /PSH/ \|\|
437	$pkt_hr->{'flags'} =~ /FIN/ \|\|
438	$pkt_hr->{'flags'} eq 'NULL') {
439
440	print STDERR "[-] err packet: bad tcp flags.\n" if $debug;
441	return 0;
442	}
443	$pkt_hr->{'frag_bit'} = 1 if $pkt_str =~ /\sDF\s+PROTO/;
444
445	### don't pickup IP options if --log-ip-options is used
446	### (they appear before the PROTO= field).
447	if ($pkt_str =~ /URGP=\S+\s+OPT\s+$(\S+)$/) {
448	$pkt_hr->{'tcp_opts'} = $1;
449	}
450
451	### make sure we have a "reasonable" packet (note that nmap
452	### can scan port 0 and iptables can report this fact)
453	unless ($pkt_hr->{'ip_len'} >= 0 and $pkt_hr->{'tos'}
454	and $pkt_hr->{'ttl'} >= 0 and $pkt_hr->{'ip_id'} >= 0
455	and $pkt_hr->{'proto'} and $pkt_hr->{'sp'} >= 0
456	and $pkt_hr->{'dp'} >= 0 and $pkt_hr->{'win'} >= 0
457	and $pkt_hr->{'flags'}) {
458	return 0;
459	}
460
461	### May 18 22:21:26 orthanc kernel: DROP IN=eth2 OUT=
462	### MAC=00:60:1d:23:d0:01:00:60:1d:23:d3:0e:08:00
463	### SRC=192.168.20.25 DST=192.168.20.1 LEN=28 TOS=0x00 PREC=0x00
464	### TTL=40 ID=47523 PROTO=UDP SPT=57339 DPT=305 LEN=8
465
466	} elsif ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+)\s+TOS=(\S+)
467	\s.TTL=(\d+)\s+ID=(\d+)\s.*\s+PROTO=UDP\s+
468	SPT=(\d+)\s+DPT=(\d+)\s+LEN=(\d+)/x) {
469
470	($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
471	$pkt_hr->{'tos'}, $pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'},
472	$pkt_hr->{'sp'}, $pkt_hr->{'dp'}, $pkt_hr->{'udp_len'})
473	= ($1,$2,$3,$4,$5,$6,$7,$8,$9);
474
475	$pkt_hr->{'proto'} = 'udp';
476
477	### make sure we have a "reasonable" packet (note that nmap
478	### can scan port 0 and iptables can report this fact)
479	unless ($pkt_hr->{'ip_len'} >= 0
480	and $pkt_hr->{'tos'} and $pkt_hr->{'ttl'} >= 0
481	and $pkt_hr->{'ip_id'} >= 0 and $pkt_hr->{'proto'}
482	and $pkt_hr->{'sp'} >= 0 and $pkt_hr->{'dp'} >= 0
483	and $pkt_hr->{'udp_len'} >= 0) {
484
485	return 0;
486	}
487
488	### Nov 27 15:45:51 orthanc kernel: DROP IN=eth1 OUT= MAC=00:a0:cc:e2:1f:f2:00:
489	### 20:78:10:70:e7:08:00 SRC=192.168.10.20 DST=192.168.10.1 LEN=84 TOS=0x00
490	### PREC=0x00 TTL=64 ID=0 DF PROTO=ICMP TYPE=8 CODE=0 ID=61055 SEQ=256
491
492	} elsif ($pkt_str =~ /SRC=($ip_re)\s+DST=($ip_re)\s+LEN=(\d+).*
493	TTL=(\d+)\s+ID=(\d+).*PROTO=ICMP\s+TYPE=(\d+)\s+
494	CODE=(\d+)\s+ID=(\d+)\s+SEQ=(\d+)/x) {
495
496	($pkt_hr->{'src'}, $pkt_hr->{'dst'}, $pkt_hr->{'ip_len'},
497	$pkt_hr->{'ttl'}, $pkt_hr->{'ip_id'}, $pkt_hr->{'itype'},
498	$pkt_hr->{'icode'}, $pkt_hr->{'icmp_id'}, $pkt_hr->{'icmp_seq'})
499	= ($1,$2,$3,$4,$5,$6,$7,$8,$9);
500
501	$pkt_hr->{'proto'} = 'icmp';
502	$pkt_hr->{'sp'} = $pkt_hr->{'dp'} = 0;
503
504	unless ($pkt_hr->{'ip_len'} >= 0 and $pkt_hr->{'ttl'} >= 0
505	and $pkt_hr->{'proto'} and $pkt_hr->{'itype'} >= 0
506	and $pkt_hr->{'icode'} >= 0 and $pkt_hr->{'ip_id'} >= 0
507	and $pkt_hr->{'icmp_seq'} >= 0) {
508
509	return 0;
510	}
511
512	} else {
513	### Sometimes the iptables log entry gets messed up due to
514	### buffering issues so we write it to the error log.
515	print STDERR "[-] err packet: no regex match.\n" if $debug;
516	return 0;
517	}
518	return 1;
519	}
520
521	sub usage() {
522	my $exitcode = shift;
523	print <<_HELP_;
524
525	nf2csv
526	[+] Version: $version (file revision: $rev_num)
527	[+] By Michael Rash (mbr\@cipherdyne.org, http://www.cipherdyne.org)
528
529	Usage: nf2csv [options]
530
531	Options:
532	-f, --fields <fields> - Restrict output to a list of
533	specfic fields.
534	-u, --uniq-lines - Only print unique lines
535	-m, --max-lines <num> - Specify the maximum number of
536	output lines to print.
537	-M, --messages-file <file> - Specify the path to the iptables
538	logfile (use in conjunction with
539	--Analyze-msgs).
540	-s, --start-line <line> - Starting line within iptables log
541	file.
542	-e, --end-line <line> - Ending line within iptables log
543	file.
544	-r, --regex <regex> - Require iptables log messages to
545	match an additional regex.
546	-n, --neg-regex <regex> - Require iptables log messages to
547	not match an additional regex.
548	-d, --debug - Run in debugging mode.
549	-V, --Version - Print the version and exit.
550	-h --help - Display usage on STDOUT and exit.
551
552	_HELP_
553	exit $exitcode;
554	}

Note: See TracBrowser for help on using the browser.

root/psad/tags/psad-2.1.1/nf2csv

Download in other formats: