Changes by Steve Colam <steve@colam.co.uk>:

- ...-asstatd.pl now accepts parameters (UDP listen port,
	  sampling rate etc.) on the command line
	
	  ****************************************************
	  * Mind the new command line syntax when upgrading! *
	  * Change your start script accordingly...          *
	  ****************************************************
	
	- hierarchical RRD structure for more efficient storage
	  (one directory per low byte of AS number)
	
	  *********************************************
	  * Use tools/migraterrdfiles.pl to move your *
	  * RRD files when upgrading!                 *
	  *********************************************

	- ...-asstatd.pl now re-reads known links file upon SIGHUP
	
Added contrib/generate-asinfo.py script to generate AS list from WHOIS
data (contributed by Thomas Mangin
<thomas.mangin@exa-networks.co.uk>).

Moved site-specific parameters of www frontend to config.inc.

New flag images from famfamfam.com.

Updated asinfo.txt.

README

@@ -1,14 +1,43 @@
-AS-Stats v1.2 (2008-12-06)
+AS-Stats v1.3 (2009-10-10)
-a simple tool to generate per-AS traffic graphs from NetFlow records
+a simple tool to generate per-AS traffic graphs from NetFlow/sFlow records
 by Manuel Kasper, Monzoon Networks AG <mkasper@monzoon.net>
 --------------------------------------------------------------------
 
 Changes
 -------
+v1.3	Changes by Steve Colam <steve@colam.co.uk>:
+			- ...-asstatd.pl now accepts parameters (UDP listen port,
+			  sampling rate etc.) on the command line
+			
+			  ****************************************************
+			  * Mind the new command line syntax when upgrading! *
+			  * Change your start script accordingly...          *
+			  ****************************************************
+			
+			- hierarchical RRD structure for more efficient storage
+			  (one directory per low byte of AS number)
+			
+			  *********************************************
+			  * Use tools/migraterrdfiles.pl to move your *
+			  * RRD files when upgrading!                 *
+			  *********************************************
+
+			- ...-asstatd.pl now re-reads known links file upon SIGHUP
+			
+		Added contrib/generate-asinfo.py script to generate AS list from WHOIS
+        data (contributed by Thomas Mangin
+     	<thomas.mangin@exa-networks.co.uk>).
+
+		Moved site-specific parameters of www frontend to config.inc.
+		
+		New flag images from famfamfam.com.
+		
+		Updated asinfo.txt.
+
 v1.2	Support for sFlow (through sflow-asstatd.pl); fix for link names
 		with upper-case characters
 		Allow inbound/outbound in graphs to be swapped (via option
-		in www/func.inc)
+		in www/config.inc)
 
 v1.1	Fix for a potential race condition surrounding $childrunning
 		(reported by Yann Gauteron; experienced on a Linux system)
@@ -20,11 +49,11 @@ How it works
 ------------
 
 A Perl script (netflow-asstatd.pl) collects NetFlow v8 AS aggregation records
-from one or more routers. It caches them for about a minute (to prevent
+or sFlow v5 samples from one or more routers. It caches them for about a
-excessive writes to RRD files), identifies the link that each record refers
+minute (to prevent excessive writes to RRD files), identifies the link that
-to (by means of the SNMP in/out interface index), maps it to a corresponding
+each record refers to (by means of the SNMP in/out interface index), maps it
-"known link" and RRD data source, and then runs RRDtool. To avoid losing
+to a corresponding "known link" and RRD data source, and then runs RRDtool. To
-new NetFlow records while the RRD files are updated, the update task is
+avoid losing new records while the RRD files are updated, the update task is
 run in a separate process.
 
 For each AS, a separate RRD file is created as needed. It contains two data
@@ -46,13 +75,17 @@ Prerequisites
 - RRDtool 1.2 (with Perl "RRDs" library)
 - web server with PHP 5
 - one or more routers than can generate NetFlow v8 AS aggregation records
+  or sFlow samples
 
 
 Installation
 ------------
+In the instructions below, "xx-asstatd.pl" refers to either netflow-asstatd.pl
+or sflow-asstatd.pl, depending on whether your routers generate NetFlow or
+sFlow data.
 
-- Copy the perl scripts netflow-asstatd.pl and rrd-extractstats.pl to the
+- Copy the perl scripts xx-asstatd.pl and rrd-extractstats.pl to the
-  machine that will collect NetFlow records
+  machine that will collect NetFlow/sFlow records
 
 - Create a "known links" file with the following information about each
   link that you want to appear in your AS stats:
@@ -65,23 +98,32 @@ Installation
   	- a human-readable description (will appear in the generated graphs)
   	- a color code for the graphs (HTML style, 6 hex digits)
   
-  See the example file provided (netflow-knownlinks) for the format.
+  See the example file provided (knownlinks) for the format.
 
 - Create a directory to hold per-AS RRD files. For each AS, about 128 KB of
   storage are required, and there could be (in theory) up to 64511 ASes.
+  AS-Stats automatically creates 256 subdirectories in this directory for
+  more efficient storage of RRD files (one directory per lower byte of
+  AS number, in hex).
 
-- Start netflow-asstatd.pl in the background (or, better yet, write a
+- Start xx-asstatd.pl in the background (or, better yet, write a
   startup script for your operating system to automatically start
-  netflow-asstatd.pl on boot):
+  xx-asstatd.pl on boot):
   
-  	nohup netflow-asstatd.pl /path/to/rrd/dir /path/to/knownlinks &
+  	nohup xx-asstatd.pl -r /path/to/rrd/dir -k /path/to/knownlinks &
 
   By default, netflow-asstatd.pl will listen on port 9000 (UDP) for NetFlow
-  datagrams. Edit $server_port in the script if you want to change that.
+  datagrams, and sflow-asstatd.pl will listen on port 6343 (UDP) for sFlow
+  datagrams. Use the -p option if you want to change that.
+  If you use sampled NetFlow or sFlow, set the sampling rate with the -s
+  option.
+  sflow-asstatd.pl also needs you to specify your own AS number with the -a
+  option for accurate classification of inbound and outbound traffic.
   It's a good idea to make sure only UDP datagrams from your trusted routers
-  will reach the machine running netflow-asstatd.pl (firewall etc.).
+  will reach the machine running xx-asstatd.pl (firewall etc.).
 
-- Have your router(s) send NetFlow v8 AS aggregation records to your machine.
+- NetFlow only:
+  Have your router(s) send NetFlow v8 AS aggregation records to your machine.
   This is typically done with commands like the following (Cisco IOS):
 
 		ip flow-cache timeout active 5
@@ -104,10 +146,16 @@ Installation
   increase in CPU usage (and NetFlow AS aggregation takes its fair share of
   CPU as well, of course).
 
+- sFlow only:
+  Have your router(s) send sFlow samples to your machine. Your routers
+  may need a software upgrade to make them include AS path information for
+  both inbound and outbound packets (this is a good thing to check if
+  your graphs only show traffic on one direction).
+
 - Wait 1-2 minutes. You should then see new RRD files popping up in the
   directory that you defined/created earlier on. If not, make sure that
-  netflow-asstatd.pl is running, not spewing out any error messages, and that
+  xx-asstatd.pl is running, not spewing out any error messages, and that
-  the NetFlow datagrams are actually reaching your machine (tcpdump...).
+  the NetFlow/sFlow datagrams are actually reaching your machine (tcpdump...).
 
 - Add a cronjob to run the following command every hour:
 
@@ -116,10 +164,11 @@ Installation
 
   That script will go through all RRD files and collect per-link summary
   stats for each AS, sort them by total traffic (descending), and write them
-  to a text file. The "top N AS" page uses this to determine which ASes to show.
+  to a text file. The "top N AS" page uses this to determine which ASes to
+  show.
   
-- Copy the contents of the "www" directory to somewhere within your web server's
+- Copy the contents of the "www" directory to somewhere within your web
-  document root and change file paths in func.inc as necessary.
+  server's document root and change file paths in config.inc as necessary.
 
 - Wait a few hours for data to accumulate. :)
 
@@ -146,7 +195,7 @@ By default, the created RRDs keep data as follows:
 	* 1 year at 1 day resolution
 
 If you want to change that, modify the getrrdfile() function in
-netflow-asstatd.pl and delete any old RRD files.
+xx-asstatd.pl and delete any old RRD files.
 
 
 To do

bin/netflow-asstatd.pl

@@ -3,21 +3,12 @@
 # $Id$
 #
 # written by Manuel Kasper, Monzoon Networks AG <mkasper@monzoon.net>
+# cli params/rrd storage/sampling mods Steve Colam <steve@colam.co.uk>
 
 use strict;
 use IO::Socket;
 use RRDs;
-
+use Getopt::Std;
-if ($#ARGV != 1) {
-	die("Usage: $0 <path to RRD file directory> <path to known links file>\n");
-}
-
-my $rrdpath = $ARGV[0];
-my $knownlinksfile = $ARGV[1];
-
-if (! -d $rrdpath) {
-	die("$rrdpath does not exist or is not a directory\n");
-}
 
 my %knownlinks;
 
@@ -33,10 +24,38 @@ my $header_len = 28;
 my $flowrec_len = 28;
 my $childrunning = 0;
 
+use vars qw/ %opt /;
+getopts('r:p:k:s:', \%opt);
+
+my $usage = "$0 [-rpks]\n".
+	"\t-r <path to RRD files>\n".
+	"\t(-p <UDP listen port - default $server_port>)\n".
+	"\t-k <path to known links file>\n".
+	"\t(-s <sampling rate - default $samplingrate>)\n";
+
+my $rrdpath = $opt{'r'};
+my $knownlinksfile = $opt{'k'};
+
+die("$usage") if (!defined($rrdpath) || !defined($knownlinksfile));
+
+die("$rrdpath does not exist or is not a directory\n") if ! -d $rrdpath;
+die("$knownlinksfile does not exist or is not a file\n") if ! -f $knownlinksfile;
+
+if (defined($opt{'s'})) {
+	$samplingrate = $opt{'s'};
+	die("Sampling rate is non numeric\n") if $samplingrate !~ /^[0-9]+$/;
+}
+
+if (defined($opt{'p'})) {
+	$server_port = $opt{'p'};
+	die("Server port is non numeric\n") if $server_port !~ /^[0-9]+$/;
+}
+
 # reap dead children
 $SIG{CHLD} = \&REAPER;
 $SIG{TERM} = \&TERM;
 $SIG{INT} = \&TERM;
+$SIG{HUP} = \&read_knownlinks;
 
 sub REAPER {
 	wait;
@@ -190,9 +209,18 @@ sub getrrdfile {
 	my $as = shift;
 	my $startts = shift;
 	$startts--;
+	
+	# we create 256 directories and store RRD files based on the lower
+	# 8 bytes of the AS number
+	my $dirname = "$rrdpath/" . sprintf("%02x", $as % 256);
+	if (! -d $dirname) {
+		# need to create directory
+		mkdir($dirname);
+	}
+	
+	my $rrdfile = "$dirname/$as.rrd";
 
 	# let's see if there's already an RRD file for this AS - if not, create one
-	my $rrdfile = "$rrdpath/$as.rrd";
 	if (! -r $rrdfile) {
 		#print "$$: creating RRD file for AS $as\n";
 		
@@ -217,13 +245,17 @@ sub getrrdfile {
 }
 
 sub read_knownlinks {
+	my %knownlinks_tmp;
 	open(KLFILE, $knownlinksfile) or die("Cannot open $knownlinksfile!");
 	while (<KLFILE>) {
 		chomp;
 		next if (/(^\s*#)|(^\s*$)/);	# empty line or comment
 		
 		my ($routerip,$ifindex,$tag,$descr,$color) = split(/\t+/);
-		$knownlinks{"${routerip}_${ifindex}"} = $tag;
+		$knownlinks_tmp{"${routerip}_${ifindex}"} = $tag;
 	}
 	close(KLFILE);
+	
+	%knownlinks = %knownlinks_tmp;
+	return;
 }

bin/rrd-extractstats.pl

@@ -3,9 +3,11 @@
 # $Id$
 #
 # written by Manuel Kasper, Monzoon Networks AG <mkasper@monzoon.net>
+# mod for rrd path sjc
 
 use strict;
 use RRDs;
+use File::Find;
 
 if ($#ARGV != 2) {
 	die("Usage: $0 <path to RRD file directory> <path to known links file> outfile\n");
@@ -24,16 +26,19 @@ my @links = values %knownlinks;
 # walk through all RRD files in the given path and extract stats for all links
 # from them; write the stats to a text file, sorted by total traffic
 
-opendir(DIR, $rrdpath);
+my @rrdfiles;
-my @rrdfiles = readdir(DIR);
+find(sub {
-closedir(DIR);
+	if (-f $_) {
+		push(@rrdfiles, $File::Find::name);
+	}
+}, $rrdpath);
 
 my $astraffic = {};
 
 $|=1;
 my $i = 0;
 foreach my $rrdfile (@rrdfiles) {
-	if ($rrdfile =~ /^(\d+).rrd$/) {
+	if ($rrdfile =~ /\/(\d+).rrd$/) {
 		my $as = $1;
 		
 		$astraffic->{$as} = gettraffic($as, time - 86400, time);
@@ -94,9 +99,12 @@ sub gettraffic {
 	
 	my $retdata = {};
 	
+	my $dirname = "$rrdpath/" . sprintf("%02x", $as % 256);
+	my $rrdfile = "$dirname/$as.rrd";
+	
 	foreach my $link (@links) {
-		push(@cmd, "DEF:${link}_in=$rrdpath/$as.rrd:${link}_in:AVERAGE");
+		push(@cmd, "DEF:${link}_in=$rrdfile:${link}_in:AVERAGE");
-		push(@cmd, "DEF:${link}_out=$rrdpath/$as.rrd:${link}_out:AVERAGE");
+		push(@cmd, "DEF:${link}_out=$rrdfile:${link}_out:AVERAGE");
 		push(@cmd, "VDEF:${link}_in_v=${link}_in,TOTAL");
 		push(@cmd, "VDEF:${link}_out_v=${link}_out,TOTAL");
 		push(@cmd, "PRINT:${link}_in_v:%lf");
@@ -142,3 +150,4 @@ sub read_knownlinks {
 	}
 	close(KLFILE);
 }
+

bin/sflow-asstatd.pl

@@ -8,21 +8,10 @@ use strict;
 use Net::sFlow;
 use IO::Socket;
 use RRDs;
-
+use Getopt::Std;
-if ($#ARGV != 1) {
-	die("Usage: $0 <path to RRD file directory> <path to known links file>\n");
-}
-
-my $rrdpath = $ARGV[0];
-my $knownlinksfile = $ARGV[1];
-
-if (! -d $rrdpath) {
-	die("$rrdpath does not exist or is not a directory\n");
-}
 
 my %knownlinks;
 
-my $myas = 8302;
 my $samplingrate = 512;
 
 my $ascache = {};
@@ -35,10 +24,41 @@ my $header_len = 28;
 my $flowrec_len = 28;
 my $childrunning = 0;
 
+use vars qw/ %opt /;
+getopts('r:p:k:a:s:', \%opt);
+
+my $usage = "$0 [-rpkas]\n".
+	"\t-r <path to RRD files>\n".
+	"\t(-p <UDP listen port - default $server_port>)\n".
+	"\t-k <path to known links file>\n".
+	"\t-a <your own AS number>\n".
+	"\t(-s <sampling rate - default $samplingrate>)\n";
+
+my $rrdpath = $opt{'r'};
+my $knownlinksfile = $opt{'k'};
+my $myas = $opt{'a'};
+
+die("$usage") if (!defined($rrdpath) || !defined($knownlinksfile) || !defined($myas));
+
+die("$rrdpath does not exist or is not a directory\n") if ! -d $rrdpath;
+die("$knownlinksfile does not exist or is not a file\n") if ! -f $knownlinksfile;
+die("Your own AS number is non numeric\n") if ($myas !~ /^[0-9]+$/);
+
+if (defined($opt{'s'})) {
+	$samplingrate = $opt{'s'};
+	die("Sampling rate is non numeric\n") if $samplingrate !~ /^[0-9]+$/;
+}
+
+if (defined($opt{'p'})) {
+	$server_port = $opt{'p'};
+	die("Server port is non numeric\n") if $server_port !~ /^[0-9]+$/;
+}
+
 # reap dead children
 $SIG{CHLD} = \&REAPER;
 $SIG{TERM} = \&TERM;
 $SIG{INT} = \&TERM;
+$SIG{HUP} = \&read_knownlinks;
 
 sub REAPER {
 	wait;
@@ -234,8 +254,17 @@ sub getrrdfile {
 	my $startts = shift;
 	$startts--;
 
+	# we create 256 directories and store RRD files based on the lower
+	# 8 bytes of the AS number
+	my $dirname = "$rrdpath/" . sprintf("%02x", $as % 256);
+	if (! -d $dirname) {
+		# need to create directory
+		mkdir($dirname);
+	}
+	
+	my $rrdfile = "$dirname/$as.rrd";
+
 	# let's see if there's already an RRD file for this AS - if not, create one
-	my $rrdfile = "$rrdpath/$as.rrd";
 	if (! -r $rrdfile) {
 		#print "$$: creating RRD file for AS $as\n";
 		
@@ -260,13 +289,17 @@ sub getrrdfile {
 }
 
 sub read_knownlinks {
+	my %knownlinks_tmp;
 	open(KLFILE, $knownlinksfile) or die("Cannot open $knownlinksfile!");
 	while (<KLFILE>) {
 		chomp;
 		next if (/(^\s*#)|(^\s*$)/);	# empty line or comment
 		
 		my ($routerip,$ifindex,$tag,$descr,$color) = split(/\t+/);
-		$knownlinks{"${routerip}_${ifindex}"} = $tag;
+		$knownlinks_tmp{"${routerip}_${ifindex}"} = $tag;
 	}
 	close(KLFILE);
+	
+	%knownlinks = %knownlinks_tmp;
+	return;
 }

contrib/generate-asinfo.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# (echo begin; echo verbose; for i in `seq 1 65535`; do echo "AS$i"; done; echo end) | netcat whois.cymru.com 43 | ./generate-asinfo.py > asinfo.txt
+
+import sys
+
+for line in sys.stdin:
+	try:
+		asn,country,_,_,data = [_.strip() for _ in line.split('|')]
+	except ValueError:
+		continue
+
+	if data == '-Private Use AS-':
+		data = 'Private Use AS'
+
+	try:
+		macro,name = data.split(' ',1)
+	except:
+		macro = data
+		name = data
+
+	if not (macro.count('-') or macro.upper() == macro or name.startswith('- ')) or macro == 'UK':
+		macro = 'UNSPECIFIED'
+		name = data
+
+ 	if name.startswith('- '):
+		name = name[2:]
+
+	print "%s\t%s\t%s\t%s" % (asn,macro,name,country)

tools/migraterrdfiles.pl

@@ -0,0 +1,40 @@
+#!/usr/bin/perl
+
+# This script moves RRD files from the old (pre-v1.3) flat directory structure
+# into subdirectories (where the name of each subdirectory is the lower
+# byte of the AS number in hex).
+
+use strict;
+
+my $rrdpath = $ARGV[0];
+die("Usage: $0 <path to RRD files>\n") if (! -d $rrdpath);
+
+opendir(DIR, $rrdpath);
+my @rrdfiles = readdir(DIR);
+closedir(DIR);
+
+my $i = 0;
+foreach my $rrdfile (@rrdfiles) {
+	if ($rrdfile =~ /^(\d+).rrd$/) {
+		my $as = $1;
+		
+		# calculate new path
+		my $dirname = "$rrdpath/" . sprintf("%02x", $as % 256);
+		if (! -d $dirname) {
+			# need to create directory
+			mkdir($dirname);
+		}
+
+		my $new_rrdfile = "$dirname/$as.rrd";
+		
+		rename("$rrdpath/$rrdfile", $new_rrdfile);
+		
+		$i++;
+		
+		if ($i % 100 == 0) {
+			print "$i...";
+		}
+	}
+}
+print "\n\n";
+print "$i files moved.\n";

www/config.inc

@@ -0,0 +1,13 @@
+<?php
+
+/* changes these values to suit your local configuration */
+
+$rrdpath = "/data/rrd";
+$daystatsfile = "/data/asstats_day.txt";
+$rrdtool = "/usr/local/bin/rrdtool";
+
+$asinfofile = "asinfo.txt";
+$knownlinksfile = "/data/knownlinks";
+$outispositive = true;
+
+?>