Make top N AS intervals configurable

* make top N AS intervals configurable (see config.inc), and allow
separate top stats file to be used per interval (enables e.g. proper
weekly/monthly stats)
* add Terabyte formatting

HISTORY.md

@@ -6,6 +6,8 @@
   your knownlinks file.
 * Added support for peer-AS stats/graphs (sFlow only).
 	* Add -n option to asstats.pl and set showpeeras=true in config.inc to enable.
+* Top N AS intervals are now configurable (see config.inc and README), and separate top stats files can be used per interval (enables e.g. proper weekly/monthly stats).
+* Added Terabyte formatting.
 
 ## 1.5
 

README.md

@@ -30,8 +30,9 @@ scripts.
 Prerequisites
 -------------
 
-- Perl 5.10
+- Perl 5.10 or newer
-- RRDtool 1.2 (with Perl "RRDs" library)
+- RRDtool 1.3 or newer (with Perl "RRDs" library)
+- File::Find::Rule module (CPAN)
 - if using sFlow: the Net::sFlow module (CPAN)
 - web server with PHP 5
 - one or more routers than can generate NetFlow v8/v9 AS aggregation records
@@ -260,14 +261,30 @@ Installation
 
 - Add a cronjob to run the following command every hour:
 
-	`rrd-extractstats.pl /path/to/rrd/dir /path/to/knownlinks \
+	`rrd-extractstats.pl /path/to/rrd/dir /path/to/knownlinks /path/to/asstats_day.txt`
-		/path/to/asstats_day.txt`
 
   That script will go through all RRD files and collect per-link summary
   stats for each AS, sort them by total traffic (descending), and write them
   to a text file. The "top N AS" page uses this to determine which ASes to
   show.
   
+  If you want an additional interval for the top N AS (e.g. top N AS in the
+  last 30 days), add another cronjob with the desired interval in hours as
+  the last argument (and another output file of course). Example:
+  
+	`rrd-extractstats.pl /path/to/rrd/dir /path/to/knownlinks /path/to/asstats_month.txt 720`
+
+  Add the interval to the top_intervals array in config.inc (see the example) so that
+  it will appear in the web interface.
+
+  Repeat for further intervals if necessary.
+  
+  It is not recommended to run more than one rrd-extractstats.pl cronjobs at the same
+  time for disk I/O reasons – add some variation in the start minute setting
+  so that the jobs can run separately.
+  For longer intervals than one day, the cronjob frequency can be adjusted as
+  well (e.g. for monthly output, it is sufficient to run the cronjob once a day).
+  
 - Copy the contents of the "www" directory to somewhere within your web
   server's document root and change file paths in config.inc as necessary.
 

bin/rrd-extractstats.pl

@@ -10,13 +10,18 @@ use RRDs;
 use File::Find;
 use File::Find::Rule;
 
-if ($#ARGV != 2) {
+if ($#ARGV < 2) {
-	die("Usage: $0 <path to RRD file directory> <path to known links file> outfile\n");
+	die("Usage: $0 <path to RRD file directory> <path to known links file> outfile [interval-hours]\n");
 }
 
 my $rrdpath = $ARGV[0];
 my $knownlinksfile = $ARGV[1];
 my $statsfile = $ARGV[2];
+my $interval = 86400;
+
+if ($ARGV[3]) {
+    $interval = $ARGV[3] * 3600;
+}
 
 my %knownlinks;
 
@@ -37,7 +42,7 @@ foreach my $rrdfile (@rrdfiles) {
 	if ($rrdfile =~ /\/(\d+).rrd$/) {
 		my $as = $1;
 		
-		$astraffic->{$as} = gettraffic($as, time - 86400, time);
+		$astraffic->{$as} = gettraffic($as, time - $interval, time);
 		$i++;
 		if ($i % 100 == 0) {
 			print "$i... ";

www/config.inc

@@ -37,5 +37,21 @@ $customlinks = array(
 	'euro-IX' => 'https://www.euro-ix.net/tools/asn_search?query=%as%'
 );
 
+/* Custom time intervals for top N AS */
+/*
+$top_intervals[] = array(
+	'hours' => 7*24,
+	'statsfile' => '/data/as-stats/asstats_week.txt',
+	'label' => '1 week',
+	//'statslabel' => '1 week'		// overrides display of "... in the last ..." - use if 'hours' parameter does not correspond with statsfile interval
+);
+$top_intervals[] = array(
+	'hours' => 30*24,
+	'statsfile' => '/data/as-stats/asstats_month.txt',
+	'label' => '30 days',
+	//'statslabel' => '30 days'		// overrides display of "... in the last ..." - use if 'hours' parameter does not correspond with statsfile interval
+);
+*/
+
 
 /* END - no closing php tag needed here (prevents problems with stray whitespace) */

www/config_defaults.inc

@@ -21,3 +21,23 @@ $asset_graph_height = 200;
 /* Defaults for other settings, introduced in recent versions */
 $vertical_label = true;			# vertical IN/OUT label in graph
 $brighten_negative = true;		# brighten the "negative" part of graphs
+$showpeeras = false;
+
+/* Time intervals for top N AS */
+$top_intervals = array(
+	array(
+		'hours' => 24,
+		'label' => '24 hours',
+		'statslabel' => '24 hours'
+	),
+	array(
+		'hours' => 4,
+		'label' => '4 hours',
+		'statslabel' => '24 hours'
+	),
+	array(
+		'hours' => 12,
+		'label' => '12 hours',
+		'statslabel' => '24 hours'
+	)
+);

www/func.inc

@@ -95,15 +95,7 @@ function getknownlinks() {
 	return $knownlinks;
 }
 
-function getasstats_top($ntop, $peer = 0) {
+function getasstats_top($ntop, $statfile) {	
-	if($peer == 0){
-		global $daystatsfile;
-		$statfile = $daystatsfile;
-	}else{
-		global $daypeerstatsfile;
-		$statfile = $daypeerstatsfile;
-	}
-	
 	/* first step: walk the data for all ASes to determine the top 5 for the given link */
 	$fd = fopen($statfile, "r");
 	if (!$fd)
@@ -148,7 +140,9 @@ function getasstats_top($ntop, $peer = 0) {
 }
 
 function format_bytes($bytes) {
-	if ($bytes >= 1073741824)
+	if ($bytes >= 1099511627776)
+		return sprintf("%.2f TB", $bytes / 1099511627776);
+	else if ($bytes >= 1073741824)
 		return sprintf("%.2f GB", $bytes / 1073741824);
 	else if ($bytes >= 1048576)
 		return sprintf("%.2f MB", $bytes / 1048576);

www/headermenu.inc

@@ -1,35 +1,22 @@
 <?php 
 $dpagename = basename($_SERVER['PHP_SELF'], ".php");
 
-if ($dpagename == "top" && !@$_GET['numhours']): 
+echo "<span>Top AS:</span> ";
-	?><a href="top.php" class="selected">Top AS</a> | <?php
+foreach ($top_intervals as $interval) {
-else:
+	echo '<a href="top.php?numhours=' . $interval['hours'] . '"';
-	?><a href="top.php">Top AS</a> | <?php 
+	if ($dpagename == "top" && @$_GET['numhours'] == $interval['hours'])
-endif;
+		echo ' class="selected"';
+	echo '>' . $interval['label'] . '</a> | ';
+}
 
 if($showpeeras == true){
 	if ($dpagename == "peerusage"):
-		?><a href="peerusage.php" class="selected">Top Peer AS</a> | <?php
+		?><a href="peerusage.php" class="selected">Peer 24 hours</a> | <?php
 	else:
-		?><a href="peerusage.php">Top AS Peer<a> | <?php
+		?><a href="peerusage.php">Peer 24 hours<a> | <?php
 	endif;
 }
 
-
-if ($dpagename == "top" && @$_GET['numhours'] == 4): 
-	?><a href="top.php?numhours=4" class="selected">4 Hour</a> | <?php
-else:
-	?><a href="top.php?numhours=4">4 Hour</a> | <?php 
-endif;
-
-
-if ($dpagename == "top" && @$_GET['numhours'] == 12): 
-	?><a href="top.php?numhours=12" class="selected">12 Hour</a> | <?php
-else:
-	?><a href="top.php?numhours=12">12 Hour</a> | <?php 
-endif;
-
-
 if ($dpagename == "history"): 
 	?><a href="history.php" class="selected">View an AS</a> | <?php
 else:

www/style.css

@@ -110,6 +110,11 @@ div#legend {
 	float: right;
 }
 
+#nav span {
+	font-weight: bold;
+	font-size: 1.15em;
+}
+
 #nav a, #nav a:visited {
 	text-decoration: none;
 	font-weight: bold;

www/top.php

@@ -15,16 +15,40 @@ if (isset($_GET['n']))
 if ($ntop > 200)
 	$ntop = 200;
 
-$topas = getasstats_top($ntop, $peerusage);
+if ($peerusage)
+	$statsfile = $daypeerstatsfile;
+else {
+	$statsfile = $daystatsfile;
+	
+	/* find more appropriate file for this interval */
+	foreach ($top_intervals as $interval) {
+		if ($interval['hours'] == @$_GET['numhours']) {
+			if (@$interval['statsfile'])
+				$statsfile = $interval['statsfile'];
+			if (@$interval['label'])
+				$label = $interval['label'];
+			if (@$interval['statslabel'])
+				$statslabel = $interval['statslabel'];
+		}
+	}
+}
+$topas = getasstats_top($ntop, $statsfile);
 
 if (@$_GET['numhours']) {
 	$start = time() - $_GET['numhours']*3600;
 	$end = time();
+	if (!$label)
+		$label = htmlspecialchars($_GET['numhours']) . " hours";
 } else {
 	$start = "";
 	$end = "";
+	if (!$label)
+		$label = "24 hours";
 }
 
+if (!$statslabel)
+	$statslabel = $label;
+
 ?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
         "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -32,7 +56,7 @@ if (@$_GET['numhours']) {
 <head>
 	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
 	<meta http-equiv="Refresh" content="300" />
-	<title>Top <?php echo $ntop; ?> AS<?php if($peerusage) echo " peer"; ?></title>
+	<title>Top <?php echo $ntop; ?> AS<?php if($peerusage) echo " peer"; ?> (<?=$label?>)</title>
 	<link rel="stylesheet" type="text/css" href="style.css" />
 </head>
 
@@ -46,7 +70,7 @@ Number of AS:
 <?php include('headermenu.inc'); ?>
 </form>
 </div>
-<div class="pgtitle">Top <?php echo $ntop; ?> AS<?php if($peerusage) echo " peer"; ?></div>
+<div class="pgtitle">Top <?php echo $ntop; ?> AS<?php if($peerusage) echo " peer"; ?> (<?=$label?>)</div>
 
 <table class="astable">
 
@@ -67,10 +91,10 @@ $class = (($i % 2) == 0) ? "even" : "odd";
 			AS<?php echo $as; ?>: <?php echo $asinfo['descr']; ?>
 		</div>
 		<div class="small">IPv4: ~ <?php echo format_bytes($nbytes[0]); ?> in / 
-			<?php echo format_bytes($nbytes[1]); ?> out in the last 24 hours</div>
+			<?php echo format_bytes($nbytes[1]); ?> out in the last <?=$statslabel?></div>
 		<?php if ($showv6): ?>
 		<div class="small">IPv6: ~ <?php echo format_bytes($nbytes[2]); ?> in / 
-			<?php echo format_bytes($nbytes[3]); ?> out in the last 24 hours</div>
+			<?php echo format_bytes($nbytes[3]); ?> out in the last <?=$statslabel?></div>
 		<?php endif; ?>
 
 <?php if (!empty($customlinks)): ?>