#!/usr/local/bin/perl
### ----------------------------------------------------------------------- ###
#
#                 Perl statistics generator for httpd log
#
#                 Martin Gleeson, gleeson@unimelb.edu.au.
#
# (c) Copyright, Martin Gleeson and The University of Melbourne, 1994-1999
#
# This program is provided free of charge to educational, charity or
# not-for-profit organisations provided the Copyright notice remains intact.
# Commercial organisations should contact the author for licensing details.
# No warranty is made, either expressed or implied. USE AT YOUR OWN RISK.
#
# Includes fixed version of newgetopt.pl from perl 5 library.
#
### ----------------------------------------------------------------------- ###
#
$version = '1.3.8';
#
### ----------------------------------------------------------------------- ###
$| = 1;

sub usage {
	$usagescreen = <<USAGE;

Usage: pwebstats -c <config file>
                 -p <page-based stats config file>
                 -s <server nickname>
                 -o <output directory>
                 -h <output header string>
                 -n <interval> (default=weekly)
                 -l <server log file>
                 -lt <server log type)
                 -t <templates directory>
              (( -Clog <cache log file> -Plog <proxy log file> ))
                 -e   exclude (remove requests/accesses from proxy stats)
                 -A <threshold for including host in stats (default=25)>
                 -R <   "       "      "     remote host in stats (default=25)>
                 -I <   "       "      "     item in stats (default=25)>
                 -D <   "       "      "     domain in stats (default=5)>
                 -patt <local hostname/IP pattern>
                 -f <path to fly> (gif-on-the fly program)
                 -v    verbose
                 -vv   very verbose (includes output of fly)
                 -debug (debug mode)
                 -X regexp of files to exclude from item/request stats
                 -d do DNS lookups on IP numbers

*  Options given on command line override those in the config file.
*  If no config file given, then all of -s, -o, -h and -l ( or -Plog 
   and -Clog) are required.
*  If -Clog and -Plog are specified rather than -l, stats will
   be summary stats for a caching proxy server
*  -Plog can be specified on its own if there is only one proxy log
*  <server nickname> must be one word, made of alphanumerics and "_".
*  The current server log file must be in Common Log Format
   or one of selected other formats (see -lt below), in a file
   specified by the -logfile option, or by the "logfile" directive
   in the config file.
*  -R applicable in proxy mode only
*  -e causes calculation of proxy stats to use a _lot_ less memory
*  -lt specified log format if not common log format. The following
   formats are supported by pwebstats:
   squid, netscape, netscape-proxy, apache, extended common log format

pwebstats, version $version

USAGE

	print STDERR $usagescreen;
	exit(1);
}

### ----------------------------------------------------------------------- ###
### Get all the configuration settings from the config file
#
&setup_configuration();

### Set up associative arrays
#
&setup_arrays();

### Initialise variables that need initialising
#
&setup_variables();

### Check and/or create directories in output area
&setup_directories();

### Create arrays dependent on information from directory setup
&generate_arrays();

&status_splash("started");

### Read in Cumulative stats
&read_cumulative_stats($server);

### Read in the log file
&read_in_logfile();

### Write out cumulative stats
&write_cumulative_stats($server);

### Print out the reports and graphs
&print_html_reports();
# &print_text_reports();
&print_graphs();

### Clean up temp files, etc.
&clean_up();

&status_splash("finished");

### done!
exit(0);

### ----------------------------------------------------------------------- ###

sub status_splash
{
	my $word = pop(@_);
	
	if( $verbose )
	{
		select(STDERR);
		$date_now = &time_now();
		print '=' x 75; print "\n";
		printf STDERR "pwebstats $version $word at $date_now.\n";
		print '=' x 75; print "\n";
	}
}

### ----------------------------------------------------------------------- ###

sub setup_configuration
{
	@options = ("c:s","p:s","s:s","o:s","h:s","n:s","l:s","t:s",
		"Clog:s","Plog:s", "A:s","R:s","I:s","D:s","patt:s",
		"v","vv","debug","X:s","e","lt:s","d");

	if (! &NGetOpt(@options)) { &usage; };

	if( $opt_c ) {
		open(CONFIG,"$opt_c") || die "Could not find/open config file: $opt_c\n";

		# OK, we've got a config file. Read it in.
		while ( <CONFIG> ) {
			chop;
			next if( /^\s*\#/ || /^\s*$/ ); # Ignore comments & blank lines
			($directive,$option) = split (/:/,$_,2);
			$config{$directive} = $option;
		}
		close(CONFIG);
	} elsif( !$opt_o || !$opt_l || !$opt_t ) {
		&usage;
	}

	# Now check for command-line overrides and assign values to variables.
	# Order for choosing values: 1) Command Line 2) Config File 3) Default

	$server         = $opt_s || $config{'server'} || 'server';
	$outdir         = $opt_o || $config{'outdir'} || &error("Output Directory not set");
	$Server_header  = $opt_h || $config{'Server_header'} || "Usage Statistics";
	$templates      = $opt_t || $config{'templates'} || &error("Templates Directory not set");
	$interval       = $opt_n || $config{'interval'} || "weekly";
	$page_config    = $opt_p || $config{'page_config'}; 
	$logfile        = $opt_l || $config{'logfile'};
	$logtype        = $opt_lt || $config{'logtype'} || "common";
	$local_patt     = $opt_patt || $config{'local_patt'};
	$cache_log_file = $opt_Clog || $config{'cache_log_file'};
	$proxy_log_file = $opt_Plog || $config{'proxy_log_file'};
	$fly_prog       = $opt_f || $config{'fly_prog'};
	$exclude_reqs   = $opt_e || $config{'exclude_reqs'};
	$dns_lookup     = $opt_d || $config{'dns_lookup'};
	$verbose        = $opt_v || $config{'verbose'};
	$flyverbose     = $opt_vv || $config{'veryverbose'};
	$verbose        = $flyverbose if( $flyverbose );
	$debug          = $opt_debug || $config{'debug'};

	# exclusion variables
	$complete_exclude_host     = $config{'complete_exclude_host'};
	$complete_exclude_url_patt = $config{'complete_exclude_url_patt'};
	$complete_exclude_user     = $config{'complete_exclude_user'};
	$exclude_from_stats        = $opt_X || $config{'exclude'} || "";

	# thresholds
	$host_threshold        = $opt_A || $config{'host_threshold'} || 25;
	$remote_host_threshold = $opt_R || $config{'remote_host_threshold'} || 25;
	$item_threshold        = $opt_I || $config{'item_threshold'} || 25;
	$domain_threshold      = $opt_D || $config{'domain_threshold'} || 5;

	if($debug) {
		select(STDERR);
		print "\nEnvironment Variables:\n";
		foreach $key (keys(%ENV)) {
			print "Key:[$key] Value:[$ENV{$key}]\n";
		}

		print "\nConfig Variables:\n";
		print "\$server         => $server\n";
		print "\$outdir         => $outdir\n";
		print "\$Server_header  => $Server_header\n";
		print "\$templates      => $templates\n";
		print "\$interval       => $interval\n";
		print "\$page_config    => $page_config\n";
		print "\$logfile        => $logfile\n";
		print "\$logtype        => $logtype\n";
		print "\$local_patt     => $local_patt\n";
		print "\$cache_log_file => $cache_log_file\n";
		print "\$proxy_log_file => $proxy_log_file\n";
		print "\$fly_prog       => $fly_prog\n";
		print "\$verbose        => $verbose\n";
		print "\$flyverbose     => $flyverbose\n";
		print "\$debug          => $debug\n";
		print "\n";
		print "\$host_threshold          => $host_threshold\n";
		print "\$remote_host_threshold   => $remote_host_threshold\n";
		print "\$item_threshold          => $item_threshold\n";
		print "\$domain_threshold        => $domain_threshold\n";
		print "\$exclude_from_stats      => $exclude_from_stats\n";
		print "\n";
	}

	# Establish the OS
	$OS = $config{'OS'};

	# FIGURE OUT THE OS WE'RE RUNNING UNDER
	# This test stolen from Lincoln Stein's CGI.pm
	# Some systems support the $^O variable.  If not
	# available then require() the Config library
	unless ($OS) {
		unless ($OS = $^O) {
			require Config;
			$OS = $Config::Config{'osname'};
		}
	}
	print STDERR "Operating System: [$OS]\n" if $debug;
}

### ----------------------------------------------------------------------- ###
#   Check for existence of directories and files
### ----------------------------------------------------------------------- ###

sub setup_directories {
	$stdin = "true" if( $logfile eq "stdin" );

	if( $logfile && ! $stdin && ! -e $logfile ) {
			print STDERR "Could not find input logfile: $logfile\n";
			&usage;
	} elsif( $cache_log_file && ! -e $cache_log_file ) {
			print STDERR "Could not find cache logfile: $cache_log_file\n";
			&usage;
	} elsif ( $proxy_log_file && ! -e $proxy_log_file ) {
			print STDERR "Could not find proxy logfile: $proxy_log_file\n";
			&usage;
	}

	if( ! -e $outdir ) {
		print STDERR "Could not find output directory: $outdir\n";
		&usage;
	}
	if( ! -e $templates ) {
		print STDERR "Could not find templates directory: $templates\n";
		&usage;
	}

	$proxy_mode = "on" if( $proxy_log_file || $logtype =~ m/squid/ || $logtype =~ m/netscape-proxy/);

	$proxy_only = "on" if( $proxy_log_file && !$cache_log_file);

	$logfile = ($newfile = readlink("$logfile")) ? $newfile : $logfile;

	$cache_log_file = ($newfile = readlink("$cache_log_file")) ? $newfile : $cache_log_file;
	$proxy_log_file = ($newfile = readlink("$proxy_log_file")) ? $newfile : $proxy_log_file;

	### ----------------------------------------------------------------------- ###
	#   Directories and Files
	### ----------------------------------------------------------------------- ###

	if($OS =~ /Win/i) {
		$temp = "c:/temp";
		mkdir("$temp",0755) if(! -e "$temp");
	} else {
		$temp="/tmp";
	}

	# various index files
	$index            = "index.html";
	$gross_index      = "g-index.html";
	$cumulative_index = "c-index.html";
	$page_index       = "page-index.html";

	# files to be read in
	$footer           = "$templates/footer.html";
	$domain_file      = "$templates/domains/domains.txt";
	$user_domain_file = "$templates/domains/my_domains.txt";

	if( ! -e $domain_file ) {
		print STDERR "Could not find domain config file: $domain_file\n";
		&usage;
	}
	if( -e $user_domain_file ) {
		$user_domains = 1;
	}

	# gif template files
	$image_templates  =  "$templates/images";

	$hourly_template  =  "$image_templates/hourly_template.gif";
	$domain_template  =  "$image_templates/domain_template.gif";

	if( $proxy_mode ) {
		$period_template = "$image_templates/growth_template_proxy.gif";
	} else {
		$period_template = "$image_templates/growth_template.gif";
	}

	# temporary files for gif creation
	$domain_tmp       = "$temp/domain.$$.tmp";
	$hosts_tmp        = "$temp/hosts.$$.tmp";
	$remote_hosts_tmp = "$temp/r_hosts.$$.tmp";
	$requests_tmp     = "$temp/requests.$$.tmp";
	$all_requests_tmp = "$temp/all_requests.$$.tmp";

	### Set up directories ###
	
	# regular stats directories
	$base = "$outdir/$plurals{$interval}";
	mkdir("$base",0755)          if(! -e "$base");
	mkdir("$base/domains",0755)  if(! -e "$base/domains");
	mkdir("$base/hosts",0755)    if(! -e "$base/hosts");
	mkdir("$base/hours",0755)    if(! -e "$base/hours");
	mkdir("$base/images",0755)   if(! -e "$base/images");
	mkdir("$base/requests",0755) if(! -e "$base/requests");
	
	# cumulative stats directories
	$base = "$outdir/cumulative/$plurals{$interval}";
	mkdir("$base",0755)          if(! -e "$base");
	mkdir("$base/domains",0755)  if(! -e "$base/domains");
	mkdir("$base/hosts",0755)    if(! -e "$base/hosts");
	mkdir("$base/hours",0755)    if(! -e "$base/hours");
	mkdir("$base/images",0755)   if(! -e "$base/images");
	mkdir("$base/requests",0755) if(! -e "$base/requests");
	
	# other directories
	mkdir("$outdir/pages",0755) if(! -e "$outdir/pages");
	mkdir("$outdir/images",0755) if(! -e "$outdir/images");

	### ----------------------------------------------------------------------- ###
	#   Programs
	### ----------------------------------------------------------------------- ###

	if ( -x $fly_prog && ! -d $fly_prog ) {
		$fly_program=$fly_prog;
	} else {
		$fly_program=&find_prog('fly');

		if ( ! -x $fly_program ) {
			print STDERR "Could not find fly\n" .
			             "     - please set fly_prog in the config " .
			             "file or use the -f option\n";
			&usage;
		}
	}
	printf STDERR "fly program: $fly_program\n" if($debug);

	$wc_prog = &find_prog('wc');
	printf STDERR "wc program: $wc_prog\n" if($debug);
	$zcat = $config{zcat_prog} || &find_prog('zcat');
	printf STDERR "zcat program: $zcat\n" if($debug);
	$NO_wc = 'true' if ( ! -x $wc_prog );

}

### ----------------------------------------------------------------------- ###

sub setup_arrays {
	%longmonths = ( '0','January', '1','February', '2','March',
		   '3','April', '4','May', '5','June', '6','July', '7','August',
		   '8','September', '9','October', '10','November', '11','December');

	%months = ( 'Jan','01', 'Feb','02', 'Mar','03', 'Apr','04',
	            'May','05', 'Jun','06', 'Jul','07', 'Aug','08',
	            'Sep','09', 'Oct','10', 'Nov','11', 'Dec','12',);

	%r_months = ( '01','Jan', '02','Feb', '03','Mar', '04','Apr',
	              '05','May', '06','Jun', '07','Jul', '08','Aug',
	              '09','Sep', '10','Oct', '11','Nov', '12','Dec');

	%hours = (  '00','12am', '01',' 1am', '02',' 2am', '03',' 3am',
	            '04',' 4am', '05',' 5am', '06',' 6am', '07',' 7am',
	            '08',' 8am', '09',' 9am', '10','10am', '11','11am',
	            '12','noon', '13',' 1pm', '14',' 2pm', '15',' 3pm',
	            '16',' 4pm', '17',' 5pm', '18',' 6pm', '19',' 7pm',
	            '20',' 8pm', '21',' 9pm', '22','10pm', '23','11pm',
	            '24','12am');

	%int_gsize = (  'daily',    '44,24', 'weekly',   '56,24',
	                'monthly',  '68,24', 'quarterly','72,24',);

	%plurals   = ( 'weekly','weeks','monthly','months','daily','days');
	%singulars = ( 'weekly','week', 'monthly','month','daily','day');
	%Titles    = ( 'weekly','Week', 'monthly','Month','daily','Day');

}

### ----------------------------------------------------------------------- ###

sub generate_arrays
{
	foreach $num ( 0 .. 9 ) {
		$num_size{$num} = &get_width("$image_templates/$num.gif");
		$date_num_size{$num} = &get_width("$image_templates/dates/$num.gif");
	}

	$num_size{'\''} = &get_width("$image_templates/10.gif");
	$date_num_size{'\''} = &get_width("$image_templates/dates/10.gif");

	foreach $mon ( keys(%months) ) {
		$mon_size{$mon} = &get_width("$image_templates/$mon.gif");
	}

	$interval_width = &get_width("$image_templates/r_$singulars{$interval}.gif");
}

### ----------------------------------------------------------------------- ###
#   Declare and initialise some variables

sub setup_variables
{

	local(%hosts);		# accesses per host, indexed by host
	local(%hostbytes);	# bytes xferred per host, indexed by host
	local(%dates);		# accesses per day, indexed by day
	local(%requests);	# accesses per file, indexed by file
	local(%times);		# accesses per hour, indexed by hour
	local(%timeshorts);	# accesses per hour:min, indexed by hour:min
	local(%timelongs);	# accesses per date:hour, indexed by date:hour
	local(@sortedhosts);	# accesses per host, sorted num by # accesses
	local(@sortedreqests);	# accesses per file, sorted num by # accesses
	local(@hour_of_the_day=('00','01','02','03','04','05','06','07','08',
	                        '09','10','11','12','13','14','15','16','17',
	                        '18','19','20','21','22','23','24'));   # hours
	local(@accesses_per_hour=('0','0','0','0','0','0','0','0','0','0','0',
	                          '0','0','0','0','0','0','0','0','0','0','0',
	                          '0','0','0'));                    # accesses per hour
	local(%d_b);		# associative array of db file
	local(%domains);	# counters for accesses by domain
	
	if( $proxy_mode ) {
		local(%remote_hosts);		# accesses per remote host, indexed by host
		local($cache_size=0);		# bytes sent from the cache
		local($cache_requests=0);	# number of requests satisfied from the cache
		local($proxy_size=0);		# bytes sent from remote sites
		local($proxy_requests=0);	# number of requests to remote sites
		local(%protocol_count);		# number of requests for each protocol proxied
		local(%protocol_size);		# number of bytes for each protocol proxied
	}
	
	#	Initialise counters, etc.
		$bytes=0;		# total number of bytes sent
		$totaldates = 0;	# number of days covered by the period
		$totalhours = 0;	# number of hours covered
		$totalhosts = 0;	# number of hosts accessing the server
		$unique_reqs = 0;	# number of unique requests satisfied by the server
		$total = 0;		# number of requests satisfied by the server
		$currDateStart = 9999999999999;	# start of the current period
		$currDateFinish = 0;	# finish of the current period
		$maxHourlyRequests = 0;	# maximum number of requests in an hour
		$address_type = 2;      # for gethostbyname
}
	

### ----------------------------------------------------------------------- ###


sub read_in_logfile
{

	if(($logtype eq "common" || $logtype eq "ncsa-extended") && ! $proxy_mode)
	{
		printf STDERR "-- Reading in log file $logfile:\n" if($verbose);
		&read_logfile($logfile);
		printf STDERR "-- Building {host,domain,item}/accesses Arrays:\n" .
		              "   Host threshold is $host_threshold\n" .
		              "   Domain threshold is $domain_threshold\n" .
		              "   Item threshold is $item_threshold\n\n" if($verbose);
		&build_arrays;
	}
	elsif( $logtype eq "squid" || $logtype eq "squid-emulated" || $logtype eq "netscape-proxy")
	{
		printf STDERR "-- Reading in log file $logfile:\n" if($verbose);
		&read_logfile($logfile);
		printf STDERR "-- Building {host,domain,item}/accesses Arrays:\n" .
		              "   Host threshold is $host_threshold\n" .
		              "   Domain threshold is $domain_threshold\n" .
		              "   Item threshold is $item_threshold\n\n" if($verbose);
		&build_arrays;
	}
	else
	{
		if(! $proxy_only)
		{
			printf STDERR "-- Reading in cache log file $cache_log_file:\n" if($verbose);
			$cache=1;	&read_logfile($cache_log_file);	undef($cache);
		}

		printf STDERR "-- Reading in proxy log file $proxy_log_file:\n" if($verbose);
		$proxy=1;	&read_logfile($proxy_log_file);

		printf STDERR "-- Building {host,domain,item}/accesses Arrays:\n" .
		              "   Host threshold is $host_threshold\n" .
		              "   Domain threshold is $domain_threshold\n" .
		              "   Item threshold is $item_threshold\n\n" if($verbose);
		&build_arrays;	undef($proxy);
	}


	### ----------------------------------------------------------------------- ###
	### Do a few calculations

	$megs      =  $bytes / 1048576 ; # 1 Megabyte = 2^20 bytes, not 10^6!!
	$avPerDay  =  $total/$totaldates if( $totaldates != 0 );
	$avPerHour =  $total/$totalhours if( $totalhours != 0 );
	$avPerMin  =  $avPerHour/60;
	
}

sub print_html_reports
{
	### ----------------------------------------------------------------------- ###
	### Read in the index page for the server and add 
	### a link to this periods's statistics.

	printf STDERR "\n-- Writing Index page.\n" if($verbose);
	&read_and_write_master_page();

	### ----------------------------------------------------------------------- ###
	### Printing: Main Stats Page

	printf STDERR "\n-- Writing stats pages & graphs:\n" .
	              "   This $singulars{$interval}'s stats page.\n" if($verbose);

	open(OUT_FILE,"> $outdir/$plurals{$interval}/$server.$period.html")
		or die "Couldn't open $outdir/$plurals{$interval}/$server.$period.html for writing: $!\n";
	select(OUT_FILE);

	print "<html><head><title>Server Usage: $Server_header</title></head><body>\n";
	print "<h1>$Server_header</h1>";

	if( $proxy_mode )
	{
		$req_pct = sprintf "%3.2f", ( 100 * ( $cache_requests / $total )) if( $total != 0);
		$byte_pct = sprintf "%3.2f", ( 100 * ( $cache_size / $bytes )) if( $bytes != 0);
		$megas = sprintf "%3.2f",$megs;
		$avh = sprintf "%3.2f",$avPerHour;
		$avm = sprintf "%3.2f",$avPerMin;

		printf "<pre>
Period covered by these statistics: <em>$totaldates days</em> ($dateStart - $dateFinish)

Total requests handled this $singulars{$interval}:   <strong>%s</strong>
<hr>\n", &commas($total);
printf "Requests Satisfied by the cache: %18s\n", &commas($cache_requests) if(!$proxy_only);
printf "Requests Proxied                 %18s\n", &commas($proxy_requests) if(!$proxy_only);
printf "Total number of requests served: %18s ", &commas($total);
printf "<strong>***</strong>\n" if(!$proxy_only);
printf "Bytes sent for cache requests:   %18s\n", &commas($cache_size) if(!$proxy_only);
printf "Bytes sent for proxy requests:   %18s\n", &commas($proxy_size) if(!$proxy_only);
printf "Total number of bytes sent:      %18s ", &commas($cache_size + $proxy_size);
printf "<strong>***</strong>" if(!$proxy_only);
printf "\n<hr>
<strong>Cache Hit Rate</strong>
Requests:                           <strong>%12s%%</strong>
Bytes:                              <strong>%12s%%</strong>\n", $req_pct, $byte_pct if(!$proxy_only);
printf "<hr>
Number of Mb sent by this server:   <strong>%12s</strong>
Number of hosts using this server:  <strong>%12s</strong>
<hr>
Average number of requests/day:     <strong>%12s</strong>
Average number of requests/hour:    <strong>%12s</strong>
Average number of requests/minute:  <strong>%12s</strong>
</pre><hr>\n", $megas, &commas($totalhosts), &commas($avPerDay), $avh, $avm;
	}
	else
	{
		printf "<pre>
Period covered by these statistics: <em>$totaldates days</em> ($dateStart - $dateFinish)

Total number of requests:           <strong>%s</strong>
Number of unique requests:          <strong>%s</strong>
Number of Mb sent:                  <strong>%3.2f</strong>
Number of bytes sent:               <strong>%s</strong>
Number of unique hosts:             <strong>%s</strong>
Average number of requests/day:     <strong>%s</strong>
Average number of requests/hour:    <strong>%3.2f</strong>
Average number of requests/minute:  <strong>%3.2f</strong>
</pre><hr>\n",
&commas($total),       &commas($unique_reqs),  $megs,       &commas($bytes),
&commas($totalhosts),  &commas($avPerDay),   $avPerHour,  $avPerMin;
	}

	if( $period != 1 )
	{
		&print_changes(OUT_FILE);
		print "<hr>\n<!-- ====================================================== -->\n";
	}

	print "<h3>Hosts accessing this server, ordered by number of accesses:";
	print "</h3>";
	print "<h4>Top Ten Hosts:</h4>\n";
	print "<pre>";
	print "Host Accessing the Server             : accesses : bytes transferred \n";
	print "---------------------------------------------------------------------\n";

	open(HOSTS,"$hosts_tmp") or die "Couldn't open $hosts_tmp for reading: $!\n";
	$i=0;
	while( <HOSTS> )
	{
		chop;
		($serv,$num)=split(/\t/);
		if( $i < 10 ){
			printf "%-37s : %8s : %17s\n",
				$serv,  &commas($num),  &commas($hostbytes{$serv});
		} else { last; }
		$i++;
	}
	close(HOSTS);

	print "</pre><p><a href=\"hosts/$server.$period.total-hosts.html\">
	       <strong>List of hosts accessing the server (more than $host_threshold times).</strong></a><br>\n";

	if( $proxy_mode )
	{
		print "<hr>\n<!-- ====================================================== -->\n";
		print "<h3>Remote sites accessed by this server, ordered by number of accesses:";
		print "</h3>";
		print "<h4>Top Ten Sites:</h4>\n";
		print "<pre>";
		print "Site Accessed by the Server             : accesses : bytes transferred \n";
		print "---------------------------------------------------------------------\n";

		open(REMOTE_HOSTS,"$remote_hosts_tmp") or die "Couldn't open $remote_hosts_tmp for reading: $!\n";
		$i=0;
		while( <REMOTE_HOSTS> )
		{
			chop;
			($serv,$num)=split(/\t/);
			if( $i < 10 )
			{
				printf "%-37s : %8s : %17s\n",
					$serv,  &commas($num),  &commas($remote_hostbytes{$serv});
			} else { last; }
			$i++;
		}
		close(REMOTE_HOSTS);

		print "</pre><p><a href=\"hosts/$server.$period.total-remote-hosts.html\">
   		    <strong>List of all sites accessed by the Server</strong></a><br>\n";
	}

	if( $proxy_mode )
	{
		print "<hr>\n<!-- ======================================================== -->";
		print "<h3>Protocols proxied, ordered by number of requests:";
		print "</h3>";
		print "<pre>";
		print "Protocol proxied     : number of requests : bytes transferred \n";
		print "<hr>\n";
	
		foreach( @sortedprotocols )
		{
			($proto,$num)=split(/\|\|/);
			printf "%-20s : %15s : %17s\n",
				$proto,  &commas($num),  &commas($protocol_size{$proto});
		}
	
		print "</pre><p>\n";
	}
	
	print "<hr>\n<!-- ======================================================== -->";
	if( $proxy_mode )
	{
		print "<h3>Accesses to remote servers, by domain of remote server:";
	}
	else
	{
		print "<h3>Hosts accessing this server, by domain:";
	}
	print "</h3>";
	print "<p>";
	print "<IMG SRC=\"images/domain.$period.gif\"";
	print " ALT=\"[Accesses by Location Graphic]\"></p>\n";
	print "<pre>\n";
	print "Domain                            : Number of Accesses : Percentage of total.\n";
	print "-----------------------------------------------------------------------------\n";
	
	$i = 0;
	foreach( @sorteddomains )
	{
		($dom,$num)=split(/\|\|/);
	
		if( $i < 5 )
		{
			$pct = sprintf "%3.1f", (( $num / $total) * 100) if ( $total != 0 );
			printf "%-33s :  %16s  : %5s\n",
					($domain_names{$dom} ? $domain_names{$dom} : $dom), &commas($num),  $pct;
			$i++;
		}
		else
		{
			$othertotal += $num;
		}
	}
	$pct = sprintf "%3.1f", (( $othertotal / $total) * 100) if ( $total != 0 );
	$ddom = "($dom)";
	$bdom = sprintf "%-7s", $ddom;
	printf "%-33s :  %16s  : %5s\n", "All Others",  &commas($othertotal),  $pct;
	
	print "</pre><p></p>\n";
	print "<p></p><p><strong>Note:</strong> ";
	print "<em>Unresolved</em> ";
	print "means IP numbers which do not resolve to hostnames.</p>";
	if( $proxy_mode )
	{
		print "<p><a href=\"domains/$server.$period.domains.html\">";
	    print "List of all domains accessed by the server.</a></p>\n";
	}
	else
	{
		print "<p><a href=\"domains/$server.$period.domains.html\">";
	    print "List of all domains accessing the server.</a></p>\n";
	}
	
	print "<hr>\n<!-- ====================================================== -->\n";
	print "<h3>Number of Accesses Per Day during this period:</h3>\n";
	print "<pre>\n";
	
	foreach $key (sort keys(%dates)){
		$_ = $key;
		($y,$m,$d)=/^(\d\d\d\d)(\d\d)(\d\d)$/;
		printf "%s %s:  %-3d\n",$d,$r_months{$m},$dates{$key};
	}
	
	print "</pre>\n";
	print "<hr>\n<!-- ====================================================== -->\n";
	print "<h3>Average Number of Accesses Per Hour of the Day:</h3>\n";
	print "<p><IMG SRC=\"images/hourly.$period.gif\"";
	print "ALT=\"[Average hourly accesses graphic]\"></p>\n";

	$peak=0;
	foreach  $key (sort keys(%timelongs)){
		if( $timelongs{$key} > $peak ){
			$peak = $timelongs{$key};
			$_ = $key;
			($y,$m,$d,$h)=/^(....)(..)(..)(..)$/;
		}
	}
	printf "<p>Peak hour: $r_months{$m} $d $hours{$h}-$hours{$h+1}: $peak</p>\n";
	
	print " <p><a href=\"hours/$server.$period.hourly.html\">[Text version]</a></p>\n";
	
	if( ! $exclude_reqs )
	{
		print "<hr>\n<!-- ====================================================== -->\n";
		print "<h3>Frequency of Items Requested:</h3>\n";
		print "<pre>\n";
		printf "<strong>%-60s</strong> : <strong>\#accesses</strong>\n",'Top Ten Items Accessed:';
	
		open(REQUESTS,"$requests_tmp") or die "Couldn't open $requests_tmp for reading: $!\n";
		$i=0;
		while( <REQUESTS> )
		{
			chop;
			($req,$num)=split(/\t/);
			if( $i < 10 ){
				printf "%-60s : %15s\n",
					$req,  &commas($num) if ( $num > 0 );
			} else { last; }
			$i++;
		}
		close(REQUESTS);
	
		print "</pre><p>";
		print "<a href=\"requests/$server.$period.total-accesses.html\">";
		print "<strong>All items accessed more than $item_threshold times.</strong></a>\n";
	}

	select(OUT_FILE);
	&add_footer(OUT_FILE);
	close(OUT_FILE);
	
	### ----------------------------------------------------------------------- ###
	#  Text form of hourly stats.
	### ----------------------------------------------------------------------- ###

	if( $verbose ) { printf STDERR "   Hourly Stats Page.\n"; }

	open(HOURLY_FILE,"> $outdir/$plurals{$interval}/hours/$server.$period.hourly.html")
		or die "Couldn't open $outdir/$plurals{$interval}/hours/$server.$period.hourly.html for writing: $!\n";
	select(HOURLY_FILE);
	print "<html><head><title>$Server_header: Hourly Accesses</title></head>";
	print "<body><h2>Average Number of Accesses Per Hour of the Day</h2>";
	print "<pre>\n";
	
	for( $i=0; $i < 21; $i+=4 ){
		printf "%s - %s: <b>%-5d</b>  %s - %s: <b>%-3d</b>  ",
		$hours{$hour_of_the_day[$i]},$hours{$hour_of_the_day[$i+1]},$accesses_per_hour[$i],
		$hours{$hour_of_the_day[$i+1]},$hours{$hour_of_the_day[$i+2]},$accesses_per_hour[$i+1];
		printf "%s - %s: <b>%-3d</b>  %s - %s: <b>%-3d</b>\n",
		$hours{$hour_of_the_day[$i+2]},$hours{$hour_of_the_day[$i+3]},$accesses_per_hour[$i+2],
		$hours{$hour_of_the_day[$i+3]},$hours{$hour_of_the_day[$i+4]},$accesses_per_hour[$i+3];
	}
	$peak=0;
	foreach  $key (sort keys(%timelongs)){
		if( $timelongs{$key} > $peak ){
			$peak = $timelongs{$key};
			$_ = $key;
			($y,$m,$d,$h)=/^(....)(..)(..)(..)$/;
		}
	}
	printf "\nPeak hour: $r_months{$m} $d $hours{$h}-$hours{$h+1}: $peak\n";
	print "</pre>\n";
	&add_footer(HOURLY_FILE);
	close(HOURLY_FILE);
	
	### ----------------------------------------------------------------------- ###
	#   Requests from all hosts.
	### ----------------------------------------------------------------------- ###
	
	if( $verbose ) { printf STDERR "   All Hosts Page.\n"; }
	
	open(TOTAL_FILE,"> $outdir/$plurals{$interval}/hosts/$server.$period.total-hosts.html")
		or die "Couldn't open $outdir/$plurals{$interval}/hosts/$server.$period.total-hosts.html for writing: $!\n";
	select(TOTAL_FILE);
	print "<html><head><title>$Server_header: Frequency of Access: Hosts</title></head><body>\n";
	print "<h1>Number of Requests from all Hosts accessing this Server</h1><hr>\n";
	print "<pre>\n";
	printf "%55s      %15s\n<hr>\n", "Requests", "Bytes Transferred";
	open(HOSTS,"$hosts_tmp") or die "Couldn't open $hosts_tmp for reading: $!\n";
	while( <HOSTS> )
	{
		chop;
		($serv,$num)=split(/\t/);
		printf "%-40s : %15s : %15s\n", $serv,  &commas($num),  &commas($hostbytes{$serv});
	}
	close(HOSTS);
	
	print "</pre>\n";
	&add_footer(TOTAL_FILE);
	close(TOTAL_FILE);
	
	### ----------------------------------------------------------------------- ###
	#   Requests from all domains.
	### ----------------------------------------------------------------------- ###
	
	if( $verbose ) { printf STDERR "   All Domains Page.\n"; }
	
	open(TOTAL_FILE,"> $outdir/$plurals{$interval}/domains/$server.$period.domains.html")
		or die "Couldn't open $outdir/$plurals{$interval}/domains/$server.$period.domains.html for writing: $!\n";
	select(TOTAL_FILE);
	print "<html><head><title>$Server_header: Frequency of Access: Domains</title></head><body>\n";
	print "<h1>Number of Requests from all Domains accessing this Server</h1><hr>\n";
	print "<pre>";
	print "Domain                            : Number of Accesses : Percentage of total.\n";
	print "-----------------------------------------------------------------------------\n";
	
	foreach( @sorteddomains )
	{
		($dom,$num)=split(/\|\|/);
	
		$pct = sprintf "%3.1f", (( $num / $total) * 100) if ( $total != 0 );
		printf "%-40s :  %16s  : %5s\n", ($domain_names{$dom} ? $domain_names{$dom} : $dom), &commas($num), $pct;
	}

	print "</pre>\n";
	&add_footer(TOTAL_FILE);
	close(TOTAL_FILE);
	
	if( $proxy_mode ){
	### ----------------------------------------------------------------------- ###
	#   Requests from all hosts.
	### ----------------------------------------------------------------------- ###
 
	if( $verbose ) { printf STDERR "   Remote sites page.\n"; }
    
		open(TOTAL_FILE,"> $outdir/$plurals{$interval}/hosts/$server.$period.total-remote-hosts.html")
			or die "Couldn't open $outdir/$plurals{$interval}/hosts/$server.$period.total-remote-hosts.html for writing: $!\n";
		select(TOTAL_FILE);
		print "<html><head><title>$Server_header: Frequency of Access to remote sites</title></head><body>\n";
		print "<h1>Number of Requests to all sites accessed by Server</h1><hr>\n";
		print "<pre>\n";
    
		open(REMOTE_HOSTS,"$remote_hosts_tmp") or die "Couldn't open $remote_hosts_tmp for reading: $!\n";
		while( <REMOTE_HOSTS> )
		{
			chop;
			($serv,$num)=split(/\t/);
			printf "%-40s : %17s : %17s\n", $serv,  &commas($num),  &commas($remote_hostbytes{$serv});
		}
		close(REMOTE_HOSTS);

		print "</pre>\n";
		&add_footer(TOTAL_FILE);
		close(TOTAL_FILE);
	}
	### ----------------------------------------------------------------------- ###
	#   Request frequency for items.
	### ----------------------------------------------------------------------- ###
	if( ! $exclude_reqs )
	{
		if( $verbose ) { printf STDERR "   Access Count Page.\n"; }

		open(TOTAL_FILE,"> $outdir/$plurals{$interval}/requests/$server.$period.total-accesses.html")
			or die "Couldn't open $outdir/$plurals{$interval}/requests/$server.$period.total-accesses.html for writing: $!\n";
		select(TOTAL_FILE);
		print "<html><head><title>$Server_header: Frequency of Access: Items</title></head><body>\n";
		print "<h1>Number of Requests for items accessed on this Server</h1><hr>\n";
		print "<pre>\n";

		open(REQUESTS,"$requests_tmp") or die "Couldn't open $requests_tmp for reading: $!\n";
		while( <REQUESTS> )
		{
			chop;
			($req,$num)=split(/\t/);
			$req =~ s/\&/\&amp\;/g;
			$req =~ s/\"/\&quot\;/g;
			printf "%-60s : %15s\n", $req,  &commas($num);
		}
		close(REQUESTS);

		print "</pre>\n";
		&add_footer(TOTAL_FILE);
		close(TOTAL_FILE);
	}
	if( ! $proxy_mode ) 
	{
		### ----------------------------------------------------------------------- ###
		#   Collection-based stats.
		### ----------------------------------------------------------------------- ###

		if( $page_config )
		{
			printf STDERR "\n   Page-based Stats:\n" if($verbose);
			if( ! -e "$page_config" ) {
				printf STDERR "    Error: page config file [$page_config]\n    doesn't exist. Continuing...\n";
			} else {
				open(PAGE_FILE,"> $outdir/pages/pages.$period.html")
					or die "Couldn't open $outdir/pages/pages.$period.html for writing: $!\n";
				select(PAGE_FILE);
				print "<html><head><title>$Server_header: Accesses to individual pages and collections:";
				print "</title></head><body>\n";
				print "<h2>Accesses to individual pages and collections</h2>\n<hr>\n";
				print "<pre>\n";
	
				printf STDERR "   Calculating ... " if($verbose);
				get_page_stats("$logfile",$page_config);
				printf STDERR "done.\n" if($verbose);
				printf STDERR "\n   Printing ... " if($verbose);
				print_page_stats(PAGE_FILE,"no");
				printf STDERR "done.\n" if($verbose);
				chdir("$outdir/pages");
				unlink("pages.html");
				if($OS =~ /Win/i) {
					system("copy pages.$period.html pages.html");
				} else {
					symlink("pages.$period.html","pages.html");
				}
	
				print "</pre>\n";
				&add_footer(PAGE_FILE);
				close(PAGE_FILE);
			}
		}
	}
}

sub print_graphs
{
	printf STDERR "\n-- Creating graphs and charts...\n" if( $verbose );
	
	printf STDERR "   [[[ Growth in accesses line graph ]]]\n" if( $verbose );
	($proxy_mode && !$proxy_only) ? &print_proxy_growth() : &print_growth();

	printf STDERR "   [[[ Domain pie chart ]]]\n" if( $verbose );
	&print_domain();
	
	printf STDERR "   [[[ Hourly bar graph ]]]\n" if( $verbose );
	&print_hourly();
}

### ----------------------------------------------------------------------- ###

sub read_logfile {

	# Subroutine for reading input files and sorting

	my $file = pop(@_);

	if( $lastline eq "NULL") {
		$last_date = 0;
	} else {
		$last_date = $lastline;
		chop($last_date) if($last_date =~ /\n$/);
	}

	print STDERR "Last: $last_date\n" if ($debug);

	if($proxy && $debug) {
		open(OTHER_PROTOCOLS,">> $outdir/other_protocols.$period.txt")
			or die "Couldn't open $outdir/other_protocols.$period.txt for appending: $!\n";
	} elsif ($cache && $debug) {
		open(OTHER_PROTOCOLS,"> $outdir/other_protocols.$period.txt")
			or die "Couldn't open $outdir/other_protocols.$period.txt for appending: $!\n";
	}

	if( $verbose ) {
		if($file =~ m/\.gz/) {
			open(COUNT,"$zcat $file \| $wc_prog -l |") or die "Couldn't run $zcat $file \| $wc_prog -l: $!\n";
			while( <COUNT> ){ chop; ($line_count) = /^\s*(\d+)\s*$/;  }
			close(COUNT);
		} elsif( $stdin ) {
			$line_count = "undef";
		} else {
			if($OS =~ /Win/i && ! $wc_prog) {
				$line_count = "undef";
			} else {
				open(COUNT,"$wc_prog -l $file |") or die "Couldn't run $wc_prog -l $file: $!\n";
				while( <COUNT> ){ chop; ($line_count) = /^\s*(\d+)\s*\S+$/;  }
				close(COUNT);
			}
		}
		$inc = sprintf "%d", ( $line_count / 50 );
		print STDERR "   The logfile has $line_count entries.\n" if($line_count ne "undef");
		print STDERR "   Processing...\n";
		print STDERR "   0%                     50%                      100%\n" if($line_count ne "undef");
		print STDERR "   |-----------------------|------------------------|\n   " if($line_count ne "undef");
		$counter=0; $hash_counter=0;
		if($debug && $line_count eq "undef") {
			print STDERR "Lines:          ";
		}
	}

        if( $file =~ m/\.gz$/ ) {
                open(LOG_FILE,"$zcat $file |") || die "Couldn't open logfile '$zcat $file': $!";
        } elsif( $stdin ) {
                open(LOG_FILE,"<&STDIN") || die "Couldn't open STDIN for reading: $!";
        } else {
                open(LOG_FILE,"$file") || die "Couldn't open logfile '$file': $!";
	}

	$lineno = 0;

	while(<LOG_FILE>) {
		$lineno++; $full_line = $_;

		next if /^#/;
		next if /^format=/;
		if(($logtype eq "common") || ($logtype eq "cern-proxy")) {
			#	split the input line into its various components
			( $host, $rfc931, $user, $longtime, $request, $type, $size)
			= /^(\S+) (\S+) (\S+) \[(.+)\] \"(.*)\" (\S+) (\S+)\s/;
			$line = $_;
		} elsif($logtype eq "ncsa-extended") {
			#	split the input line into its various components
			( $host, $rfc931, $user, $longtime, $request, $type, $size, $user_agent, $referer, $download_time)
                	= /^(\S+) (\S+) (\S+) \[(.+)\] \"(.*)\" (\S+) (\S+) \"(.+)\" \"(.+)\" \S+\s*/;
			$line = $_;
		} elsif( $logtype eq "squid-emulated" ) {
			($host, $rfc931, $user, $longtime, $request, $type, $size)
				= /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"([^\"]*)\" (\S+) (\S+)$/;
		} elsif($logtype eq "netscape-proxy") {
			#	split the input line into its various components
			($host,$rfc931,$user,$longtime,$request,$type,$size,$type_r,$size_r,$c_body,$p_body,
				$h1,$h2,$h3,$h4,$download_time)
			= /^(\S+) (\S+) (\S+) \[(.+)\] \"(.+)\" (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+) (\S+)$/;
			$line = $_;
		} elsif( $logtype eq "squid" ) {
			undef($cache); undef($proxy);
			@line = split(/\s+/,$_,10);
			$line = $_;
		
			$time = $line[0];
			$elapsed = $line[1];
			$host = $line[2];
			$codes = $line[3];
			$size = $line[4];
			$htype = $line[5];
			$url = $line[6];
			$ident = $line[7];
			$hiercodes = $line[8];
			$ContentType = $line[9];
	
			($seconds,$milliseconds) = split(/\./,$time);
			if(!$seconds || $seconds !~ /\d\d\d+/) {
				chomp;
				print STDERR "Error: line $lineno: no time on line [$_]\n";
				next;
			}
			($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($seconds);
			$year += 1900;
			$mon  = "0" . "$mon" if($mon  < 10); $mon++;
			$mday = "0" . "$mday" if($mday < 10);
			$hour = "0" . "$hour" if($hour < 10);
			$min  = "0" . "$min"  if($min  < 10);
			$sec  = "0" . "$sec"  if($sec  < 10);
	
			$longtime = "$mday/$r_months{$mon}/$year:$hour:$min:$sec +0000";
	
			($type,$code,$remotetype) = split(/\//,$codes);
			$request = "$htype $url HTTP/1.0";
		}

		if($logtype eq "squid" || $logtype eq "squid-emulated") {
			if( $type eq "TCP_DENIED" ){
				$proxy = 1;
				$code = "401";
			} elsif( $type =~ /TCP_HIT/  ||  ($type =~ /UDP_HIT_OBJ/)
			|| ($type =~ /SIBLING_HIT/) || ($type =~ /TCP_IMS_HIT/)
			|| ($hiercodes =~ /SIBLING_HIT/) || ($hiercodes =~ /PARENT_HIT/)
			|| ($type =~ /TCP_REF_FAIL_HIT/) || ($type =~ /TCP_REFRESH_HIT/)
			|| ($type =~ /PARENT_HIT/)) {
				$cache = 1;
				$proxy = 0;
				$code = "200";
			} else {
				$proxy = 1;
				$cache = 0;
				$code = "200";
			}
			$type = $code;
			$neighbours{$host} = 1 if(!$neighbours{$host} && $request =~ m/ICP_QUERY/);
			$request =~ s/ICP_QUERY/GET/g;
		} elsif($logtype eq "netscape-proxy") {
			if($type_r ne "304") { $proxy = 1; $cache = 0; }
			else { $proxy = 0; $cache = 1; }
		}
	
		$URL = $request;
		$URL =~ m#[A-Z_]* ?([^ ]+) ?(HTTP\/\d.\d)?$#;
		$URL = $1;

		# do the url and username exclude checks
		next if( $complete_exclude_url_patt && $URL =~ m/$complete_exclude_url_patt/i );
		next if( $complete_exclude_user && $user =~ m/$complete_exclude_user/i );

		# do the hostname exclude check before & after converting IP -> FQDN
		next if( $complete_exclude_host && $host =~ m/$complete_exclude_host/i );

		# convert IP into hostname
		if( $dns_lookup ) {
			if( $hostnames{$host} ) { $name = $hostnames{$host}; $host = $name;}
			else {
				if($host =~ /\d+\.\d+\.\d+\.\d+/) {
					@address = split(/\./,$host);
					$addpacked = pack('C4',@address);
					($name,$aliases,$addrtype,$length,@addrs)
						= gethostbyaddr($addpacked,$address_type);
					$name = $host if(!$name);
				} else {
					$name = $host;
				}
				$name = "\L$name";
				$hostnames{$host} = $name;
				$host = $name;
			}
		}

		# check validity of hostname
		if($host !~ /\d+\.\d+\.\d+\.\d+/) {
			$remainder = $host;
			$remainder =~ s/[a-zA-Z0-9\.\-]//g;
						# not really concerned about underscores
						# the main problem is with idiots like '*.blah.com'
			$remainder =~ s/_//g unless $config{'care_about_underscores'};

			if($remainder) {
				print "Error: illegal character";
				printf "%s", (length($remainder) == 1) ? "" : "s";
				print " in hostname $host [$remainder] skipping (line $lineno).\n";
				next;
			}
		}

		next if( $complete_exclude_host && $host =~ m/$complete_exclude_host/i );

		($datetime) = split(/ /,$longtime,1);
		($date, $time) = split(/:/,$datetime,2);
		($hour, $minute, $second) = split (/:/,$time);
		$second =~ s/\ \+.+$//;
		$timeshort = "$hour:$minute";
		( $dd, $mm, $yy ) = split (/\//,$date);
		$month = $months{$mm};
		$datelong =  "$yy$month$dd";
		$timelong = "$datelong$hour";
		$curr_date =  "$yy$month$dd$hour$minute$second";

		if( $verbose && ($line_count ne "undef")) {
			$counter++;
			if( $counter >= $inc ) {
				$counter = 0;
				$hash_counter ++;
 				printf STDERR "\#" if($hash_counter <= 50);
			}
		}

		if( $foundline ) {
			&increment_counters();
		} else {
			if( $curr_date > $last_date) {
				$foundline = "true";
				&increment_counters();
			} else {
				next;
			}
		}
	}
	close(LOG_FILE);
	
	$totalhosts = keys(%hosts);
	$totalhours = keys(%timelongs);
	$totaldates = keys(%dates);
	$total_remote_hosts = keys(%remote_hosts);
	$unique_reqs = keys(%requests) if(!$exclude_reqs);

	if ($dateFinish == 0) {$dateFinish = $dateStart;}
	$lastline = $curr_date; 
	
	if( $hash_counter < 50 ) {
		while( $hash_counter <= 50 ) {
			$hash_counter++;
 			if( $verbose ) { printf STDERR "\#" if($hash_counter <= 50); }
		}
	}
 	if( $verbose ){
 		printf STDERR "\n   Finished.\n\n";
 	}

	if( $total == 0 ) {
		print "No intelligible lines produced from the read of your logfile\n" .
		      "Either:\na) Something is wrong with your logfile (it should be in " .
		      "common log format), or\n b) you have re-run a period without adjusting \n" .
		      "the line in the file $outdir/$server.lastline.\n" .
		      "\nPwebstats now aborting this run.\n\n";
		exit(0);
	}
}   #	End read_logfile();

sub increment_counters
{
	$size = 0 if ($size eq "-");

	if($type eq "404")
	{
		$error_total++;
		next;
	}

	if ( ($datelong =~ m/\[/i) || ( $datelong !~ m/^\d?\d\d\d\d\d\d\d$/i )){
		next; 
	}

	#	find start & finish dates of the logging period
	if( $datelong < $currDateStart )
	{ 
		$currDateStart = $datelong;
		$dateStart = $date;
		$dateStart =~ s/\//\ /g;
	}
	elsif( $datelong > $currDateFinish ) {
		$currDateFinish = $datelong;
		$dateFinish = $date;
		$dateFinish =~ s/\//\ /g;
	}
		
	$_ = $request;
		
	($method, $request, $http_level) = /^(\S+)\s+(.+)\s*(HTTP\/\d\.\d)?$/;
	$request =~ s/HTTP\/1\..//;
	$request =~ s/\s+$//;
	
	if($proxy_mode)
	{

		$_ = $request;
		if($method =~ /CONNECT/) {
			($remote_host,$port) = /^(\w+):\d+$/;
		} else {
			($protocol,$remote_host) = /^(\w+):\/\/([^\/]+)\/?.*$/;
		}
		$remote_host =~ s/\:\d+$//;
		if( $protocol =~ m/http|ftp|file|gopher|news|wais/i )
		{
			$protocol_count{$protocol} += 1;
			$protocol_size{$protocol} += $size;
		}
		else
		{
			if( /^\/http/ )
			{
				$protocol_count{internal_icons} += 1;
				$protocol_size{internal_icons} += $size;
			}
			else
			{
				if( $request ne "")
				{
					$protocol_count{other} += 1;
					$protocol_size{other} += $size;
					print OTHER_PROTOCOLS "Proxy: $lineno: $_, $request\n" if ($proxy && $debug);
					print OTHER_PROTOCOLS "Cache: $lineno: $_, $request\n" if ($cache && $debug);
				}
			}
		}
	}

	#	Plug the values into associative arrays (or increment if already there)
	
	$total++;
	$bytes += $size;
	$hosts{$host}++;
	$hostbytes{$host} += $size;
	$timeshorts{$timeshort}++;
	$timelongs{$timelong}++;
	$times{$hour}++;
	$dates{$datelong}++;


	$requests{$request}++ if(($request !~ m/$exclude_from_stats/) && (!$exclude_reqs));
	if( $page_config )
	{
		$all_requests{$request}++;
		$requests_hosts{$request} .= "$host," if( $requests_hosts{$request} !~ m/$host/ );
		if( $local_patt && ($host =~ m/$local_patt/i))
		{
			$requests_local{$request}++;
			$local_hosts{$request} .= "$host," if( $local_hosts{$request} !~ m/$host/ );;
		}
	}

	if( $proxy_mode )
	{
		if( $cache && ! $proxy_only ){
			$cache_size += $size;
			$cache_requests++;
		}
		elsif( $proxy ){
			$proxy_size += $size;
			$proxy_requests++;
		}
	}

	if( ! $proxy_mode )
	{
		if(($host =~ m/$local_patt/i) || ($host !~ /\./))
		{
			$domains{loc}++;
		}
		else
		{
			$_ = $host;
			if( /\d+\.\d+\.\d+/ )
			{
				$domains{unres} ++;
			}
			else
			{
				($domain) = /.+\.(\w+)$/;
				$domain = "\L$domain";
				$domains{$domain}++;
			}
		}
	}
	else
	{
		if(($remote_host =~ m/$local_patt/i) || ($remote_host !~ /\./))
		{
			$domains{loc}++;
		}
		else
		{
			$_ = $remote_host;
			if( /^\d+\.\d+\.\d+\.\d+$/ )
			{
				$domains{unres} ++;
			}
			else
			{
				($domain) = /.+\.(\w+)$/;
				$domain = "\L$domain";
				$domains{$domain}++;
			}
		}
		$remote_hosts{$remote_host}++;
		$remote_total++;

		$remote_hostbytes{$remote_host} += $size;
	}

}

### ----------------------------------------------------------------------- ###
# Build various arrays

sub build_arrays
{

	### --------------------------------------------------- ###
	# Build a sorted array of hosts/accesses

	printf STDERR "   Building Hosts/Accesses Array..." if($verbose);
	open(HOSTS,"> $hosts_tmp") or die "Couldn't open $hosts_tmp for writing: $!\n";
	for( sort { $hosts{$b} <=> $hosts{$a}; } keys %hosts )
	{
		print HOSTS "$_\t$hosts{$_}\n" if ($hosts{$_} >= $host_threshold);
	}
	close(HOSTS);
 	printf STDERR "done.\n" if($verbose);

	### --------------------------------------------------- ###
	# Build an assoc. array of domains and real names

	open(DOMAINS,"$domain_file") or die "Couldn't open $domain_file for reading: $!\n";
	while( <DOMAINS> )
	{
		chop;
		($discard, $domain, $string) = split(/\t/);
		chop($domain);
		$string =~ s/\"//g;
		$domain =~ s/\.//g;
		$domain_names{$domain}="$string";
	}
	close(DOMAINS);

	$domain_names{loc} = "Local Domain";
	$domain_names{unres} = "Unresolved";

	printf STDERR "   Building Domains/Accesses Array... " if($verbose);
	for( sort { $domains{$b} <=> $domains{$a}; } keys %domains )
	{
		push(@sorteddomains,"$_||$domains{$_}")
			if ($domains{$_} >= $domain_threshold);
	}
 	printf STDERR "done.\n" if($verbose);

	### --------------------------------------------------- ###

	if( $proxy_mode )
	{
	 	printf STDERR "   Building Remote Hosts/Accesses Array..." if($verbose);
		foreach $key (keys %remote_hosts)
		{
			if( $remote_hosts{$key} < $remote_host_threshold ) { delete($remote_hosts{$key}); }
		}
		printf STDERR "  " if($verbose);
		open(REMOTE_HOSTS,"> $remote_hosts_tmp") or die "Couldn't open $remote_hosts_tmp for writing: $!\n";
		for( sort { $remote_hosts{$b} <=> $remote_hosts{$a}; } keys %remote_hosts )
		{
			print REMOTE_HOSTS "$_\t$remote_hosts{$_}\n";
		}
		close(REMOTE_HOSTS);
	 	printf STDERR "done.\n" if($verbose);
	}

	### --------------------------------------------------- ###
	if( $proxy_mode )
	{
		if( ! $exclude_reqs )
		{
			printf STDERR "  " if($verbose);
			open(REQUESTS,"> $requests_tmp") or die "Couldn't open $requests_tmp for writing: $!\n";
			for( sort { $requests{$b} <=> $requests{$a}; } keys %requests )
			{
				print REQUESTS "$_\t$requests{$_}\n" if ($requests{$_} >= $item_threshold);
			}
			close(REQUESTS);
 			printf STDERR "done.\n" if($verbose);
		}
	}
	else
	{
		printf STDERR "   Building Requests/Accesses Array..." if($verbose);
		open(REQUESTS,"> $requests_tmp") or die "Couldn't open $requests_tmp for writing: $!\n";
		for( sort { $requests{$b} <=> $requests{$a}; } keys %requests )
		{
			print REQUESTS "$_\t$requests{$_}\n" if ($requests{$_} >= $item_threshold);
		}
		close(REQUESTS);
 		printf STDERR "done.\n" if($verbose);

		if($debug)
		{
			open(REQUESTS,"> $all_requests_tmp") or die "Couldn't open $all_requests_tmp for writing: $!\n";
			for( sort { $all_requests{$b} <=> $all_requests{$a}; } keys %all_requests )
			{
				print REQUESTS "$_\t$all_requests{$_}\n";
			}
			close(REQUESTS);
		}
	}

	### --------------------------------------------------- ###

	if( $proxy_mode )
	{
	 	printf STDERR "   Building Protocol Summary Array... " if($verbose);
		for( sort { $protocol_count{$b} <=> $protocol_count{$a}; } keys %protocol_count )
		{
			push(@sortedprotocols,"$_||$protocol_count{$_}") if ($protocol_count{$_} >= $protocol_threshold);
		}
 		printf STDERR "done.\n" if($verbose);
	}

	### --------------------------------------------------- ###

 	printf STDERR "   Summarising Requests by Hour... " if($verbose);
	$maxHourlyRequests="0";
	foreach $key (sort keys(%times))
	{
		$value = sprintf "%d", $times{$key}/$totaldates;

		if($value > $maxHourlyRequests)
		{
		     $maxHourlyRequests = $value;
		}
		$accesses_per_hour[$key] = "$value";
	}
 	printf STDERR "done.\n" if($verbose);
}

### ----------------------------------------------------------------------- ###

sub print_changes {

	# Subroutine for writing out changes in stats from last period

	$output=pop(@_);
	select($output);
	$change_in_requests= ((($total - $tot_reqs) / $tot_reqs ) * 100) unless ( $tot_reqs == 0);
	$change_in_bytes= ((($bytes - $bytes_sent) / $bytes_sent ) * 100) unless ( $bytes_sent == 0);
	$change_in_hosts= ((($totalhosts - $unique_hosts) / $unique_hosts) * 100) unless ( $unique_hosts == 0);
	print "<pre><strong>Changes since last summary period:</strong>\n";
	printf "In total requests to this server:        %3d%%\n",$change_in_requests;
	printf "In total bytes sent by this server:      %3d%%\n",$change_in_bytes;
	printf "In number of hosts accessing the server: %3d%%\n",$change_in_hosts;
	print "</pre>\n";

}

### ----------------------------------------------------------------------- ###
sub write_cumulative_stats {

	# Subroutine for writing out cumulative stats from database file

	printf STDERR "\n-- Writing Out Cumulative Details.\n\n" if($verbose);
	
	open(CURR,"> $outdir/$server.current") or die "Couldn't open $outdir/$server.current for writing: $!\n";
	print CURR "$period:$dateFinish";
	close(CURR);

	open(LASTLINE,"> $outdir/$server.lastline") or die "Couldn't open $outdir/$server.lastline for writing: $!\n";
	print LASTLINE $lastline;
	close(LASTLINE);

	if ( -e "$outdir/$server.db"){
        open(LASTSTATS,">> $outdir/$server.db") or die "Couldn't open $outdir/$server.db for appending: $!\n";
	} else {
        open(LASTSTATS,"> $outdir/$server.db") or die "Couldn't open $outdir/$server.db for writing: $!\n";
	}
	# format for pervious period's stats:
	# total reqs, bytes sent, unique hosts accessing server,
	# date starting stats period, date ending stats peroid, period number
	if( $proxy_mode && ! $proxy_only ){
		print LASTSTATS "$period:$total:$cache_requests:$proxy_requests:$bytes:$cache_size:$proxy_size:$totalhosts:$dateStart:$dateFinish\n";
	}
	else
	{
		print LASTSTATS "$period:$total:$bytes:$totalhosts:$dateStart:$dateFinish\n";
	}
	close(LASTSTATS);
}


### ----------------------------------------------------------------------- ###
sub read_cumulative_stats {

	# Subroutine for reading in cumulative stats from database file
	printf STDERR "-- Reading in previous $plurals{$interval}'" .
              " details.\n" if( $verbose );

	if( -e "$outdir/$server.current"  )
	{
		open(CURR,"$outdir/$server.current") or die "Couldn't open $outdir/$server.current for reading: $!\n";
		while( <CURR> )
		{
			($period,$date_end) = split(/:/);
		}
		close(CURR);
		$period++;
	}
	else
	{
		$period = 1;
		if( $verbose ){ printf STDERR "   [First Run - No Previous Periods]\n\n";}
	}
	if( -e "$outdir/$server.db" )
	{
		open(LASTSTATS,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		# format for pervious period's stats:
		# total reqs, bytes sent, unique hosts accessing server
		while( <LASTSTATS> )
		{
			if( $proxy_mode && ! $proxy_only )
			{
				($period_no,$tot_reqs,$cache_re,$proxy_re,$bytes_sent,$cache_si,
				$proxy_si, $unique_hosts,$start_date,$finish_date)=split(/:/);
			}
			else
			{
				($period_no,$tot_reqs,$bytes_sent,
				$unique_hosts,$start_date,$finish_date)=split(/:/);
			}
			if( $proxy_mode && ! $proxy_only )
			{
				$d_b{$period_no}="$tot_reqs:$cache_re:$proxy_re:$bytes_sent:$cache_si:$proxy_si;$unique_hosts:$start_date:$finish_date";
			}
			else
			{
				$d_b{$period_no}="$tot_reqs:$bytes_sent:$unique_hosts:$start_date:$finish_date";
			}
		}
		close(LASTSTATS);
	}
	else
	{
		$tot_reqs=$total; $bytes_sent=$bytes; $unique_hosts=$unique_reqs;
	}
	if( -e "$outdir/$server.lastline" )
	{
		open(LASTLINE,"$outdir/$server.lastline") or die "Couldn't open $outdir/$server.lastline for reading: $!\n";
		$lastline = <LASTLINE>;
		close(LASTLINE);
	}
	else
	{
		$lastline = "NULL";
	}
}

### ----------------------------------------------------------------------- ###
# Subroutines for writing to output files and sorting
### ----------------------------------------------------------------------- ###

sub read_and_write_master_page {
	
	if( ! -e "$outdir/$index")
	{
		open(INDEX,"> $outdir/$index") or die "Couldn't open $outdir/$index for writing: $!\n";
		print INDEX "<html>\n<head>\n<title>Usage Statistics: $Server_header</title>\n</head>\n";
		print INDEX "<body>\n<h1 align=\"center\">Usage Statistics:<br>$Server_header</h1>\n";
		print INDEX "<p align=\"center\"><img src=\"images/growth.gif\" alt=\"[Growth Chart]\"></p>\n";
		print INDEX "<ul>\n   <li><p><a href=\"$gross_index\">Gross Statistics</a><br>\n";
		print INDEX "             Raw Stats: accesses per hour, day, $singulars{$interval}, host, request, etc.</p>\n";
		if( $page_config )
		{
		print INDEX "         <li><p><a href=\"$page_index\">Collection Statistics</a><br>\n";
		print INDEX "             Specific Stats: accesses per page, collections of pages, etc.</p>\n";
		}
		print INDEX "</ul>\n<p></p>\n";
		&add_footer(INDEX);
		close(INDEX);
	}
	if( -e "$outdir/$gross_index" )
	{
		if(-z "$outdir/$gross_index") {
			open MASTER,">$outdir/$gross_index" or die "Couldn't open $outdir/$gross_index for writing: $!\n";
			opendir DIR_HANDLE, "$outdir/$plurals{$interval}" or die "Can't open $outdir/$plurals{$interval}";
			my @file_list= grep /\.html$/ , readdir DIR_HANDLE;
			print MASTER <<EOD;
<html><head><title>Server Usage: $Server_header</title></head>
<body>
<h1>$Server_header</h1>
<p><IMG SRC="$plurals{$interval}/images/$interval.gif" alt="[$interval usage chart]"></p>
<p>
<table cellspacing="2" cellpadding="1" border="0">
<!--Periods-->
<tr><td align="left"><kbd><a href="$plurals{$interval}/$server.$period.html">
EOD
			printf MASTER "$Titles{$interval} $period</a></kbd></td><td align=\"right\"> <kbd> ($dateStart - $dateFinish)</kbd></td><td align=\"right\"> <kbd> %8d</kbd><br></td></tr>\n",$total;

			foreach $filename (@file_list) {
				($discard,$period_number,$discard)=split /\./, $filename;
				open(THEFILEHANDLE,"$outdir/$plurals{$interval}/$filename")
					or die "Couldn't open $outdir/$plurals{$interval}/$filename: $!\n";
				my $c=0;
				while(<THEFILEHANDLE>){
					if (/^Period covered/ ){
						$c++;
						($discard,$the_period) = split /\<\/em\>/;
						chomp $the_period;
					}
					if (/^Total.+requests/ ){
						$c++;
						s/,//g;
						($reqs) = /(\d+)/;
					}
					if(c==2){
						break;
					}
				}
				close(THEFILEHANDLE);
				$info=[$period_number,$plurals{$interval}."/".$filename,$the_period,$reqs];
				push @list_of_file_info , $info;
			}

			@sorted_list_of_file_info = sort {  $$b[0] <=> $$a[0] } @list_of_file_info ;

			foreach $ref (@sorted_list_of_file_info){
    				my ($period_number,$href,$the_period,$reqs)=@$ref;
    				printf MASTER "<tr><td align=\"left\"><kbd><a href=\"%s\">$Titles{$interval} %s</a></kbd></td><td align=\"right\"> <kbd>%s</kbd></td><td align=\"right\"><kbd>%s</kbd><br></td></tr>\n",$href,$period_number,$the_period,$reqs;
			}
			print MASTER "</table>\n";
			&add_footer(MASTER);
		} else {
			open(MASTER,"$outdir/$gross_index") or die "Couldn't open $outdir/$gross_index for reading: $!\n";
			open(TMP,"> $temp/master.tmp.$$")  or die "Couldn't open $temp/master.tmp.$$ for writing: $!\n";
		
			while(<MASTER>) {
				if ( $_ =~ /\<\!\-\-Periods\-\-\>/) {
					print TMP "<!--Periods-->\n";
					print TMP "<tr><td align=\"left\"><kbd><a href=\"$plurals{$interval}/$server.$period.html\">";
					printf TMP "$Titles{$interval} $period</a></kbd></td><td align=\"right\"> <kbd> ($dateStart - $dateFinish)</kbd></td><td align=\"right\"> <kbd> %8d</kbd><br></td></tr>\n",$total;
				}
				else
				{
					print TMP $_;
				}
			}
			close(MASTER); close(TMP);
    	
			open(TMP,"$temp/master.tmp.$$") or die "Couldn't open $temp/master.tmp.$$ for reading: $!\n";
			open(MASTER,">$outdir/$gross_index") or die "Couldn't open $outdir/$gross_index for writing: $!\n";
			while( <TMP> ){ print MASTER $_; }
			close(MASTER); close(TMP);
		}
	}
	else{
		open(MASTER,"> $outdir/$gross_index") or die "Couldn't open $outdir/$gross_index for writing: $!\n";
		select(MASTER);
		print "<html><head><title>Server Usage: $Server_header</title></head>";
		print "<body>\n<h1>$Server_header</h1>\n";
		print "<p><IMG SRC=\"$plurals{$interval}/images/$interval.gif\" alt=\"\[$interval usage chart\]\"></p>\n";
		print "<p>\n<table cellspacing=\"2\" cellpadding=\"1\" border=\"0\">\n";
		print "<!--Periods-->\n";
		print "<tr><td align=\"left\"><kbd><a href=\"$plurals{$interval}/$server.$period.html\">";
		printf "$Titles{$interval} $period</a></kbd></td><td align=\"right\"> <kbd> ($dateStart - $dateFinish)</kbd></td><td align=\"right\"> <kbd> %8d</kbd><br></td></tr>\n",$total;
		print "</table>\n";
		&add_footer(MASTER);
		close(MASTER);
	}
	if( $page_config )
	{
		if( -e "$outdir/$page_index" )
		{
			open(MASTER,"$outdir/$page_index") or die "Couldn't open $outdir/$page_index for reading: $!\n";
			open(TMP,"> $temp/master.tmp.$$") or die "Couldn't open $temp/master.tmp.$$ for writing: $!\n";
		
			while( <MASTER> ){
    				if ( $_ =~ /\<\!\-\-Periods\-\-\>/)
    				{
    					print TMP "<!--Periods-->\n";
    					print TMP "<li><kbd><a href=\"pages/pages.$period.html\">";
    					print TMP "$Titles{$interval} $period</a> ($dateStart - $dateFinish)</kbd></li>\n";
    				}
    				else
    				{
    					print TMP $_;
    				}
			}
    			close(MASTER); close(TMP);
    	
    			open(TMP,"$temp/master.tmp.$$") or die "Couldn't open $temp/master.tmp.$$ for reading: $!\n";
			open(MASTER,">$outdir/$page_index") or die "Couldn't open $outdir/$page_index for writing: $!\n";
    			while( <TMP> ){ print MASTER $_; }
    			close(MASTER); close(TMP);
		}
		else
		{
			open(MASTER,"> $outdir/$page_index") or die "Couldn't open $outdir/$page_index for writing: $!\n";
			select(MASTER);
			print "<html><head><title>Server Usage: $Server_header</title></head>";
			print "<body>\n<h1 align=\"center\">$Server_header</h1>\n";
			print "<h3>Page and collection based Statistics</h3>\n";
			print "<ul>\n";
			print "<!--Periods-->\n";
			print "<li><kbd><a href=\"pages/pages.$period.html\">";
			print "$Titles{$interval} $period</a> ($dateStart - $dateFinish)</kbd></li>\n";
			print "</ul>\n";
			&add_footer(MASTER);
			close(MASTER);
		}
	}
	else
	{
		chdir("$outdir");
		unlink("$index");
		if($OS =~ /Win/i) {
			local($thedir)=$outdir;
			$thedir =~ s/\//\\/g;
			print STDERR "thedir = [$thedir]\n";
			system("copy \"$outdir\\$gross_index\" \"$thedir\\$index\"");
			print STDERR "cmd = [copy \"$thedir\\$gross_index\" \"$thedir\\$index\"]\n";
		} else {
			symlink("$gross_index","$index");
		}
	}
	unlink("$temp/master.tmp.$$") if ( -e "$temp/master.tmp.$$" );
}

### ----------------------------------------------------------------------- ###
#
#  get_page_stats(logfile,page-config-file)  creates an associative 
#    array of hits indexed by page (or pattern).
#
# Format for page-config-file:
#       Page (or pattern) : Description of Page : indent-level
#
#   indent-level is 0,1,2,3... for heirarchical collections (subsections).
#
### ----------------------------------------------------------------------- ###

sub get_page_stats{
	local($log_file);
	local($config);

	$log_file = shift(@_);
 	$config = shift(@_);
 
 	open(CONFIG,"$config") or die "Couldn't open $config for reading: $!\n";
 	while( <CONFIG> )
 	{
 		next if( /^#/ || /^$/ );
 		chop;
		($page, $description,$level,$url) = split(/:/,$_,4);
 		$page_descriptions{$page} = $description;
 		$page_levels{$page} = $level;
 		$page_urls{$page} = $url unless($url eq "");
 		push(@page_array, $page);
 	}
 	close(CONFIG);
 
	# initialise the array - we want to report no accesses
 	foreach $page (@page_array)
 	{
		$page_hits{$page} = 0;
	}

	# for each pattern, step through the array of requests/accesses
	# and tally up the hits.
 	foreach $page (@page_array)
 	{
		next if ( $page eq "--");
		foreach $url (keys(%all_requests))
		{
			if( $url =~ m#^$page$# )
			{
				$page_hits{$page} += $all_requests{$url};
				$local_hits{$page} += $requests_local{$url};

				$hosts = $requests_hosts{$url};
				chop($hosts) if($hosts =~ /,$/);
				@line_hosts = split(/,/,$hosts);
				foreach $host (@line_hosts)
				{
					$hosts_list{$page} .= "$host," if($hosts_list{$page} !~ /$host/i);
				}
				undef(@hosts);
			}
		}
	}
	undef(@hosts);

	foreach $item (keys(%hosts_list))
	{
		chop($hosts_list{$item});
		@hosts = split(/,/,$hosts_list{$item});
		$hosts_pages{$item} = $#hosts +1;
	}

}
### ----------------------------------------------------------------------- ###
#  print_page_stats(page-array,file)  prints out stats for the
#  associative array returned by get_page_stats to <file>.
### ----------------------------------------------------------------------- ###
sub print_page_stats{
 
	my ($outfile,$table) = (@_);

	printf $outfile "%-55s:%6s:%7s:%9s\n", "Collection or Page", " #hits", " %Local", "visitors";
	for($i=0;$i<55;$i++){$line1.="-";} for($i=0;$i<6;$i++){$line2.="-";$line3.="-";$line4.="-";}
	$line4.="-";$line4.="-";$line4.="-";$line3.="-";
	printf $outfile "$line1 $line2 $line3 $line4\n";

	foreach $page (@page_array)
	{
		$width = "";
		for( $i=0; $i < $page_levels{$page}; $i++)
		{ 
			$width .= "    ";
		}
		if( $page_hits{$page} ){
			$local_pct = sprintf "%3.1d", ($local_hits{$page}/$page_hits{$page}*100) if( $page_hits{$page} != 0 );
			if( $page_urls{$page} )
			{
				$page_desc = "<a href=\"$page_urls{$page}\">";
				$page_desc .= sprintf "%s",
					&escape_chars($page_descriptions{$page});
				$page_desc .= "</a>";
				$plen = 55 - length($width) - length($page_descriptions{$page});
				if($plen>0){$padding = "";for($i=0;$i<$plen;$i++){$padding.=" ";} }
				$desc = "$width" . "$page_desc" . "$padding";
				if($table eq "yes")
				{
					printf $outfile "<tr><td>%s</td><td>%6d</td><td>%5s%%</td><td>%8s</td></tr>\n",
						$desc, $page_hits{$page},$local_pct, $hosts_pages{$page};
				}
				else
				{
					printf $outfile "%s:%6d:%5s%% :%8s\n",
						$desc, $page_hits{$page},$local_pct, $hosts_pages{$page};
				}
			}
			else 
			{
				$page_desc = $page_descriptions{$page};
				$desc = "$width" . "$page_desc";
				if($table eq "yes")
				{
					printf $outfile "<tr><td>%-55s</td><td>%6d</td><td>%5s%%</td><td>%8s</td></tr>\n",
						$desc, $page_hits{$page},$local_pct, $hosts_pages{$page};
				}
				else
				{
					printf $outfile "%-55s:%6d:%5s%% :%8s\n",
						$desc, $page_hits{$page},$local_pct, $hosts_pages{$page};
				}
			}
		}
		elsif( $page eq "--" )
		{
			printf $outfile "\n";
		}
	}
}

### ----------------------------------------------------------------------- ###
sub add_footer
{
	#   Adds a standard footer to the filehandle given as an argument

	$outfile= pop(@_);
	open(FOOTER,$footer) or die "Couldn't open $footer for reading: $!\n";
	while (<FOOTER>)
	{
		$_ =~ s/VERSION/$version/g if($_ =~ m/VERSION/);
		print $outfile $_ ;
	}
	close(FOOTER);
}

### ----------------------------------------------------------------------- ###
sub commas
{
	# turn a number string 1234567 into 1,234,567

	local($_)=@_;
	$_ = sprintf "%.0f", $_;
	1 while s/(.*\w)(\w\w\w)/$1,$2/;
	s/ //g;
	$_;
}

### ----------------------------------------------------------------------- ###
sub find_prog
{
	# find the path to a program if it's in your PATH, or return nothing.

	local( $prog ) = @_;
	local( $path ) = $ENV{ 'PATH' };

	if($OS =~ /Win/i) {
		print STDERR "Searching for $prog\n" if $debug;
		foreach $dir ( split( /;/, $path ) ) {
			print STDERR "Searching in $dir\n" if $debug;
			local( $path ) = "$dir\$prog";
			if( -x $path && ! -d $path ) {
				print STDERR "Found $path\n" if $debug;
				return $path;
			}
			local( $path ) = "$dir\${prog}.EXE";
			$path = "\U$path";
			print STDERR "Found $path!\n" if( -e $path && $debug);
			if( -x $path && ! -d $path ) {
				print STDERR "Found $path\n" if $debug;
				return $path;
			}
		}
	} else {
		print STDERR "Searching for $prog\n" if $debug;
		foreach $dir ( split( /:/, $path ) ) {
			print STDERR "Searching in $dir\n" if $debug;
			local( $path ) = "$dir/$prog";
			if( -x $path && ! -d $path ) {
				print STDERR "Found $path\n" if $debug;
				return $path;
			}
		}
	}
	return '';
}

### ----------------------------------------------------------------------- ###

sub time_now
{
	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst);

	($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
	$year += 1900; $hour = "0" . $hour if($hour < 10);
	$min = "0" . $min if($min < 10); $sec = "0" . $sec if($sec < 10);

	$now = "$hour:$min:$sec on $mday $longmonths{$mon} $year";

	$now;
}

### ----------------------------------------------------------------------- ###

sub error
{
	my $err_message = pop(@_);
	
	print STDERR "Error: $err_message\n\n";
	&usage();
}

### ----------------------------------------------------------------------- ###

sub get_width
{
	my $arg = pop(@_);
	my $tmp_gw = "$temp/fly.$$";

	&error("Couldn't find image: $arg\n") if( ! -e $arg );
	open(FLY,"> $tmp_gw") or die "Couldn't open $tmp_gw for reading: $!\n";
	print FLY "existing $arg\n";
	print FLY "sizex\n";
	print FLY "sizey\n";
	close(FLY);

	if($OS =~ /Win/i) {
		open(OUT, "$fly_program -i $tmp_gw -o $temp/temp.gif |")
			or die "Couldn't run $fly_program -i $tmp_gw -o $temp/temp.gif: $!\n";
	} else {
		open(OUT, "$fly_program -i $tmp_gw -o /dev/null |")
			or die "Couldn't run $fly_program -i $tmp_gw -o /dev/null: $!\n";
	}
	while( <OUT> )
	{
		($x) = /is\ (\d+)$/ if /Size\ -\ X/;
	}
	close(OUT);
	unlink($tmp_gw);

	$x;
}

### ----------------------------------------------------------------------- ###

sub escape_chars
{
	my $string = pop(@_);
	$string =~ s/&/\&amp\;/g;
	$string =~ s/>/\&gt\;/g;
	$string =~ s/</\&lt\;/g;
	$string =~ s/\"/\&quot\;/g;
	return $string;
}

### ----------------------------------------------------------------------- ###

sub clean_up
{
	unlink("$domain_tmp")       unless( $debug );
	unlink("$hosts_tmp")        unless( $debug );
	unlink("$remote_hosts_tmp") unless( $debug );
	unlink("$requests_tmp")     unless( $debug );
}

### ----------------------------------------------------------------------- ###

sub print_proxy_growth{

	# Subroutine for printing growth proxy usage chart

	$fly_control = "$temp/fly.$singulars{$interval}.$period.$$";
	$growth_control = "$temp/fly.growth.$$";
	open(GROWTH,"> $growth_control") or die "Couldn't open $growth_control for writing: $!\n";
	print GROWTH "new\nsize 195,85\n";
	print GROWTH "copyresized 35,45,45,55,0,0,195,185,$period_template\n";
	print GROWTH "rect 0,0,194,84,0,0,0\n";

	open(FLY,"> $fly_control") or die "Couldn't open $fly_control for writing: $!\n";
	select(FLY);
	$total_colour="0,0,128";
	$proxy_colour="205,0,0";
	$cache_colour="0,100,0";
	$black="0,0,0";
	print "existing $period_template\n";

	$startx=29; $starty=208; $height=168;
	$maxrequests=0; $width = 389;

	$gstartx=0; $gstarty=85; $g_height=85;
	$maxrequests=0; $g_width = 195;
	
	if(-e "$outdir/$server.db") {
		open(DB,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		while( <DB> ) {
			($period_no,$total_requests,$cache_r,$proxy_r,$total_bytes,$cache_s,$proxy_s,
			$unique_hosts,$start_date,$finish_date)=split(/:/);
			if($total_requests > $maxrequests ) {
				$maxrequests = $total_requests;
			}
		}
		close(DB);
	}
	if($total > $maxrequests ) {
		$maxrequests = $total;
	}
	$horizontal_increment = ($width / $period ) if( $period > 0);
	#$horizontal_increment = sprintf "%d", $horizontal_increment;
	$g_horizontal_increment = ($g_width / $period ) if( $period > 0);

	#print CACHE requests line
	$x = $startx;
	$gx = $gstartx;

	print "line $startx,$starty";
	print GROWTH "line $gstartx,$gstarty";
	if(-e "$outdir/$server.db") {
		open(DB,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		while ( <DB> ) {
			($period_no,$total_requests,$cache_r,$proxy_r,$total_bytes,$cache_s,$proxy_s,
			$unique_hosts,$start_date,$finish_date)=split(/:/);
			$x += $horizontal_increment;  $gx += $g_horizontal_increment;
			$y = ($height -(($cache_r / $maxrequests ) * $height) ) + 40 if($maxrequests>0);
			$gy = ($g_height -(($total_requests / $maxrequests ) * $g_height) ) if($maxrequests>0);
			printf ",%d,%d,$cache_colour\n",$x,$y;
			printf "line %d,%d",$x,$y;
			printf GROWTH ",%d,%d,$cache_colour\n",$gx,$gy;
			printf GROWTH "line %d,%d",$gx,$gy;
		}
		close(DB);
	}
	$x = ( $width + $startx );  $gx += $g_horizontal_increment;
	# $x += $horizontal_increment;  $gx += $g_horizontal_increment;
	$y = ($height - (($cache_requests / $maxrequests ) * $height) ) + 40 if($maxrequests>0);
	$gy = ($g_height -(($total / $maxrequests ) * $g_height) ) if($maxrequests>0);
	printf ",%d,%d,$cache_colour\n",$x,$y;
	printf "line 388,244,418,244,$cache_colour\n";
	printf GROWTH ",%d,%d,$cache_colour\n",$gx,$gy;
	
	#print PROXY requests line
	$x = $startx;
	$gx = $gstartx;
	print "line $startx,$starty";
	print GROWTH "line $gstartx,$gstarty";
	if(-e "$outdir/$server.db") {
		open(DB,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		while ( <DB> ) {
			($period_no,$total_requests,$cache_r,$proxy_r,$total_bytes,$cache_s,$proxy_s,
			$unique_hosts,$start_date,$finish_date)=split(/:/);
			$x += $horizontal_increment;  $gx += $g_horizontal_increment;
			$y = ($height -(($proxy_r / $maxrequests ) * $height) ) + 40 if($maxrequests>0);
			$gy = ($g_height -(($total_requests / $maxrequests ) * $g_height) ) if($maxrequests>0);
			printf ",%d,%d,$proxy_colour\n",$x,$y;
			printf GROWTH ",%d,%d,$proxy_colour\n",$gx,$gy;
			printf "line %d,%d",$x,$y;
			printf GROWTH "line %d,%d",$gx,$gy;
		}
		close(DB);
	}
	$x = ( $width + $startx );  $gx += $g_horizontal_increment;
	# $x += $horizontal_increment;  $gx += $g_horizontal_increment;
	$y = ($height - (($proxy_requests / $maxrequests ) * $height) ) + 40 if($maxrequests>0);
	$gy = ($g_height -(($total / $maxrequests ) * $g_height) ) if($maxrequests>0);
	printf ",%d,%d,$proxy_colour\n",$x,$y;
	printf "line 248,244,278,244,$proxy_colour\n";
	printf GROWTH ",%d,%d,$proxy_colour\n",$gx,$gy;

	#print total requests line
	$x = $startx;
	$gx = $gstartx;
	print "line $startx,$starty";
	print GROWTH "line $gstartx,$gstarty";
	if(-e "$outdir/$server.db") {
		open(DB,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		while ( <DB> ) {
			($period_no,$total_requests,$cache_r,$proxy_r,$total_bytes,$cache_s,$proxy_s,
			$unique_hosts,$start_date,$finish_date)=split(/:/);
			$x += $horizontal_increment;  $gx += $g_horizontal_increment;
			$y = ($height -(($total_requests / $maxrequests ) * $height) ) + 40 if($maxrequests>0);
			$gy = ($g_height -(($total_requests / $maxrequests ) * $g_height) ) if($maxrequests>0);
			printf ",%d,%d,$total_colour\n",$x,$y;
			printf GROWTH ",%d,%d,$total_colour\n",$gx,$gy;
			printf "line %d,%d",$x,$y;
			printf GROWTH "line %d,%d",$gx,$gy;
		}
		close(DB);
	}
	$x = ( $width + $startx );  $gx += $g_horizontal_increment;
	# $x += $horizontal_increment;  $gx += $g_horizontal_increment;
	$y = ($height - (($total / $maxrequests ) * $height) ) + 40 if($maxrequests>0);
	$gy = ($g_height -(($total / $maxrequests ) * $g_height) ) if($maxrequests>0);
	printf ",%d,%d,$total_colour\n",$x,$y;
	printf "line 112,244,142,244,$total_colour\n";
	printf GROWTH ",%d,%d,$total_colour\n",$gx,$gy;
	
	# calculate and print peak requests
	$_ = $maxrequests;
	$x=135;
	while( $_ ne "" ){
		s/^(.)//;
		$num=$1;
		$width = $num_size{$num};
		print "copy $x,259,0,0,$width,12,$image_templates/$num.gif\n";
		$x+=$width;
	}
	# calculate and print this period's total requests
	$_=$total;
	$x=303;
	print "copy $x,254,0,0,$interval_width,20,$image_templates/r_$singulars{$interval}.gif\n";
	$x += $interval_width;

	while( $_ ne "" ){
		s/^(.)//;
		$num=$1;
		$width = $num_size{$num};
		print "copy $x,259,0,0,$width,12,$image_templates/$num.gif\n";
		$x+=$width;
	}
	
	# print interval
	print "copy 261,210,0,0,$int_gsize{$interval},$image_templates/i_$interval.gif\n";
	
	# print original start date
	if( $d_b{1} ) { ($discard,$discard,$discard,$discard,$discard,$discard,$discard,$dateorig,$discard)=split(/:/,$d_b{1}); }
	else { $dateorig = $dateStart; }
	if( $dateorig=~ /\n$/ ) { chop($dateorig); }
	($day,$mon,$yea)=split(/ /,$dateorig);
	$mon=~ tr/A-Z/a-z/;
	print "copy 10,214,0,0,24,16,$image_templates/dates/$mon.gif\n";
	($y1,$y2) = $yea =~ m/^\d\d(.)(.)$/;
	$x = 34;
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/10.gif\n", $date_num_size{"'"};
	$x += $date_num_size{"'"};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y1.gif\n", $date_num_size{"$y1"};
	$x += $date_num_size{$y1};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y2.gif\n", $date_num_size{"$y2"};

	# print finish date
	($day,$mon,$yea)=split(/ /,$dateFinish);
	$mon=~ tr/A-Z/a-z/;
	print "copy 397,214,0,0,24,16,$image_templates/dates/$mon.gif\n";
	($y1,$y2) = $yea =~ m/^\d\d(.)(.)$/;
	$x = 423;
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/10101010101010101010.gif\n", $date_num_size{"'"};
	$x += $date_num_size{"'"};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y1.gif\n", $date_num_size{"$y1"};
	$x += $date_num_size{$y1};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y2.gif\n", $date_num_size{"$y2"};

	# caclulate and print horizontal scale
	$x1 = $startx; $x2 = $startx; $y1 = 206; $y2= 209;
	for( $i=0; $i<= $period_no; $i++ ){
		printf "line %d,%d,%d,%d,$black\n",$x1,$y1,$x2,$y2;
		$x1 += $horizontal_increment;
		$x2 += $horizontal_increment;
	}
	

	# calculate and print vertical scale
	$_ = $maxrequests;
	$x=0;
	while( $_ ne "" ){
		s/^(.)//;
		$x += 1;
	}
	$factor = 10**($x-1);

	$remainder = $maxrequests % $factor;
	$temp2 = $maxrequests / $factor;
	$divisions = sprintf "%d", $temp2;
	$x1 = 29; $x2 = 31; $height=170;
	$vertical_increment = (( $factor / $maxrequests ) * $height ) if($maxrequests>0);
	$y2 = $y1 = ( 39 + (( $remainder / $maxrequests ) * $height )) if($maxrequests>0);
	for( $i=0; $i < $divisions; $i++ ){
		printf "line %d,%d,%d,%d,$black\n",$x1,$y1,$x2,$y2;
		$y1 += $vertical_increment;
		$y2 += $vertical_increment;
	}

	print "interlace\n";
	print GROWTH "interlace\n";
	close(FLY);
	close(GROWTH);
	if( $flyverbose ) {
		open(FOO,"$fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif |")
			or die "Couldn't run $fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif: $!\n";
		while( <FOO> ){ print STDERR "        $_"; }
	} else {
		open(FOO,"$fly_program -q -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif |")
			or die "Couldn't run $fly_program -q -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif: $!";
	}
	close(FOO);	
	if( $flyverbose ) {
		open(BAR,"$fly_program -i $growth_control -o $outdir/images/growth.$period.gif |")
			or die "Couldn't run $fly_program -i $growth_control -o $outdir/images/growth.$period.gif: $!\n";
		while ( <BAR> ){ print STDERR  "        $_"; }
	} else {
		open(BAR,"$fly_program -q -i $growth_control -o $outdir/images/growth.$period.gif |")
			or die "Couldn't run $fly_program -q -i $growth_control -o $outdir/images/growth.$period.gif: $!\n";
	}
	close(BAR);	

	unlink("$outdir/$plurals{$interval}/images/$interval.gif");
	chdir("$outdir/$plurals{$interval}/images");
	if($OS =~ /Win/i) {
		system("copy $interval.$period.gif $interval.gif");
	} else {
		symlink("$interval.$period.gif","$interval.gif");
	}
	unlink("$outdir/images/growth.gif");
	chdir("$outdir/images");
	if($OS =~ /Win/i) {
		system("copy growth.$period.gif growth.gif");
	} else {
		symlink("growth.$period.gif","growth.gif");
	}

	unlink("$fly_control") if( ! $debug );
	unlink("$growth_control") if( ! $debug );
}

### ----------------------------------------------------------------------- ###
sub print_growth{

	# Subroutine for printing normal growth usage chart

	$fly_control = "$temp/fly.$interval.$$";
	open(FLY,"> $fly_control") or die "Couldn't open $fly_control for writing: $!\n";
	$growth_control = "$temp/fly.growth.$$";
	open(GROWTH,"> $growth_control") or die "Couldn't open $growth_control for writing: $!\n";

	
	print GROWTH "new\nsize 195,85\n";
	print GROWTH "copyresized 35,45,45,55,0,0,195,185,$period_template\n";
	print GROWTH "rect 0,0,194,84,0,0,0\n";
	select(FLY);
	$black="0,0,0";
	print "existing $period_template\n";

	$startx=29; $starty=209; $height=170;
	$maxrequests=0; $width = 390;
	
	$gstartx=0; $gstarty=85; $g_height=85;
	$maxrequests=0; $g_width = 195;
	
	if(-e "$outdir/$server.db") {
		open(DB,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		while( <DB> ){
			($period_no,$total_requests,$total_bytes,
			$unique_hosts,$start_date,$finish_date)=split(/:/);
			if($total_requests > $maxrequests ){
				$maxrequests = $total_requests;
			}
		}
		close(DB);
	} else {
		print "Couldn't find $outdir/$server.db\n" if $debug;
	}
	if($total > $maxrequests ){
		$maxrequests = $total;
	}
	
	$horizontal_increment = ($width / $period );
	$g_horizontal_increment = ($g_width / $period );

	$x = $startx;
	$gx = $gstartx;

	print "line $startx,$starty";
	print GROWTH "line $gstartx,$gstarty";
	if(-e "$outdir/$server.db") {
		open(DB,"$outdir/$server.db") or die "Couldn't open $outdir/$server.db for reading: $!\n";
		while ( <DB> ){
			($period_no,$total_requests,$total_bytes,
			$unique_hosts,$start_date,$finish_date)=split(/:/);
			$x += $horizontal_increment;  $gx += $g_horizontal_increment;
			$y = (($height -(($total_requests / $maxrequests ) * $height) )) + 40 if($maxrequests>0);
			$gy = (($g_height -(($total_requests / $maxrequests ) * $g_height) )) if($maxrequests>0);
			printf ",%d,%d,$black\n",$x,$y;
			printf GROWTH ",%d,%d,$black\n",$gx,$gy;
			printf "line %d,%d",$x,$y;
			printf GROWTH "line %d,%d",$gx,$gy;
		}
		close(DB);
	}
	printf ",%d,%d,$black\n",$x,$y;
	printf GROWTH ",%d,%d,$black\n",$gx,$gy;
	
	# calculate and print peak requests
	$_ = $maxrequests;
	$x=135;
	while( $_ ne "" ){
		s/^(.)//;
		$num=$1;
		$width = $num_size{$num};
		print "copy $x,239,0,0,$width,12,$image_templates/$num.gif\n";
		$x+=$width;
	}
	# calculate and print this period's total requests
	$_=$total;
	$x=303;
	print "copy $x,234,0,0,$interval_width,20,$image_templates/r_$singulars{$interval}.gif\n";
	$x += $interval_width;

	while( $_ ne "" ){
		s/^(.)//;
		$num=$1;
		$width = $num_size{$num};
		print "copy $x,239,0,0,$width,12,$image_templates/$num.gif\n";
		$x+=$width;
	}
	
	# print interval
	print "copy 261,210,0,0,$int_gsize{$interval},$image_templates/i_$interval.gif\n";
	
	# print original start date
	if( $d_b{1} ) { ($a,$b,$c,$dateorig,$d)=split(/:/,$d_b{1}); }
	else { $dateorig = $dateStart; }
	($day,$mon,$yea)=split(/ /,$dateorig);
	$mon=~ tr/A-Z/a-z/;
	print "copy 10,214,0,0,24,16,$image_templates/dates/$mon.gif\n";
	($y1,$y2) = $yea =~ m/^\d\d(.)(.)$/;
	$x = 34;
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/10.gif\n", $date_num_size{"'"};
	$x += $date_num_size{"'"};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y1.gif\n", $date_num_size{"$y1"};
	$x += $date_num_size{$y1};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y2.gif\n", $date_num_size{"$y2"};

	# print finish date
	($day,$mon,$yea)=split(/ /,$dateFinish);
	$mon=~ tr/A-Z/a-z/;
	print "copy 397,214,0,0,24,16,$image_templates/dates/$mon.gif\n";
	($y1,$y2) = $yea =~ m/^\d\d(.)(.)$/;
	$x = 423;
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/10.gif\n", $date_num_size{"'"};
	$x += $date_num_size{"'"};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y1.gif\n", $date_num_size{"$y1"};
	$x += $date_num_size{$y1};
	printf "copy $x,214,0,0,%s,16,$image_templates/dates/$y2.gif\n", $date_num_size{"$y2"};
	
	# caclulate and print horizontal scale
	$x1 = $startx; $x2 = $startx; $y1 = 206; $y2= 209;
	for( $i=0; $i<=$period_no; $i++ ){
		printf "line %d,$y1,%d,$y2,$black\n",$x1,$x2;
		$x1 += $horizontal_increment;
		$x2 += $horizontal_increment;
	}
	

	# calculate and print vertical scale
	$_ = $maxrequests;
	$x=0;
	while( $_ ne "" ){
		s/^(.)//;
		$x += 1;
	}
	$factor = 10 ** ($x -1);
	
	$remainder = $maxrequests % $factor;
	$temp2 = $maxrequests / $factor;
	$divisions = sprintf "%d", $temp2;
	$x1 = 29; $x2 = 31; $height=170;
	$vertical_increment = (( $factor / $maxrequests ) * $height ) if($maxrequests>0);
	$y2 = $y1 = ( 39 + (( $remainder / $maxrequests ) * $height )) if($maxrequests>0);
	for( $i=0; $i < $divisions; $i++ ){
		printf "line %d,%d,%d,%d,$black\n",$x1,$y1,$x2,$y2;
		$y1 += $vertical_increment;
		$y2 += $vertical_increment;
	}

	print "interlace\n";
	print GROWTH "interlace\n";
	close(FLY);
	close(GROWTH);
	if( $flyverbose ) {
		open(FOO,"$fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif |")
			or die "Couldn't run $fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif: $!\n";
		while ( <FOO> ){ print STDERR  "        $_"; }
	} else {
		open(FOO,"$fly_program -q -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif |")
			or die "Couldn't run $fly_program -q -i $fly_control -o $outdir/$plurals{$interval}/images/$interval.$period.gif: $!";
	}
	close(FOO);	
	if( $flyverbose ) {
		open(BAR,"$fly_program -i $growth_control -o $outdir/images/growth.$period.gif |")
			or die "Couldn't run $fly_program -i $growth_control -o $outdir/images/growth.$period.gif: $!\n";
		while ( <BAR> ){ print STDERR  "        $_"; }
	} else {
		open(BAR,"$fly_program -q -i $growth_control -o $outdir/images/growth.$period.gif |")
			or die "Couldn't run $fly_program -q -i $growth_control -o $outdir/images/growth.$period.gif: $!\n";
	}
	close(BAR);	

	unlink("$outdir/$plurals{$interval}/images/$interval.gif");
	chdir("$outdir/$plurals{$interval}/images");
	if($OS =~ /Win/i) {
		system("copy $interval.$period.gif $interval.gif");
	} else {
		symlink("$interval.$period.gif","$interval.gif");
	}

	unlink("$outdir/images/growth.gif");
	chdir("$outdir/images");
	if($OS =~ /Win/i) {
		system("copy growth.$period.gif growth.gif");
	} else {
		symlink("growth.$period.gif","growth.gif");
	}

	unlink("$fly_control") if( ! $debug );
	unlink("$growth_control") if( ! $debug );
}

### ----------------------------------------------------------------------- ###
sub print_hourly{

	# Subroutine for printing hourly usage chart

	$fly_control = "$temp/fly.hourly.$$";
	open(FLY,"> $fly_control") or die "Couldn't open $fly_control for writing: $!\n";
	$black="0,0,0";
	print FLY "existing $hourly_template\n";
	$x1=20; $x2=35; $y2=177;
	for( $i=0; $i < 24; $i++ ) {

		if( $maxHourlyRequests > 0 )
		{
			$y1 = ( 40 +(137-(137*($accesses_per_hour[$i]/$maxHourlyRequests))));
		}
		else
		{
			$y1 = ( 40 +(137-(137*($accesses_per_hour[$i]))));
		}
	
		if( $accesses_per_hour[$i] != $maxHourlyRequests ) {
			$colour = "102,153,204";
		}
		else{
			$colour = "238,0,0";
		}
		printf FLY "frect %d,%d,%d,%d,%s\n",$x1,$y1,$x2,$y2,$colour;
		printf FLY "rect %d,%d,%d,%d,%s\n",$x1,$y1,$x2,$y2,$black;
		$x1 += 15; $x2 += 15;
	}
	$_ =  sprintf "%d",$maxHourlyRequests;
	$x = 85;
	while( $_ ne "" ){
		s/^(.)//;
		$num=$1;
		$width = $num_size{$num};
		print "copy $x,41,0,0,$width,12,$image_templates/$num.gif\n";
		$x+=$width;
	}
	print "interlace\n";
	close(FLY);
	if( $flyverbose )
	{
		open(FOO,"$fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/hourly.$period.gif |")
			or die "Couldn't run $fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/hourly.$period.gif: $!\n";
		while( <FOO> ){ print STDERR "        $_"; }
	}
	else
	{
		open(FOO,"$fly_program -q -i $fly_control -o $outdir/$plurals{$interval}/images/hourly.$period.gif |")
			or die "Couldn't run $fly_program -q -i $fly_control -o $outdir/$plurals{$interval}/images/hourly.$period.gif:$!\n";
	}
	close(FOO);
	
	unlink($fly_control) if( ! $debug );
}

### ----------------------------------------------------------------------- ###
sub print_domain{

	# Subroutine for printing domain pie chart

    # prints and allocates wedges in the pie for the top five domains
    # accessing the server (or accessed by the server

	# we're doing things with circles here, so we need pi
	$pi = atan2(1,1) * 4;
	$pi_over_180 = $pi / 180;
	
	#open the control file for writing
	$fly_control = "$temp/fly.domain.$$";
	open(FLY,"> $fly_control") or die "Couldn't open $fly_control for writing: $!\n";
	print FLY "existing $domain_template\n";
	
	# x,y coords for the centre of the circle, and radius & diameter
	$cx = 134; $cy = 164;
	$width = $height = $radius = 114;
	$diameter = 2 * $radius;
	
	# work out the degrees that each wedge of the pie passes through
	$degrees{0} = 0;
	for( $i=0; $i < 5; $i++ )
	{
		$_ = $sorteddomains[$i];
		($dom,$num)=split(/\|\|/);
		$total_top += $num;
		$degrees{$i+1} = sprintf "%d", (( $num / $total ) * 360 );
		$degrees{$i+1} += $degrees{$i};
	}
	# last wedge is total of all others
	$degrees{$i+1} = sprintf "%d", (( ($total - $total_top) / $total ) * 360 ) if ($total > 0);
	$degrees{$i+1} += $degrees{$i};
	$degrees{6} = 360;	
	
	# coords for filling in the colours in the boxes
	@x = (260,260,260,260,260,260);
	@y = (50,90,130,170,210,250);

	# colours in the boxes and each wedge
	@colours = ("0,51,102","0,139,0","255,255,255","205,0,0","139,0,0","255,255,0");
	$black="0,0,0";

	# draw the wedges, and fill them with the appropriate colour
	for ( $i = 0; $i < 6 ; $i++ )
	{
		$_ = $sorteddomains[$i];
		($dom,$num)=split(/\|\|/);
		if( $num > 0 || $i == 5 )
		{
			# draw & fill boxes

			if( $i == 5 ) {
				$dom = "others" if ( $dom eq "" );
				$x= $x[$i]+35; $y = $y[$i]+5;
				printf FLY "string 0,0,0,$x,$y,large,%s\n", 'All Others';
			}
			else {
				$x= $x[$i]+35; $y = $y[$i]+5;
				printf FLY "string 0,0,0,$x,$y,large,%s\n", ( $domain_names{$dom} ? $domain_names{$dom} : $dom);
			}
			$x1 = $x[$i]; $y1 = $y[$i];        $x2 = $x[$i]+30; $y2 = $y[$i];
			$x3 = $x[$i]+30; $y3 = $y[$i]+25;  $x4 = $x[$i]; $y4 = $y[$i]+25;
			print FLY "poly $black,$x1,$y1,$x2,$y2,$x3,$y3,$x4,$y4,$x1,$y1\n";
			$x = $x[$i]+10; $y = $y[$i]+10;
			print FLY "fill $x,$y,$colours[$i]\n";

			# draw wedge
			$x = sprintf "%d", ($cx + ( $radius * cos($degrees{$i} * $pi_over_180)));
			$y = sprintf "%d", ($cy - ( $radius * sin($degrees{$i} * $pi_over_180)));
			print FLY "line $cx,$cy,$x,$y,$colours[$i]\n";
			$x = sprintf "%d", ($cx + ( $radius * cos($degrees{$i+1} * $pi_over_180)));
			$y = sprintf "%d", ($cy - ( $radius * sin($degrees{$i+1} * $pi_over_180)));
			print FLY "line $cx,$cy,$x,$y,$colours[$i]\n";
	
			# fill wedge
			$mid_deg =  ( (($degrees{$i} * $pi_over_180) + ($degrees{$i+1} * $pi_over_180)) / 2);
			$x = sprintf "%d", ($cx + ( $radius * cos($mid_deg)));
			$y = sprintf "%d", ($cy - ( $radius * sin($mid_deg)));
			$point_x = sprintf "%d", ( ($x + $cx) / 2);
			$point_y = sprintf "%d", ( ($y + $cy) / 2);
			print FLY "fill $point_x,$point_y,$colours[$i]\n";
		}
	}

	print FLY "interlace\n";
	close(FLY);
	open(FOO,"$fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/domain.$period.gif | ")
		or die or die "Couldn't run $fly_program -i $fly_control -o $outdir/$plurals{$interval}/images/domain.$period.gif: $!\n";
	if( $flyverbose ) { while( <FOO> ){ print STDERR "        $_"; } }
	close(FOO);

	unlink($fly_control) if( ! $debug );
}

### ----------------------------------------------------------------------- ###

# newgetopt.pl -- new options parsing
#
#
# Included here for bug fix purposes - Martin Gleeson
#
#
# SCCS Status     : @(#)@ newgetopt.pl	1.13
# Author          : Johan Vromans
# Created On      : Tue Sep 11 15:00:12 1990
# Last Modified By: Johan Vromans
# Last Modified On: Tue Jun  2 11:24:03 1992
# Update Count    : 75
# Status          : Okay

# This package implements a new getopt function. This function adheres
# to the new syntax (long option names, no bundling).
#
# Arguments to the function are:
#
#  - a list of possible options. These should designate valid perl
#    identifiers, optionally followed by an argument specifier ("="
#    for mandatory arguments or ":" for optional arguments) and an
#    argument type specifier: "n" or "i" for integer numbers, "f" for
#    real (fix) numbers or "s" for strings.
#    If an "@" sign is appended, the option is treated as an array.
#    Value(s) are not set, but pushed.
#
#  - if the first option of the list consists of non-alphanumeric
#    characters only, it is interpreted as a generic option starter.
#    Everything starting with one of the characters from the starter
#    will be considered an option.
#    Likewise, a double occurrence (e.g. "--") signals end of
#    the options list.
#    The default value for the starter is "-", "--" or "+".
#
# Upon return, the option variables, prefixed with "opt_", are defined
# and set to the respective option arguments, if any.
# Options that do not take an argument are set to 1. Note that an
# option with an optional argument will be defined, but set to '' if
# no actual argument has been supplied.
# A return status of 0 (false) indicates that the function detected
# one or more errors.
#
# Special care is taken to give a correct treatment to optional arguments.
#
# E.g. if option "one:i" (i.e. takes an optional integer argument),
# then the following situations are handled:
#
#    -one -two		-> $opt_one = '', -two is next option
#    -one -2		-> $opt_one = -2
#
# Also, assume "foo=s" and "bar:s" :
#
#    -bar -xxx		-> $opt_bar = '', '-xxx' is next option
#    -foo -bar		-> $opt_foo = '-bar'
#    -foo --		-> $opt_foo = '--'
#
# HISTORY 
# 2-Jun-1992		Johan Vromans	
#    Do not use //o to allow multiple NGetOpt calls with different delimeters.
#    Prevent typeless option from using previous $array state.
#    Prevent empty option from being eaten as a (negative) number.

# 25-May-1992		Johan Vromans	
#    Add array options. "foo=s@" will return an array @opt_foo that
#    contains all values that were supplied. E.g. "-foo one -foo -two" will
#    return @opt_foo = ("one", "-two");
#    Correct bug in handling options that allow for a argument when followed
#    by another option.

# 4-May-1992		Johan Vromans	
#    Add $ignorecase to match options in either case.
#    Allow '' option.

# 19-Mar-1992		Johan Vromans	
#    Allow require from packages.
#    NGetOpt is now defined in the package that requires it.
#    @ARGV and $opt_... are taken from the package that calls it.
#    Use standard (?) option prefixes: -, -- and +.

# 20-Sep-1990		Johan Vromans	
#    Set options w/o argument to 1.
#    Correct the dreadful semicolon/require bug.


#{   package newgetopt;
#    $debug = 0;			# for debugging
#    $ignorecase = 0;		# ignore case when matching options
#}

sub NGetOpt {

    @newgetopt'optionlist = @_;
    *newgetopt'ARGV = *ARGV;

    package newgetopt;

    local ($[) = 0;
    local ($genprefix) = "(--|-|\\+)";
    local ($argend) = "--";
    local ($error) = 0;
    local ($opt, $arg, $type, $mand, %opctl);
    local ($pkg) = (caller)[0];

    print STDERR "NGetOpt 1.13 -- called from $pkg\n" if $debug;

    # See if the first element of the optionlist contains option
    # starter characters.
    if ( $optionlist[0] =~ /^\W+$/ ) {
	$genprefix = shift (@optionlist);
	# Turn into regexp.
	$genprefix =~ s/(\W)/\\$1/g;
	$genprefix = "[" . $genprefix . "]";
	undef $argend;
    }

    # Verify correctness of optionlist.
    %opctl = ();
    foreach $opt ( @optionlist ) {
	$opt =~ tr/A-Z/a-z/ if $ignorecase;
	if ( $opt !~ /^(\w*)([=:][infse]@?)?$/ ) {
	    print STDERR ("Error in option spec: \"", $opt, "\"\n");
	    $error++;
	    next;
	}
	$opctl{$1} = defined $2 ? $2 : "";
    }

    return 0 if $error;

    if ( $debug ) {
	local ($arrow, $k, $v);
	$arrow = "=> ";
	while ( ($k,$v) = each(%opctl) ) {
	    print STDERR ($arrow, "\$opctl{\"$k\"} = \"$v\"\n");
	    $arrow = "   ";
	}
    }

    # Process argument list

    while ( $#ARGV >= 0 ) {

	# >>> See also the continue block <<<

	# Get next argument
	$opt = shift (@ARGV);
	print STDERR ("=> option \"", $opt, "\"\n") if $debug;
	$arg = undef;

	# Check for exhausted list.
	if ( $opt =~ /^$genprefix/ ) {
	    # Double occurrence is terminator
	    return ($error == 0) 
		if ($opt eq "$+$+") || ((defined $argend) && $opt eq $argend);
	    $opt = $';		# option name (w/o prefix)
	}
	else {
	    # Apparently not an option - push back and exit.
	    unshift (@ARGV, $opt);
	    return ($error == 0);
	}

	# Look it up.
	$opt =~ tr/A-Z/a-z/ if $ignorecase;
	unless  ( defined ( $type = $opctl{$opt} ) ) {
	    print STDERR ("Unknown option: ", $opt, "\n");
	    $error++;
	    next;
	}

	# Determine argument status.
	print STDERR ("=> found \"$type\" for ", $opt, "\n") if $debug;

	# If it is an option w/o argument, we're almost finished with it.
	if ( $type eq "" ) {
	    $arg = 1;		# supply explicit value
	    $array = 0;
	    next;
	}

	# Get mandatory status and type info.
	($mand, $type, $array) = $type =~ /^(.)(.)(@?)$/;

	# Check if the argument list is exhausted.
	if ( $#ARGV < 0 ) {

	    # Complain if this option needs an argument.
	    if ( $mand eq "=" ) {
		print STDERR ("Option ", $opt, " requires an argument\n");
		$error++;
	    }
	    if ( $mand eq ":" ) {
		$arg = $type eq "s" ? "" : 0;
	    }
	    next;
	}

	# Get (possibly optional) argument.
	$arg = shift (@ARGV);

	# Check if it is a valid argument. A mandatory string takes
	# anything. 
	if ( "$mand$type" ne "=s" && $arg =~ /^$genprefix/ ) {

	    # Check for option list terminator.
	    if ( $arg eq "$+$+" || 
		 ((defined $argend) && $arg eq $argend)) {
		# Push back so the outer loop will terminate.
		unshift (@ARGV, $arg);
		# Complain if an argument is required.
		if ($mand eq "=") {
		    print STDERR ("Option ", $opt, " requires an argument\n");
		    $error++;
		    undef $arg;	# don't assign it
		}
		else {
		    # Supply empty value.
		    $arg = $type eq "s" ? "" : 0;
		}
		next;
	    }

	    # Maybe the optional argument is the next option?
	    if ( $mand eq ":" && ($' eq "" || $' =~ /[a-zA-Z_]/) ) {
		# Yep. Push back.
		unshift (@ARGV, $arg);
		$arg = $type eq "s" ? "" : 0;
		next;
	    }
	}

	if ( $type eq "n" || $type eq "i" ) { # numeric/integer
	    if ( $arg !~ /^-?[0-9]+$/ ) {
		print STDERR ("Value \"", $arg, "\" invalid for option ",
			      $opt, " (number expected)\n");
		$error++;
		undef $arg;	# don't assign it
	    }
	    next;
	}

	if ( $type eq "f" ) { # fixed real number, int is also ok
	    if ( $arg !~ /^-?[0-9.]+$/ ) {
		print STDERR ("Value \"", $arg, "\" invalid for option ",
			      $opt, " (real number expected)\n");
		$error++;
		undef $arg;	# don't assign it
	    }
	    next;
	}

	if ( $type eq "s" ) { # string
	    next;
	}

    }
    continue {
	if ( defined $arg ) {
	    if ( $array ) {
		print STDERR ('=> push (@', $pkg, '\'opt_', $opt, ", \"$arg\")\n")
		    if $debug;
	        eval ('push(@' . $pkg . '\'opt_' . $opt . ", \$arg);");
	    }
	    else {
		print STDERR ('=> $', $pkg, '\'opt_', $opt, " = \"$arg\"\n")
		    if $debug;
	        #   eval ('$' . $pkg . '\'opt_' . $opt . " = \$arg;"); # here is the error!!!
			eval ('$' . $pkg . '\'opt_' . $opt . " = \"$arg\";");
	    }
	}
    }

    return ($error == 0);
}
1;
