Filter log parsing update

* Share filter log parsing code instead of using copy/paste/code duplication.
* Reworked the JavaScript a little so it could also be shared
* Fix a large number of bugs, especially in the AJAX-based dynamic log viewer.
* Picks up some more detail from the logs, and more accurately determines the protocol of a given log entry.
* Adds a CLI log parser (filterparser.php)
* Removed some redundant/unused code
* Code cleanup/style fixes
* Added support for finding logged rdr rules from miniupnpd
NOTE: Due to the dynamic nature of upnp rules, the rule may not be present when checked.

3 files changed, 81 insertions, 355 deletions

diff --git a/usr/local/www/widgets/include/log.inc b/usr/local/www/widgets/include/log.inc
deleted file mode 100644
index 08d4205..0000000
--- a/usr/local/www/widgets/include/log.inc
+++ /dev/null
@@ -1,170 +0,0 @@
-<?php
-
-//set variable for custom title
-$log_title = "Firewall Logs";
-$log_title_link = "diag_logs_filter.php";
-
-//set variables for log
-$filter_logfile = "{$g['varlog_path']}/filter.log";
-$nentries = 5;
-$filterlog = conv_clog_filter($filter_logfile, $nentries);
-
-/* AJAX related routines */
-	handle_ajax();
-	
-
-/* format filter logs */	
-function conv_clog_filter($logfile, $tail = 8) {
-	global $config, $nentries, $g;
-	$logarr = "";
-	/* make interface/port table */
-	$iftable = array();
-	$iflist = get_configured_interface_with_descr();
-	foreach ($iflist as $ifl => $ifdesc)
-		$iftable[get_real_interface($ifl)] = $ifdesc;
-
-	$sor = isset($config['syslog']['reverse']) ? "-r" : "";
-
-	exec("/usr/sbin/fifolog_reader {$logfile} | /usr/bin/tail {$sor} -n 500", $logarr);
-
-	$filterlog = array();
-
-	$counter = 1;
-
-	foreach ($logarr as $logent) {
-
-		if($counter > $nentries)
-			break;
-
-		$log_split = "";
-
-
-		preg_match("/(\b(?:\d{1,3}\.){3}\d{1,3}(\.\w+)?)\s.*\s(\b(?:\d{1,3}\.){3}\d{1,3}(\.\w+)?)/", $logent, $log_split);
-
-		$flent['src'] 		= convert_port_period_to_colon($log_split[1]);
-		$flent['dst'] 		= convert_port_period_to_colon($log_split[3]);
-
-		preg_match("/(.*)\s.*\spf:\s.*\srule\s(.*)\(match\)\:\s(.*)\s\w+\son\s(\w+)\:\s(.*)\s>\s(.*)\:\s.*/", $logent, $log_split);
-
-		$beforeupper = $logent;
-		$logent = strtoupper($logent);
-
-		if(stristr(strtoupper($logent), "UDP") == true)
-			$flent['proto'] = "UDP";
-		else if(stristr(strtoupper($logent), "TCP") == true)
-			$flent['proto'] = "TCP";
-		else if(stristr(strtoupper($logent), "ICMP") == true)
-			$flent['proto'] = "ICMP";
-		else if(stristr(strtoupper($logent), "HSRP") == true)
-			$flent['proto'] = "HSRP";
-		else if(stristr(strtoupper($logent), "ESP") == true)
-			$flent['proto'] = "ESP";
-		else if(stristr(strtoupper($logent), "AH") == true)
-			$flent['proto'] = "AH";
-		else if(stristr(strtoupper($logent), "GRE") == true)
-			$flent['proto'] = "GRE";
-		else if(stristr(strtoupper($logent), "IGMP") == true)
-			$flent['proto'] = "IGMP";
-		else if(stristr(strtoupper($logent), "CARP") == true)
-			$flent['proto'] = "CARP";
-		else if(stristr(strtoupper($logent), "VRRP") == true)
-			$flent['proto'] = "VRRP";
-		else if(stristr(strtoupper($logent), "PFSYNC") == true)
-			$flent['proto'] = "PFSYNC";
-		else if(stristr($logent, "sack") == true)
-			$flent['proto'] = "TCP";
-		else
-			$flent['proto'] = "TCP";
-
-		$flent['time'] 		= $log_split[1];
-		$flent['act'] 		= $log_split[3];
-		$flent['interface'] 	=  empty($iftable[$log_split[4]]) ? $log_split[4] : $iftable[$log_split[4]];
-
-		$tmp = split("/", $log_split[2]);
-		$flent['rulenum'] = $tmp[0];
-
-		$shouldadd = true;
-
-		if(trim($flent['src']) == "")
-			$shouldadd = false;
-		if(trim($flent['dst']) == "")
-			$shouldadd = false;
-		if(trim($flent['time']) == "")
-			$shouldadd = false;
-
-		if($shouldadd == true) {
-			$counter++;
-			$filterlog[] = $flent;
-		} else {
-			if($g['debug']) {
-				log_error("There was a error parsing rule: $beforeupper .   Please report to mailing list or forum.");
-			}
-		}
-
-	}
-
-	return $filterlog;
-}
-
-function convert_port_period_to_colon($addr) {
-	$addr_split = split("\.", $addr);
-	if($addr_split[4] == "")
-		$newvar = $addr_split[0] . "." . $addr_split[1] . "." . $addr_split[2] . "." . $addr_split[3];
-	else
-		$newvar = $addr_split[0] . "." . $addr_split[1] . "." . $addr_split[2] . "." . $addr_split[3] . ":" . $addr_split[4];
-	if($newvar == "...")
-		return $addr;
-	return $newvar;
-}
-
-function format_ipf_ip($ipfip) {
-	list($ip,$port) = explode(",", $ipfip);
-	if (!$port)
-		return $ip;
-
-	return $ip . ", port " . $port;
-}
-
-/* AJAX specific handlers */
-function handle_ajax() {
-	if($_GET['getrulenum'] or $_POST['getrulenum']) {
-		if($_GET['getrulenum'])
-			$rulenum = $_GET['getrulenum'];
-		if($_POST['getrulenum'])
-			$rulenum = $_POST['getrulenum'];
-		$rule = `pfctl -vvsr | grep @{$rulenum}`;
-		echo "The rule that triggered this action is:\n\n{$rule}";
-		exit;
-	}
-
-	if($_GET['lastsawtime'] or $_POST['lastsawtime']) {
-		global $filter_logfile,$filterent;
-		if($_GET['lastsawtime'])
-			$lastsawtime = $_GET['lastsawtime'];
-		if($_POST['lastsawtime'])
-			$lastsawtime = $_POST['lastsawtime'];
-		/*  compare lastsawrule's time stamp to filter logs.
-		 *  afterwards return the newer records so that client
-                 *  can update AJAX interface screen.
-		 */
-		$new_rules = "";
-		$filterlog = conv_clog_filter($filter_logfile, 8);
-		foreach($filterlog as $log_row) {
-			$time_regex = "";
-			preg_match("/.*([0-9][0-9]:[0-9][0-9]:[0-9][0-9])/", $log_row['time'], $time_regex);
-			$row_time = strtotime($time_regex[1]);
-			if (strstr(strtolower($log_row['act']), "p"))
-				$img = "<img border='0' src='/themes/metallic/images/icons/icon_pass.gif'>";
-			else if(strstr(strtolower($filterent['act']), "r"))
-				$img = "<img border='0' src='/themes/metallic/images/icons/icon_reject.gif'>";
-			else
-				$img = "<img border='0' src='/themes/metallic/images/icons/icon_block.gif'>";
-			//echo "{$time_regex[1]} - $row_time > $lastsawtime<p>";
-			if($row_time > $lastsawtime)
-				$new_rules .= "{$img}||{$log_row['time']}||{$log_row['interface']}||{$log_row['src']}||{$log_row['dst']}||{$log_row['proto']}||" . time() . "||\n";
-		}
-		echo $new_rules;
-		exit;
-	}
-}
-?>
diff --git a/usr/local/www/widgets/javascript/log.js b/usr/local/www/widgets/javascript/log.js
deleted file mode 100644
index 4528882..0000000
--- a/usr/local/www/widgets/javascript/log.js
+++ /dev/null
@@ -1,155 +0,0 @@
-
-lastsawtime = '<?php echo time(); ?>;';
-var lines = Array();
-var timer;
-var updateDelay = 30000;
-var isBusy = false;
-var isPaused = true;
-
-<?php
-	if(isset($config['syslog']['reverse']))
-		echo "	var isReverse = true;\n";
-	else
-		echo "	var isReverse = false;\n";
-?>
-
-if (typeof getURL == 'undefined') {
-	getURL = function(url, callback) {
-		if (!url)
-			throw 'No URL for getURL';
-		try {
-			if (typeof callback.operationComplete == 'function')
-				callback = callback.operationComplete;
-		} catch (e) {}
-			if (typeof callback != 'function')
-				throw 'No callback function for getURL';
-		var http_request = null;
-		if (typeof XMLHttpRequest != 'undefined') {
-		    http_request = new XMLHttpRequest();
-		}
-		else if (typeof ActiveXObject != 'undefined') {
-			try {
-				http_request = new ActiveXObject('Msxml2.XMLHTTP');
-			} catch (e) {
-				try {
-					http_request = new ActiveXObject('Microsoft.XMLHTTP');
-				} catch (e) {}
-			}
-		}
-		if (!http_request)
-			throw 'Both getURL and XMLHttpRequest are undefined';
-		http_request.onreadystatechange = function() {
-			if (http_request.readyState == 4) {
-				callback( { success : true,
-				  content : http_request.responseText,
-				  contentType : http_request.getResponseHeader("Content-Type") } );
-			}
-		}
-		http_request.open('GET', url, true);
-		http_request.send(null);
-	}
-}
-
-function outputrule(req) {
-	alert(req.content);
-}
-function fetch_new_rules() {
-	if(isPaused)
-		return;
-	if(isBusy)
-		return;
-	isBusy = true;
-	getURL('diag_logs_filter_dynamic.php?lastsawtime=' + lastsawtime, fetch_new_rules_callback);
-}
-function fetch_new_rules_callback(callback_data) {
-	if(isPaused)
-		return;
-
-	var data_split;
-	var new_data_to_add = Array();
-	var data = callback_data.content;
-
-	data_split = data.split("\n");
-
-	for(var x=0; x<data_split.length-1; x++) {
-		/* loop through rows */
-		row_split = data_split[x].split("||");
-		var line = '';
-		line = '<div class="log-entry">';
-		line += '  <span class="log-action-mini" nowrap>&nbsp;' + row_split[0] + '&nbsp;</span>';
-		line += '  <span class="log-interface-mini" nowrap>' + row_split[2] + '</span>';
-		line += '  <span class="log-source-mini" nowrap>' + row_split[3] + '</span>';
-		line += '  <span class="log-destination-mini" nowrap>' + row_split[4] + '</span>';
-		line += '  <span class="log-protocol-mini" nowrap>' + row_split[5] + '</span>';
-		line += '</tr></div>';
-		lastsawtime = row_split[6];
-		new_data_to_add[new_data_to_add.length] = line;
-	}
-	update_div_rows(new_data_to_add);
-	isBusy = false;
-}
-function update_div_rows(data) {
-	if(isPaused)
-		return;
-
-	var isIE = navigator.appName.indexOf('Microsoft') != -1;
-	var isSafari = navigator.userAgent.indexOf('Safari') != -1;
-	var isOpera = navigator.userAgent.indexOf('Opera') != -1;
-	var rulestable = document.getElementById('log');
-	var rows = rulestable.getElementsByTagName('div');
-	var showanim = 1;
-	if (isIE) {
-		showanim = 0;
-	}
-	//alert(data.length);
-	for(var x=0; x<data.length; x++) {
-		var numrows = rows.length;
-		var appearatrow;
-		/*    if reverse logging is enabled we need to show the
-		 *    records in a reverse order with new items appearing
-         *    on the top
-         */
-		//if(isReverse == false) {
-		//	for (var i = 2; i < numrows; i++) {
-		//		nextrecord = i + 1;
-		//		if(nextrecord < numrows)
-		//			rows[i].innerHTML = rows[nextrecord].innerHTML;
-		//	}
-		//	appearatrow = numrows - 1;
-		//} else {
-			for (var i = numrows; i > 0; i--) {
-				nextrecord = i + 1;
-				if(nextrecord < numrows)
-					rows[nextrecord].innerHTML = rows[i].innerHTML;
-			}
-			appearatrow = 1;
-		//}
-		var item = document.getElementById('firstrow');
-		if(x == data.length-1) {
-			/* nothing */
-			showanim = false;
-		} else {
-			showanim = false;
-		}
-		if (showanim) {
-			rows[appearatrow].style.display = 'none';
-			rows[appearatrow].innerHTML = data[x];
-			new Effect.Appear(rows[appearatrow]);
-		} else {
-			rows[appearatrow].innerHTML = data[x];
-		}
-	}
-	/* rechedule AJAX interval */
-	timer = setInterval('fetch_new_rules()', updateDelay);
-}
-function toggle_pause() {
-	if(isPaused) {
-		isPaused = false;
-		fetch_new_rules();
-	} else {
-		isPaused = true;
-	}
-}
-/* start local AJAX engine */
-lastsawtime = '<?php echo time(); ?>;';
-timer = setInterval('fetch_new_rules()', updateDelay);
diff --git a/usr/local/www/widgets/widgets/log.widget.php b/usr/local/www/widgets/widgets/log.widget.php
index 521cd05..4533548 100644
--- a/usr/local/www/widgets/widgets/log.widget.php
+++ b/usr/local/www/widgets/widgets/log.widget.php
@@ -34,9 +34,73 @@
 require_once("guiconfig.inc");
 require_once("pfsense-utils.inc");
 require_once("functions.inc");
-require_once("/usr/local/www/widgets/include/log.inc");
+
+/* In an effort to reduce duplicate code, many shared functions have been moved here. */
+require_once("filter_log.inc");
+
+//set variable for custom title
+$log_title = "Firewall Logs";
+$log_title_link = "diag_logs_filter.php";
+
+if($_POST['filterlogentries']) {
+	$config['widgets']['filterlogentries'] = $_POST['filterlogentries'];
+	write_config("Saved Filter Log Entries via Dashboard");
+	Header("Location: /");
+}
+
+$nentries = isset($config['widgets']['filterlogentries']) ? $config['widgets']['filterlogentries'] : 5;
+
+//set variables for log
+$filter_logfile = "{$g['varlog_path']}/filter.log";
+$filterlog = conv_log_filter($filter_logfile, $nentries);
+
+/* AJAX related routines */
+handle_ajax($nentries, $nentries + 20);
+
 ?>
 
+<script language="javascript">
+lastsawtime = '<?php echo time(); ?>';
+var lines = Array();
+var timer;
+var updateDelay = 30000;
+var isBusy = false;
+var isPaused = false;
+var nentries = <?php echo $nentries; ?>;
+
+<?php
+if(isset($config['syslog']['reverse']))
+	echo "var isReverse = true;\n";
+else
+	echo "var isReverse = false;\n";
+?>
+
+/* Called by the AJAX updater */
+function format_log_line(row) {
+	var line = '';
+	line = '  <span class="log-action-mini" nowrap>&nbsp;' + row[0] + '&nbsp;</span>';
+	line += '  <span class="log-interface-mini" nowrap>' + row[2] + '</span>';
+	line += '  <span class="log-source-mini" nowrap>' + row[3] + '</span>';
+	line += '  <span class="log-destination-mini" nowrap>' + row[4] + '</span>';
+	line += '  <span class="log-protocol-mini" nowrap>' + row[5] + '</span>';
+	return line;
+}
+</script>
+<script src="/javascript/filter_log.js" type="text/javascript"></script>
+<input type="hidden" id="log-config" name="log-config" value="">
+
+<div id="log-settings" name="log-settings" class="widgetconfigdiv" style="display:none;">
+	<form action="/widgets/widgets/log.widget.php" method="post" name="iforma">
+		Number of lines to display: 
+		<select name="filterlogentries" class="formfld unknown" id="filterlogentries">
+		<?php for ($i = 1; $i <= 20; $i++) { ?>
+			<option value="<?php echo $i;?>" <?php if ($nentries == $i) echo "SELECTED";?>><?php echo $i;?></option>
+		<?php } ?>
+		</select><br/>
+        <input id="submita" name="submita" type="submit" class="formbtn" value="Save" />
+	</form>
+</div>
+
 <div class="log-header">
     <span class="log-action-mini-header">Act</span>
     <span class="log-interface-mini-header">IF</span>
@@ -45,36 +109,23 @@ require_once("/usr/local/www/widgets/include/log.inc");
     <span class="log-protocol-mini-header">Prot</span>
 </div>
 <?php $counter=0; foreach ($filterlog as $filterent): ?>
-<?php
-	if(isset($config['syslog']['reverse'])) {
-		/* honour reverse logging setting */
-		if($counter == 0)
-			$activerow = " id=\"firstrow\"";
-		else
-			$activerow = "";
-
-	} else {
-		/* non-reverse logging */
-		if($counter == count($filterlog))
-			$activerow = " id=\"firstrow\"";
-		else
-			$activerow = "";
-	}
-?>
-<div class="log-entry-mini" <?php echo $activerow; ?> style="clear:both;">
+<div class="log-entry-mini" <?php echo is_first_row($counter, count($filterlog)); ?> style="clear:both;">
 	<span class="log-action-mini" nowrap>
+	&nbsp;<a href="#" onClick="javascript:getURL('diag_logs_filter.php?getrulenum=<?php echo "{$filterent['rulenum']},{$filterent['act']}"; ?>', outputrule);"><img border="0" src="<?php echo find_action_image($filterent['act']);?>" alt="<?php echo $filterent['act'];?>" title="<?php echo $filterent['act'];?>" /></a>&nbsp;</span>
+	<span class="log-interface-mini"><?php echo htmlspecialchars($filterent['interface']);?>&nbsp;</span>
+	<span class="log-source-mini"><?php echo htmlspecialchars($filterent['src']);?>&nbsp;</span>
+	<span class="log-destination-mini"><?php echo htmlspecialchars($filterent['dst']);?>&nbsp;</span>
 	<?php
-		if (strstr(strtolower($filterent['act']), "p"))
-			$img = "/themes/metallic/images/icons/icon_pass.gif";
-		else if(strstr(strtolower($filterent['act']), "r"))
-			$img = "/themes/metallic/images/icons/icon_reject.gif";
-		else
-			$img = "/themes/metallic/images/icons/icon_block.gif";
+	if ($filterent['proto'] == "TCP")
+		$filterent['proto'] .= ":{$filterent['tcpflags']}";
 	?>
-	&nbsp;<img border="0" src="<?=$img;?>">&nbsp;</span>
-	<span class="log-interface-mini" ><?=htmlspecialchars(convert_real_interface_to_friendly_interface_name($filterent['interface']));?>&nbsp;</span>
-	<span class="log-source-mini" ><?=htmlspecialchars($filterent['src']);?>&nbsp;</span>
-	<span class="log-destination-mini" ><?=htmlspecialchars($filterent['dst']);?>&nbsp;</span>
-	<span class="log-protocol-mini" ><?=htmlspecialchars($filterent['proto']);?>&nbsp;</span>
+	<span class="log-protocol-mini"><?php echo htmlspecialchars($filterent['proto']);?>&nbsp;</span>
 </div>
-<?php $counter++; endforeach; ?>
\ No newline at end of file
+<?php $counter++; endforeach; ?>
+
+<!-- needed to display the widget settings menu -->
+<script language="javascript" type="text/javascript">
+	selectIntLink = "log-configure";
+	textlink = document.getElementById(selectIntLink);
+	textlink.style.display = "inline";
+</script>