Merge pull request #2693 from NOYB/Diagnostics_/_Tables_-_URL_Table_Aliases

4 files changed, 134 insertions, 49 deletions

diff --git a/src/etc/inc/pfsense-utils.inc b/src/etc/inc/pfsense-utils.inc
index 416a89c..e364afb 100644
--- a/src/etc/inc/pfsense-utils.inc
+++ b/src/etc/inc/pfsense-utils.inc
@@ -1919,7 +1919,7 @@ function update_alias_names_upon_change($section, $field, $new_alias_name, $orig
 
 }
 
-function parse_aliases_file($filename, $type = "url", $max_items = -1) {
+function parse_aliases_file($filename, $type = "url", $max_items = -1, $kflc = false) {
 	/*
 	 * $filename = file to process for example blocklist like DROP:  http://www.spamhaus.org/drop/drop.txt
 	 * $type = if set to 'url' then subnets and ips will be returned,
@@ -1935,31 +1935,36 @@ function parse_aliases_file($filename, $type = "url", $max_items = -1) {
 		return null;
 	}
 	$items = array();
+	$comments = array();
 	/* NOTE: fgetss() is not a typo RTFM before being smart */
 	while (($fc = fgetss($fd)) !== FALSE) {
 		$tmp = trim($fc, " \t\n\r");
 		if (empty($tmp)) {
 			continue;
 		}
-		$tmp_str = strstr($tmp, '#', true);
-		if (!empty($tmp_str)) {
-			$tmp = $tmp_str;
-		}
-		$tmp_str = strstr($tmp, ' ', true);
-		if (!empty($tmp_str)) {
-			$tmp = $tmp_str;
-		}
-		$valid = ($type == "url" && (is_ipaddr($tmp) || is_subnet($tmp))) ||
-			 ($type == "url_ports" && (is_port($tmp) || is_portrange($tmp)));
-		if ($valid) {
-			$items[] = $tmp;
-			if (count($items) == $max_items) {
-				break;
+		if (($kflc) && (strpos($tmp, '#') === 0)) {	// Keep Full Line Comments (lines beginning with #).
+			$comments[] = $tmp;
+		} else {
+			$tmp_str = strstr($tmp, '#', true);
+			if (!empty($tmp_str)) {
+				$tmp = $tmp_str;
+			}
+			$tmp_str = strstr($tmp, ' ', true);
+			if (!empty($tmp_str)) {
+				$tmp = $tmp_str;
+			}
+			$valid = ($type == "url" && (is_ipaddr($tmp) || is_subnet($tmp))) ||
+				($type == "url_ports" && (is_port($tmp) || is_portrange($tmp)));
+			if ($valid) {
+				$items[] = $tmp;
+				if (count($items) == $max_items) {
+					break;
+				}
 			}
 		}
 	}
 	fclose($fd);
-	return $items;
+	return array_merge($comments, $items);
 }
 
 function update_alias_url_data() {
@@ -2166,13 +2171,14 @@ function process_alias_urltable($name, $url, $freq, $forceupdate=false, $validat
 		unlink_if_exists($tmp_urltable_filename);
 		$verify_ssl = isset($config['system']['checkaliasesurlcert']);
 		if (download_file($url, $tmp_urltable_filename, $verify_ssl)) {
-			mwexec("/usr/bin/sed -i \"\" -E 's/\;.*//g; /^[[:space:]]*($|#)/d' " . escapeshellarg($tmp_urltable_filename));
+			// Convert lines that begin with '$' or ';' to comments '#' instead of deleting them.
+			mwexec("/usr/bin/sed -i \"\" -E 's/^[[:space:]]*($|#|;)/#/g; /^#/!s/\;.*//g;' ". escapeshellarg($tmp_urltable_filename));
 			if (alias_get_type($name) == "urltable_ports") {
-				$ports = parse_aliases_file($tmp_urltable_filename, "url_ports", "-1");
-				$ports = group_ports($ports);
+				$ports = parse_aliases_file($tmp_urltable_filename, "url_ports", "-1", true);
+				$ports = group_ports($ports, true);
 				file_put_contents($urltable_filename, implode("\n", $ports));
 			} else {
-				$urltable = parse_aliases_file($tmp_urltable_filename, "url", "-1");
+				$urltable = parse_aliases_file($tmp_urltable_filename, "url", "-1", true);
 				file_put_contents($urltable_filename, implode("\n", $urltable));
 			}
 			unlink_if_exists($tmp_urltable_filename);
diff --git a/src/etc/inc/util.inc b/src/etc/inc/util.inc
index 0c77758..e1f5c80 100644
--- a/src/etc/inc/util.inc
+++ b/src/etc/inc/util.inc
@@ -1106,14 +1106,17 @@ function is_portoralias($port) {
 }
 
 /* create ranges of sequential port numbers (200:215) and remove duplicates */
-function group_ports($ports) {
+function group_ports($ports, $kflc = false) {
 	if (!is_array($ports) || empty($ports)) {
 		return;
 	}
 
 	$uniq = array();
+	$comments = array();
 	foreach ($ports as $port) {
-		if (is_portrange($port)) {
+		if (($kflc) && (strpos($port, '#') === 0)) {	// Keep Full Line Comments (lines beginning with #).
+			$comments[] = $port;
+		} else if (is_portrange($port)) {
 			list($begin, $end) = explode(":", $port);
 			if ($begin > $end) {
 				$aux = $begin;
@@ -1155,7 +1158,7 @@ function group_ports($ports) {
 		}
 	}
 
-	return $result;
+	return array_merge($comments, $result);
 }
 
 /* returns true if $val is a valid shaper bandwidth value */
diff --git a/src/etc/rc.update_urltables b/src/etc/rc.update_urltables
index c4dfeb1..887dfac 100755
--- a/src/etc/rc.update_urltables
+++ b/src/etc/rc.update_urltables
@@ -32,11 +32,24 @@ if (count($todo) > 0) {
 		sleep($wait);
 	}
 
+	// Set whether or not to force the table update before it's time.
+	if (!empty($argv[2]) && ($argv[2] == "forceupdate")) {
+		$forceupdate = true;
+	} else {
+		$forceupdate = false;
+	}
+
 	log_error("{$argv[0]}: Starting URL table alias updates");
 
 	$filter_reload = false;
 	foreach ($todo as $t) {
-		$r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
+
+		// Update a specifically named URL table only.
+		if (!empty($argv[3]) && ($argv[3] != $t['name'])) {
+			continue;
+		}
+
+		$r = process_alias_urltable($t['name'], $t['url'], $t['freq'], $forceupdate);
 		if ($r == 1) {
 			$result = "";
 			// TODO: Change it when pf supports tables with ports
diff --git a/src/usr/local/www/diag_tables.php b/src/usr/local/www/diag_tables.php
index 2fe1b6d..4543899 100644
--- a/src/usr/local/www/diag_tables.php
+++ b/src/usr/local/www/diag_tables.php
@@ -68,12 +68,34 @@ require_once("guiconfig.inc");
 
 // Set default table
 $tablename = "sshlockout";
-$bogons = false;
 
 if ($_REQUEST['type']) {
 	$tablename = $_REQUEST['type'];
 }
 
+// Gather selected alias metadata.
+if (isset($config['aliases']['alias'])) {
+	foreach ($config['aliases']['alias'] as $alias) {
+		if ( $alias['name'] == $tablename ) {
+			$tmp = array();
+			$tmp['type'] = $alias['type'];
+			$tmp['name'] = $alias['name'];
+			$tmp['url']  = $alias['url'];
+			$tmp['freq'] = $alias['updatefreq'];
+			break;
+		}
+	}
+}
+
+# Determine if selected alias is either a bogons or URL table.
+if (($tablename == "bogons") || ($tablename == "bogonsv6")) {
+	$bogons = true;
+} else if (preg_match('/urltable/i', $tmp['type'])) {
+	$urltable = true;
+} else {
+	$bogons = $urltable = false;
+}
+
 if ($_REQUEST['delete']) {
 	if (is_ipaddr($_REQUEST['delete']) || is_subnet($_REQUEST['delete'])) {
 		exec("/sbin/pfctl -t " . escapeshellarg($_REQUEST['type']) . " -T delete " . escapeshellarg($_REQUEST['delete']), $delete);
@@ -93,27 +115,34 @@ if ($_POST['clearall']) {
 	unset($entries);
 }
 
-if (($tablename == "bogons") || ($tablename == "bogonsv6")) {
-	$bogons = true;
+if ($_POST['Download'] && ($bogons || $urltable)) {
 
-	if ($_POST['Download']) {
-		mwexec_bg("/etc/rc.update_bogons.sh now");
-		$maxtimetowait = 0;
-		$loading = true;
-		while ($loading == true) {
-			$isrunning = `/bin/ps awwwux | /usr/bin/grep -v grep | /usr/bin/grep bogons`;
-			if ($isrunning == "") {
-				$loading = false;
-			}
-			$maxtimetowait++;
-			if ($maxtimetowait > 89) {
-				$loading = false;
-			}
-			sleep(1);
+	if ($bogons) {				// If selected table is either bogons or bogonsv6.
+		$mwexec_bg_cmd = '/etc/rc.update_bogons.sh now';
+		$table_type = 'bogons';
+		$db_name = 'bogons';
+	} else if ($urltable) {		//  If selected table is a URL table alias.
+		$mwexec_bg_cmd = '/etc/rc.update_urltables now forceupdate ' . $tablename;
+		$table_type = 'urltables';
+		$db_name = $tablename;
+	}
+
+	mwexec_bg($mwexec_bg_cmd);
+	$maxtimetowait = 0;
+	$loading = true;
+	while ($loading == true) {
+		$isrunning = `/bin/ps awwwux | /usr/bin/grep -v grep | /usr/bin/grep $table_type`;
+		if ($isrunning == "") {
+			$loading = false;
 		}
-		if ($maxtimetowait < 90) {
-			$savemsg = gettext("The bogons database has been updated.");
+		$maxtimetowait++;
+		if ($maxtimetowait > 89) {
+			$loading = false;
 		}
+		sleep(1);
+	}
+	if ($maxtimetowait < 90) {
+		$savemsg = sprintf(gettext("The %s database has been updated."), $db_name);
 	}
 }
 
@@ -144,8 +173,8 @@ $group->add(new Form_Select(
 	array_combine($tables, $tables)
 ));
 
-if ($bogons || !empty($entries)) {
-	if ($bogons) {
+if ($bogons || $urltable || !empty($entries)) {
+	if ($bogons || $urltable) {
 		$group->add(new Form_Button(
 			'Download',
 			'Update'
@@ -162,12 +191,24 @@ $section->add($group);
 $form->add($section);
 print $form;
 
-if ($bogons || !empty($entries)) {
+if ($bogons || $urltable || !empty($entries)) {
 ?>
 <div>
 	<div class="infoblock blockopen">
 <?php
-	$last_updated = exec('/usr/bin/grep -i -m 1 -E "^# last updated" /etc/' . escapeshellarg($tablename) . '|cut -d"(" -f2|tr -d ")" ');
+	if ($bogons) {
+		$table_file = '/etc/' . escapeshellarg($tablename);
+	} else if ($urltable) {
+		$table_file = '/var/db/aliastables/' . escapeshellarg($tablename) . '.txt';
+	} else {
+		$table_file = '';
+	}
+
+	$datestrregex = '(Mon|Tue|Wed|Thr|Fri|Sat|Sun).* GMT';
+	$datelineregex = 'last.*' . $datestrregex;
+
+	$last_updated = exec('/usr/bin/grep -i -m 1 -E "^# ' . $datelineregex . '" ' . $table_file . '|/usr/bin/grep -i -m 1 -E -o "' . $datestrregex . '"');
+
 	if ($last_updated != "") {
 		$last_update_msg = sprintf(gettext("Table last updated on %s."), $last_updated);
 	} else {
@@ -176,7 +217,22 @@ if ($bogons || !empty($entries)) {
 
 	$records_count_msg = sprintf(gettext("%s records."), number_format(count($entries), 0, gettext("."), gettext(",")));
 
-	print_info_box($last_update_msg . "&nbsp; &nbsp; " . $records_count_msg, 'info', false);
+	# Display up to 10 comment lines (lines that begin with '#').
+	unset($comment_lines);
+	$res = exec('/usr/bin/grep -i -m 10 -E "^#" ' . $table_file, $comment_lines);
+
+	foreach ($comment_lines as $comment_line) {
+		$table_comments .= "$comment_line" . "<br />";
+	}
+
+	if ($table_comments) {
+		print_info_box($last_update_msg . " &nbsp; &nbsp; " . $records_count_msg . " &nbsp; &nbsp; " . 
+		'<span style="display:none" class="infoblock">' . ' ' . gettext("Hide table comments.") . '<br />' . $table_comments . '</span>' .
+		'<span style="display:none"   id="showtblcom">' . ' ' . gettext("Show table comments.") . '</span>' .
+		'' , 'info', false);
+	} else {
+		print_info_box($last_update_msg . "&nbsp; &nbsp; " . $records_count_msg, 'info', false);
+	}
 ?>
 	</div>
 </div>
@@ -187,6 +243,13 @@ if ($bogons || !empty($entries)) {
 <script type="text/javascript">
 //<![CDATA[
 events.push(function() {
+
+	$('#showtblcom').show();
+
+	$('[id^="showinfo1"]').click(function() {
+			$('#showtblcom').toggle();
+	});
+
 	$('a[data-entry]').on('click', function() {
 		var el = $(this);
 
@@ -249,7 +312,7 @@ if (empty($entries)) {
 							<?=$entry?>
 						</td>
 						<td>
-							<?php if (!$bogons): ?>
+							<?php if (!$bogons && !$urltable): ?>
 								<a class="btn btn-xs btn-default" data-entry="<?=htmlspecialchars($entry)?>"><?=gettext("Remove")?></a>
 							<?php endif ?>
 						</td>