Diagnostics / Tables - URL Table Aliases

Handle URL table aliases similar to bogons tables.  Suppress record remove buttons.
Update (download) only the selected URL table alias instead of all of them.
Retain full line comments in downloaded URL table files so we can view and use them.
When URL table file full line comments are present, provide view option in info box.
Increase ability to extract last update date from wider variety of URL table file comments.

3 files changed, 46 insertions, 24 deletions

diff --git a/src/etc/inc/pfsense-utils.inc b/src/etc/inc/pfsense-utils.inc
index 416a89c..e364afb 100644
--- a/src/etc/inc/pfsense-utils.inc
+++ b/src/etc/inc/pfsense-utils.inc
@@ -1919,7 +1919,7 @@ function update_alias_names_upon_change($section, $field, $new_alias_name, $orig
 
 }
 
-function parse_aliases_file($filename, $type = "url", $max_items = -1) {
+function parse_aliases_file($filename, $type = "url", $max_items = -1, $kflc = false) {
 	/*
 	 * $filename = file to process for example blocklist like DROP:  http://www.spamhaus.org/drop/drop.txt
 	 * $type = if set to 'url' then subnets and ips will be returned,
@@ -1935,31 +1935,36 @@ function parse_aliases_file($filename, $type = "url", $max_items = -1) {
 		return null;
 	}
 	$items = array();
+	$comments = array();
 	/* NOTE: fgetss() is not a typo RTFM before being smart */
 	while (($fc = fgetss($fd)) !== FALSE) {
 		$tmp = trim($fc, " \t\n\r");
 		if (empty($tmp)) {
 			continue;
 		}
-		$tmp_str = strstr($tmp, '#', true);
-		if (!empty($tmp_str)) {
-			$tmp = $tmp_str;
-		}
-		$tmp_str = strstr($tmp, ' ', true);
-		if (!empty($tmp_str)) {
-			$tmp = $tmp_str;
-		}
-		$valid = ($type == "url" && (is_ipaddr($tmp) || is_subnet($tmp))) ||
-			 ($type == "url_ports" && (is_port($tmp) || is_portrange($tmp)));
-		if ($valid) {
-			$items[] = $tmp;
-			if (count($items) == $max_items) {
-				break;
+		if (($kflc) && (strpos($tmp, '#') === 0)) {	// Keep Full Line Comments (lines beginning with #).
+			$comments[] = $tmp;
+		} else {
+			$tmp_str = strstr($tmp, '#', true);
+			if (!empty($tmp_str)) {
+				$tmp = $tmp_str;
+			}
+			$tmp_str = strstr($tmp, ' ', true);
+			if (!empty($tmp_str)) {
+				$tmp = $tmp_str;
+			}
+			$valid = ($type == "url" && (is_ipaddr($tmp) || is_subnet($tmp))) ||
+				($type == "url_ports" && (is_port($tmp) || is_portrange($tmp)));
+			if ($valid) {
+				$items[] = $tmp;
+				if (count($items) == $max_items) {
+					break;
+				}
 			}
 		}
 	}
 	fclose($fd);
-	return $items;
+	return array_merge($comments, $items);
 }
 
 function update_alias_url_data() {
@@ -2166,13 +2171,14 @@ function process_alias_urltable($name, $url, $freq, $forceupdate=false, $validat
 		unlink_if_exists($tmp_urltable_filename);
 		$verify_ssl = isset($config['system']['checkaliasesurlcert']);
 		if (download_file($url, $tmp_urltable_filename, $verify_ssl)) {
-			mwexec("/usr/bin/sed -i \"\" -E 's/\;.*//g; /^[[:space:]]*($|#)/d' " . escapeshellarg($tmp_urltable_filename));
+			// Convert lines that begin with '$' or ';' to comments '#' instead of deleting them.
+			mwexec("/usr/bin/sed -i \"\" -E 's/^[[:space:]]*($|#|;)/#/g; /^#/!s/\;.*//g;' ". escapeshellarg($tmp_urltable_filename));
 			if (alias_get_type($name) == "urltable_ports") {
-				$ports = parse_aliases_file($tmp_urltable_filename, "url_ports", "-1");
-				$ports = group_ports($ports);
+				$ports = parse_aliases_file($tmp_urltable_filename, "url_ports", "-1", true);
+				$ports = group_ports($ports, true);
 				file_put_contents($urltable_filename, implode("\n", $ports));
 			} else {
-				$urltable = parse_aliases_file($tmp_urltable_filename, "url", "-1");
+				$urltable = parse_aliases_file($tmp_urltable_filename, "url", "-1", true);
 				file_put_contents($urltable_filename, implode("\n", $urltable));
 			}
 			unlink_if_exists($tmp_urltable_filename);
diff --git a/src/etc/inc/util.inc b/src/etc/inc/util.inc
index 0c77758..e1f5c80 100644
--- a/src/etc/inc/util.inc
+++ b/src/etc/inc/util.inc
@@ -1106,14 +1106,17 @@ function is_portoralias($port) {
 }
 
 /* create ranges of sequential port numbers (200:215) and remove duplicates */
-function group_ports($ports) {
+function group_ports($ports, $kflc = false) {
 	if (!is_array($ports) || empty($ports)) {
 		return;
 	}
 
 	$uniq = array();
+	$comments = array();
 	foreach ($ports as $port) {
-		if (is_portrange($port)) {
+		if (($kflc) && (strpos($port, '#') === 0)) {	// Keep Full Line Comments (lines beginning with #).
+			$comments[] = $port;
+		} else if (is_portrange($port)) {
 			list($begin, $end) = explode(":", $port);
 			if ($begin > $end) {
 				$aux = $begin;
@@ -1155,7 +1158,7 @@ function group_ports($ports) {
 		}
 	}
 
-	return $result;
+	return array_merge($comments, $result);
 }
 
 /* returns true if $val is a valid shaper bandwidth value */
diff --git a/src/etc/rc.update_urltables b/src/etc/rc.update_urltables
index c4dfeb1..887dfac 100755
--- a/src/etc/rc.update_urltables
+++ b/src/etc/rc.update_urltables
@@ -32,11 +32,24 @@ if (count($todo) > 0) {
 		sleep($wait);
 	}
 
+	// Set whether or not to force the table update before it's time.
+	if (!empty($argv[2]) && ($argv[2] == "forceupdate")) {
+		$forceupdate = true;
+	} else {
+		$forceupdate = false;
+	}
+
 	log_error("{$argv[0]}: Starting URL table alias updates");
 
 	$filter_reload = false;
 	foreach ($todo as $t) {
-		$r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
+
+		// Update a specifically named URL table only.
+		if (!empty($argv[3]) && ($argv[3] != $t['name'])) {
+			continue;
+		}
+
+		$r = process_alias_urltable($t['name'], $t['url'], $t['freq'], $forceupdate);
 		if ($r == 1) {
 			$result = "";
 			// TODO: Change it when pf supports tables with ports