Add a new alias type, urltable, which downloads a file of IP/CIDR addresses and loads them into a pf persist table instead of importing the addresses directly into a traditional alias. This allows for using huge tables of addresses that would otherwise break the GUI and/or fail to load into pf. Part of ticket #512

9 files changed, 185 insertions, 9 deletions

diff --git a/etc/inc/filter.inc b/etc/inc/filter.inc
index 12d07ef..e8a7631 100644
--- a/etc/inc/filter.inc
+++ b/etc/inc/filter.inc
@@ -488,7 +488,7 @@ function filter_generate_aliases() {
 			$aliasnesting = array();
 			$aliasaddrnesting = array();
 			$addrlist = filter_generate_nested_alias($aliased['name'], $aliased['address'], $aliasnesting, $aliasaddrnesting);
-			if($aliased['type'] == "host" || $aliased['type'] == "network") {
+			if($aliased['type'] == "host" || $aliased['type'] == "network" || $aliased['type'] == "url") {
 				$tableaddrs = "{$addrlist}{$extralias}";
 				if(empty($tableaddrs))
 					$aliases .= "table <{$aliased['name']}> persist\n";
@@ -516,6 +516,12 @@ function filter_generate_aliases() {
 				}
 				$aliases .= "table <{$aliased['name']}> { {$newaddress}{$extralias} } \n";
                                 $aliases .= "{$aliased['name']} = \"<{$aliased['name']}>\"\n";
+			} elseif($aliased['type'] == "urltable") {
+				$urlfn = alias_expand_urltable($aliased['name']);
+				if ($urlfn) {
+					$aliases .= "table <{$aliased['name']}> persist file \"{$urlfn}\"\n";
+					$aliases .= "{$aliased['name']} = \"<{$aliased['name']}>\"\n";
+				}
 			} else 
 				$aliases .= "{$aliased['name']} = \"{ {$aliased['address']}{$extralias} }\"\n";
 		}
diff --git a/etc/inc/pfsense-utils.inc b/etc/inc/pfsense-utils.inc
index 007a27e..b21ec6f 100644
--- a/etc/inc/pfsense-utils.inc
+++ b/etc/inc/pfsense-utils.inc
@@ -1877,5 +1877,42 @@ function pfs_version_compare($cur_time, $cur_text, $remote) {
 	}
 	return $v;
 }
+function process_alias_urltable($name, $url, $freq, $forceupdate=false) {
+	$urltable_prefix = "/var/db/aliastables/";
+	$urltable_filename = $urltable_prefix . $name . ".txt";
+
+	// Make the aliases directory if it doesn't exist
+	if (!file_exists($urltable_prefix)) {
+		mkdir($urltable_prefix);
+	} elseif (!is_dir($urltable_prefix)) {
+		unlink($urltable_prefix);
+		mkdir($urltable_prefix);
+	}
+
+	// If the file doesn't exist or is older than update_freq days, fetch a new copy.
+	if (!file_exists($urltable_filename)
+		|| ((time() - filemtime($urltable_filename)) > ($freq * 86400))
+		|| $forceupdate) {
+
+		// Try to fetch the URL supplied
+		conf_mount_rw();
+		unlink_if_exists($urltable_filename . ".tmp");
+		// Use fetch to grab data since these may be large files, we don't want to process them through PHP if we can help it.
+		mwexec("/usr/bin/fetch -q -o " . escapeshellarg($urltable_filename . ".tmp") . " " . escapeshellarg($url));
+		// Remove comments. Might need some grep-fu to only allow lines that look like IPs/subnets
+		mwexec("/usr/bin/grep -v '^#' " . escapeshellarg($urltable_filename . ".tmp") . " > " . escapeshellarg($urltable_filename));
+		unlink_if_exists($urltable_filename . ".tmp");
+		conf_mount_ro();
+		if (filesize($urltable_filename)) {
+			return true;
+		} else {
+			// If it's unfetchable or an empty file, bail
+			return false;
+		}
+	} else {
+		// File exists, and it doesn't need updated.
+		return -1;
+	}
+}
 
 ?>
diff --git a/etc/inc/util.inc b/etc/inc/util.inc
index ea18345..d71c512 100644
--- a/etc/inc/util.inc
+++ b/etc/inc/util.inc
@@ -851,6 +851,22 @@ function alias_expand($name) {
 		return null;
 }
 
+function alias_expand_urltable($name) {
+	global $config;
+	$urltable_prefix = "/var/db/aliastables/";
+	$urltable_filename = $urltable_prefix . $name . ".txt";
+
+	foreach ($config['aliases']['alias'] as $alias) {
+		if (($alias['type'] == 'urltable') && ($alias['name'] == $name)) {
+			if (is_URL($alias["url"]) && file_exists($urltable_filename))
+				return $urltable_filename;
+			else if (process_alias_urltable($name, $alias["url"], 0, true))
+				return $urltable_filename;
+		}
+	}
+	return null;
+}
+
 /* find out whether two subnets overlap */
 function check_subnets_overlap($subnet1, $bits1, $subnet2, $bits2) {
 
diff --git a/etc/rc.update_urltables b/etc/rc.update_urltables
new file mode 100644
index 0000000..fcb60d3
--- /dev/null
+++ b/etc/rc.update_urltables
@@ -0,0 +1,49 @@
+#!/usr/local/bin/php -q
+<?php
+require_once("config.inc");
+require_once("util.inc");
+require_once("pfsense-utils.inc");
+
+if (!is_array($config['aliases']['alias'])) {
+	// No aliases
+	exit;
+}
+
+// Gather list of urltable aliases
+$todo = array();
+foreach ($config['aliases']['alias'] as $alias) {
+	if ($alias['type'] == 'urltable') {
+		$tmp = array();
+		$tmp['name'] = $alias['name'];
+		$tmp['url']  = $alias['url'];
+		$tmp['freq'] = $alias['updatefreq'];
+		$todo[] = $tmp;
+	}
+}
+
+if (count($todo) > 0) {
+	log_error("{$argv[0]}: Starting up.");
+
+	if ($argv[1] != "now") {
+		// Wait a little before updating.
+		$wait = mt_rand(5, 60);
+		log_error("{$argv[0]}: Sleeping for {$wait} seconds.");
+		sleep($wait);
+	}
+
+	log_error("{$argv[0]}: Starting URL table alias updates");
+
+	foreach ($todo as $t) {
+		$r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
+		if ($r == 1) {
+			$result = "";
+			exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result);
+			log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}");
+		} elseif ($r == -1) {
+			log_error("{$argv[0]}: {$t['name']} does not need updated.");
+		} else {
+			log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}");
+		}
+	}
+}
+?>
\ No newline at end of file
diff --git a/usr/local/www/firewall_aliases.php b/usr/local/www/firewall_aliases.php
index 14168a0..d5a6a5f 100755
--- a/usr/local/www/firewall_aliases.php
+++ b/usr/local/www/firewall_aliases.php
@@ -183,6 +183,12 @@ include("head.inc");
   </td>
   <td class="listr" ondblclick="document.location='firewall_aliases_edit.php?id=<?=$i;?>';">
       <?php
+	if ($alias["url"]) {
+		echo $alias["url"] . "<br/>";
+	}
+	if ($alias["aliasurl"]) {
+		echo $alias["aliasurl"] . "<br/>";
+	}
 	$tmpaddr = explode(" ", $alias['address']);
 	$addresses = implode(", ", array_slice($tmpaddr, 0, 10));
 	echo $addresses;
diff --git a/usr/local/www/firewall_aliases_edit.php b/usr/local/www/firewall_aliases_edit.php
index d26fbb0..9c8025d 100755
--- a/usr/local/www/firewall_aliases_edit.php
+++ b/usr/local/www/firewall_aliases_edit.php
@@ -103,7 +103,10 @@ if (isset($id) && $a_aliases[$id]) {
 		if($ifdesc == $pconfig['descr']) 
 			$input_errors[] = "Sorry, an interface is already named {$pconfig['descr']}.";
 
-
+	if($a_aliases[$id]['type'] == "urltable") {
+		$pconfig['address'] = $a_aliases[$id]['url'];
+		$pconfig['updatefreq'] = $a_aliases[$id]['updatefreq'];
+	}
 	if($a_aliases[$id]['aliasurl'] <> "") {
 		$pconfig['type'] = "url";
 		if(is_array($a_aliases[$id]['aliasurl'])) {
@@ -122,7 +125,6 @@ if (isset($id) && $a_aliases[$id]) {
 }
 
 if ($_POST) {
-
 	unset($input_errors);
 
 	/* input validation */
@@ -168,7 +170,27 @@ if ($_POST) {
 	$address = array();
 	$final_address_details = array();
 	$alias['name'] = $_POST['name'];
-	if($_POST['type'] == "url") {
+
+	if ($_POST['type'] == "urltable") {
+		$address = "";
+		$isfirst = 0;
+
+		/* item is a url type */
+		if ($_POST['address0']) {
+			/* fetch down and add in */
+			$isfirst = 0;
+			$address = "";
+			$alias['url'] = $_POST['address0'];
+			$alias['updatefreq'] = $_POST['address_subnet0'] ? $_POST['address_subnet0'] : 7;
+			if (!is_URL($alias['url']) || empty($alias['url'])) {
+				$input_errors[] = "You must provide a valid URL.";
+				$dont_update = true;
+			} elseif (! process_alias_urltable($alias['name'], $alias['url'], 0, true)) {
+				$input_errors[] = "Unable to fetch usable data.";
+				$dont_update = true;
+			}
+		}
+	} elseif($_POST['type'] == "url") {
 		$isfirst = 0;
 		$address_count = 2;
 
@@ -192,6 +214,9 @@ if ($_POST) {
 					$file_contents = str_replace("#", "\n#", $file_contents);
 					$file_contents_split = split("\n", $file_contents);
 					foreach($file_contents_split as $fc) {
+						// Stop at 3000 items, aliases larger than that tend to break both pf and the WebGUI.
+						if ($address_count >= 3000)
+							break;
 						$tmp = trim($fc);
 						if(stristr($fc, "#")) {
 							$tmp_split = split("#", $tmp);
@@ -201,6 +226,7 @@ if ($_POST) {
 						if(!empty($tmp) && (is_ipaddr($tmp) || is_subnet($tmp))) {
 							$address[] = $tmp;
 							$isfirst = 1;
+							$address_count++;
 						}
 					}
 					if($isfirst == 0) {
@@ -256,7 +282,7 @@ if ($_POST) {
 	}
 
 	if (!$input_errors) {
-		$alias['address'] = implode(" ", $address);
+		$alias['address'] = is_array($address) ? implode(" ", $address) : $address;
 		$alias['descr'] = mb_convert_encoding($_POST['descr'],"HTML-ENTITIES","auto");
 		$alias['type'] = $_POST['type'];
 		$alias['detail'] = implode("||", $final_address_details);
@@ -311,6 +337,7 @@ if ($_POST) {
 	//we received input errors, copy data to prevent retype
 	else
 	{
+		$pconfig['name'] = $_POST['name'];
 		$pconfig['descr'] = mb_convert_encoding($_POST['descr'],"HTML-ENTITIES","auto");
 		$pconfig['address'] = implode(" ", $address);
 		$pconfig['type'] = $_POST['type'];
@@ -378,6 +405,15 @@ function typesel_change() {
 				eval(comd);
 			}
 			break;
+
+		case 5:	/* urltable */
+			var cmd;
+			newrows = totalrows;
+			for(i=0; i<newrows; i++) {
+				comd = 'document.iform.address_subnet' + i + '.disabled = 0;';
+				eval(comd);
+			}
+			break;
 	}
 }
 
@@ -399,12 +435,14 @@ $ip_str = gettext("IP");
 $ports_str = gettext("Port(s)");
 $port_str = gettext("Port");
 $url_str = gettext("URL");
+$urltable_str = gettext("URL Table");
 $update_freq_str = gettext("Update Freq.");
 
 $networks_help = gettext("Networks are specified in CIDR format.  Select the CIDR mask that pertains to each entry. /32 specifies a single host, /24 specifies 255.255.255.0, etc. Hostnames (FQDNs) may also be specified, using a /32 mask. You may also enter an IP range such as 192.168.1.1-192.168.1.254 and a list of CIDR networks will be derived to fill the range.");
 $hosts_help = gettext("Enter as many hosts as you would like.  Hosts must be specified by their IP address.");
 $ports_help = gettext("Enter as many ports as you wish.  Port ranges can be expressed by seperating with a colon.");
-$url_help = gettext("Enter as many URLs as you wish. After saving {$g['product_name']} will download the URL and import the items into the alias.");
+$url_help = gettext("Enter as many URLs as you wish. After saving {$g['product_name']} will download the URL and import the items into the alias. Use only with small sets of IP addresses (less than 3000).");
+$urltable_help = gettext("Enter a single URL containing a large number of IPs and/or Subnets. After saving {$g['product_name']} will download the URL and create a table file containing these addresses. This will work with large numbers of addresses (30,000+) or small numbers.");
 
 $openvpn_str = gettext("Username");
 $openvpn_user_str = gettext("OpenVPN Users");
@@ -422,30 +460,48 @@ function update_box_type() {
 		document.getElementById ("twocolumn").firstChild.data = "{$cidr_str}";
 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
 		document.getElementById ("itemhelp").firstChild.data = "{$networks_help}";
+		document.getElementById ("addrowbutton").style.display = 'block';
 	} else if(selected == '{$hosts_str}') {
 		document.getElementById ("addressnetworkport").firstChild.data = "{$hosts_str}";
 		document.getElementById ("onecolumn").firstChild.data = "{$ip_str}";
 		document.getElementById ("twocolumn").firstChild.data = "";
 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
 		document.getElementById ("itemhelp").firstChild.data = "{$hosts_help}";
+		document.getElementById ("addrowbutton").style.display = 'block';
 	} else if(selected == '{$ports_str}') {
 		document.getElementById ("addressnetworkport").firstChild.data = "{$ports_str}";
 		document.getElementById ("onecolumn").firstChild.data = "{$port_str}";
 		document.getElementById ("twocolumn").firstChild.data = "";
 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
 		document.getElementById ("itemhelp").firstChild.data = "{$ports_help}";
+		document.getElementById ("addrowbutton").style.display = 'block';
 	} else if(selected == '{$url_str}') {
 		document.getElementById ("addressnetworkport").firstChild.data = "{$url_str}";
 		document.getElementById ("onecolumn").firstChild.data = "{$url_str}";
 		document.getElementById ("twocolumn").firstChild.data = "";
 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
 		document.getElementById ("itemhelp").firstChild.data = "{$url_help}";
+		document.getElementById ("addrowbutton").style.display = 'block';
 	} else if(selected == '{$openvpn_user_str}') {
 		document.getElementById ("addressnetworkport").firstChild.data = "{$openvpn_user_str}";
 		document.getElementById ("onecolumn").firstChild.data = "{$openvpn_str}";
 		document.getElementById ("twocolumn").firstChild.data = "{$openvpn_freq}";
 		document.getElementById ("threecolumn").firstChild.data = "{$description_str}";
 		document.getElementById ("itemhelp").firstChild.data = "{$openvpn_help}";
+		document.getElementById ("addrowbutton").style.display = 'block';
+	} else if(selected == '{$urltable_str}') {
+		if ((typeof(totalrows) == "undefined") || (totalrows < 1)) {
+			addRowTo('maintable', 'formfldalias');
+			typesel_change();
+			add_alias_control(this);
+		}
+		document.getElementById ("addressnetworkport").firstChild.data = "{$url_str}";
+		document.getElementById ("onecolumn").firstChild.data = "{$url_str}";
+		document.getElementById ("twocolumn").firstChild.data = "{$update_freq_str}";
+		document.getElementById ("threecolumn").firstChild.data = "";
+		document.getElementById ("threecolumn").style.display = 'none';
+		document.getElementById ("itemhelp").firstChild.data = "{$urltable_help}";
+		document.getElementById ("addrowbutton").style.display = 'none';
 	}
 }
 </script>
@@ -525,6 +581,7 @@ EOD;
         <option value="port" <?php if ($pconfig['type'] == "port") echo "selected"; ?>>Port(s)</option>
         <option value="openvpn" <?php if ($pconfig['type'] == "openvpn") echo "selected"; ?>>OpenVPN Users</option>
         <option value="url" <?php if ($pconfig['type'] == "url") echo "selected"; ?>>URL</option>
+        <option value="urltable" <?php if ($pconfig['type'] == "urltable") echo "selected"; ?>>URL Table</option>
       </select>
     </td>
   </tr>
@@ -572,7 +629,7 @@ EOD;
 			        <select name="address_subnet<?php echo $tracker; ?>" class="formselect" id="address_subnet<?php echo $tracker; ?>">
 				<option></option>
 			          <?php for ($i = 32; $i >= 1; $i--): ?>
-			          <option value="<?=$i;?>" <?php if ($i == $address_subnet) echo "selected"; ?>><?=$i;?></option>
+			          <option value="<?=$i;?>" <?php if (($i == $address_subnet) || ($i == $pconfig['updatefreq'])) echo "selected"; ?>><?=$i;?></option>
 			          <?php endfor; ?>
 			        </select>
 			      </td>
@@ -594,8 +651,8 @@ EOD;
 
         </tfoot>
 		  </table>
-			<a onclick="javascript:addRowTo('maintable', 'formfldalias'); typesel_change(); add_alias_control(this); return false;" href="#">
-        <img border="0" src="/themes/<?= $g['theme']; ?>/images/icons/icon_plus.gif" alt="" title="add another entry" />
+			<div id="addrowbutton"><a onclick="javascript:addRowTo('maintable', 'formfldalias'); typesel_change(); add_alias_control(this); return false;" href="#">
+        <img border="0" src="/themes/<?= $g['theme']; ?>/images/icons/icon_plus.gif" alt="" title="add another entry" /></div>
       </a>
 		</td>
   </tr>
diff --git a/usr/local/www/firewall_nat_edit.php b/usr/local/www/firewall_nat_edit.php
index 932b02e..8b36fb8 100755
--- a/usr/local/www/firewall_nat_edit.php
+++ b/usr/local/www/firewall_nat_edit.php
@@ -544,6 +544,7 @@ if($config['aliases']['alias'] <> "")
                         case "host":
                         case "network":
                         case "openvpn":
+			case "urltable":
                                 if($addrisfirst == 1) $aliasesaddr .= ",";
                                 $aliasesaddr .= "'" . $alias_name['name'] . "'";
                                 $addrisfirst = 1;
diff --git a/usr/local/www/firewall_rules_edit.php b/usr/local/www/firewall_rules_edit.php
index e88ac6a..236a23a 100755
--- a/usr/local/www/firewall_rules_edit.php
+++ b/usr/local/www/firewall_rules_edit.php
@@ -1307,6 +1307,7 @@ include("head.inc");
 			case "host":
 			case "network":
 			case "openvpn":
+			case "urltable":
 				if($addrisfirst == 1) $aliasesaddr .= ",";
 				$aliasesaddr .= "'" . $alias_name['name'] . "'";
 				$addrisfirst = 1;
diff --git a/usr/local/www/guiconfig.inc b/usr/local/www/guiconfig.inc
index 7d2b4c2..0de2f2d 100755
--- a/usr/local/www/guiconfig.inc
+++ b/usr/local/www/guiconfig.inc
@@ -947,6 +947,9 @@ function rule_popup($src,$srcport,$dst,$dstport){
                 	$alias_details = explode ("||", $alias_name['detail']);
                 	$alias_objects_with_details = "";
                 	$counter = 0;
+			if ($alias_name['url']) {
+				$alias_objects_with_details .= $alias_name['url'] . "<br/>";
+			}
                 	foreach($alias_addresses as $alias_ports_address)
                 	{
                         	$alias_objects_with_details .= $alias_addresses[$counter];