Add a new alias type, urltable, which downloads a file of IP/CIDR addresses and loads them into a pf persist table instead of importing the addresses directly into a traditional alias. This allows for using huge tables of addresses that would otherwise break the GUI and/or fail to load into pf. Part of ticket #512

4 files changed, 109 insertions, 1 deletions

diff --git a/etc/inc/filter.inc b/etc/inc/filter.inc
index 12d07ef..e8a7631 100644
--- a/etc/inc/filter.inc
+++ b/etc/inc/filter.inc
@@ -488,7 +488,7 @@ function filter_generate_aliases() {
 			$aliasnesting = array();
 			$aliasaddrnesting = array();
 			$addrlist = filter_generate_nested_alias($aliased['name'], $aliased['address'], $aliasnesting, $aliasaddrnesting);
-			if($aliased['type'] == "host" || $aliased['type'] == "network") {
+			if($aliased['type'] == "host" || $aliased['type'] == "network" || $aliased['type'] == "url") {
 				$tableaddrs = "{$addrlist}{$extralias}";
 				if(empty($tableaddrs))
 					$aliases .= "table <{$aliased['name']}> persist\n";
@@ -516,6 +516,12 @@ function filter_generate_aliases() {
 				}
 				$aliases .= "table <{$aliased['name']}> { {$newaddress}{$extralias} } \n";
                                 $aliases .= "{$aliased['name']} = \"<{$aliased['name']}>\"\n";
+			} elseif($aliased['type'] == "urltable") {
+				$urlfn = alias_expand_urltable($aliased['name']);
+				if ($urlfn) {
+					$aliases .= "table <{$aliased['name']}> persist file \"{$urlfn}\"\n";
+					$aliases .= "{$aliased['name']} = \"<{$aliased['name']}>\"\n";
+				}
 			} else 
 				$aliases .= "{$aliased['name']} = \"{ {$aliased['address']}{$extralias} }\"\n";
 		}
diff --git a/etc/inc/pfsense-utils.inc b/etc/inc/pfsense-utils.inc
index 007a27e..b21ec6f 100644
--- a/etc/inc/pfsense-utils.inc
+++ b/etc/inc/pfsense-utils.inc
@@ -1877,5 +1877,42 @@ function pfs_version_compare($cur_time, $cur_text, $remote) {
 	}
 	return $v;
 }
+function process_alias_urltable($name, $url, $freq, $forceupdate=false) {
+	$urltable_prefix = "/var/db/aliastables/";
+	$urltable_filename = $urltable_prefix . $name . ".txt";
+
+	// Make the aliases directory if it doesn't exist
+	if (!file_exists($urltable_prefix)) {
+		mkdir($urltable_prefix);
+	} elseif (!is_dir($urltable_prefix)) {
+		unlink($urltable_prefix);
+		mkdir($urltable_prefix);
+	}
+
+	// If the file doesn't exist or is older than update_freq days, fetch a new copy.
+	if (!file_exists($urltable_filename)
+		|| ((time() - filemtime($urltable_filename)) > ($freq * 86400))
+		|| $forceupdate) {
+
+		// Try to fetch the URL supplied
+		conf_mount_rw();
+		unlink_if_exists($urltable_filename . ".tmp");
+		// Use fetch to grab data since these may be large files, we don't want to process them through PHP if we can help it.
+		mwexec("/usr/bin/fetch -q -o " . escapeshellarg($urltable_filename . ".tmp") . " " . escapeshellarg($url));
+		// Remove comments. Might need some grep-fu to only allow lines that look like IPs/subnets
+		mwexec("/usr/bin/grep -v '^#' " . escapeshellarg($urltable_filename . ".tmp") . " > " . escapeshellarg($urltable_filename));
+		unlink_if_exists($urltable_filename . ".tmp");
+		conf_mount_ro();
+		if (filesize($urltable_filename)) {
+			return true;
+		} else {
+			// If it's unfetchable or an empty file, bail
+			return false;
+		}
+	} else {
+		// File exists, and it doesn't need updated.
+		return -1;
+	}
+}
 
 ?>
diff --git a/etc/inc/util.inc b/etc/inc/util.inc
index ea18345..d71c512 100644
--- a/etc/inc/util.inc
+++ b/etc/inc/util.inc
@@ -851,6 +851,22 @@ function alias_expand($name) {
 		return null;
 }
 
+function alias_expand_urltable($name) {
+	global $config;
+	$urltable_prefix = "/var/db/aliastables/";
+	$urltable_filename = $urltable_prefix . $name . ".txt";
+
+	foreach ($config['aliases']['alias'] as $alias) {
+		if (($alias['type'] == 'urltable') && ($alias['name'] == $name)) {
+			if (is_URL($alias["url"]) && file_exists($urltable_filename))
+				return $urltable_filename;
+			else if (process_alias_urltable($name, $alias["url"], 0, true))
+				return $urltable_filename;
+		}
+	}
+	return null;
+}
+
 /* find out whether two subnets overlap */
 function check_subnets_overlap($subnet1, $bits1, $subnet2, $bits2) {
 
diff --git a/etc/rc.update_urltables b/etc/rc.update_urltables
new file mode 100644
index 0000000..fcb60d3
--- /dev/null
+++ b/etc/rc.update_urltables
@@ -0,0 +1,49 @@
+#!/usr/local/bin/php -q
+<?php
+require_once("config.inc");
+require_once("util.inc");
+require_once("pfsense-utils.inc");
+
+if (!is_array($config['aliases']['alias'])) {
+	// No aliases
+	exit;
+}
+
+// Gather list of urltable aliases
+$todo = array();
+foreach ($config['aliases']['alias'] as $alias) {
+	if ($alias['type'] == 'urltable') {
+		$tmp = array();
+		$tmp['name'] = $alias['name'];
+		$tmp['url']  = $alias['url'];
+		$tmp['freq'] = $alias['updatefreq'];
+		$todo[] = $tmp;
+	}
+}
+
+if (count($todo) > 0) {
+	log_error("{$argv[0]}: Starting up.");
+
+	if ($argv[1] != "now") {
+		// Wait a little before updating.
+		$wait = mt_rand(5, 60);
+		log_error("{$argv[0]}: Sleeping for {$wait} seconds.");
+		sleep($wait);
+	}
+
+	log_error("{$argv[0]}: Starting URL table alias updates");
+
+	foreach ($todo as $t) {
+		$r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
+		if ($r == 1) {
+			$result = "";
+			exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result);
+			log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}");
+		} elseif ($r == -1) {
+			log_error("{$argv[0]}: {$t['name']} does not need updated.");
+		} else {
+			log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}");
+		}
+	}
+}
+?>
\ No newline at end of file