diff options
author | Stephen Beaver <sbeaver@netgate.com> | 2016-02-29 11:55:20 -0500 |
---|---|---|
committer | Stephen Beaver <sbeaver@netgate.com> | 2016-02-29 11:55:20 -0500 |
commit | 191136b34a00ac3e6b43302eabd9e713ac020e87 (patch) | |
tree | 335152433b9d1a74f4f189bb33bc685f25e234ab /src | |
parent | 4313cf55a9c2ca5ddf6dc101b638ee4d92af1850 (diff) | |
parent | f66221675e9e37ddad037fe2d29b85dc12cb8253 (diff) | |
download | pfsense-191136b34a00ac3e6b43302eabd9e713ac020e87.zip pfsense-191136b34a00ac3e6b43302eabd9e713ac020e87.tar.gz |
Merge pull request #2693 from NOYB/Diagnostics_/_Tables_-_URL_Table_Aliases
Diffstat (limited to 'src')
-rw-r--r-- | src/etc/inc/pfsense-utils.inc | 46 | ||||
-rw-r--r-- | src/etc/inc/util.inc | 9 | ||||
-rwxr-xr-x | src/etc/rc.update_urltables | 15 | ||||
-rw-r--r-- | src/usr/local/www/diag_tables.php | 113 |
4 files changed, 134 insertions, 49 deletions
diff --git a/src/etc/inc/pfsense-utils.inc b/src/etc/inc/pfsense-utils.inc index 416a89c..e364afb 100644 --- a/src/etc/inc/pfsense-utils.inc +++ b/src/etc/inc/pfsense-utils.inc @@ -1919,7 +1919,7 @@ function update_alias_names_upon_change($section, $field, $new_alias_name, $orig } -function parse_aliases_file($filename, $type = "url", $max_items = -1) { +function parse_aliases_file($filename, $type = "url", $max_items = -1, $kflc = false) { /* * $filename = file to process for example blocklist like DROP: http://www.spamhaus.org/drop/drop.txt * $type = if set to 'url' then subnets and ips will be returned, @@ -1935,31 +1935,36 @@ function parse_aliases_file($filename, $type = "url", $max_items = -1) { return null; } $items = array(); + $comments = array(); /* NOTE: fgetss() is not a typo RTFM before being smart */ while (($fc = fgetss($fd)) !== FALSE) { $tmp = trim($fc, " \t\n\r"); if (empty($tmp)) { continue; } - $tmp_str = strstr($tmp, '#', true); - if (!empty($tmp_str)) { - $tmp = $tmp_str; - } - $tmp_str = strstr($tmp, ' ', true); - if (!empty($tmp_str)) { - $tmp = $tmp_str; - } - $valid = ($type == "url" && (is_ipaddr($tmp) || is_subnet($tmp))) || - ($type == "url_ports" && (is_port($tmp) || is_portrange($tmp))); - if ($valid) { - $items[] = $tmp; - if (count($items) == $max_items) { - break; + if (($kflc) && (strpos($tmp, '#') === 0)) { // Keep Full Line Comments (lines beginning with #). + $comments[] = $tmp; + } else { + $tmp_str = strstr($tmp, '#', true); + if (!empty($tmp_str)) { + $tmp = $tmp_str; + } + $tmp_str = strstr($tmp, ' ', true); + if (!empty($tmp_str)) { + $tmp = $tmp_str; + } + $valid = ($type == "url" && (is_ipaddr($tmp) || is_subnet($tmp))) || + ($type == "url_ports" && (is_port($tmp) || is_portrange($tmp))); + if ($valid) { + $items[] = $tmp; + if (count($items) == $max_items) { + break; + } } } } fclose($fd); - return $items; + return array_merge($comments, $items); } function update_alias_url_data() { @@ -2166,13 +2171,14 @@ function process_alias_urltable($name, $url, $freq, $forceupdate=false, $validat unlink_if_exists($tmp_urltable_filename); $verify_ssl = isset($config['system']['checkaliasesurlcert']); if (download_file($url, $tmp_urltable_filename, $verify_ssl)) { - mwexec("/usr/bin/sed -i \"\" -E 's/\;.*//g; /^[[:space:]]*($|#)/d' " . escapeshellarg($tmp_urltable_filename)); + // Convert lines that begin with '$' or ';' to comments '#' instead of deleting them. + mwexec("/usr/bin/sed -i \"\" -E 's/^[[:space:]]*($|#|;)/#/g; /^#/!s/\;.*//g;' ". escapeshellarg($tmp_urltable_filename)); if (alias_get_type($name) == "urltable_ports") { - $ports = parse_aliases_file($tmp_urltable_filename, "url_ports", "-1"); - $ports = group_ports($ports); + $ports = parse_aliases_file($tmp_urltable_filename, "url_ports", "-1", true); + $ports = group_ports($ports, true); file_put_contents($urltable_filename, implode("\n", $ports)); } else { - $urltable = parse_aliases_file($tmp_urltable_filename, "url", "-1"); + $urltable = parse_aliases_file($tmp_urltable_filename, "url", "-1", true); file_put_contents($urltable_filename, implode("\n", $urltable)); } unlink_if_exists($tmp_urltable_filename); diff --git a/src/etc/inc/util.inc b/src/etc/inc/util.inc index 0c77758..e1f5c80 100644 --- a/src/etc/inc/util.inc +++ b/src/etc/inc/util.inc @@ -1106,14 +1106,17 @@ function is_portoralias($port) { } /* create ranges of sequential port numbers (200:215) and remove duplicates */ -function group_ports($ports) { +function group_ports($ports, $kflc = false) { if (!is_array($ports) || empty($ports)) { return; } $uniq = array(); + $comments = array(); foreach ($ports as $port) { - if (is_portrange($port)) { + if (($kflc) && (strpos($port, '#') === 0)) { // Keep Full Line Comments (lines beginning with #). + $comments[] = $port; + } else if (is_portrange($port)) { list($begin, $end) = explode(":", $port); if ($begin > $end) { $aux = $begin; @@ -1155,7 +1158,7 @@ function group_ports($ports) { } } - return $result; + return array_merge($comments, $result); } /* returns true if $val is a valid shaper bandwidth value */ diff --git a/src/etc/rc.update_urltables b/src/etc/rc.update_urltables index c4dfeb1..887dfac 100755 --- a/src/etc/rc.update_urltables +++ b/src/etc/rc.update_urltables @@ -32,11 +32,24 @@ if (count($todo) > 0) { sleep($wait); } + // Set whether or not to force the table update before it's time. + if (!empty($argv[2]) && ($argv[2] == "forceupdate")) { + $forceupdate = true; + } else { + $forceupdate = false; + } + log_error("{$argv[0]}: Starting URL table alias updates"); $filter_reload = false; foreach ($todo as $t) { - $r = process_alias_urltable($t['name'], $t['url'], $t['freq']); + + // Update a specifically named URL table only. + if (!empty($argv[3]) && ($argv[3] != $t['name'])) { + continue; + } + + $r = process_alias_urltable($t['name'], $t['url'], $t['freq'], $forceupdate); if ($r == 1) { $result = ""; // TODO: Change it when pf supports tables with ports diff --git a/src/usr/local/www/diag_tables.php b/src/usr/local/www/diag_tables.php index 2fe1b6d..4543899 100644 --- a/src/usr/local/www/diag_tables.php +++ b/src/usr/local/www/diag_tables.php @@ -68,12 +68,34 @@ require_once("guiconfig.inc"); // Set default table $tablename = "sshlockout"; -$bogons = false; if ($_REQUEST['type']) { $tablename = $_REQUEST['type']; } +// Gather selected alias metadata. +if (isset($config['aliases']['alias'])) { + foreach ($config['aliases']['alias'] as $alias) { + if ( $alias['name'] == $tablename ) { + $tmp = array(); + $tmp['type'] = $alias['type']; + $tmp['name'] = $alias['name']; + $tmp['url'] = $alias['url']; + $tmp['freq'] = $alias['updatefreq']; + break; + } + } +} + +# Determine if selected alias is either a bogons or URL table. +if (($tablename == "bogons") || ($tablename == "bogonsv6")) { + $bogons = true; +} else if (preg_match('/urltable/i', $tmp['type'])) { + $urltable = true; +} else { + $bogons = $urltable = false; +} + if ($_REQUEST['delete']) { if (is_ipaddr($_REQUEST['delete']) || is_subnet($_REQUEST['delete'])) { exec("/sbin/pfctl -t " . escapeshellarg($_REQUEST['type']) . " -T delete " . escapeshellarg($_REQUEST['delete']), $delete); @@ -93,27 +115,34 @@ if ($_POST['clearall']) { unset($entries); } -if (($tablename == "bogons") || ($tablename == "bogonsv6")) { - $bogons = true; +if ($_POST['Download'] && ($bogons || $urltable)) { - if ($_POST['Download']) { - mwexec_bg("/etc/rc.update_bogons.sh now"); - $maxtimetowait = 0; - $loading = true; - while ($loading == true) { - $isrunning = `/bin/ps awwwux | /usr/bin/grep -v grep | /usr/bin/grep bogons`; - if ($isrunning == "") { - $loading = false; - } - $maxtimetowait++; - if ($maxtimetowait > 89) { - $loading = false; - } - sleep(1); + if ($bogons) { // If selected table is either bogons or bogonsv6. + $mwexec_bg_cmd = '/etc/rc.update_bogons.sh now'; + $table_type = 'bogons'; + $db_name = 'bogons'; + } else if ($urltable) { // If selected table is a URL table alias. + $mwexec_bg_cmd = '/etc/rc.update_urltables now forceupdate ' . $tablename; + $table_type = 'urltables'; + $db_name = $tablename; + } + + mwexec_bg($mwexec_bg_cmd); + $maxtimetowait = 0; + $loading = true; + while ($loading == true) { + $isrunning = `/bin/ps awwwux | /usr/bin/grep -v grep | /usr/bin/grep $table_type`; + if ($isrunning == "") { + $loading = false; } - if ($maxtimetowait < 90) { - $savemsg = gettext("The bogons database has been updated."); + $maxtimetowait++; + if ($maxtimetowait > 89) { + $loading = false; } + sleep(1); + } + if ($maxtimetowait < 90) { + $savemsg = sprintf(gettext("The %s database has been updated."), $db_name); } } @@ -144,8 +173,8 @@ $group->add(new Form_Select( array_combine($tables, $tables) )); -if ($bogons || !empty($entries)) { - if ($bogons) { +if ($bogons || $urltable || !empty($entries)) { + if ($bogons || $urltable) { $group->add(new Form_Button( 'Download', 'Update' @@ -162,12 +191,24 @@ $section->add($group); $form->add($section); print $form; -if ($bogons || !empty($entries)) { +if ($bogons || $urltable || !empty($entries)) { ?> <div> <div class="infoblock blockopen"> <?php - $last_updated = exec('/usr/bin/grep -i -m 1 -E "^# last updated" /etc/' . escapeshellarg($tablename) . '|cut -d"(" -f2|tr -d ")" '); + if ($bogons) { + $table_file = '/etc/' . escapeshellarg($tablename); + } else if ($urltable) { + $table_file = '/var/db/aliastables/' . escapeshellarg($tablename) . '.txt'; + } else { + $table_file = ''; + } + + $datestrregex = '(Mon|Tue|Wed|Thr|Fri|Sat|Sun).* GMT'; + $datelineregex = 'last.*' . $datestrregex; + + $last_updated = exec('/usr/bin/grep -i -m 1 -E "^# ' . $datelineregex . '" ' . $table_file . '|/usr/bin/grep -i -m 1 -E -o "' . $datestrregex . '"'); + if ($last_updated != "") { $last_update_msg = sprintf(gettext("Table last updated on %s."), $last_updated); } else { @@ -176,7 +217,22 @@ if ($bogons || !empty($entries)) { $records_count_msg = sprintf(gettext("%s records."), number_format(count($entries), 0, gettext("."), gettext(","))); - print_info_box($last_update_msg . " " . $records_count_msg, 'info', false); + # Display up to 10 comment lines (lines that begin with '#'). + unset($comment_lines); + $res = exec('/usr/bin/grep -i -m 10 -E "^#" ' . $table_file, $comment_lines); + + foreach ($comment_lines as $comment_line) { + $table_comments .= "$comment_line" . "<br />"; + } + + if ($table_comments) { + print_info_box($last_update_msg . " " . $records_count_msg . " " . + '<span style="display:none" class="infoblock">' . ' ' . gettext("Hide table comments.") . '<br />' . $table_comments . '</span>' . + '<span style="display:none" id="showtblcom">' . ' ' . gettext("Show table comments.") . '</span>' . + '' , 'info', false); + } else { + print_info_box($last_update_msg . " " . $records_count_msg, 'info', false); + } ?> </div> </div> @@ -187,6 +243,13 @@ if ($bogons || !empty($entries)) { <script type="text/javascript"> //<![CDATA[ events.push(function() { + + $('#showtblcom').show(); + + $('[id^="showinfo1"]').click(function() { + $('#showtblcom').toggle(); + }); + $('a[data-entry]').on('click', function() { var el = $(this); @@ -249,7 +312,7 @@ if (empty($entries)) { <?=$entry?> </td> <td> - <?php if (!$bogons): ?> + <?php if (!$bogons && !$urltable): ?> <a class="btn btn-xs btn-default" data-entry="<?=htmlspecialchars($entry)?>"><?=gettext("Remove")?></a> <?php endif ?> </td> |