<?php
require_once(dirname(__FILE__).'/_plugin.php');
class Default_Plugin extends Plugin
{

	/**
	 * @var boolean $savefullcontent Do we save the full page content to the database ?
	 * */
	var $savefullcontent = true;

	/**
	 * @var $basehref  boolean Default value for the base href of a page
	 */
	var $basehref=false;

	/**
	 * @var boolean $debug_memory Do we want to enable memory usage debugging ?
	 */
	var $debug_memory = false;

	/**
	 * @var boolean $debug Do we want to enable general debugging ?
	 */
	var $debug = false;

	/**
	 * Parse the page data
	 *
	 * @param string $pageData The content of the page to parse
	 * @param string $url The url of the page that we are parsing (to be able to handle relative links)
	 *
	 * @return string The content of the page that we actually want included in our index
	 */
	function Parse($pageData, $url)
	{
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// We can't parse pdf files so check if the file is a pdf and
		// if it is skip it
		if (substr($pageData, 0, 4) == '%PDF') {
			$return = false;
			return $return;
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// Don't need to cleanup the html here since ExtractNewLinks does this for us
		//$this->CleanupHTML($pageData);

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		$this->ExtractBaseHref($pageData);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		$this->ExtractTitle($pageData);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		if ($this->debug) {
			error_log(sprintf("The base href for %s is %s\n", $url, $this->basehref), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		// We have to get the links before we strip it down to the body so
		// we can take base hrefs into consideration
		if ($this->debug) {
			error_log(sprintf("Pagcontent to be checked for links at url %s: %s\n", $url, $pageData), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		$this->links = $this->ExtractNewLinks($pageData, $url);

		if ($this->debug) {
			error_log(sprintf("Extracted links @ LINE %d: %s\n", __LINE__, print_r($this->links, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		$this->RemoveFFIgnoreTags($pageData);
		$this->CleanupComments($pageData);

		if ($this->debug) {
			error_log(sprintf("PAGEDATA @ LINE %d: %s\n", __LINE__, print_r($pageData, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// We replace any tags which create whitespace with a space
		$pageData = preg_replace('%</?(?:br|tr|td|table|p|div|span|img|option|li|ul|select)[^>]*?>%si', ' ', $pageData);

		if ($this->debug) {
			error_log(sprintf("PAGEDATA @ LINE %d: %s\n", __LINE__, print_r($pageData, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		$this->ExtractDescription($pageData);

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// We don't want the title in the page content since it is indexed separately
		$pageData = preg_replace('%<title>.*?</title>%si', ' ', $pageData);

		// Strip unimportant tags from the page content before indexing
		$pageData = strip_tags($pageData, "<meta>");

		// remove extra spacing to cut down on the amount of data we're storing.
		$pageData = preg_replace('/\s+/', ' ', $pageData);

		if ($this->debug) {
			error_log(sprintf("Extracted links @ LINE %d: %s\n", __LINE__, print_r($this->links, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
			error_log(sprintf("Page data @ LINE %d: %s\n", __LINE__, print_r($pageData, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		return $pageData;
	}

	/**
	 * Extract the <base href=""></base> from the page and store it in $this->basehref (or false if there is none)
	 *
	 * @pararm string $pageData The content of the page to try and get the base href from
	 *
	 * @return void
	 */
	function ExtractBaseHref($pageData)
	{
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		$this->basehref = false;
		$basematches = array();
		preg_match('%<base href\s*=\s*["\']\s*([^"\'>]*)["\']*\s*/?>%is', $pageData, $basematches);
		if (isset($basematches[1])) {
			$this->basehref = $basematches[1];
			// If the base href does not have a trailing /, add one
			if (substr($this->basehref, -1) != '/') {
				$this->basehref .= '/';
			}
		}
		unset($basematches);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
	}

	/**
	 * Extract the pages title <title></title> from the page and store it in $this->title
	 *
	 * @pararm string $pageData The content of the page to try and get the page title from. If none is found then the
	 *                          language variable defaultPageTitle will be used
	 *
	 * @return void
	 */
	function ExtractTitle($pageData)
	{
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		// Try and get the pages title
		$titleArray = array();
		if (preg_match("%<title>([^<]*)</title>%si", $pageData, $titleArray)) {
			// The page as a title tag, extract it
			$this->title = $titleArray[1];
		} else {
			// Page contains no title, give it the default title
			$this->title = GetLang('defaultPageTitle');
		}
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// Truncate the title and description if we need to
		if (strlen($this->title) > 255) {
			$this->title = substr($this->title, 0, 252) . "...";
		}
	}

	function ExtractDescription($pageData)
	{
		$num_words = 30;

		$pageData = substr(preg_replace('/\s+/', ' ', $pageData), 0, 2000);

		$tempArray = explode(" ", $pageData);

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		$this->desc = implode(' ', array_slice($tempArray, 0, $num_words));

		$this->desc = trim($this->desc);

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		if (strlen($this->desc) > 255) {
			$this->desc = substr($this->desc, 0, 252) . "...";
		}
	}

	function ExtractNewLinks(&$pageData, $url='')
	{
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// Do we index sub domains?
		$indexSubdomains = $GLOBALS["FF_CFG"]["site"]["indexSubDomains"];

		if ($indexSubdomains) {
			// Get the base URL for pattern matching
			$baseURL = $GLOBALS["FF_CFG"]["site"]["baseURL"];
		} else {
			$url_parts = parse_url($GLOBALS["FF_CFG"]["site"]["siteURL"]);
			$baseURL = $url_parts['host'];
		}

		$spiderURLs = array();
		$finalURLs = array();

		$this->CleanupHTML($pageData);

		if ($this->debug) {
			error_log(sprintf("Pagcontent after calling cleanuphtml %s\n", $pageData), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		// Get all the regular links out of the page
		$matches = array();
		preg_match_all('%<a.+href\s*=\s*["\']*([^"\' >]+?)["\']*\s*.+>%isU', $pageData, $matches);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		// Clean up the links (remove duplicates etc)
		$links = array_unique($matches[1]);

		if ($this->debug) {
			error_log(sprintf("Links extracted from the content %s\n", print_r($links, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		// Get all the links to frames from the page we grabbed.
		$matches = array();
		preg_match_all('%<frame[^>]*src\s*=\s*[\'\"]?([:\%/?=&;\\,.\w-]*)[#\'\" ]?[^>]*>%is', $pageData, $matches);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		// Remove the match from the page data
		$links = array_unique(array_merge($links, $matches[1]));
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		if ($this->debug) {
			error_log(sprintf("Links before being normalised %s\n", print_r($links, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		$spiderURLs = $this->NormaliseURLs($links, $url);

		if ($this->debug) {
			error_log(sprintf("Links after being normalised %s\n", print_r($spiderURLs, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		// Remove duplicate entries and bad paths
		foreach ($spiderURLs as $url) {
			// Only index extensions that have been chosen
			$url_parts = parse_url($url);

			if (isset($url_parts['path'])) {
				if ($this->debug) {
					error_log(sprintf("Final check, are we looking at a url with a valid extension for %s ?\n", $url), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}

				if (!$this->IsValidExtension($url)) {
					if ($this->debug) {
						error_log(sprintf("Skipping url %s which has an invalid extension\n", $url), 3, TEMP_DIRECTORY.'/default_plugin.log');
					}
					continue;
				}
				if ($this->debug) {
					error_log(sprintf("Extension is ok %s ?\n", $url), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
			}

			if ($this->debug) {
				error_log(sprintf("Final check, are we looking at a url on a valid host for %s ?\n", $url), 3, TEMP_DIRECTORY.'/default_plugin.log');
			}
			if ($indexSubdomains) {
				if ($this->debug) {
					$pos = preg_match('%'.preg_quote($url_parts['host']).'$%', $baseURL);
					error_log(sprintf("    Index subdomains is on\n"), 3, TEMP_DIRECTORY.'/default_plugin.log');
					error_log(sprintf("    Strpos of host in baseurl is %s\n", var_export($pos, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				// If we are indexing subdomains, check to make sure
				// the subdomain is in the url's host
				if (!preg_match('%'.preg_quote($baseURL).'$%', $url_parts['host'])) {
					if ($this->debug) {
						error_log(sprintf("Skipping url %s - url not a (sub)domain we care about\n", $url), 3, TEMP_DIRECTORY.'/default_plugin.log');
					}
					continue;
				}
				if ($this->debug) {
					error_log("Url is ok to keep\n", 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
			} else {
				if ($this->debug) {
					error_log(sprintf("    Index subdomains is off\n"), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				// If we are not indexing subdomains, make sure the
				// host starts with our base
				if (strpos($url_parts['host'], $baseURL) !== 0) {
					if ($this->debug) {
						error_log(sprintf("Skipping url %s - not indexing subdomains %s VS %s \n", $url, $url_parts['host'], $baseURL), 3, TEMP_DIRECTORY.'/default_plugin.log');
					}
					continue;
				}
			}

			$finalURLs[] = $url;
		}

		if ($this->debug) {
			error_log(sprintf("Final URLS %s\n", print_r($finalURLs, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		$finalURLs = array_unique($finalURLs);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
		// Now that we have the URL's on this page, return them
		return $finalURLs;
	}

	function RemoveFFIgnoreTags(&$pageData)
	{
		if (strpos($pageData, 'FF_IGNORE') === false) {
			if ($this->debug) {
				error_log("LOOKING FOR FF_IGNORE BUT DIDNT FIND IT: \n", 3, TEMP_DIRECTORY.'/default_plugin.log');
				error_log("CALLED FROM".print_r(debug_backtrace(), true)."\n", 3, TEMP_DIRECTORY.'/default_plugin.log');
			}
			return;
		}
		// Remove any FF_IGNORE tags
		if ($this->debug) {
			error_log("BEFORE FF REMOVE IGNORE TAGS: ".$pageData, 3, TEMP_DIRECTORY.'/default_plugin.log');
		}
		$pageData = preg_replace('%<!--\s*FF_IGNORE\s*-->.*?<!--\s*FF_END_IGNORE\s*-->%is', '', $pageData);
		if ($this->debug) {
			error_log("AFTER FF REMOVE IGNORE TAGS: ".$pageData, 3, TEMP_DIRECTORY.'/default_plugin.log');
		}
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
	}

	function CleanupHTML(&$pageData)
	{
		if ($this->debug) {
			error_log(sprintf("PAGEDATA @ LINE %d: %s\n", __LINE__, print_r($pageData, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		// Has this page got any script tags?
		$pageData = preg_replace('%<script[^>]*>.*?</script>%is', '', $pageData);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		if ($this->debug) {
			error_log(sprintf("PAGEDATA @ LINE %d: %s\n", __LINE__, print_r($pageData, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		$pageData = preg_replace('%<style[^>]*>.*?</style>%is', '', $pageData);
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}

		if ($this->debug) {
			error_log(sprintf("PAGEDATA @ LINE %d: %s\n", __LINE__, print_r($pageData, true)), 3, TEMP_DIRECTORY.'/default_plugin.log');
		}

		$pageData = str_replace('&nbsp;', ' ', $pageData);
		$pageData = str_replace('&amp;', '&', $pageData);

		// Remove extra whitespace
		$pageData = preg_replace('%\r%', "\n", $pageData);
		$pageData = preg_replace('%(\s)\s+%s', '\\1', $pageData);
		$pageData = trim($pageData);
	}

	function CleanupComments(&$pageData)
	{
		// Has this page got any comment tags?
		if ($this->debug) {
			error_log("BEFORE FF REMOVE COMMENTS: ".$pageData, 3, TEMP_DIRECTORY.'/default_plugin.log');
		}
		$pageData = preg_replace('%<!--.*?-->%is', '', $pageData);
		if ($this->debug) {
			error_log("AFTER FF REMOVE COMMENTS: ".$pageData, 3, TEMP_DIRECTORY.'/default_plugin.log');
		}
		if ($this->debug_memory) {
			error_log(sprintf("%s|%d|%s\n", __FILE__, __LINE__, number_format((memory_get_usage(true)/1024/1024), 2)), 3, TEMP_DIRECTORY.'/memory.log');
		}
	}

	/**
	* NormaliseURLs takes an array of urls and modifies them to try and make
	* sure that any 2 equivilant urls are actually the same in our array
	*
	* @param array $links The array of links to check
	* @param string $url The url of the page the links were on (for taking
	*                    relativity into consideration
	*
	* @return array The normalised array of links
	*/
	function NormaliseURLs($links, $url)
	{
		$return_urls = array();

		// Make sure we are going to be able to get a host from the url
		if (substr($url, 0, 4) != 'http') {
			$url='http://'.$url;
		}

		foreach ($links as $link) {
			$link = trim($link);
			// Skip blank links
			if ($link == '') {
				continue;
			}

			// Skip javascript links
			if (substr($link, 0, 11) == 'javascript:') {
				if ($this->debug) {
					error_log(sprintf("Skipping javascript: link %s\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				continue;
			}

			// Skip mailto links
			if (substr($link, 0, 7) == 'mailto:') {
				if ($this->debug) {
					error_log(sprintf("Skipping mailto: link %s\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				continue;
			}

			// Skip links within the page
			if (substr($link, 0, 1) == '#') {
				if ($this->debug) {
					error_log(sprintf("Skipping # link %s\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				continue;
			}

			// Skip links with just the protocol
			if (strlen($link) <= 8 && (substr($link, 0, 7) == 'http://' || substr($link, 0, 8) == 'https://')) {
				if ($this->debug) {
					error_log(sprintf("Skipping http(s):// link %s\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				continue;
			}

			$link_parts = parse_url($link);

			if ($this->debug) {
				error_log(sprintf("Normalising %s\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
			}

			// If it doesn't have a path it's a link fragment (i.e. #top etc) so skip it
			if (!isset($link_parts['path'])) {
				if ($this->debug) {
					error_log(sprintf("    %s doesn't have a path, skipping\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}
				continue;
			}

			// Cleanup the scheme
			if (!isset($link_parts['scheme']) || $link_parts['scheme'] == '') {
				if ($this->debug) {
					error_log(sprintf("    %s doesn't have a protocol, adding one\n", $link), 3, TEMP_DIRECTORY.'/default_plugin.log');
				}

				$link_parts['scheme'] = 'http';
			}
			$link_parts['scheme'] = strtolower($link_parts['scheme']);

			// Cleanup the host
			if (!isset($link_parts['host']) || $link_parts['host'] == '') {
				// If there is no host it's a relative link
				if ($this->basehref != '') {
					// If it's got a basehref use that
					$url_parts = parse_url($this->basehref);
					$link_parts['host'] = $url_parts['host'];
					$link_parts['scheme'] = $url_parts['scheme'];
					if ($link_parts['path']{0} == '/') {
						$link_parts['path'] = $url_parts['path'];
					} else {
						$link_parts['path'] = $url_parts['path'] . $link_parts['path'];
					}
					unset($url_parts);
					if ($this->debug) {
						error_log(sprintf("    %s doesn't have a host, setting the host to %s @ line %d\n", $link, $link_parts['host'], __LINE__), 3, TEMP_DIRECTORY.'/default_plugin.log');
					}
				} elseif ($link_parts['path'] != '' && $link_parts['path']{0} == '/') {
					$url_parts = parse_url($url);
					$link_parts['host'] = $url_parts['host'];
					unset($url_parts);
					if ($this->debug) {
						error_log(sprintf("    %s doesn't have a host, setting the host to %s (url is %s) @ line %d\n", $link, $link_parts['host'], $url, __LINE__), 3, TEMP_DIRECTORY.'/default_plugin.log');
					}
				} else {
					$url_parts = parse_url($url);

					if (!isset($url_parts['path'])) {
						$url_parts['path'] = '';
					}

					// Only pull off the last path component if it has a . in it
					$last_part = substr($url_parts['path'], strrpos($url_parts['path'], '/'));

					if (strpos($last_part, '.') !== false) {
						$url_parts['path'] = substr($url_parts['path'], 0, strrpos($url_parts['path'], '/')+1);
					}

					if (substr($url_parts['path'], -1) !== '/') {
						$url_parts['path'] .= '/';
					}

					$link_parts['host'] = $url_parts['host'];
					if (isset($url_parts['port'])) {
						$link_parts['port'] = $url_parts['port'];
					}
					$link_parts['path'] = $url_parts['path'] . $link_parts['path'];
					unset($url_parts);
					if ($this->debug) {
						if (!isset($link_parts['port'])) {
							$link_parts['port'] = '80';
						}
						error_log(sprintf("    %s doesn't have a host, setting the host to %s @ line %d\n", $link, $link_parts['host'].':'.$link_parts['port'], __LINE__), 3, TEMP_DIRECTORY.'/default_plugin.log');
					}
				}
			}
			$link_parts['host'] = strtolower($link_parts['host']);

			// Cleanup the port
			if (isset($link_parts['port']) && $link_parts['port'] == '80') {
				$link_parts['port'] = '';
			} elseif (!isset($link_parts['port'])) {
				$link_parts['port'] = '';
			}

			// Cleanup the path
			if (!isset($link_parts['path'])) {
				$link_parts['path'] = '';
			}

			// Cleanup the query string
			if (!isset($link_parts['query'])) {
				$link_parts['query'] = '';
			}

			// Resolve any . or .. occurances in the path
			$path_parts = explode('/', $link_parts['path']);
			$good_path_parts = array();
			while (!empty($path_parts)) {
				$part = array_shift($path_parts);

				if ($part == '.') {
					continue;
				}

				if ($part == '..') {
					array_pop($good_path_parts);
					continue;
				}

				array_push($good_path_parts, $part);
			}
			unset($path_parts);

			$link_parts['path'] = implode('/', $good_path_parts);

			// Reconstruct the normalised url
			$final_url = $link_parts['scheme'] . '://'.$link_parts['host'];
			if ($link_parts['port'] != '') {
				$final_url .= ':'.$link_parts['port'];
			}
			$final_url .= $link_parts['path'];

			if ($link_parts['query'] != '') {
				$final_url .= '?'.$link_parts['query'];
			}

			// Add the normalised url to the return array
			$return_urls[] = $this->RemoveExtraQueryStrings($final_url);
		}

		//sort($return_urls);

		return $return_urls;
	}

}
?>
