h/plugins/util/urlnorm.py

134 lines
4.9 KiB
Python
Raw Normal View History

"""
URI Normalization function:
* Always provide the URI scheme in lowercase characters.
* Always provide the host, if any, in lowercase characters.
* Only perform percent-encoding where it is essential.
* Always use uppercase A-through-F characters when percent-encoding.
* Prevent dot-segments appearing in non-relative URI paths.
* For schemes that define a default authority, use an empty authority if the
default is desired.
* For schemes that define an empty path to be equivalent to a path of "/",
use "/".
* For schemes that define a port, use an empty port if the default is desired
* All portions of the URI must be utf-8 encoded NFC from Unicode strings
implements:
http://gbiv.com/protocols/uri/rev-2002/rfc2396bis.html#canonical-form
http://www.intertwingly.net/wiki/pie/PaceCanonicalIds
inspired by:
Tony J. Ibbs, http://starship.python.net/crew/tibs/python/tji_url.py
Mark Nottingham, http://www.mnot.net/python/urlnorm.py
"""
__license__ = "Python"
2010-08-30 03:35:27 +00:00
import re
import unicodedata
import urlparse
from urllib import quote, unquote
default_port = {
'http': 80,
}
2010-08-30 03:35:27 +00:00
class Normalizer(object):
def __init__(self, regex, normalize_func):
self.regex = regex
self.normalize = normalize_func
normalizers = ( Normalizer( re.compile(r'(?:https?://)?(?:[a-zA-Z0-9\-]+\.)?(?:amazon|amzn){1}\.(?P<tld>[a-zA-Z\.]{2,})\/(gp/(?:product|offer-listing|customer-media/product-gallery)/|exec/obidos/tg/detail/-/|o/ASIN/|dp/|(?:[A-Za-z0-9\-]+)/dp/)?(?P<ASIN>[0-9A-Za-z]{10})'),
lambda m: r'http://amazon.%s/dp/%s' % (m.group('tld'), m.group('ASIN'))),
2010-08-30 03:35:27 +00:00
Normalizer( re.compile(r'.*waffleimages\.com.*/([0-9a-fA-F]{40})'),
lambda m: r'http://img.waffleimages.com/%s' % m.group(1) ),
Normalizer( re.compile(r'(?:youtube.*?(?:v=|/v/)|youtu\.be/|yooouuutuuube.*?id=)([-_a-z0-9]+)'),
lambda m: r'http://youtube.com/watch?v=%s' % m.group(1) ),
)
2010-08-30 03:35:27 +00:00
def normalize(url):
"""Normalize a URL."""
2010-03-04 03:25:13 +00:00
scheme, auth, path, query, fragment = urlparse.urlsplit(url.strip())
2010-08-30 03:35:27 +00:00
userinfo, host, port = re.search('([^@]*@)?([^:]*):?(.*)', auth).groups()
# Always provide the URI scheme in lowercase characters.
scheme = scheme.lower()
# Always provide the host, if any, in lowercase characters.
host = host.lower()
2010-08-30 03:35:27 +00:00
if host and host[-1] == '.':
host = host[:-1]
if host and host.startswith("www."):
if not scheme:
scheme = "http"
host = host[4:]
elif path and path.startswith("www."):
2010-08-30 03:35:27 +00:00
if not scheme:
scheme = "http"
path = path[4:]
# Only perform percent-encoding where it is essential.
# Always use uppercase A-through-F characters when percent-encoding.
# All portions of the URI must be utf-8 encoded NFC from Unicode strings
def clean(string):
2010-08-30 03:35:27 +00:00
string = unicode(unquote(string), 'utf-8', 'replace')
2010-03-04 03:25:13 +00:00
return unicodedata.normalize('NFC', string).encode('utf-8')
2010-08-30 03:35:27 +00:00
path = quote(clean(path), "~:/?#[]@!$&'()*+,;=")
fragment = quote(clean(fragment), "~")
# note care must be taken to only encode & and = characters as values
2010-08-30 03:35:27 +00:00
query = "&".join(["=".join([quote(clean(t), "~:/?#[]@!$'()*+,;=")
2010-03-04 03:25:13 +00:00
for t in q.split("=", 1)]) for q in query.split("&")])
# Prevent dot-segments appearing in non-relative URI paths.
2010-03-04 03:25:13 +00:00
if scheme in ["", "http", "https", "ftp", "file"]:
2010-08-30 03:35:27 +00:00
output = []
for input in path.split('/'):
2010-08-30 03:35:27 +00:00
if input == "":
if not output:
output.append(input)
elif input == ".":
pass
2010-08-30 03:35:27 +00:00
elif input == "..":
if len(output) > 1:
output.pop()
else:
output.append(input)
2010-08-30 03:35:27 +00:00
if input in ["", ".", ".."]:
output.append("")
path = '/'.join(output)
# For schemes that define a default authority, use an empty authority if
# the default is desired.
2010-08-30 03:35:27 +00:00
if userinfo in ["@", ":@"]:
userinfo = ""
# For schemes that define an empty path to be equivalent to a path of "/",
# use "/".
2010-08-30 03:35:27 +00:00
if path == "" and scheme in ["http", "https", "ftp", "file"]:
path = "/"
# For schemes that define a port, use an empty port if the default is
# desired
if port and scheme in default_port.keys():
if port.isdigit():
2010-08-30 03:35:27 +00:00
port = str(int(port))
if int(port) == default_port[scheme]:
port = ''
# Put it all back together again
2010-08-30 03:35:27 +00:00
auth = (userinfo or "") + host
if port:
auth += ":" + port
if url.endswith("#") and query == "" and fragment == "":
2010-03-04 03:25:13 +00:00
path += "#"
2010-08-30 03:35:27 +00:00
normal_url = urlparse.urlunsplit((scheme, auth, path, query,
fragment)).replace("http:///", "http://")
for norm in normalizers:
m = norm.regex.match(normal_url)
if m:
return norm.normalize(m)
return normal_url