[PATCH 1/6] Generalized and improved the proxy URL parsing regex
Vratislav Podzimek
vpodzime at redhat.com
Tue Apr 1 08:03:34 UTC 2014
On Mon, 2014-03-31 at 16:05 -0400, David Shea wrote:
> Look out it's a regex patch.
>
> Add a regex to match IPv6 addresses. Move the hostname regex into
> regexes.py and add a test.
>
> Modify the proxy url parsing regex in the following ways:
> - handle IPv6 literals
> - handle percent encoded characters (encoded and decoded in iutil)
> - change the character restrictions to what is actually allowed in a
> URL
> - add match groups for the query and fragment portions of URLs
> - change the existing match groups so we only match what is needed and
> don't include, for example, a colon before the port number
> - remove the redundant match group for @username:password
> - split the whole thing into pieces so that hopefully no one ever has
> to think about the whole awful thing at once
> ---
> pyanaconda/iutil.py | 40 ++++----
> pyanaconda/network.py | 18 +---
> pyanaconda/regexes.py | 59 +++++++++++-
> tests/regex_tests/hostname_test.py | 190 +++++++++++++++++++++++++++++++++++++
> tests/regex_tests/proxy_test.py | 119 -----------------------
> tests/regex_tests/url_test.py | 183 +++++++++++++++++++++++++++++++++++
> 6 files changed, 451 insertions(+), 158 deletions(-)
> create mode 100644 tests/regex_tests/hostname_test.py
> delete mode 100644 tests/regex_tests/proxy_test.py
> create mode 100644 tests/regex_tests/url_test.py
>
> diff --git a/pyanaconda/iutil.py b/pyanaconda/iutil.py
> index ad3ce50..cc2e945 100644
> --- a/pyanaconda/iutil.py
> +++ b/pyanaconda/iutil.py
> @@ -31,10 +31,11 @@ import string
> import types
> from threading import Thread
> from Queue import Queue, Empty
> +from urllib import quote, unquote
>
> from pyanaconda.flags import flags
> from pyanaconda.constants import DRACUT_SHUTDOWN_EJECT, ROOT_PATH, TRANSLATIONS_UPDATE_DIR, UNSUPPORTED_HW
> -from pyanaconda.regexes import PROXY_URL_PARSE
> +from pyanaconda.regexes import URL_PARSE
>
> import logging
> log = logging.getLogger("anaconda")
> @@ -481,16 +482,17 @@ class ProxyString(object):
> """
> # NOTE: If this changes, update tests/regex/proxy.py
> #
> - # proxy=[protocol://][username[:password]@]host[:port][path]
> + # proxy=[protocol://][username[:password]@]host[:port][path][?query][#fragment]
> # groups
> # 1 = protocol
> - # 2 = username:password@
> - # 3 = username
> - # 4 = password
> - # 5 = hostname
> - # 6 = port
> - # 7 = extra
> - m = PROXY_URL_PARSE.match(self.url)
> + # 2 = username
> + # 3 = password
> + # 4 = hostname
> + # 5 = port
> + # 6 = path
> + # 7 = query
> + # 8 = fragment
> + m = URL_PARSE.match(self.url)
What about using the groupdict() here instead of those positional
groups?
> if not m:
> raise ProxyStringError("malformed url, cannot parse it.")
>
> @@ -500,18 +502,16 @@ class ProxyString(object):
> else:
> self.protocol = "http://"
>
> + if m.group(2):
> + self.username = unquote(m.group(2))
> +
> if m.group(3):
> - self.username = m.group(3)
> + self.password = unquote(m.group(3))
>
> if m.group(4):
> - # Skip the leading colon
> - self.password = m.group(4)[1:]
> -
> - if m.group(5):
> - self.host = m.group(5)
> - if m.group(6):
> - # Skip the leading colon
> - self.port = m.group(6)[1:]
> + self.host = m.group(4)
> + if m.group(5):
> + self.port = m.group(5)
> else:
> raise ProxyStringError("url has no host component")
>
> @@ -521,8 +521,8 @@ class ProxyString(object):
> """ Parse the components of a proxy url into url and noauth_url
> """
> if self.username or self.password:
> - self.proxy_auth = "%s:%s@" % (self.username or "",
> - self.password or "")
> + self.proxy_auth = "%s:%s@" % (quote(self.username) or "",
> + quote(self.password) or "")
>
> self.url = self.protocol + self.proxy_auth + self.host + ":" + self.port
> self.noauth_url = self.protocol + self.host + ":" + self.port
> diff --git a/pyanaconda/network.py b/pyanaconda/network.py
> index 2a90bdf..4227ff1 100644
> --- a/pyanaconda/network.py
> +++ b/pyanaconda/network.py
> @@ -24,7 +24,6 @@
> # David Cantrell <dcantrell at redhat.com>
> # Radek Vykydal <rvykydal at redhat.com>
>
> -import string
> import shutil
> from pyanaconda import iutil
> import socket
> @@ -45,6 +44,7 @@ from pyanaconda import nm
> from pyanaconda import constants
> from pyanaconda.flags import flags, can_touch_runtime_system
> from pyanaconda.i18n import _
> +from pyanaconda.regexes import HOSTNAME_PATTERN_WITHOUT_ANCHORS
>
> from gi.repository import NetworkManager
>
> @@ -59,10 +59,6 @@ ipv6ConfFile = "/etc/sysctl.d/anaconda.conf"
> ifcfgLogFile = "/tmp/ifcfg.log"
> DEFAULT_HOSTNAME = "localhost.localdomain"
>
> -# part of a valid hostname between two periods (cannot start nor end with '-')
> -# for more info about '(?!-)' and '(?<!-)' see 're' module documentation
> -HOSTNAME_PART_RE = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
> -
> ifcfglog = None
>
> network_connected = None
> @@ -109,17 +105,7 @@ def sanityCheckHostname(hostname):
> if len(hostname) > 255:
> return (False, _("Hostname must be 255 or fewer characters in length."))
>
> - validStart = string.ascii_letters + string.digits
> -
> - if hostname[0] not in validStart:
> - return (False, _("Hostname must start with a valid character in the "
> - "ranges 'a-z', 'A-Z', or '0-9'"))
> -
> - if hostname.endswith("."):
> - # hostname can end with '.', but the regexp used below would not match
> - hostname = hostname[:-1]
> -
> - if not all(HOSTNAME_PART_RE.match(part) for part in hostname.split(".")):
> + if not (re.match('^' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + '$', hostname)):
> return (False, _("Hostnames can only contain the characters 'a-z', "
> "'A-Z', '0-9', '-', or '.', parts between periods "
> "must contain something and cannot start or end with "
> diff --git a/pyanaconda/regexes.py b/pyanaconda/regexes.py
> index 5d8fd9b..560457a 100644
> --- a/pyanaconda/regexes.py
> +++ b/pyanaconda/regexes.py
> @@ -70,8 +70,61 @@ GROUPLIST_SIMPLE_VALID = re.compile(r'^\s*(' + _USERNAME_BASE + r'(\s*,\s*' + _U
> # be validated with GROUPNAME_VALID.
> GROUPLIST_FANCY_PARSE = re.compile(r'^(?:\s*)(?P<name>.*?)\s*(?:\((?P<gid>\d+)\))?(?:\s*)$')
>
> -# Proxy parsing
> -PROXY_URL_PARSE = re.compile("([A-Za-z]+://)?(([A-Za-z0-9]+)(:[^:@]+)?@)?([^:/]+)(:[0-9]+)?(/.*)?")
> -
> # IPv4 address without anchors
> IPV4_PATTERN_WITHOUT_ANCHORS = r'(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
> +
> +# IPv6 address without anchors
> +# Adapted from the IPv6address ABNF definition in RFC 3986, so it has all those
> +# IPv4 compatibility bits too. All groups are non-capturing to make it easy to
> +# use in an expression with groups and completely impossible to read
> +IPV6_PATTERN_WITHOUT_ANCHORS = r'(?:(?:(?:[0-9a-fA-F]{1,4}:){6})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:::(?:(?:[0-9a-fA-F]{1,4}:){5})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:(?:[0-9a-fA-F]{1,4})?::(?:(?:[0-9a-fA-F]{1,4}:){4})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,1}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){3})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,2}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){2})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,3}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){1})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,4}(?:[0-9a-fA-F]{1,4}))?::(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> + r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,5}(?:[0-9a-fA-F]{1,4}))?::(?:[0-9a-fA-F]{1,4}))|' + \
> + r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,6}(?:[0-9a-fA-F]{1,4}))?::)'
> +
> +# Hostname validation
> +# A hostname consists of sections separated by periods. Each of these sections
> +# must be between 1 and 63 characters, contain only alphanumeric characters or
> +# hyphens, and may not start or end with a hyphen. The whole string cannot start
> +# with a period, but it can end with one.
> +# This regex uses negative lookahead and lookback assertions to enforce the
> +# hyphen rules and make it way more confusing
> +HOSTNAME_PATTERN_WITHOUT_ANCHORS = r'(?:(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.(?!-)[A-Za-z0-9-]{1,63}(?<!-))*\.?)'
> +
> +# URL Hostname
> +# This matches any hostname, IPv4 literal or properly encased IPv6 literal
> +# This does not match the "IPvFuture" form because come the hell on
> +URL_HOSTNAME_PATTERN_WITHOUT_ANCHORS = r'(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')|(?:\[' + IPV6_PATTERN_WITHOUT_ANCHORS + r'])|(?:' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + ')'
> +
> +# Matches the "scheme" defined by RFC 3986
> +URL_SCHEME_PATTERN_WITHOUT_ANCHORS = r'[A-Za-z][A-Za-z0-9+.-]*'
> +
> +# Matches any unreserved or percent-encoded character
> +URL_NORMAL_CHAR = r'[A-Za-z0-9._~-]|(?:%[0-9A-Fa-f]{2})'
> +
> +# The above but also matches 'sub-delims' and :, @ and /
> +URL_PATH_CHAR = URL_NORMAL_CHAR + "|[!$&'()*+,;=:@/]"
> +
> +# Parse a URL
> +# Parses a URL of the form [protocol://][username[:password]@]host[:port][path][?query][#fragment]
> +# into the following named groups:
> +# 1: protocol (e.g., http://)
> +# 2: username
> +# 3: password
> +# 4: host
> +# 5: port
> +# 6: path
> +# 7: query
> +# 8: fragment
> +URL_PARSE = re.compile(r'^(?P<protocol>' + URL_SCHEME_PATTERN_WITHOUT_ANCHORS + r'://)?' + \
> + r'(?:(?P<username>(?:' + URL_NORMAL_CHAR + r')*)(?::(?P<password>(?:' + URL_NORMAL_CHAR + r')*))?@)?' + \
> + r'(?P<host>' + URL_HOSTNAME_PATTERN_WITHOUT_ANCHORS + ')' + \
> + r'(?::(?P<port>[0-9]+))?' + \
> + r'(?P<path>/(?:' + URL_PATH_CHAR + r')*)?' + \
> + r'(?:\?(?P<query>(?:' + URL_PATH_CHAR + r'|\?)*))?' + \
> + r'(?:#(?P<fragment>(?:' + URL_PATH_CHAR + r'|\?)*))?$')
You can drop all those backslashes at the line ends here. And maybe we
could use brackets instead of all those backslashes in the IPv6 regexp
as well?
--
Vratislav Podzimek
Anaconda Rider | Red Hat, Inc. | Brno - Czech Republic
More information about the anaconda-patches
mailing list