[PATCH 1/6] Generalized and improved the proxy URL parsing regex

Vratislav Podzimek vpodzime at redhat.com
Tue Apr 1 08:03:34 UTC 2014


On Mon, 2014-03-31 at 16:05 -0400, David Shea wrote:
> Look out it's a regex patch.
> 
> Add a regex to match IPv6 addresses. Move the hostname regex into
> regexes.py and add a test.
> 
> Modify the proxy url parsing regex in the following ways:
>   - handle IPv6 literals
>   - handle percent encoded characters (encoded and decoded in iutil)
>   - change the character restrictions to what is actually allowed in a
>     URL
>   - add match groups for the query and fragment portions of URLs
>   - change the existing match groups so we only match what is needed and
>     don't include, for example, a colon before the port number
>   - remove the redundant match group for @username:password
>   - split the whole thing into pieces so that hopefully no one ever has
>     to think about the whole awful thing at once
> ---
>  pyanaconda/iutil.py                |  40 ++++----
>  pyanaconda/network.py              |  18 +---
>  pyanaconda/regexes.py              |  59 +++++++++++-
>  tests/regex_tests/hostname_test.py | 190 +++++++++++++++++++++++++++++++++++++
>  tests/regex_tests/proxy_test.py    | 119 -----------------------
>  tests/regex_tests/url_test.py      | 183 +++++++++++++++++++++++++++++++++++
>  6 files changed, 451 insertions(+), 158 deletions(-)
>  create mode 100644 tests/regex_tests/hostname_test.py
>  delete mode 100644 tests/regex_tests/proxy_test.py
>  create mode 100644 tests/regex_tests/url_test.py
> 
> diff --git a/pyanaconda/iutil.py b/pyanaconda/iutil.py
> index ad3ce50..cc2e945 100644
> --- a/pyanaconda/iutil.py
> +++ b/pyanaconda/iutil.py
> @@ -31,10 +31,11 @@ import string
>  import types
>  from threading import Thread
>  from Queue import Queue, Empty
> +from urllib import quote, unquote
>  
>  from pyanaconda.flags import flags
>  from pyanaconda.constants import DRACUT_SHUTDOWN_EJECT, ROOT_PATH, TRANSLATIONS_UPDATE_DIR, UNSUPPORTED_HW
> -from pyanaconda.regexes import PROXY_URL_PARSE
> +from pyanaconda.regexes import URL_PARSE
>  
>  import logging
>  log = logging.getLogger("anaconda")
> @@ -481,16 +482,17 @@ class ProxyString(object):
>          """
>          # NOTE: If this changes, update tests/regex/proxy.py
>          #
> -        # proxy=[protocol://][username[:password]@]host[:port][path]
> +        # proxy=[protocol://][username[:password]@]host[:port][path][?query][#fragment]
>          # groups
>          # 1 = protocol
> -        # 2 = username:password@
> -        # 3 = username
> -        # 4 = password
> -        # 5 = hostname
> -        # 6 = port
> -        # 7 = extra
> -        m = PROXY_URL_PARSE.match(self.url)
> +        # 2 = username
> +        # 3 = password
> +        # 4 = hostname
> +        # 5 = port
> +        # 6 = path
> +        # 7 = query
> +        # 8 = fragment
> +        m = URL_PARSE.match(self.url)
What about using the groupdict() here instead of those positional
groups?

>          if not m:
>              raise ProxyStringError("malformed url, cannot parse it.")
>  
> @@ -500,18 +502,16 @@ class ProxyString(object):
>          else:
>              self.protocol = "http://"
>  
> +        if m.group(2):
> +            self.username = unquote(m.group(2))
> +
>          if m.group(3):
> -            self.username = m.group(3)
> +            self.password = unquote(m.group(3))
>  
>          if m.group(4):
> -            # Skip the leading colon
> -            self.password = m.group(4)[1:]
> -
> -        if m.group(5):
> -            self.host = m.group(5)
> -            if m.group(6):
> -                # Skip the leading colon
> -                self.port = m.group(6)[1:]
> +            self.host = m.group(4)
> +            if m.group(5):
> +                self.port = m.group(5)
>          else:
>              raise ProxyStringError("url has no host component")
>  
> @@ -521,8 +521,8 @@ class ProxyString(object):
>          """ Parse the components of a proxy url into url and noauth_url
>          """
>          if self.username or self.password:
> -            self.proxy_auth = "%s:%s@" % (self.username or "",
> -                                          self.password or "")
> +            self.proxy_auth = "%s:%s@" % (quote(self.username) or "",
> +                                          quote(self.password) or "")
>  
>          self.url = self.protocol + self.proxy_auth + self.host + ":" + self.port
>          self.noauth_url = self.protocol + self.host + ":" + self.port
> diff --git a/pyanaconda/network.py b/pyanaconda/network.py
> index 2a90bdf..4227ff1 100644
> --- a/pyanaconda/network.py
> +++ b/pyanaconda/network.py
> @@ -24,7 +24,6 @@
>  #            David Cantrell <dcantrell at redhat.com>
>  #            Radek Vykydal <rvykydal at redhat.com>
>  
> -import string
>  import shutil
>  from pyanaconda import iutil
>  import socket
> @@ -45,6 +44,7 @@ from pyanaconda import nm
>  from pyanaconda import constants
>  from pyanaconda.flags import flags, can_touch_runtime_system
>  from pyanaconda.i18n import _
> +from pyanaconda.regexes import HOSTNAME_PATTERN_WITHOUT_ANCHORS
>  
>  from gi.repository import NetworkManager
>  
> @@ -59,10 +59,6 @@ ipv6ConfFile = "/etc/sysctl.d/anaconda.conf"
>  ifcfgLogFile = "/tmp/ifcfg.log"
>  DEFAULT_HOSTNAME = "localhost.localdomain"
>  
> -# part of a valid hostname between two periods (cannot start nor end with '-')
> -# for more info about '(?!-)' and '(?<!-)' see 're' module documentation
> -HOSTNAME_PART_RE = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
> -
>  ifcfglog = None
>  
>  network_connected = None
> @@ -109,17 +105,7 @@ def sanityCheckHostname(hostname):
>      if len(hostname) > 255:
>          return (False, _("Hostname must be 255 or fewer characters in length."))
>  
> -    validStart = string.ascii_letters + string.digits
> -
> -    if hostname[0] not in validStart:
> -        return (False, _("Hostname must start with a valid character in the "
> -                         "ranges 'a-z', 'A-Z', or '0-9'"))
> -
> -    if hostname.endswith("."):
> -        # hostname can end with '.', but the regexp used below would not match
> -        hostname = hostname[:-1]
> -
> -    if not all(HOSTNAME_PART_RE.match(part) for part in hostname.split(".")):
> +    if not (re.match('^' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + '$', hostname)):
>          return (False, _("Hostnames can only contain the characters 'a-z', "
>                           "'A-Z', '0-9', '-', or '.', parts between periods "
>                           "must contain something and cannot start or end with "
> diff --git a/pyanaconda/regexes.py b/pyanaconda/regexes.py
> index 5d8fd9b..560457a 100644
> --- a/pyanaconda/regexes.py
> +++ b/pyanaconda/regexes.py
> @@ -70,8 +70,61 @@ GROUPLIST_SIMPLE_VALID = re.compile(r'^\s*(' + _USERNAME_BASE + r'(\s*,\s*' + _U
>  # be validated with GROUPNAME_VALID.
>  GROUPLIST_FANCY_PARSE = re.compile(r'^(?:\s*)(?P<name>.*?)\s*(?:\((?P<gid>\d+)\))?(?:\s*)$')
>  
> -# Proxy parsing
> -PROXY_URL_PARSE = re.compile("([A-Za-z]+://)?(([A-Za-z0-9]+)(:[^:@]+)?@)?([^:/]+)(:[0-9]+)?(/.*)?")
> -
>  # IPv4 address without anchors
>  IPV4_PATTERN_WITHOUT_ANCHORS = r'(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
> +
> +# IPv6 address without anchors
> +# Adapted from the IPv6address ABNF definition in RFC 3986, so it has all those
> +# IPv4 compatibility bits too. All groups are non-capturing to make it easy to
> +# use in an expression with groups and completely impossible to read
> +IPV6_PATTERN_WITHOUT_ANCHORS = r'(?:(?:(?:[0-9a-fA-F]{1,4}:){6})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:::(?:(?:[0-9a-fA-F]{1,4}:){5})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:(?:[0-9a-fA-F]{1,4})?::(?:(?:[0-9a-fA-F]{1,4}:){4})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,1}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){3})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,2}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){2})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,3}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){1})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,4}(?:[0-9a-fA-F]{1,4}))?::(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
> +                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,5}(?:[0-9a-fA-F]{1,4}))?::(?:[0-9a-fA-F]{1,4}))|' + \
> +                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,6}(?:[0-9a-fA-F]{1,4}))?::)'
> +
> +# Hostname validation
> +# A hostname consists of sections separated by periods. Each of these sections
> +# must be between 1 and 63 characters, contain only alphanumeric characters or
> +# hyphens, and may not start or end with a hyphen. The whole string cannot start
> +# with a period, but it can end with one.
> +# This regex uses negative lookahead and lookback assertions to enforce the
> +# hyphen rules and make it way more confusing
> +HOSTNAME_PATTERN_WITHOUT_ANCHORS = r'(?:(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.(?!-)[A-Za-z0-9-]{1,63}(?<!-))*\.?)'
> +
> +# URL Hostname
> +# This matches any hostname, IPv4 literal or properly encased IPv6 literal
> +# This does not match the "IPvFuture" form because come the hell on
> +URL_HOSTNAME_PATTERN_WITHOUT_ANCHORS = r'(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')|(?:\[' + IPV6_PATTERN_WITHOUT_ANCHORS + r'])|(?:' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + ')'
> +
> +# Matches the "scheme" defined by RFC 3986
> +URL_SCHEME_PATTERN_WITHOUT_ANCHORS = r'[A-Za-z][A-Za-z0-9+.-]*'
> +
> +# Matches any unreserved or percent-encoded character
> +URL_NORMAL_CHAR = r'[A-Za-z0-9._~-]|(?:%[0-9A-Fa-f]{2})'
> +
> +# The above but also matches 'sub-delims' and :, @ and /
> +URL_PATH_CHAR = URL_NORMAL_CHAR + "|[!$&'()*+,;=:@/]"
> +
> +# Parse a URL
> +# Parses a URL of the form [protocol://][username[:password]@]host[:port][path][?query][#fragment]
> +# into the following named groups:
> +#   1: protocol (e.g., http://)
> +#   2: username
> +#   3: password
> +#   4: host
> +#   5: port
> +#   6: path
> +#   7: query
> +#   8: fragment
> +URL_PARSE = re.compile(r'^(?P<protocol>' + URL_SCHEME_PATTERN_WITHOUT_ANCHORS + r'://)?' + \
> +                       r'(?:(?P<username>(?:' + URL_NORMAL_CHAR + r')*)(?::(?P<password>(?:' + URL_NORMAL_CHAR + r')*))?@)?' + \
> +                       r'(?P<host>' + URL_HOSTNAME_PATTERN_WITHOUT_ANCHORS + ')' + \
> +                       r'(?::(?P<port>[0-9]+))?' + \
> +                       r'(?P<path>/(?:' + URL_PATH_CHAR + r')*)?' + \
> +                       r'(?:\?(?P<query>(?:' + URL_PATH_CHAR + r'|\?)*))?' + \
> +                       r'(?:#(?P<fragment>(?:' + URL_PATH_CHAR + r'|\?)*))?$')
You can drop all those backslashes at the line ends here. And maybe we
could use brackets instead of all those backslashes in the IPv6 regexp
as well?

-- 
Vratislav Podzimek

Anaconda Rider | Red Hat, Inc. | Brno - Czech Republic



More information about the anaconda-patches mailing list