[PATCH 1/6] Generalized and improved the proxy URL parsing regex

David Shea dshea at redhat.com
Mon Mar 31 20:05:30 UTC 2014


Look out it's a regex patch.

Add a regex to match IPv6 addresses. Move the hostname regex into
regexes.py and add a test.

Modify the proxy url parsing regex in the following ways:
  - handle IPv6 literals
  - handle percent encoded characters (encoded and decoded in iutil)
  - change the character restrictions to what is actually allowed in a
    URL
  - add match groups for the query and fragment portions of URLs
  - change the existing match groups so we only match what is needed and
    don't include, for example, a colon before the port number
  - remove the redundant match group for @username:password
  - split the whole thing into pieces so that hopefully no one ever has
    to think about the whole awful thing at once
---
 pyanaconda/iutil.py                |  40 ++++----
 pyanaconda/network.py              |  18 +---
 pyanaconda/regexes.py              |  59 +++++++++++-
 tests/regex_tests/hostname_test.py | 190 +++++++++++++++++++++++++++++++++++++
 tests/regex_tests/proxy_test.py    | 119 -----------------------
 tests/regex_tests/url_test.py      | 183 +++++++++++++++++++++++++++++++++++
 6 files changed, 451 insertions(+), 158 deletions(-)
 create mode 100644 tests/regex_tests/hostname_test.py
 delete mode 100644 tests/regex_tests/proxy_test.py
 create mode 100644 tests/regex_tests/url_test.py

diff --git a/pyanaconda/iutil.py b/pyanaconda/iutil.py
index ad3ce50..cc2e945 100644
--- a/pyanaconda/iutil.py
+++ b/pyanaconda/iutil.py
@@ -31,10 +31,11 @@ import string
 import types
 from threading import Thread
 from Queue import Queue, Empty
+from urllib import quote, unquote
 
 from pyanaconda.flags import flags
 from pyanaconda.constants import DRACUT_SHUTDOWN_EJECT, ROOT_PATH, TRANSLATIONS_UPDATE_DIR, UNSUPPORTED_HW
-from pyanaconda.regexes import PROXY_URL_PARSE
+from pyanaconda.regexes import URL_PARSE
 
 import logging
 log = logging.getLogger("anaconda")
@@ -481,16 +482,17 @@ class ProxyString(object):
         """
         # NOTE: If this changes, update tests/regex/proxy.py
         #
-        # proxy=[protocol://][username[:password]@]host[:port][path]
+        # proxy=[protocol://][username[:password]@]host[:port][path][?query][#fragment]
         # groups
         # 1 = protocol
-        # 2 = username:password@
-        # 3 = username
-        # 4 = password
-        # 5 = hostname
-        # 6 = port
-        # 7 = extra
-        m = PROXY_URL_PARSE.match(self.url)
+        # 2 = username
+        # 3 = password
+        # 4 = hostname
+        # 5 = port
+        # 6 = path
+        # 7 = query
+        # 8 = fragment
+        m = URL_PARSE.match(self.url)
         if not m:
             raise ProxyStringError("malformed url, cannot parse it.")
 
@@ -500,18 +502,16 @@ class ProxyString(object):
         else:
             self.protocol = "http://"
 
+        if m.group(2):
+            self.username = unquote(m.group(2))
+
         if m.group(3):
-            self.username = m.group(3)
+            self.password = unquote(m.group(3))
 
         if m.group(4):
-            # Skip the leading colon
-            self.password = m.group(4)[1:]
-
-        if m.group(5):
-            self.host = m.group(5)
-            if m.group(6):
-                # Skip the leading colon
-                self.port = m.group(6)[1:]
+            self.host = m.group(4)
+            if m.group(5):
+                self.port = m.group(5)
         else:
             raise ProxyStringError("url has no host component")
 
@@ -521,8 +521,8 @@ class ProxyString(object):
         """ Parse the components of a proxy url into url and noauth_url
         """
         if self.username or self.password:
-            self.proxy_auth = "%s:%s@" % (self.username or "",
-                                          self.password or "")
+            self.proxy_auth = "%s:%s@" % (quote(self.username) or "",
+                                          quote(self.password) or "")
 
         self.url = self.protocol + self.proxy_auth + self.host + ":" + self.port
         self.noauth_url = self.protocol + self.host + ":" + self.port
diff --git a/pyanaconda/network.py b/pyanaconda/network.py
index 2a90bdf..4227ff1 100644
--- a/pyanaconda/network.py
+++ b/pyanaconda/network.py
@@ -24,7 +24,6 @@
 #            David Cantrell <dcantrell at redhat.com>
 #            Radek Vykydal <rvykydal at redhat.com>
 
-import string
 import shutil
 from pyanaconda import iutil
 import socket
@@ -45,6 +44,7 @@ from pyanaconda import nm
 from pyanaconda import constants
 from pyanaconda.flags import flags, can_touch_runtime_system
 from pyanaconda.i18n import _
+from pyanaconda.regexes import HOSTNAME_PATTERN_WITHOUT_ANCHORS
 
 from gi.repository import NetworkManager
 
@@ -59,10 +59,6 @@ ipv6ConfFile = "/etc/sysctl.d/anaconda.conf"
 ifcfgLogFile = "/tmp/ifcfg.log"
 DEFAULT_HOSTNAME = "localhost.localdomain"
 
-# part of a valid hostname between two periods (cannot start nor end with '-')
-# for more info about '(?!-)' and '(?<!-)' see 're' module documentation
-HOSTNAME_PART_RE = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
-
 ifcfglog = None
 
 network_connected = None
@@ -109,17 +105,7 @@ def sanityCheckHostname(hostname):
     if len(hostname) > 255:
         return (False, _("Hostname must be 255 or fewer characters in length."))
 
-    validStart = string.ascii_letters + string.digits
-
-    if hostname[0] not in validStart:
-        return (False, _("Hostname must start with a valid character in the "
-                         "ranges 'a-z', 'A-Z', or '0-9'"))
-
-    if hostname.endswith("."):
-        # hostname can end with '.', but the regexp used below would not match
-        hostname = hostname[:-1]
-
-    if not all(HOSTNAME_PART_RE.match(part) for part in hostname.split(".")):
+    if not (re.match('^' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + '$', hostname)):
         return (False, _("Hostnames can only contain the characters 'a-z', "
                          "'A-Z', '0-9', '-', or '.', parts between periods "
                          "must contain something and cannot start or end with "
diff --git a/pyanaconda/regexes.py b/pyanaconda/regexes.py
index 5d8fd9b..560457a 100644
--- a/pyanaconda/regexes.py
+++ b/pyanaconda/regexes.py
@@ -70,8 +70,61 @@ GROUPLIST_SIMPLE_VALID = re.compile(r'^\s*(' + _USERNAME_BASE + r'(\s*,\s*' + _U
 # be validated with GROUPNAME_VALID.
 GROUPLIST_FANCY_PARSE = re.compile(r'^(?:\s*)(?P<name>.*?)\s*(?:\((?P<gid>\d+)\))?(?:\s*)$')
 
-# Proxy parsing
-PROXY_URL_PARSE = re.compile("([A-Za-z]+://)?(([A-Za-z0-9]+)(:[^:@]+)?@)?([^:/]+)(:[0-9]+)?(/.*)?")
-
 # IPv4 address without anchors
 IPV4_PATTERN_WITHOUT_ANCHORS = r'(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)'
+
+# IPv6 address without anchors
+# Adapted from the IPv6address ABNF definition in RFC 3986, so it has all those
+# IPv4 compatibility bits too. All groups are non-capturing to make it easy to
+# use in an expression with groups and completely impossible to read
+IPV6_PATTERN_WITHOUT_ANCHORS = r'(?:(?:(?:[0-9a-fA-F]{1,4}:){6})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:::(?:(?:[0-9a-fA-F]{1,4}:){5})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:(?:[0-9a-fA-F]{1,4})?::(?:(?:[0-9a-fA-F]{1,4}:){4})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,1}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){3})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,2}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){2})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,3}(?:[0-9a-fA-F]{1,4}))?::(?:(?:[0-9a-fA-F]{1,4}:){1})(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,4}(?:[0-9a-fA-F]{1,4}))?::(?:(?:(?:[0-9a-fA-F]{1,4}):(?:[0-9a-fA-F]{1,4}))|(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')))|' + \
+                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,5}(?:[0-9a-fA-F]{1,4}))?::(?:[0-9a-fA-F]{1,4}))|' + \
+                               r'(?:(?:(?:[0-9a-fA-F]{1,4}:){,6}(?:[0-9a-fA-F]{1,4}))?::)'
+
+# Hostname validation
+# A hostname consists of sections separated by periods. Each of these sections
+# must be between 1 and 63 characters, contain only alphanumeric characters or
+# hyphens, and may not start or end with a hyphen. The whole string cannot start
+# with a period, but it can end with one.
+# This regex uses negative lookahead and lookback assertions to enforce the
+# hyphen rules and make it way more confusing
+HOSTNAME_PATTERN_WITHOUT_ANCHORS = r'(?:(?!-)[A-Za-z0-9-]{1,63}(?<!-)(?:\.(?!-)[A-Za-z0-9-]{1,63}(?<!-))*\.?)'
+
+# URL Hostname
+# This matches any hostname, IPv4 literal or properly encased IPv6 literal
+# This does not match the "IPvFuture" form because come the hell on
+URL_HOSTNAME_PATTERN_WITHOUT_ANCHORS = r'(?:' + IPV4_PATTERN_WITHOUT_ANCHORS + r')|(?:\[' + IPV6_PATTERN_WITHOUT_ANCHORS + r'])|(?:' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + ')'
+
+# Matches the "scheme" defined by RFC 3986
+URL_SCHEME_PATTERN_WITHOUT_ANCHORS = r'[A-Za-z][A-Za-z0-9+.-]*'
+
+# Matches any unreserved or percent-encoded character
+URL_NORMAL_CHAR = r'[A-Za-z0-9._~-]|(?:%[0-9A-Fa-f]{2})'
+
+# The above but also matches 'sub-delims' and :, @ and /
+URL_PATH_CHAR = URL_NORMAL_CHAR + "|[!$&'()*+,;=:@/]"
+
+# Parse a URL
+# Parses a URL of the form [protocol://][username[:password]@]host[:port][path][?query][#fragment]
+# into the following named groups:
+#   1: protocol (e.g., http://)
+#   2: username
+#   3: password
+#   4: host
+#   5: port
+#   6: path
+#   7: query
+#   8: fragment
+URL_PARSE = re.compile(r'^(?P<protocol>' + URL_SCHEME_PATTERN_WITHOUT_ANCHORS + r'://)?' + \
+                       r'(?:(?P<username>(?:' + URL_NORMAL_CHAR + r')*)(?::(?P<password>(?:' + URL_NORMAL_CHAR + r')*))?@)?' + \
+                       r'(?P<host>' + URL_HOSTNAME_PATTERN_WITHOUT_ANCHORS + ')' + \
+                       r'(?::(?P<port>[0-9]+))?' + \
+                       r'(?P<path>/(?:' + URL_PATH_CHAR + r')*)?' + \
+                       r'(?:\?(?P<query>(?:' + URL_PATH_CHAR + r'|\?)*))?' + \
+                       r'(?:#(?P<fragment>(?:' + URL_PATH_CHAR + r'|\?)*))?$')
diff --git a/tests/regex_tests/hostname_test.py b/tests/regex_tests/hostname_test.py
new file mode 100644
index 0000000..fa2bfc6
--- /dev/null
+++ b/tests/regex_tests/hostname_test.py
@@ -0,0 +1,190 @@
+#!/usr/bin/python
+# vim:set fileencoding=utf-8
+#
+# Copyright (C) 2014  Red Hat, Inc.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# the GNU General Public License v.2, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY expressed or implied, including the implied warranties of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.  You should have received a copy of the
+# GNU General Public License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
+# source code or documentation are not subject to the GNU General Public
+# License and may only be used or replicated with the express permission of
+# Red Hat, Inc.
+#
+# Red Hat Author(s): David Shea <dshea at redhat.com>
+#
+import unittest
+import re
+
+from pyanaconda.regexes import HOSTNAME_PATTERN_WITHOUT_ANCHORS, IPV4_PATTERN_WITHOUT_ANCHORS,\
+        IPV6_PATTERN_WITHOUT_ANCHORS
+
+def _run_tests(testcase, expression, goodlist, badlist):
+    got_error = False
+    for good in goodlist:
+        try:
+            testcase.assertIsNotNone(expression.match(good))
+        except AssertionError:
+            got_error = True
+            print("Good string %s did not match expression" % good)
+
+    for bad in badlist:
+        try:
+            testcase.assertIsNone(expression.match(bad))
+        except AssertionError:
+            got_error = True
+            print("Bad string %s matched expression" % bad)
+
+    if got_error:
+        testcase.fail()
+
+class HostnameRegexTestCase(unittest.TestCase):
+    def hostname_test(self):
+        good_tests = [
+                '0',
+                'a',
+                'A',
+                'hostname',
+                'host-name',
+                'host.name',
+                'host.name.with.oneverylongsectionthatisexactly63characterslong-and-contains-se',
+                '3numberstart',
+                'numberend3',
+                'first.3numberstart',
+                'first.3numberend',
+                'dot.end.'
+                ]
+
+        bad_tests = [
+                '.',
+                '..',
+                'too..many.dots',
+                '-hypenstart',
+                'hyphenend-',
+                'first.-hyphenstart',
+                'first.hyphenend-',
+                'bad,character',
+                '.dot.start',
+                'host.name.with.oneverylongsectionthatisexactly64characterslong-and-contains-sev',
+                'Ășnicode'
+                ]
+
+        hostname_re = re.compile('^' + HOSTNAME_PATTERN_WITHOUT_ANCHORS + '$')
+        _run_tests(self, hostname_re, good_tests, bad_tests)
+
+class IPv4RegexTestCase(unittest.TestCase):
+    def ipv4_test(self):
+        good_tests = [
+                '1.2.3.4',
+                '0.0.0.0',
+                '10.20.30.40',
+                '255.255.255.255',
+                '249.249.249.249'
+                ]
+
+        bad_tests = [
+                '1.2.3.',
+                '1.2.3',
+                '256.2.3.4',
+                'a.b.c.d',
+                '1.2.3.400'
+                '....',
+                '1..2.3'
+                ]
+
+        ipv4_re = re.compile('^(' + IPV4_PATTERN_WITHOUT_ANCHORS + ')$')
+        _run_tests(self, ipv4_re, good_tests, bad_tests)
+
+class IPv6RegexTestCase(unittest.TestCase):
+    def ipv6_test(self):
+        good_tests = [
+                '0000:0000:0000:0000:0000:0000:0000:0000',
+                '0000:0000:0000:0000:0000:0000:1.2.3.4',
+                '::a:b:c:d:e:f:1',
+                '::a:b:c:d:e:255.255.255.255',
+                '1::a:b:c:d:e:f',
+                '1::a:b:c:d:255.255.255.255',
+                '1:12::a:b:c:d:e',
+                '1:12::a:b:c:10.20.30.40',
+                '12::a:b:c:d:e',
+                '12::a:b:c:10.20.30.40',
+                '1:12:123::a:b:c:d',
+                '1:12:123::a:b:100.200.250.249',
+                '12:123::a:b:c:d',
+                '12:123::a:b:100.200.250.249',
+                '123::a:b:c:d',
+                '123::a:b:100.200.250.249',
+                '::a:b:c:d',
+                '::a:b:100.200.250.249',
+                '1:12:123:1234::a:b:c',
+                '1:12:123:1234::a:1.20.30.99',
+                '12:123:1234::a:b:c',
+                '12:123:1234::a:1.20.30.99',
+                '123:1234::a:b:c',
+                '123:1234::a:1.20.30.99',
+                '1234::a:b:c',
+                '1234::a:1.20.30.99',
+                '::a:b:c',
+                '::a:1.20.30.99',
+                '1:12:123:1234:abcd::a:b',
+                '1:12:123:1234:abcd::0.0.0.0',
+                '12:123:1234:abcd::a:b',
+                '12:123:1234:abcd::0.0.0.0',
+                '123:1234:abcd::a:b',
+                '123:1234:abcd::0.0.0.0',
+                '1234:abcd::a:b',
+                '1234:abcd::0.0.0.0',
+                'abcd::a:b',
+                'abcd::0.0.0.0',
+                '::a:b',
+                '::0.0.0.0',
+                '1:12:123:1234:dead:beef::aaaa',
+                '12:123:1234:dead:beef::aaaa',
+                '123:1234:dead:beef::aaaa',
+                '1234:dead:beef::aaaa',
+                'dead:beef::aaaa',
+                'beef::aaaa',
+                '::aaaa',
+                '::'
+                ]
+
+        bad_tests = [
+                # Too many bits
+                '0000:0000:0000:0000:0000:0000:0000:0000:0000'
+                '0000:0000:0000:0000:0000:0000:0000:1.2.3.4',
+                '0000:0000:0000:0000:0000:0000:1.2.3.4.5',
+                # Not enough bits
+                '0000:0000:0000:0000:0000:0000:0000',
+                '0000:0000:0000:0000:0000:1.2.3.4',
+                # zero-length contractions
+                '0000::0000:0000:0000:0000:0000:1.2.3.4',
+                '0000:0000::0000:0000:0000:0000:1.2.3.4',
+                '0000:0000:0000::0000:0000:0000:1.2.3.4',
+                '0000:0000:0000:0000::0000:0000:1.2.3.4',
+                '0000:0000:0000:0000:0000::0000:1.2.3.4',
+                '0000:0000:0000:0000:0000:0000::1.2.3.4',
+                '123::4567:89:a:bcde:f0f0:aaaa:8',
+                '123:4567::89:a:bcde:f0f0:aaaa:8',
+                '123:4567:89::a:bcde:f0f0:aaaa:8',
+                '123:4567:89:a:bcde::f0f0:aaaa:8',
+                '123:4567:89:a:bcde:f0f0::aaaa:8',
+                '123:4567:89:a:bcde:f0f0:aaaa::8',
+                # too many contractions
+                'a::b::c',
+                '::a::b',
+                'a::b::',
+                # invalid numbers
+                '00000::0000',
+                'defg::',
+                '12345::abcd',
+                'ffff::0x1e'
+                ]
+
+        ipv6_re = re.compile('^(' + IPV6_PATTERN_WITHOUT_ANCHORS + ')$')
+        _run_tests(self, ipv6_re, good_tests, bad_tests)
diff --git a/tests/regex_tests/proxy_test.py b/tests/regex_tests/proxy_test.py
deleted file mode 100644
index 6fae463..0000000
--- a/tests/regex_tests/proxy_test.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#
-# Copyright (C) 2010-2013  Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use,
-# modify, copy, or redistribute it subject to the terms and conditions of
-# the GNU General Public License v.2, or (at your option) any later version.
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY expressed or implied, including the implied warranties of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
-# Public License for more details.  You should have received a copy of the
-# GNU General Public License along with this program; if not, write to the
-# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
-# source code or documentation are not subject to the GNU General Public
-# License and may only be used or replicated with the express permission of
-# Red Hat, Inc.
-#
-# Red Hat Author(s): Brian C. Lane <bcl at redhat.com>
-#
-
-import unittest
-
-from pyanaconda.regexes import PROXY_URL_PARSE
-
-class ProxyRegexTestCase(unittest.TestCase):
-    def proxy_regex_test(self):
-        """
-        Run a list of possible proxy= values through the regex and check for
-        correct results.
-
-        tests are in the form of: (proxy string, match.groups() tuple)
-        """
-        tests = [ ( "proxy.host",
-                      (None, None, None, None, 'proxy.host', None, None) ),
-
-                  ( "proxy.host:3128",
-                      (None, None, None, None, 'proxy.host', ':3128', None) ),
-
-                  ( "user:password at proxy.host",
-                      (None, 'user:password@', 'user', ':password', 'proxy.host', None, None) ),
-
-                  ( "user at proxy.host",
-                      (None, 'user@', 'user', None, 'proxy.host', None, None) ),
-
-                  ( "user:password at proxy.host:3128",
-                      (None, 'user:password@', 'user', ':password', 'proxy.host', ':3128', None) ),
-
-                  ( "user at proxy.host:3128",
-                      (None, 'user@', 'user', None, 'proxy.host', ':3128', None) ),
-
-                  ( "proxy.host/blah/blah",
-                      (None, None, None, None, 'proxy.host', None, '/blah/blah') ),
-
-                  ( "proxy.host:3128/blah/blah",
-                      (None, None, None, None, 'proxy.host', ':3128', '/blah/blah') ),
-
-                  ( "user:password at proxy.host/blah/blah",
-                      (None, 'user:password@', 'user', ':password', 'proxy.host', None, '/blah/blah') ),
-
-                  ( "user at proxy.host/blah/blah",
-                      (None, 'user@', 'user', None, 'proxy.host', None, '/blah/blah') ),
-
-                  ( "user:password at proxy.host:3128/blah/blah",
-                      (None, 'user:password@', 'user', ':password', 'proxy.host', ':3128', "/blah/blah") ),
-
-                  ( "user at proxy.host:3128/blah/blah",
-                      (None, 'user@', 'user', None, 'proxy.host', ':3128', "/blah/blah") ),
-
-
-
-                  ( "http://proxy.host",
-                      ('http://', None, None, None, 'proxy.host', None, None) ),
-
-                  ( "http://proxy.host:3128",
-                      ('http://', None, None, None, 'proxy.host', ':3128', None) ),
-
-                  ( "http://user:password@proxy.host",
-                      ('http://', 'user:password@', 'user', ':password', 'proxy.host', None, None) ),
-
-                  ( "http://user@proxy.host",
-                      ('http://', 'user@', 'user', None, 'proxy.host', None, None) ),
-
-                  ( "http://user:password@proxy.host:3128",
-                      ('http://', 'user:password@', 'user', ':password', 'proxy.host', ':3128', None) ),
-
-                  ( "http://user@proxy.host:3128",
-                      ('http://', 'user@', 'user', None, 'proxy.host', ':3128', None) ),
-
-                  ( "http://proxy.host/blah/blah",
-                      ('http://', None, None, None, 'proxy.host', None, '/blah/blah') ),
-
-                  ( "http://proxy.host:3128/blah/blah",
-                      ('http://', None, None, None, 'proxy.host', ':3128', '/blah/blah') ),
-
-                  ( "http://user:password@proxy.host/blah/blah",
-                      ("http://", 'user:password@', 'user', ':password', 'proxy.host', None, '/blah/blah') ),
-
-                  ( "http://user@proxy.host/blah/blah",
-                      ("http://", 'user@', 'user', None, 'proxy.host', None, '/blah/blah') ),
-
-                  ( "http://user:password@proxy.host:3128/blah/blah",
-                      ("http://", 'user:password@', 'user', ':password', 'proxy.host', ':3128', '/blah/blah') ),
-
-                  ( "http://user@proxy.host:3128/blah/blah",
-                      ("http://", 'user@', 'user', None, 'proxy.host', ':3128', '/blah/blah') ),
-
-                ]
-
-
-        got_error = False
-        for proxy, result in tests:
-            try:
-                self.assertEqual(PROXY_URL_PARSE.match(proxy).groups(), result)
-            except AssertionError:
-                got_error = True
-                print("Proxy parse error: `%s' did not parse as `%s'" % (proxy, result))
-
-        if got_error:
-            self.fail()
diff --git a/tests/regex_tests/url_test.py b/tests/regex_tests/url_test.py
new file mode 100644
index 0000000..3c77f7d
--- /dev/null
+++ b/tests/regex_tests/url_test.py
@@ -0,0 +1,183 @@
+#
+# Copyright (C) 2010-2013  Red Hat, Inc.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# the GNU General Public License v.2, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY expressed or implied, including the implied warranties of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.  You should have received a copy of the
+# GNU General Public License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
+# source code or documentation are not subject to the GNU General Public
+# License and may only be used or replicated with the express permission of
+# Red Hat, Inc.
+#
+# Red Hat Author(s): Brian C. Lane <bcl at redhat.com>
+#                    David Shea <dshea at redhat.com>
+#
+
+import unittest
+
+from pyanaconda.regexes import URL_PARSE
+
+class URLRegexTestCase(unittest.TestCase):
+    def url_regex_test(self):
+        """
+        Run a list of possible URL values through the regex and check for
+        correct results.
+
+        tests are in the form of: (URL string, match.groups() tuple)
+        """
+        tests = [ ( "proxy.host",
+                      (None, None, None, 'proxy.host', None, None, None, None) ),
+
+                  ( "proxy.host:3128",
+                      (None, None, None, 'proxy.host', '3128', None, None, None) ),
+
+                  ( "user:password at proxy.host",
+                      (None, 'user', 'password', 'proxy.host', None, None, None, None) ),
+
+                  ( "user at proxy.host",
+                      (None, 'user', None, 'proxy.host', None, None, None, None) ),
+
+                  ( "user:password at proxy.host:3128",
+                      (None, 'user', 'password', 'proxy.host', '3128', None, None, None) ),
+
+                  ( "user at proxy.host:3128",
+                      (None, 'user', None, 'proxy.host', '3128', None, None, None) ),
+
+                  ( "proxy.host/blah/blah",
+                      (None, None, None, 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "proxy.host:3128/blah/blah",
+                      (None, None, None, 'proxy.host', '3128', '/blah/blah', None, None) ),
+
+                  ( "user:password at proxy.host/blah/blah",
+                      (None, 'user', 'password', 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "user at proxy.host/blah/blah",
+                      (None, 'user', None, 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "user:password at proxy.host:3128/blah/blah",
+                      (None, 'user', 'password', 'proxy.host', '3128', "/blah/blah", None, None) ),
+
+                  ( "user at proxy.host:3128/blah/blah",
+                      (None, 'user', None, 'proxy.host', '3128', "/blah/blah", None, None) ),
+
+
+
+                  ( "http://proxy.host",
+                      ('http://', None, None, 'proxy.host', None, None, None, None) ),
+
+                  ( "http://proxy.host:3128",
+                      ('http://', None, None, 'proxy.host', '3128', None, None, None) ),
+
+                  ( "http://user:password@proxy.host",
+                      ('http://', 'user', 'password', 'proxy.host', None, None, None, None) ),
+
+                  ( "http://user@proxy.host",
+                      ('http://', 'user', None, 'proxy.host', None, None, None, None) ),
+
+                  ( "http://user:password@proxy.host:3128",
+                      ('http://', 'user', 'password', 'proxy.host', '3128', None, None, None) ),
+
+                  ( "http://user@proxy.host:3128",
+                      ('http://', 'user', None, 'proxy.host', '3128', None, None, None) ),
+
+                  ( "http://proxy.host/blah/blah",
+                      ('http://', None, None, 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "http://proxy.host:3128/blah/blah",
+                      ('http://', None, None, 'proxy.host', '3128', '/blah/blah', None, None) ),
+
+                  ( "http://user:password@proxy.host/blah/blah",
+                      ("http://", 'user', 'password', 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "http://%75ser:password@proxy.host/blah/blah",
+                      ("http://", '%75ser', 'password', 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "http://user:%70assword@proxy.host/blah/blah",
+                      ("http://", 'user', '%70assword', 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "http://user@proxy.host/blah/blah",
+                      ("http://", 'user', None, 'proxy.host', None, '/blah/blah', None, None) ),
+
+                  ( "http://user@proxy.host/blah/bla%68",
+                      ("http://", 'user', None, 'proxy.host', None, '/blah/bla%68', None, None) ),
+
+                  ( "http://user:password@proxy.host:3128/blah/blah",
+                      ("http://", 'user', 'password', 'proxy.host', '3128', '/blah/blah', None, None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', None, None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah?query",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', "query", None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah?query?",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', "query?", None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah?query=whatever",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', "query=whatever", None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah?query=whate%76er",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', "query=whate%76er", None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah?",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', "", None) ),
+
+                  ( "http://user@proxy.host:3128/blah/blah#fragment",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', None, "fragment") ),
+
+                  ( "http://user@proxy.host:3128/blah/blah#",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', None, "") ),
+
+                  ( "http://user@proxy.host:3128/blah/blah#fragm%65nt",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', None, "fragm%65nt") ),
+
+                  ( "http://user@proxy.host:3128/blah/blah?query=whatever#fragment",
+                      ("http://", 'user', None, 'proxy.host', '3128', '/blah/blah', "query=whatever", "fragment") ),
+
+                  # Invalid schemes
+                  ( "0http://proxy.host/", None),
+                  ( "h~ttp://proxy.host/", None),
+
+                  # Invalid usernames and passwords
+                  ( "http://%x5ser@proxy.host/", None),
+                  ( "http://*ser@proxy.host/", None),
+                  ( "http://user:p%xxssword@proxy.host/", None),
+                  ( "http://user:p@ssword@proxy.host/", None),
+
+                  # Invalid paths
+                  ( "http://user:password@proxy.host/%xxlah/blah", None),
+                  ( "http://user:password@proxy.host/[]lah/blah", None),
+
+                  # Invalid queries
+                  ( "http://proxy.host/blah/blah?quer%xx", None),
+                  ( "http://proxy.host/blah/blah?que[]y", None),
+
+                  # Invalid fragments
+                  ( "http://proxy.host/blah/blah#fragment#", None),
+                  ( "http://proxy.host/blah/blah#%xxragment", None),
+                ]
+
+
+        got_error = False
+        for proxy, result in tests:
+            match = URL_PARSE.match(proxy)
+            if match:
+                match= match.groups()
+            else:
+                match = None
+
+            try:
+                self.assertEqual(match, result)
+            except AssertionError:
+                got_error = True
+                print("Proxy parse error: `%s' did not parse as `%s': %s" % (proxy, result, match))
+
+        if got_error:
+            self.fail()
-- 
1.9.0



More information about the anaconda-patches mailing list