WebTest not handling unicode urlquote URLs well

[carn1x]

I've been wrestling with this for a while and I still can't figure out who the culprit is, however I suppose WebTest should be responsible since it is the package pulling the various components together, and may be Web Test's handling of the URL. Still unsure... In any case, take the following unit tests:

class WebTestUnicodeUrlTests(WebTest):

    def setUp(self):
        self.event_ascii = EventFactory(url=u'eee')
        self.event_unicode = EventFactory(url=u'ééé')

    def test_ascii_url(self):
        url = reverse('vanity_url_catcher', args=[self.event_ascii.url])
        print url  # /eee
        self.app.get(url, status=200)

    def test_unicode_url(self):
        url = reverse('vanity_url_catcher', args=[self.event_unicode.url])
        print url  # /%C3%A9%C3%A9%C3%A9
        self.app.get(url, status=200)

    def test_ascii_url_via_baserequest(self):
        url = reverse('vanity_url_catcher', args=[self.event_ascii.url])
        print url  # /eee
        req = BaseRequest.blank(url)
        self.assertEqual(req.url, url)

    def test_unicode_url_via_baserequest(self):
        url = reverse('vanity_url_catcher', args=[self.event_unicode.url])
        print url  # /%C3%A9%C3%A9%C3%A9
        req = BaseRequest.blank(url)
        self.assertEqual(req.url, url)

The result of which is:

ERROR: test_unicode_url (frontend.generic.tests.test_views.WebTestUnicodeUrlTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/code/frontend/generic/tests/test_views.py", line 58, in test_unicode_url
    self.app.get(url, status=200)
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/django_webtest/__init__.py", line 124, in get
    response = super(DjangoTestApp, self).get(url, **all_kwargs_but_url)
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webtest/app.py", line 330, in get
    expect_errors=expect_errors)
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/django_webtest/__init__.py", line 86, in do_request
    expect_errors)
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webtest/app.py", line 648, in do_request
    utils._RequestCookieAdapter(req))
  File "/usr/lib/python2.7/cookielib.py", line 1659, in extract_cookies
    if self._policy.set_ok(cookie, request):
  File "/usr/lib/python2.7/cookielib.py", line 941, in set_ok
    if not fn(cookie, request):
  File "/usr/lib/python2.7/cookielib.py", line 962, in set_ok_verifiability
    if request.is_unverifiable() and is_third_party(request):
  File "/usr/lib/python2.7/cookielib.py", line 716, in is_third_party
    req_host = request_host(request)
  File "/usr/lib/python2.7/cookielib.py", line 605, in request_host
    url = request.get_full_url()
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webtest/utils.py", line 111, in get_full_url
    return self._request.url
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webob/request.py", line 504, in url
    url = self.path_url
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webob/request.py", line 476, in path_url
    bpath_info = bytes_(self.path_info, self.url_encoding)
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webob/descriptors.py", line 68, in fget
    return req.encget(key, encattr=encattr)
  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webob/request.py", line 175, in encget
    return val.decode(encoding)
  File "/virtualenv/frontend/lib/python2.7/encodings/utf_8.py", line 16, in decode
    return codecs.utf_8_decode(input, errors, True)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 1-3: ordinal not in range(128)

----------------------------------------------------------------------
Ran 4 tests in 4.683s

FAILED (errors=1)

Is there a way around this? I believe I've tried all combinations I can think of to convert the url to different formats but nothing makes a difference, or simply falls foul of the immediate str(url) call in webtest.app:

  File "/virtualenv/frontend/local/lib/python2.7/site-packages/webtest/app.py", line 310, in get
    url = str(url)

[gawel]

I don't understand why utf_8_decode raise an ascii encode error. No clue about this one.
Maybe @bertjwregeer has one.

[NomadDemon]

problem is still not solved for unicode urls, i had to change decode "coding" on webob/request to force this to work

[interDist]

The problem seems to be in WebOb (and still not solved 6 years later...) :

  File "/root/.venvs/my_env/lib/python3.10/site-packages/webtest/app.py", line 319, in get
    req = self.RequestClass.blank(url, environ)
  File "/root/.venvs/my_env/lib/python3.10/site-packages/webob/request.py", line 1347, in blank
    env = environ_from_url(path)
  File "/root/.venvs/my_env/lib/python3.10/site-packages/webob/request.py", line 1444, in environ_from_url
    path_info = url_unquote(path)
  File "/root/.venvs/my_env/lib/python3.10/site-packages/webob/compat.py", line 110, in url_unquote
    return unquote(s.encode('ascii')).decode('latin-1')
UnicodeEncodeError: 'ascii' codec can't encode character '\u0109' in position 1: ordinal not in range(128)