Annotation of pkgsrc/lang/python27/patches/patch-Lib_urlparse.py, revision 1.4

1.4     ! gutterid    1: $NetBSD: patch-Lib_urlparse.py,v 1.3 2022/02/25 22:41:32 gutteridge Exp $
1.1       gutterid    2: 
                      3: Fix CVE-2021-23336: Add `separator` argument to parse_qs; warn with default
                      4: Via Fedora:
                      5: https://src.fedoraproject.org/rpms/python2.7/blob/rawhide/f/00359-CVE-2021-23336.patch
                      6: 
1.3       gutterid    7: Fix CVE-2022-0391: urlparse does not sanitize URLs containing ASCII newline and tabs
                      8: Via Fedora:
                      9: https://src.fedoraproject.org/rpms/python2.7/raw/40dd05e5d77dbfa81777c9f84b704bc2239bf710/f/00377-CVE-2022-0391.patch
                     10: 
1.4     ! gutterid   11: Fix CVE-2023-24329: Add more sanitizing to respect the "Remove any leading C0 control or space from input" rule
        !            12: Via Fedora:
        !            13: https://src.fedoraproject.org/rpms/python2.7/c/3f00cdccd59ef2955a7f4b4c42bb59c631cce4c1.patch
        !            14: 
1.1       gutterid   15: --- Lib/urlparse.py.orig       2020-04-19 21:13:39.000000000 +0000
                     16: +++ Lib/urlparse.py
1.4     ! gutterid   17: @@ -26,9 +26,14 @@ scenarios for parsing, and for backward
        !            18:  parsing quirks from older RFCs are retained. The testcases in
        !            19:  test_urlparse.py provides a good indicator of parsing behavior.
        !            20:  
        !            21: +The WHATWG URL Parser spec should also be considered.  We are not compliant with
        !            22: +it either due to existing user code API behavior expectations (Hyrum's Law).
        !            23: +It serves as a useful guide when making changes.
        !            24: +
1.1       gutterid   25:  """
                     26:  
                     27:  import re
                     28: +import os
                     29:  
                     30:  __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
                     31:             "urlsplit", "urlunsplit", "parse_qs", "parse_qsl"]
1.4     ! gutterid   32: @@ -62,6 +67,13 @@ scheme_chars = ('abcdefghijklmnopqrstuvw
1.3       gutterid   33:                  '0123456789'
                     34:                  '+-.')
                     35:  
1.4     ! gutterid   36: +# Leading and trailing C0 control and space to be stripped per WHATWG spec.
        !            37: +# == "".join([chr(i) for i in range(0, 0x20 + 1)])
        !            38: +_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f '
        !            39: +
1.3       gutterid   40: +# Unsafe bytes to be removed per WHATWG spec
                     41: +_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n']
                     42: +
                     43:  MAX_CACHE_SIZE = 20
                     44:  _parse_cache = {}
                     45:  
1.4     ! gutterid   46: @@ -184,12 +196,21 @@ def _checknetloc(netloc):
1.3       gutterid   47:                               "under NFKC normalization"
                     48:                               % netloc)
                     49:  
                     50: +def _remove_unsafe_bytes_from_url(url):
                     51: +    for b in _UNSAFE_URL_BYTES_TO_REMOVE:
                     52: +        url = url.replace(b, "")
                     53: +    return url
                     54: +
                     55:  def urlsplit(url, scheme='', allow_fragments=True):
                     56:      """Parse a URL into 5 components:
                     57:      <scheme>://<netloc>/<path>?<query>#<fragment>
                     58:      Return a 5-tuple: (scheme, netloc, path, query, fragment).
                     59:      Note that we don't break the components up in smaller bits
                     60:      (e.g. netloc is a single string) and we don't expand % escapes."""
                     61: +    url = _remove_unsafe_bytes_from_url(url)
                     62: +    scheme = _remove_unsafe_bytes_from_url(scheme)
1.4     ! gutterid   63: +    url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE)
        !            64: +    scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE)
1.3       gutterid   65:      allow_fragments = bool(allow_fragments)
                     66:      key = url, scheme, allow_fragments, type(url), type(scheme)
                     67:      cached = _parse_cache.get(key, None)
1.4     ! gutterid   68: @@ -382,7 +403,8 @@ def unquote(s):
1.1       gutterid   69:              append(item)
                     70:      return ''.join(res)
                     71:  
                     72: -def parse_qs(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
                     73: +def parse_qs(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None,
                     74: +             separator=None):
                     75:      """Parse a query given as a string argument.
                     76:  
                     77:          Arguments:
1.4     ! gutterid   78: @@ -405,14 +427,23 @@ def parse_qs(qs, keep_blank_values=0, st
1.1       gutterid   79:      """
                     80:      dict = {}
                     81:      for name, value in parse_qsl(qs, keep_blank_values, strict_parsing,
                     82: -                                 max_num_fields):
                     83: +                                 max_num_fields, separator):
                     84:          if name in dict:
                     85:              dict[name].append(value)
                     86:          else:
                     87:              dict[name] = [value]
                     88:      return dict
                     89:  
                     90: -def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None):
                     91: +class _QueryStringSeparatorWarning(RuntimeWarning):
                     92: +    """Warning for using default `separator` in parse_qs or parse_qsl"""
                     93: +
                     94: +# The default "separator" for parse_qsl can be specified in a config file.
                     95: +# It's cached after first read.
1.2       gutterid   96: +_QS_SEPARATOR_CONFIG_FILENAME = '@PKG_SYSCONFDIR@/python/urllib.cfg'
1.1       gutterid   97: +_default_qs_separator = None
                     98: +
                     99: +def parse_qsl(qs, keep_blank_values=0, strict_parsing=0, max_num_fields=None,
                    100: +              separator=None):
                    101:      """Parse a query given as a string argument.
                    102:  
                    103:      Arguments:
1.4     ! gutterid  104: @@ -434,15 +465,72 @@ def parse_qsl(qs, keep_blank_values=0, s
1.1       gutterid  105:  
                    106:      Returns a list, as G-d intended.
                    107:      """
                    108: +
                    109: +    if (not separator or (not isinstance(separator, (str, bytes)))) and separator is not None:
                    110: +        raise ValueError("Separator must be of type string or bytes.")
                    111: +
                    112: +    # Used when both "&" and ";" act as separators. (Need a non-string value.)
                    113: +    _legacy = object()
                    114: +
                    115: +    if separator is None:
                    116: +        global _default_qs_separator
                    117: +        separator = _default_qs_separator
                    118: +        envvar_name = 'PYTHON_URLLIB_QS_SEPARATOR'
                    119: +        if separator is None:
                    120: +            # Set default separator from environment variable
                    121: +            separator = os.environ.get(envvar_name)
                    122: +            config_source = 'environment variable'
                    123: +        if separator is None:
                    124: +            # Set default separator from the configuration file
                    125: +            try:
                    126: +                file = open(_QS_SEPARATOR_CONFIG_FILENAME)
                    127: +            except EnvironmentError:
                    128: +                pass
                    129: +            else:
                    130: +                with file:
                    131: +                    import ConfigParser
                    132: +                    config = ConfigParser.ConfigParser()
                    133: +                    config.readfp(file)
                    134: +                    separator = config.get('parse_qs', envvar_name)
                    135: +                    _default_qs_separator = separator
                    136: +                config_source = _QS_SEPARATOR_CONFIG_FILENAME
                    137: +        if separator is None:
                    138: +            # The default is '&', but warn if not specified explicitly
                    139: +            if ';' in qs:
                    140: +                from warnings import warn
                    141: +                warn("The default separator of urlparse.parse_qsl and "
                    142: +                    + "parse_qs was changed to '&' to avoid a web cache "
                    143: +                    + "poisoning issue (CVE-2021-23336). "
                    144: +                    + "By default, semicolons no longer act as query field "
                    145: +                    + "separators. "
                    146: +                    + "See https://access.redhat.com/articles/5860431 for "
                    147: +                    + "more details.",
                    148: +                    _QueryStringSeparatorWarning, stacklevel=2)
                    149: +            separator = '&'
                    150: +        elif separator == 'legacy':
                    151: +            separator = _legacy
                    152: +        elif len(separator) != 1:
                    153: +            raise ValueError(
                    154: +                '{} (from {}) must contain '.format(envvar_name, config_source)
                    155: +                + '1 character, or "legacy". See '
                    156: +                + 'https://access.redhat.com/articles/5860431 for more details.'
                    157: +            )
                    158: +
                    159:      # If max_num_fields is defined then check that the number of fields
                    160:      # is less than max_num_fields. This prevents a memory exhaustion DOS
                    161:      # attack via post bodies with many fields.
                    162:      if max_num_fields is not None:
                    163: -        num_fields = 1 + qs.count('&') + qs.count(';')
                    164: +        if separator is _legacy:
                    165: +            num_fields = 1 + qs.count('&') + qs.count(';')
                    166: +        else:
                    167: +            num_fields = 1 + qs.count(separator)
                    168:          if max_num_fields < num_fields:
                    169:              raise ValueError('Max number of fields exceeded')
                    170:  
                    171: -    pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
                    172: +    if separator is _legacy:
                    173: +        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
                    174: +    else:
                    175: +        pairs = [s1 for s1 in qs.split(separator)]
                    176:      r = []
                    177:      for name_value in pairs:
                    178:          if not name_value and not strict_parsing:

CVSweb <webmaster@jp.NetBSD.org>