mirror of
https://github.com/lazylibrarian/LazyLibrarian.git
synced 2026-01-23 05:08:37 -05:00
907 lines
32 KiB
Python
907 lines
32 KiB
Python
# -*- coding: utf-8 -*-
|
||
|
||
"""
|
||
requests.models
|
||
~~~~~~~~~~~~~~~
|
||
|
||
This module contains the primary objects that power Requests.
|
||
"""
|
||
|
||
import collections
|
||
import datetime
|
||
|
||
# Import encoding now, to avoid implicit import later.
|
||
# Implicit import within threads may cause LookupError when standard library is in a ZIP,
|
||
# such as in Embedded Python. See https://github.com/kennethreitz/requests/issues/3578.
|
||
import encodings.idna
|
||
|
||
from io import BytesIO, UnsupportedOperation
|
||
from .hooks import default_hooks
|
||
from .structures import CaseInsensitiveDict
|
||
|
||
from .auth import HTTPBasicAuth
|
||
from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
|
||
from .packages import idna
|
||
from .packages.urllib3.fields import RequestField
|
||
from .packages.urllib3.filepost import encode_multipart_formdata
|
||
from .packages.urllib3.util import parse_url
|
||
from .packages.urllib3.exceptions import (
|
||
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
|
||
from .exceptions import (
|
||
HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
|
||
ContentDecodingError, ConnectionError, StreamConsumedError)
|
||
from ._internal_utils import to_native_string, unicode_is_ascii
|
||
from .utils import (
|
||
guess_filename, get_auth_from_url, requote_uri,
|
||
stream_decode_response_unicode, to_key_val_list, parse_header_links,
|
||
iter_slices, guess_json_utf, super_len, check_header_validity)
|
||
from .compat import (
|
||
cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
|
||
is_py2, chardet, builtin_str, basestring)
|
||
from .compat import json as complexjson
|
||
from .status_codes import codes
|
||
|
||
#: The set of HTTP status codes that indicate an automatically
|
||
#: processable redirect.
|
||
REDIRECT_STATI = (
|
||
codes.moved, # 301
|
||
codes.found, # 302
|
||
codes.other, # 303
|
||
codes.temporary_redirect, # 307
|
||
codes.permanent_redirect, # 308
|
||
)
|
||
|
||
DEFAULT_REDIRECT_LIMIT = 30
|
||
CONTENT_CHUNK_SIZE = 10 * 1024
|
||
ITER_CHUNK_SIZE = 512
|
||
|
||
|
||
class RequestEncodingMixin(object):
|
||
@property
|
||
def path_url(self):
|
||
"""Build the path URL to use."""
|
||
|
||
url = []
|
||
|
||
p = urlsplit(self.url)
|
||
|
||
path = p.path
|
||
if not path:
|
||
path = '/'
|
||
|
||
url.append(path)
|
||
|
||
query = p.query
|
||
if query:
|
||
url.append('?')
|
||
url.append(query)
|
||
|
||
return ''.join(url)
|
||
|
||
@staticmethod
|
||
def _encode_params(data):
|
||
"""Encode parameters in a piece of data.
|
||
|
||
Will successfully encode parameters when passed as a dict or a list of
|
||
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
|
||
if parameters are supplied as a dict.
|
||
"""
|
||
|
||
if isinstance(data, (str, bytes)):
|
||
return data
|
||
elif hasattr(data, 'read'):
|
||
return data
|
||
elif hasattr(data, '__iter__'):
|
||
result = []
|
||
for k, vs in to_key_val_list(data):
|
||
if isinstance(vs, basestring) or not hasattr(vs, '__iter__'):
|
||
vs = [vs]
|
||
for v in vs:
|
||
if v is not None:
|
||
result.append(
|
||
(k.encode('utf-8') if isinstance(k, str) else k,
|
||
v.encode('utf-8') if isinstance(v, str) else v))
|
||
return urlencode(result, doseq=True)
|
||
else:
|
||
return data
|
||
|
||
@staticmethod
|
||
def _encode_files(files, data):
|
||
"""Build the body for a multipart/form-data request.
|
||
|
||
Will successfully encode files when passed as a dict or a list of
|
||
tuples. Order is retained if data is a list of tuples but arbitrary
|
||
if parameters are supplied as a dict.
|
||
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
|
||
or 4-tuples (filename, fileobj, contentype, custom_headers).
|
||
"""
|
||
if (not files):
|
||
raise ValueError("Files must be provided.")
|
||
elif isinstance(data, basestring):
|
||
raise ValueError("Data must not be a string.")
|
||
|
||
new_fields = []
|
||
fields = to_key_val_list(data or {})
|
||
files = to_key_val_list(files or {})
|
||
|
||
for field, val in fields:
|
||
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
|
||
val = [val]
|
||
for v in val:
|
||
if v is not None:
|
||
# Don't call str() on bytestrings: in Py3 it all goes wrong.
|
||
if not isinstance(v, bytes):
|
||
v = str(v)
|
||
|
||
new_fields.append(
|
||
(field.decode('utf-8') if isinstance(field, bytes) else field,
|
||
v.encode('utf-8') if isinstance(v, str) else v))
|
||
|
||
for (k, v) in files:
|
||
# support for explicit filename
|
||
ft = None
|
||
fh = None
|
||
if isinstance(v, (tuple, list)):
|
||
if len(v) == 2:
|
||
fn, fp = v
|
||
elif len(v) == 3:
|
||
fn, fp, ft = v
|
||
else:
|
||
fn, fp, ft, fh = v
|
||
else:
|
||
fn = guess_filename(v) or k
|
||
fp = v
|
||
|
||
if isinstance(fp, (str, bytes, bytearray)):
|
||
fdata = fp
|
||
else:
|
||
fdata = fp.read()
|
||
|
||
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
|
||
rf.make_multipart(content_type=ft)
|
||
new_fields.append(rf)
|
||
|
||
body, content_type = encode_multipart_formdata(new_fields)
|
||
|
||
return body, content_type
|
||
|
||
|
||
class RequestHooksMixin(object):
|
||
def register_hook(self, event, hook):
|
||
"""Properly register a hook."""
|
||
|
||
if event not in self.hooks:
|
||
raise ValueError('Unsupported event specified, with event name "%s"' % (event))
|
||
|
||
if isinstance(hook, collections.Callable):
|
||
self.hooks[event].append(hook)
|
||
elif hasattr(hook, '__iter__'):
|
||
self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
|
||
|
||
def deregister_hook(self, event, hook):
|
||
"""Deregister a previously registered hook.
|
||
Returns True if the hook existed, False if not.
|
||
"""
|
||
|
||
try:
|
||
self.hooks[event].remove(hook)
|
||
return True
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
class Request(RequestHooksMixin):
|
||
"""A user-created :class:`Request <Request>` object.
|
||
|
||
Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
|
||
|
||
:param method: HTTP method to use.
|
||
:param url: URL to send.
|
||
:param headers: dictionary of headers to send.
|
||
:param files: dictionary of {filename: fileobject} files to multipart upload.
|
||
:param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
|
||
:param json: json for the body to attach to the request (if files or data is not specified).
|
||
:param params: dictionary of URL parameters to append to the URL.
|
||
:param auth: Auth handler or (user, pass) tuple.
|
||
:param cookies: dictionary or CookieJar of cookies to attach to this request.
|
||
:param hooks: dictionary of callback hooks, for internal usage.
|
||
|
||
Usage::
|
||
|
||
>>> import requests
|
||
>>> req = requests.Request('GET', 'http://httpbin.org/get')
|
||
>>> req.prepare()
|
||
<PreparedRequest [GET]>
|
||
"""
|
||
|
||
def __init__(self, method=None, url=None, headers=None, files=None,
|
||
data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
|
||
|
||
# Default empty dicts for dict params.
|
||
data = [] if data is None else data
|
||
files = [] if files is None else files
|
||
headers = {} if headers is None else headers
|
||
params = {} if params is None else params
|
||
hooks = {} if hooks is None else hooks
|
||
|
||
self.hooks = default_hooks()
|
||
for (k, v) in list(hooks.items()):
|
||
self.register_hook(event=k, hook=v)
|
||
|
||
self.method = method
|
||
self.url = url
|
||
self.headers = headers
|
||
self.files = files
|
||
self.data = data
|
||
self.json = json
|
||
self.params = params
|
||
self.auth = auth
|
||
self.cookies = cookies
|
||
|
||
def __repr__(self):
|
||
return '<Request [%s]>' % (self.method)
|
||
|
||
def prepare(self):
|
||
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
|
||
p = PreparedRequest()
|
||
p.prepare(
|
||
method=self.method,
|
||
url=self.url,
|
||
headers=self.headers,
|
||
files=self.files,
|
||
data=self.data,
|
||
json=self.json,
|
||
params=self.params,
|
||
auth=self.auth,
|
||
cookies=self.cookies,
|
||
hooks=self.hooks,
|
||
)
|
||
return p
|
||
|
||
|
||
class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
||
"""The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
|
||
containing the exact bytes that will be sent to the server.
|
||
|
||
Generated from either a :class:`Request <Request>` object or manually.
|
||
|
||
Usage::
|
||
|
||
>>> import requests
|
||
>>> req = requests.Request('GET', 'http://httpbin.org/get')
|
||
>>> r = req.prepare()
|
||
<PreparedRequest [GET]>
|
||
|
||
>>> s = requests.Session()
|
||
>>> s.send(r)
|
||
<Response [200]>
|
||
"""
|
||
|
||
def __init__(self):
|
||
#: HTTP verb to send to the server.
|
||
self.method = None
|
||
#: HTTP URL to send the request to.
|
||
self.url = None
|
||
#: dictionary of HTTP headers.
|
||
self.headers = None
|
||
# The `CookieJar` used to create the Cookie header will be stored here
|
||
# after prepare_cookies is called
|
||
self._cookies = None
|
||
#: request body to send to the server.
|
||
self.body = None
|
||
#: dictionary of callback hooks, for internal usage.
|
||
self.hooks = default_hooks()
|
||
#: integer denoting starting position of a readable file-like body.
|
||
self._body_position = None
|
||
|
||
def prepare(self, method=None, url=None, headers=None, files=None,
|
||
data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
|
||
"""Prepares the entire request with the given parameters."""
|
||
|
||
self.prepare_method(method)
|
||
self.prepare_url(url, params)
|
||
self.prepare_headers(headers)
|
||
self.prepare_cookies(cookies)
|
||
self.prepare_body(data, files, json)
|
||
self.prepare_auth(auth, url)
|
||
|
||
# Note that prepare_auth must be last to enable authentication schemes
|
||
# such as OAuth to work on a fully prepared request.
|
||
|
||
# This MUST go after prepare_auth. Authenticators could add a hook
|
||
self.prepare_hooks(hooks)
|
||
|
||
def __repr__(self):
|
||
return '<PreparedRequest [%s]>' % (self.method)
|
||
|
||
def copy(self):
|
||
p = PreparedRequest()
|
||
p.method = self.method
|
||
p.url = self.url
|
||
p.headers = self.headers.copy() if self.headers is not None else None
|
||
p._cookies = _copy_cookie_jar(self._cookies)
|
||
p.body = self.body
|
||
p.hooks = self.hooks
|
||
p._body_position = self._body_position
|
||
return p
|
||
|
||
def prepare_method(self, method):
|
||
"""Prepares the given HTTP method."""
|
||
self.method = method
|
||
if self.method is not None:
|
||
self.method = to_native_string(self.method.upper())
|
||
|
||
def prepare_url(self, url, params):
|
||
"""Prepares the given HTTP URL."""
|
||
#: Accept objects that have string representations.
|
||
#: We're unable to blindly call unicode/str functions
|
||
#: as this will include the bytestring indicator (b'')
|
||
#: on python 3.x.
|
||
#: https://github.com/kennethreitz/requests/pull/2238
|
||
if isinstance(url, bytes):
|
||
url = url.decode('utf8')
|
||
else:
|
||
url = unicode(url) if is_py2 else str(url)
|
||
|
||
# Remove leading whitespaces from url
|
||
url = url.lstrip()
|
||
|
||
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
|
||
# `data` etc to work around exceptions from `url_parse`, which
|
||
# handles RFC 3986 only.
|
||
if ':' in url and not url.lower().startswith('http'):
|
||
self.url = url
|
||
return
|
||
|
||
# Support for unicode domain names and paths.
|
||
try:
|
||
scheme, auth, host, port, path, query, fragment = parse_url(url)
|
||
except LocationParseError as e:
|
||
raise InvalidURL(*e.args)
|
||
|
||
if not scheme:
|
||
error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")
|
||
error = error.format(to_native_string(url, 'utf8'))
|
||
|
||
raise MissingSchema(error)
|
||
|
||
if not host:
|
||
raise InvalidURL("Invalid URL %r: No host supplied" % url)
|
||
|
||
# In general, we want to try IDNA encoding every hostname, as that
|
||
# allows users to automatically get the correct behaviour. However,
|
||
# we’re quite strict about IDNA encoding, so certain valid hostnames
|
||
# may fail to encode. On failure, we verify the hostname meets a
|
||
# minimum standard of only containing ASCII characters, and not starting
|
||
# with a wildcard (*), before allowing the unencoded hostname through.
|
||
try:
|
||
host = idna.encode(host, uts46=True).decode('utf-8')
|
||
except (UnicodeError, idna.IDNAError):
|
||
if not unicode_is_ascii(host) or host.startswith(u'*'):
|
||
raise InvalidURL('URL has an invalid label.')
|
||
|
||
# Carefully reconstruct the network location
|
||
netloc = auth or ''
|
||
if netloc:
|
||
netloc += '@'
|
||
netloc += host
|
||
if port:
|
||
netloc += ':' + str(port)
|
||
|
||
# Bare domains aren't valid URLs.
|
||
if not path:
|
||
path = '/'
|
||
|
||
if is_py2:
|
||
if isinstance(scheme, str):
|
||
scheme = scheme.encode('utf-8')
|
||
if isinstance(netloc, str):
|
||
netloc = netloc.encode('utf-8')
|
||
if isinstance(path, str):
|
||
path = path.encode('utf-8')
|
||
if isinstance(query, str):
|
||
query = query.encode('utf-8')
|
||
if isinstance(fragment, str):
|
||
fragment = fragment.encode('utf-8')
|
||
|
||
if isinstance(params, (str, bytes)):
|
||
params = to_native_string(params)
|
||
|
||
enc_params = self._encode_params(params)
|
||
if enc_params:
|
||
if query:
|
||
query = '%s&%s' % (query, enc_params)
|
||
else:
|
||
query = enc_params
|
||
|
||
url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
|
||
self.url = url
|
||
|
||
def prepare_headers(self, headers):
|
||
"""Prepares the given HTTP headers."""
|
||
|
||
self.headers = CaseInsensitiveDict()
|
||
if headers:
|
||
for header in headers.items():
|
||
# Raise exception on invalid header value.
|
||
check_header_validity(header)
|
||
name, value = header
|
||
self.headers[to_native_string(name)] = value
|
||
|
||
def prepare_body(self, data, files, json=None):
|
||
"""Prepares the given HTTP body data."""
|
||
|
||
# Check if file, fo, generator, iterator.
|
||
# If not, run through normal process.
|
||
|
||
# Nottin' on you.
|
||
body = None
|
||
content_type = None
|
||
|
||
if not data and json is not None:
|
||
# urllib3 requires a bytes-like body. Python 2's json.dumps
|
||
# provides this natively, but Python 3 gives a Unicode string.
|
||
content_type = 'application/json'
|
||
body = complexjson.dumps(json)
|
||
if not isinstance(body, bytes):
|
||
body = body.encode('utf-8')
|
||
|
||
is_stream = all([
|
||
hasattr(data, '__iter__'),
|
||
not isinstance(data, (basestring, list, tuple, collections.Mapping))
|
||
])
|
||
|
||
try:
|
||
length = super_len(data)
|
||
except (TypeError, AttributeError, UnsupportedOperation):
|
||
length = None
|
||
|
||
if is_stream:
|
||
body = data
|
||
|
||
if getattr(body, 'tell', None) is not None:
|
||
# Record the current file position before reading.
|
||
# This will allow us to rewind a file in the event
|
||
# of a redirect.
|
||
try:
|
||
self._body_position = body.tell()
|
||
except (IOError, OSError):
|
||
# This differentiates from None, allowing us to catch
|
||
# a failed `tell()` later when trying to rewind the body
|
||
self._body_position = object()
|
||
|
||
if files:
|
||
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
|
||
|
||
if length:
|
||
self.headers['Content-Length'] = builtin_str(length)
|
||
else:
|
||
self.headers['Transfer-Encoding'] = 'chunked'
|
||
else:
|
||
# Multi-part file uploads.
|
||
if files:
|
||
(body, content_type) = self._encode_files(files, data)
|
||
else:
|
||
if data:
|
||
body = self._encode_params(data)
|
||
if isinstance(data, basestring) or hasattr(data, 'read'):
|
||
content_type = None
|
||
else:
|
||
content_type = 'application/x-www-form-urlencoded'
|
||
|
||
self.prepare_content_length(body)
|
||
|
||
# Add content-type if it wasn't explicitly provided.
|
||
if content_type and ('content-type' not in self.headers):
|
||
self.headers['Content-Type'] = content_type
|
||
|
||
self.body = body
|
||
|
||
def prepare_content_length(self, body):
|
||
"""Prepare Content-Length header based on request method and body"""
|
||
if body is not None:
|
||
length = super_len(body)
|
||
if length:
|
||
# If length exists, set it. Otherwise, we fallback
|
||
# to Transfer-Encoding: chunked.
|
||
self.headers['Content-Length'] = builtin_str(length)
|
||
elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None:
|
||
# Set Content-Length to 0 for methods that can have a body
|
||
# but don't provide one. (i.e. not GET or HEAD)
|
||
self.headers['Content-Length'] = '0'
|
||
|
||
def prepare_auth(self, auth, url=''):
|
||
"""Prepares the given HTTP auth data."""
|
||
|
||
# If no Auth is explicitly provided, extract it from the URL first.
|
||
if auth is None:
|
||
url_auth = get_auth_from_url(self.url)
|
||
auth = url_auth if any(url_auth) else None
|
||
|
||
if auth:
|
||
if isinstance(auth, tuple) and len(auth) == 2:
|
||
# special-case basic HTTP auth
|
||
auth = HTTPBasicAuth(*auth)
|
||
|
||
# Allow auth to make its changes.
|
||
r = auth(self)
|
||
|
||
# Update self to reflect the auth changes.
|
||
self.__dict__.update(r.__dict__)
|
||
|
||
# Recompute Content-Length
|
||
self.prepare_content_length(self.body)
|
||
|
||
def prepare_cookies(self, cookies):
|
||
"""Prepares the given HTTP cookie data.
|
||
|
||
This function eventually generates a ``Cookie`` header from the
|
||
given cookies using cookielib. Due to cookielib's design, the header
|
||
will not be regenerated if it already exists, meaning this function
|
||
can only be called once for the life of the
|
||
:class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
|
||
to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
|
||
header is removed beforehand.
|
||
"""
|
||
if isinstance(cookies, cookielib.CookieJar):
|
||
self._cookies = cookies
|
||
else:
|
||
self._cookies = cookiejar_from_dict(cookies)
|
||
|
||
cookie_header = get_cookie_header(self._cookies, self)
|
||
if cookie_header is not None:
|
||
self.headers['Cookie'] = cookie_header
|
||
|
||
def prepare_hooks(self, hooks):
|
||
"""Prepares the given hooks."""
|
||
# hooks can be passed as None to the prepare method and to this
|
||
# method. To prevent iterating over None, simply use an empty list
|
||
# if hooks is False-y
|
||
hooks = hooks or []
|
||
for event in hooks:
|
||
self.register_hook(event, hooks[event])
|
||
|
||
|
||
class Response(object):
|
||
"""The :class:`Response <Response>` object, which contains a
|
||
server's response to an HTTP request.
|
||
"""
|
||
|
||
__attrs__ = [
|
||
'_content', 'status_code', 'headers', 'url', 'history',
|
||
'encoding', 'reason', 'cookies', 'elapsed', 'request'
|
||
]
|
||
|
||
def __init__(self):
|
||
super(Response, self).__init__()
|
||
|
||
self._content = False
|
||
self._content_consumed = False
|
||
|
||
#: Integer Code of responded HTTP Status, e.g. 404 or 200.
|
||
self.status_code = None
|
||
|
||
#: Case-insensitive Dictionary of Response Headers.
|
||
#: For example, ``headers['content-encoding']`` will return the
|
||
#: value of a ``'Content-Encoding'`` response header.
|
||
self.headers = CaseInsensitiveDict()
|
||
|
||
#: File-like object representation of response (for advanced usage).
|
||
#: Use of ``raw`` requires that ``stream=True`` be set on the request.
|
||
# This requirement does not apply for use internally to Requests.
|
||
self.raw = None
|
||
|
||
#: Final URL location of Response.
|
||
self.url = None
|
||
|
||
#: Encoding to decode with when accessing r.text.
|
||
self.encoding = None
|
||
|
||
#: A list of :class:`Response <Response>` objects from
|
||
#: the history of the Request. Any redirect responses will end
|
||
#: up here. The list is sorted from the oldest to the most recent request.
|
||
self.history = []
|
||
|
||
#: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
|
||
self.reason = None
|
||
|
||
#: A CookieJar of Cookies the server sent back.
|
||
self.cookies = cookiejar_from_dict({})
|
||
|
||
#: The amount of time elapsed between sending the request
|
||
#: and the arrival of the response (as a timedelta).
|
||
#: This property specifically measures the time taken between sending
|
||
#: the first byte of the request and finishing parsing the headers. It
|
||
#: is therefore unaffected by consuming the response content or the
|
||
#: value of the ``stream`` keyword argument.
|
||
self.elapsed = datetime.timedelta(0)
|
||
|
||
#: The :class:`PreparedRequest <PreparedRequest>` object to which this
|
||
#: is a response.
|
||
self.request = None
|
||
|
||
def __getstate__(self):
|
||
# Consume everything; accessing the content attribute makes
|
||
# sure the content has been fully read.
|
||
if not self._content_consumed:
|
||
self.content
|
||
|
||
return dict(
|
||
(attr, getattr(self, attr, None))
|
||
for attr in self.__attrs__
|
||
)
|
||
|
||
def __setstate__(self, state):
|
||
for name, value in state.items():
|
||
setattr(self, name, value)
|
||
|
||
# pickled objects do not have .raw
|
||
setattr(self, '_content_consumed', True)
|
||
setattr(self, 'raw', None)
|
||
|
||
def __repr__(self):
|
||
return '<Response [%s]>' % (self.status_code)
|
||
|
||
def __bool__(self):
|
||
"""Returns true if :attr:`status_code` is 'OK'."""
|
||
return self.ok
|
||
|
||
def __nonzero__(self):
|
||
"""Returns true if :attr:`status_code` is 'OK'."""
|
||
return self.ok
|
||
|
||
def __iter__(self):
|
||
"""Allows you to use a response as an iterator."""
|
||
return self.iter_content(128)
|
||
|
||
@property
|
||
def ok(self):
|
||
try:
|
||
self.raise_for_status()
|
||
except HTTPError:
|
||
return False
|
||
return True
|
||
|
||
@property
|
||
def is_redirect(self):
|
||
"""True if this Response is a well-formed HTTP redirect that could have
|
||
been processed automatically (by :meth:`Session.resolve_redirects`).
|
||
"""
|
||
return ('location' in self.headers and self.status_code in REDIRECT_STATI)
|
||
|
||
@property
|
||
def is_permanent_redirect(self):
|
||
"""True if this Response one of the permanent versions of redirect"""
|
||
return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))
|
||
|
||
@property
|
||
def apparent_encoding(self):
|
||
"""The apparent encoding, provided by the chardet library"""
|
||
return chardet.detect(self.content)['encoding']
|
||
|
||
def iter_content(self, chunk_size=1, decode_unicode=False):
|
||
"""Iterates over the response data. When stream=True is set on the
|
||
request, this avoids reading the content at once into memory for
|
||
large responses. The chunk size is the number of bytes it should
|
||
read into memory. This is not necessarily the length of each item
|
||
returned as decoding can take place.
|
||
|
||
chunk_size must be of type int or None. A value of None will
|
||
function differently depending on the value of `stream`.
|
||
stream=True will read data as it arrives in whatever size the
|
||
chunks are received. If stream=False, data is returned as
|
||
a single chunk.
|
||
|
||
If decode_unicode is True, content will be decoded using the best
|
||
available encoding based on the response.
|
||
"""
|
||
|
||
def generate():
|
||
# Special case for urllib3.
|
||
if hasattr(self.raw, 'stream'):
|
||
try:
|
||
for chunk in self.raw.stream(chunk_size, decode_content=True):
|
||
yield chunk
|
||
except ProtocolError as e:
|
||
raise ChunkedEncodingError(e)
|
||
except DecodeError as e:
|
||
raise ContentDecodingError(e)
|
||
except ReadTimeoutError as e:
|
||
raise ConnectionError(e)
|
||
else:
|
||
# Standard file-like object.
|
||
while True:
|
||
chunk = self.raw.read(chunk_size)
|
||
if not chunk:
|
||
break
|
||
yield chunk
|
||
|
||
self._content_consumed = True
|
||
|
||
if self._content_consumed and isinstance(self._content, bool):
|
||
raise StreamConsumedError()
|
||
elif chunk_size is not None and not isinstance(chunk_size, int):
|
||
raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size))
|
||
# simulate reading small chunks of the content
|
||
reused_chunks = iter_slices(self._content, chunk_size)
|
||
|
||
stream_chunks = generate()
|
||
|
||
chunks = reused_chunks if self._content_consumed else stream_chunks
|
||
|
||
if decode_unicode:
|
||
chunks = stream_decode_response_unicode(chunks, self)
|
||
|
||
return chunks
|
||
|
||
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
|
||
"""Iterates over the response data, one line at a time. When
|
||
stream=True is set on the request, this avoids reading the
|
||
content at once into memory for large responses.
|
||
|
||
.. note:: This method is not reentrant safe.
|
||
"""
|
||
|
||
pending = None
|
||
|
||
for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode):
|
||
|
||
if pending is not None:
|
||
chunk = pending + chunk
|
||
|
||
if delimiter:
|
||
lines = chunk.split(delimiter)
|
||
else:
|
||
lines = chunk.splitlines()
|
||
|
||
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
|
||
pending = lines.pop()
|
||
else:
|
||
pending = None
|
||
|
||
for line in lines:
|
||
yield line
|
||
|
||
if pending is not None:
|
||
yield pending
|
||
|
||
@property
|
||
def content(self):
|
||
"""Content of the response, in bytes."""
|
||
|
||
if self._content is False:
|
||
# Read the contents.
|
||
if self._content_consumed:
|
||
raise RuntimeError(
|
||
'The content for this response was already consumed')
|
||
|
||
if self.status_code == 0 or self.raw is None:
|
||
self._content = None
|
||
else:
|
||
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
|
||
|
||
self._content_consumed = True
|
||
# don't need to release the connection; that's been handled by urllib3
|
||
# since we exhausted the data.
|
||
return self._content
|
||
|
||
@property
|
||
def text(self):
|
||
"""Content of the response, in unicode.
|
||
|
||
If Response.encoding is None, encoding will be guessed using
|
||
``chardet``.
|
||
|
||
The encoding of the response content is determined based solely on HTTP
|
||
headers, following RFC 2616 to the letter. If you can take advantage of
|
||
non-HTTP knowledge to make a better guess at the encoding, you should
|
||
set ``r.encoding`` appropriately before accessing this property.
|
||
"""
|
||
|
||
# Try charset from content-type
|
||
content = None
|
||
encoding = self.encoding
|
||
|
||
if not self.content:
|
||
return str('')
|
||
|
||
# Fallback to auto-detected encoding.
|
||
if self.encoding is None:
|
||
encoding = self.apparent_encoding
|
||
|
||
# Decode unicode from given encoding.
|
||
try:
|
||
content = str(self.content, encoding, errors='replace')
|
||
except (LookupError, TypeError):
|
||
# A LookupError is raised if the encoding was not found which could
|
||
# indicate a misspelling or similar mistake.
|
||
#
|
||
# A TypeError can be raised if encoding is None
|
||
#
|
||
# So we try blindly encoding.
|
||
content = str(self.content, errors='replace')
|
||
|
||
return content
|
||
|
||
def json(self, **kwargs):
|
||
"""Returns the json-encoded content of a response, if any.
|
||
|
||
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
|
||
:raises ValueError: If the response body does not contain valid json.
|
||
"""
|
||
|
||
if not self.encoding and self.content and len(self.content) > 3:
|
||
# No encoding set. JSON RFC 4627 section 3 states we should expect
|
||
# UTF-8, -16 or -32. Detect which one to use; If the detection or
|
||
# decoding fails, fall back to `self.text` (using chardet to make
|
||
# a best guess).
|
||
encoding = guess_json_utf(self.content)
|
||
if encoding is not None:
|
||
try:
|
||
return complexjson.loads(
|
||
self.content.decode(encoding), **kwargs
|
||
)
|
||
except UnicodeDecodeError:
|
||
# Wrong UTF codec detected; usually because it's not UTF-8
|
||
# but some other 8-bit codec. This is an RFC violation,
|
||
# and the server didn't bother to tell us what codec *was*
|
||
# used.
|
||
pass
|
||
return complexjson.loads(self.text, **kwargs)
|
||
|
||
@property
|
||
def links(self):
|
||
"""Returns the parsed header links of the response, if any."""
|
||
|
||
header = self.headers.get('link')
|
||
|
||
# l = MultiDict()
|
||
l = {}
|
||
|
||
if header:
|
||
links = parse_header_links(header)
|
||
|
||
for link in links:
|
||
key = link.get('rel') or link.get('url')
|
||
l[key] = link
|
||
|
||
return l
|
||
|
||
def raise_for_status(self):
|
||
"""Raises stored :class:`HTTPError`, if one occurred."""
|
||
|
||
http_error_msg = ''
|
||
if isinstance(self.reason, bytes):
|
||
# We attempt to decode utf-8 first because some servers
|
||
# choose to localize their reason strings. If the string
|
||
# isn't utf-8, we fall back to iso-8859-1 for all other
|
||
# encodings. (See PR #3538)
|
||
try:
|
||
reason = self.reason.decode('utf-8')
|
||
except UnicodeDecodeError:
|
||
reason = self.reason.decode('iso-8859-1')
|
||
else:
|
||
reason = self.reason
|
||
|
||
if 400 <= self.status_code < 500:
|
||
http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url)
|
||
|
||
elif 500 <= self.status_code < 600:
|
||
http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url)
|
||
|
||
if http_error_msg:
|
||
raise HTTPError(http_error_msg, response=self)
|
||
|
||
def close(self):
|
||
"""Releases the connection back to the pool. Once this method has been
|
||
called the underlying ``raw`` object must not be accessed again.
|
||
|
||
*Note: Should not normally need to be called explicitly.*
|
||
"""
|
||
if not self._content_consumed:
|
||
self.raw.close()
|
||
|
||
release_conn = getattr(self.raw, 'release_conn', None)
|
||
if release_conn is not None:
|
||
release_conn()
|