Added delete option to database storage.
This commit is contained in:
parent
308604a33c
commit
963b5bc68b
1868 changed files with 192402 additions and 13278 deletions
|
@ -7,30 +7,37 @@ requests.models
|
|||
This module contains the primary objects that power Requests.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
from io import BytesIO, UnsupportedOperation
|
||||
# Import encoding now, to avoid implicit import later.
|
||||
# Implicit import within threads may cause LookupError when standard library is in a ZIP,
|
||||
# such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
|
||||
import encodings.idna
|
||||
|
||||
from urllib3.fields import RequestField
|
||||
from urllib3.filepost import encode_multipart_formdata
|
||||
from urllib3.util import parse_url
|
||||
from urllib3.exceptions import (
|
||||
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
|
||||
|
||||
from io import UnsupportedOperation
|
||||
from .hooks import default_hooks
|
||||
from .structures import CaseInsensitiveDict
|
||||
|
||||
from .auth import HTTPBasicAuth
|
||||
from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
|
||||
from .packages.urllib3.fields import RequestField
|
||||
from .packages.urllib3.filepost import encode_multipart_formdata
|
||||
from .packages.urllib3.util import parse_url
|
||||
from .packages.urllib3.exceptions import (
|
||||
DecodeError, ReadTimeoutError, ProtocolError, LocationParseError)
|
||||
from .exceptions import (
|
||||
HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
|
||||
ContentDecodingError, ConnectionError, StreamConsumedError)
|
||||
from ._internal_utils import to_native_string, unicode_is_ascii
|
||||
from .utils import (
|
||||
guess_filename, get_auth_from_url, requote_uri,
|
||||
stream_decode_response_unicode, to_key_val_list, parse_header_links,
|
||||
iter_slices, guess_json_utf, super_len, to_native_string,
|
||||
check_header_validity)
|
||||
iter_slices, guess_json_utf, super_len, check_header_validity)
|
||||
from .compat import (
|
||||
cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
|
||||
Callable, Mapping,
|
||||
cookielib, urlunparse, urlsplit, urlencode, str, bytes,
|
||||
is_py2, chardet, builtin_str, basestring)
|
||||
from .compat import json as complexjson
|
||||
from .status_codes import codes
|
||||
|
@ -148,8 +155,12 @@ class RequestEncodingMixin(object):
|
|||
|
||||
if isinstance(fp, (str, bytes, bytearray)):
|
||||
fdata = fp
|
||||
else:
|
||||
elif hasattr(fp, 'read'):
|
||||
fdata = fp.read()
|
||||
elif fp is None:
|
||||
continue
|
||||
else:
|
||||
fdata = fp
|
||||
|
||||
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
|
||||
rf.make_multipart(content_type=ft)
|
||||
|
@ -167,10 +178,10 @@ class RequestHooksMixin(object):
|
|||
if event not in self.hooks:
|
||||
raise ValueError('Unsupported event specified, with event name "%s"' % (event))
|
||||
|
||||
if isinstance(hook, collections.Callable):
|
||||
if isinstance(hook, Callable):
|
||||
self.hooks[event].append(hook)
|
||||
elif hasattr(hook, '__iter__'):
|
||||
self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
|
||||
self.hooks[event].extend(h for h in hook if isinstance(h, Callable))
|
||||
|
||||
def deregister_hook(self, event, hook):
|
||||
"""Deregister a previously registered hook.
|
||||
|
@ -193,9 +204,13 @@ class Request(RequestHooksMixin):
|
|||
:param url: URL to send.
|
||||
:param headers: dictionary of headers to send.
|
||||
:param files: dictionary of {filename: fileobject} files to multipart upload.
|
||||
:param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place.
|
||||
:param data: the body to attach to the request. If a dictionary or
|
||||
list of tuples ``[(key, value)]`` is provided, form-encoding will
|
||||
take place.
|
||||
:param json: json for the body to attach to the request (if files or data is not specified).
|
||||
:param params: dictionary of URL parameters to append to the URL.
|
||||
:param params: URL parameters to append to the URL. If a dictionary or
|
||||
list of tuples ``[(key, value)]`` is provided, form-encoding will
|
||||
take place.
|
||||
:param auth: Auth handler or (user, pass) tuple.
|
||||
:param cookies: dictionary or CookieJar of cookies to attach to this request.
|
||||
:param hooks: dictionary of callback hooks, for internal usage.
|
||||
|
@ -203,13 +218,14 @@ class Request(RequestHooksMixin):
|
|||
Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> req = requests.Request('GET', 'http://httpbin.org/get')
|
||||
>>> req = requests.Request('GET', 'https://httpbin.org/get')
|
||||
>>> req.prepare()
|
||||
<PreparedRequest [GET]>
|
||||
"""
|
||||
|
||||
def __init__(self, method=None, url=None, headers=None, files=None,
|
||||
data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
|
||||
def __init__(self,
|
||||
method=None, url=None, headers=None, files=None, data=None,
|
||||
params=None, auth=None, cookies=None, hooks=None, json=None):
|
||||
|
||||
# Default empty dicts for dict params.
|
||||
data = [] if data is None else data
|
||||
|
@ -262,8 +278,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
Usage::
|
||||
|
||||
>>> import requests
|
||||
>>> req = requests.Request('GET', 'http://httpbin.org/get')
|
||||
>>> req = requests.Request('GET', 'https://httpbin.org/get')
|
||||
>>> r = req.prepare()
|
||||
>>> r
|
||||
<PreparedRequest [GET]>
|
||||
|
||||
>>> s = requests.Session()
|
||||
|
@ -285,9 +302,12 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
self.body = None
|
||||
#: dictionary of callback hooks, for internal usage.
|
||||
self.hooks = default_hooks()
|
||||
#: integer denoting starting position of a readable file-like body.
|
||||
self._body_position = None
|
||||
|
||||
def prepare(self, method=None, url=None, headers=None, files=None,
|
||||
data=None, params=None, auth=None, cookies=None, hooks=None, json=None):
|
||||
def prepare(self,
|
||||
method=None, url=None, headers=None, files=None, data=None,
|
||||
params=None, auth=None, cookies=None, hooks=None, json=None):
|
||||
"""Prepares the entire request with the given parameters."""
|
||||
|
||||
self.prepare_method(method)
|
||||
|
@ -314,6 +334,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
p._cookies = _copy_cookie_jar(self._cookies)
|
||||
p.body = self.body
|
||||
p.hooks = self.hooks
|
||||
p._body_position = self._body_position
|
||||
return p
|
||||
|
||||
def prepare_method(self, method):
|
||||
|
@ -322,18 +343,31 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
if self.method is not None:
|
||||
self.method = to_native_string(self.method.upper())
|
||||
|
||||
@staticmethod
|
||||
def _get_idna_encoded_host(host):
|
||||
import idna
|
||||
|
||||
try:
|
||||
host = idna.encode(host, uts46=True).decode('utf-8')
|
||||
except idna.IDNAError:
|
||||
raise UnicodeError
|
||||
return host
|
||||
|
||||
def prepare_url(self, url, params):
|
||||
"""Prepares the given HTTP URL."""
|
||||
#: Accept objects that have string representations.
|
||||
#: We're unable to blindly call unicode/str functions
|
||||
#: as this will include the bytestring indicator (b'')
|
||||
#: on python 3.x.
|
||||
#: https://github.com/kennethreitz/requests/pull/2238
|
||||
#: https://github.com/psf/requests/pull/2238
|
||||
if isinstance(url, bytes):
|
||||
url = url.decode('utf8')
|
||||
else:
|
||||
url = unicode(url) if is_py2 else str(url)
|
||||
|
||||
# Remove leading whitespaces from url
|
||||
url = url.lstrip()
|
||||
|
||||
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
|
||||
# `data` etc to work around exceptions from `url_parse`, which
|
||||
# handles RFC 3986 only.
|
||||
|
@ -356,10 +390,16 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
if not host:
|
||||
raise InvalidURL("Invalid URL %r: No host supplied" % url)
|
||||
|
||||
# Only want to apply IDNA to the hostname
|
||||
try:
|
||||
host = host.encode('idna').decode('utf-8')
|
||||
except UnicodeError:
|
||||
# In general, we want to try IDNA encoding the hostname if the string contains
|
||||
# non-ASCII characters. This allows users to automatically get the correct IDNA
|
||||
# behaviour. For strings containing only ASCII characters, we need to also verify
|
||||
# it doesn't start with a wildcard (*), before allowing the unencoded hostname.
|
||||
if not unicode_is_ascii(host):
|
||||
try:
|
||||
host = self._get_idna_encoded_host(host)
|
||||
except UnicodeError:
|
||||
raise InvalidURL('URL has an invalid label.')
|
||||
elif host.startswith(u'*'):
|
||||
raise InvalidURL('URL has an invalid label.')
|
||||
|
||||
# Carefully reconstruct the network location
|
||||
|
@ -419,7 +459,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
# Nottin' on you.
|
||||
body = None
|
||||
content_type = None
|
||||
length = None
|
||||
|
||||
if not data and json is not None:
|
||||
# urllib3 requires a bytes-like body. Python 2's json.dumps
|
||||
|
@ -431,17 +470,28 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
|
||||
is_stream = all([
|
||||
hasattr(data, '__iter__'),
|
||||
not isinstance(data, (basestring, list, tuple, dict))
|
||||
not isinstance(data, (basestring, list, tuple, Mapping))
|
||||
])
|
||||
|
||||
try:
|
||||
length = super_len(data)
|
||||
except (TypeError, AttributeError, UnsupportedOperation):
|
||||
length = None
|
||||
|
||||
if is_stream:
|
||||
try:
|
||||
length = super_len(data)
|
||||
except (TypeError, AttributeError, UnsupportedOperation):
|
||||
length = None
|
||||
|
||||
body = data
|
||||
|
||||
if getattr(body, 'tell', None) is not None:
|
||||
# Record the current file position before reading.
|
||||
# This will allow us to rewind a file in the event
|
||||
# of a redirect.
|
||||
try:
|
||||
self._body_position = body.tell()
|
||||
except (IOError, OSError):
|
||||
# This differentiates from None, allowing us to catch
|
||||
# a failed `tell()` later when trying to rewind the body
|
||||
self._body_position = object()
|
||||
|
||||
if files:
|
||||
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
|
||||
|
||||
|
@ -470,17 +520,16 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
self.body = body
|
||||
|
||||
def prepare_content_length(self, body):
|
||||
if hasattr(body, 'seek') and hasattr(body, 'tell'):
|
||||
curr_pos = body.tell()
|
||||
body.seek(0, 2)
|
||||
end_pos = body.tell()
|
||||
self.headers['Content-Length'] = builtin_str(max(0, end_pos - curr_pos))
|
||||
body.seek(curr_pos, 0)
|
||||
elif body is not None:
|
||||
l = super_len(body)
|
||||
if l:
|
||||
self.headers['Content-Length'] = builtin_str(l)
|
||||
elif (self.method not in ('GET', 'HEAD')) and (self.headers.get('Content-Length') is None):
|
||||
"""Prepare Content-Length header based on request method and body"""
|
||||
if body is not None:
|
||||
length = super_len(body)
|
||||
if length:
|
||||
# If length exists, set it. Otherwise, we fallback
|
||||
# to Transfer-Encoding: chunked.
|
||||
self.headers['Content-Length'] = builtin_str(length)
|
||||
elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None:
|
||||
# Set Content-Length to 0 for methods that can have a body
|
||||
# but don't provide one. (i.e. not GET or HEAD)
|
||||
self.headers['Content-Length'] = '0'
|
||||
|
||||
def prepare_auth(self, auth, url=''):
|
||||
|
@ -546,10 +595,9 @@ class Response(object):
|
|||
]
|
||||
|
||||
def __init__(self):
|
||||
super(Response, self).__init__()
|
||||
|
||||
self._content = False
|
||||
self._content_consumed = False
|
||||
self._next = None
|
||||
|
||||
#: Integer Code of responded HTTP Status, e.g. 404 or 200.
|
||||
self.status_code = None
|
||||
|
@ -561,7 +609,7 @@ class Response(object):
|
|||
|
||||
#: File-like object representation of response (for advanced usage).
|
||||
#: Use of ``raw`` requires that ``stream=True`` be set on the request.
|
||||
# This requirement does not apply for use internally to Requests.
|
||||
#: This requirement does not apply for use internally to Requests.
|
||||
self.raw = None
|
||||
|
||||
#: Final URL location of Response.
|
||||
|
@ -593,16 +641,19 @@ class Response(object):
|
|||
#: is a response.
|
||||
self.request = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.close()
|
||||
|
||||
def __getstate__(self):
|
||||
# Consume everything; accessing the content attribute makes
|
||||
# sure the content has been fully read.
|
||||
if not self._content_consumed:
|
||||
self.content
|
||||
|
||||
return dict(
|
||||
(attr, getattr(self, attr, None))
|
||||
for attr in self.__attrs__
|
||||
)
|
||||
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
|
||||
|
||||
def __setstate__(self, state):
|
||||
for name, value in state.items():
|
||||
|
@ -616,11 +667,23 @@ class Response(object):
|
|||
return '<Response [%s]>' % (self.status_code)
|
||||
|
||||
def __bool__(self):
|
||||
"""Returns true if :attr:`status_code` is 'OK'."""
|
||||
"""Returns True if :attr:`status_code` is less than 400.
|
||||
|
||||
This attribute checks if the status code of the response is between
|
||||
400 and 600 to see if there was a client error or a server error. If
|
||||
the status code, is between 200 and 400, this will return True. This
|
||||
is **not** a check to see if the response code is ``200 OK``.
|
||||
"""
|
||||
return self.ok
|
||||
|
||||
def __nonzero__(self):
|
||||
"""Returns true if :attr:`status_code` is 'OK'."""
|
||||
"""Returns True if :attr:`status_code` is less than 400.
|
||||
|
||||
This attribute checks if the status code of the response is between
|
||||
400 and 600 to see if there was a client error or a server error. If
|
||||
the status code, is between 200 and 400, this will return True. This
|
||||
is **not** a check to see if the response code is ``200 OK``.
|
||||
"""
|
||||
return self.ok
|
||||
|
||||
def __iter__(self):
|
||||
|
@ -629,6 +692,13 @@ class Response(object):
|
|||
|
||||
@property
|
||||
def ok(self):
|
||||
"""Returns True if :attr:`status_code` is less than 400, False if not.
|
||||
|
||||
This attribute checks if the status code of the response is between
|
||||
400 and 600 to see if there was a client error or a server error. If
|
||||
the status code is between 200 and 400, this will return True. This
|
||||
is **not** a check to see if the response code is ``200 OK``.
|
||||
"""
|
||||
try:
|
||||
self.raise_for_status()
|
||||
except HTTPError:
|
||||
|
@ -644,12 +714,17 @@ class Response(object):
|
|||
|
||||
@property
|
||||
def is_permanent_redirect(self):
|
||||
"""True if this Response one of the permanent versions of redirect"""
|
||||
"""True if this Response one of the permanent versions of redirect."""
|
||||
return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))
|
||||
|
||||
@property
|
||||
def next(self):
|
||||
"""Returns a PreparedRequest for the next request in a redirect chain, if there is one."""
|
||||
return self._next
|
||||
|
||||
@property
|
||||
def apparent_encoding(self):
|
||||
"""The apparent encoding, provided by the chardet library"""
|
||||
"""The apparent encoding, provided by the chardet library."""
|
||||
return chardet.detect(self.content)['encoding']
|
||||
|
||||
def iter_content(self, chunk_size=1, decode_unicode=False):
|
||||
|
@ -707,7 +782,7 @@ class Response(object):
|
|||
|
||||
return chunks
|
||||
|
||||
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None):
|
||||
def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None):
|
||||
"""Iterates over the response data, one line at a time. When
|
||||
stream=True is set on the request, this avoids reading the
|
||||
content at once into memory for large responses.
|
||||
|
@ -744,18 +819,14 @@ class Response(object):
|
|||
|
||||
if self._content is False:
|
||||
# Read the contents.
|
||||
try:
|
||||
if self._content_consumed:
|
||||
raise RuntimeError(
|
||||
'The content for this response was already consumed')
|
||||
if self._content_consumed:
|
||||
raise RuntimeError(
|
||||
'The content for this response was already consumed')
|
||||
|
||||
if self.status_code == 0:
|
||||
self._content = None
|
||||
else:
|
||||
self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
|
||||
|
||||
except AttributeError:
|
||||
if self.status_code == 0 or self.raw is None:
|
||||
self._content = None
|
||||
else:
|
||||
self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''
|
||||
|
||||
self._content_consumed = True
|
||||
# don't need to release the connection; that's been handled by urllib3
|
||||
|
@ -801,9 +872,10 @@ class Response(object):
|
|||
return content
|
||||
|
||||
def json(self, **kwargs):
|
||||
"""Returns the json-encoded content of a response, if any.
|
||||
r"""Returns the json-encoded content of a response, if any.
|
||||
|
||||
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
|
||||
:raises ValueError: If the response body does not contain valid json.
|
||||
"""
|
||||
|
||||
if not self.encoding and self.content and len(self.content) > 3:
|
||||
|
@ -844,11 +916,18 @@ class Response(object):
|
|||
return l
|
||||
|
||||
def raise_for_status(self):
|
||||
"""Raises stored :class:`HTTPError`, if one occurred."""
|
||||
"""Raises :class:`HTTPError`, if one occurred."""
|
||||
|
||||
http_error_msg = ''
|
||||
if isinstance(self.reason, bytes):
|
||||
reason = self.reason.decode('utf-8', 'ignore')
|
||||
# We attempt to decode utf-8 first because some servers
|
||||
# choose to localize their reason strings. If the string
|
||||
# isn't utf-8, we fall back to iso-8859-1 for all other
|
||||
# encodings. (See PR #3538)
|
||||
try:
|
||||
reason = self.reason.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
reason = self.reason.decode('iso-8859-1')
|
||||
else:
|
||||
reason = self.reason
|
||||
|
||||
|
@ -870,4 +949,6 @@ class Response(object):
|
|||
if not self._content_consumed:
|
||||
self.raw.close()
|
||||
|
||||
return self.raw.release_conn()
|
||||
release_conn = getattr(self.raw, 'release_conn', None)
|
||||
if release_conn is not None:
|
||||
release_conn()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue