"""HTTP wrapper for apitools. This library wraps the underlying http library we use, which is currently :mod:`httplib2`. """ import collections import contextlib import logging import socket import time import httplib2 import six from six.moves import http_client from six.moves.urllib import parse from gcloud.streaming.exceptions import BadStatusCodeError from gcloud.streaming.exceptions import RequestError from gcloud.streaming.exceptions import RetryAfterError from gcloud.streaming.util import calculate_wait_for_retry # 308 and 429 don't have names in httplib. RESUME_INCOMPLETE = 308 TOO_MANY_REQUESTS = 429 _REDIRECT_STATUS_CODES = ( http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER, http_client.TEMPORARY_REDIRECT, RESUME_INCOMPLETE, ) _RETRYABLE_EXCEPTIONS = ( http_client.BadStatusLine, http_client.IncompleteRead, http_client.ResponseNotReady, socket.error, httplib2.ServerNotFoundError, ValueError, RequestError, BadStatusCodeError, RetryAfterError, ) class _ExceptionRetryArgs( collections.namedtuple( '_ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries', 'max_retry_wait'])): """Bundle of information for retriable exceptions. :type http: :class:`httplib2.Http` (or conforming alternative) :param http: instance used to perform requests. :type http_request: :class:`Request` :param http_request: the request whose response was a retriable error :type exc: :class:`Exception` subclass :param exc: the exception being raised. :type num_retries: integer :param num_retries: Number of retries consumed; used for exponential backoff. """ @contextlib.contextmanager def _httplib2_debug_level(http_request, level, http=None): """Temporarily change the value of httplib2.debuglevel, if necessary. If http_request has a `loggable_body` distinct from `body`, then we need to prevent httplib2 from logging the full body. This sets httplib2.debuglevel for the duration of the `with` block; however, that alone won't change the value of existing HTTP connections. If an httplib2.Http object is provided, we'll also change the level on any cached connections attached to it. :type http_request: :class:`Request` :param http_request: the request to be logged. :type level: integer :param level: the debuglevel for logging. :type http: :class:`httplib2.Http`, or ``None`` :param http: the instance on whose connections to set the debuglevel. """ if http_request.loggable_body is None: yield return old_level = httplib2.debuglevel http_levels = {} httplib2.debuglevel = level if http is not None: for connection_key, connection in http.connections.items(): # httplib2 stores two kinds of values in this dict, connection # classes and instances. Since the connection types are all # old-style classes, we can't easily distinguish by connection # type -- so instead we use the key pattern. if ':' not in connection_key: continue http_levels[connection_key] = connection.debuglevel connection.set_debuglevel(level) yield httplib2.debuglevel = old_level if http is not None: for connection_key, old_level in http_levels.items(): http.connections[connection_key].set_debuglevel(old_level) class Request(object): """Encapsulates the data for an HTTP request. :type url: str :param url: the URL for the request :type http_method: str :param http_method: the HTTP method to use for the request :type headers: mapping or None :param headers: headers to be sent with the request :type body: str :param body: body to be sent with the request """ def __init__(self, url='', http_method='GET', headers=None, body=''): self.url = url self.http_method = http_method self.headers = headers or {} self.__body = None self.__loggable_body = None self.body = body @property def loggable_body(self): """Request body for logging purposes :rtype: str """ return self.__loggable_body @loggable_body.setter def loggable_body(self, value): """Update request body for logging purposes :type value: str :param value: updated body :raises: :exc:`RequestError` if the request does not have a body. """ if self.body is None: raise RequestError( 'Cannot set loggable body on request with no body') self.__loggable_body = value @property def body(self): """Request body :rtype: str """ return self.__body @body.setter def body(self, value): """Update the request body Handles logging and length measurement. :type value: str :param value: updated body """ self.__body = value if value is not None: # Avoid calling len() which cannot exceed 4GiB in 32-bit python. body_length = getattr( self.__body, 'length', None) or len(self.__body) self.headers['content-length'] = str(body_length) else: self.headers.pop('content-length', None) # This line ensures we don't try to print large requests. if not isinstance(value, (type(None), six.string_types)): self.loggable_body = '' def _process_content_range(content_range): """Convert a 'Content-Range' header into a length for the response. Helper for :meth:`Response.length`. :type content_range: str :param content_range: the header value being parsed. :rtype: integer :returns: the length of the response chunk. """ _, _, range_spec = content_range.partition(' ') byte_range, _, _ = range_spec.partition('/') start, _, end = byte_range.partition('-') return int(end) - int(start) + 1 # Note: currently the order of fields here is important, since we want # to be able to pass in the result from httplib2.request. _ResponseTuple = collections.namedtuple( 'HttpResponse', ['info', 'content', 'request_url']) class Response(_ResponseTuple): """Encapsulates data for an HTTP response. """ __slots__ = () def __len__(self): return self.length @property def length(self): """Length of this response. Exposed as an attribute since using ``len()`` directly can fail for responses larger than ``sys.maxint``. :rtype: integer or long """ if 'content-encoding' in self.info and 'content-range' in self.info: # httplib2 rewrites content-length in the case of a compressed # transfer; we can't trust the content-length header in that # case, but we *can* trust content-range, if it's present. return _process_content_range(self.info['content-range']) elif 'content-length' in self.info: return int(self.info.get('content-length')) elif 'content-range' in self.info: return _process_content_range(self.info['content-range']) return len(self.content) @property def status_code(self): """HTTP status code :rtype: integer """ return int(self.info['status']) @property def retry_after(self): """Retry interval (if set). :rtype: integer :returns: interval in seconds """ if 'retry-after' in self.info: return int(self.info['retry-after']) @property def is_redirect(self): """Does this response contain a redirect :rtype: boolean :returns: True if the status code indicates a redirect and the 'location' header is present. """ return (self.status_code in _REDIRECT_STATUS_CODES and 'location' in self.info) def _check_response(response): """Validate a response :type response: :class:`Response` :param response: the response to validate :raises: :exc:`gcloud.streaming.exceptions.RequestError` if response is None, :exc:`gcloud.streaming.exceptions.BadStatusCodeError` if response status code indicates an error, or :exc:`gcloud.streaming.exceptions.RetryAfterError` if response indicates a retry interval. """ if response is None: # Caller shouldn't call us if the response is None, but handle anyway. raise RequestError( 'Request did not return a response.') elif (response.status_code >= 500 or response.status_code == TOO_MANY_REQUESTS): raise BadStatusCodeError.from_response(response) elif response.retry_after: raise RetryAfterError.from_response(response) def _reset_http_connections(http): """Rebuild all http connections in the httplib2.Http instance. httplib2 overloads the map in http.connections to contain two different types of values: { scheme string: connection class } and { scheme + authority string : actual http connection } Here we remove all of the entries for actual connections so that on the next request httplib2 will rebuild them from the connection types. :type http: :class:`httplib2.Http` :param http: the instance whose connections are to be rebuilt """ if getattr(http, 'connections', None): for conn_key in list(http.connections.keys()): if ':' in conn_key: del http.connections[conn_key] def _make_api_request_no_retry(http, http_request, redirections=5, check_response_func=_check_response): """Send an HTTP request via the given http instance. This wrapper exists to handle translation between the plain httplib2 request/response types and the Request and Response types above. :type http: :class:`httplib2.Http` :param http: an instance which impelements the `Http` API. :type http_request: :class:`Request` :param http_request: the request to send. :type redirections: integer :param redirections: Number of redirects to follow. :type check_response_func: function taking (response, content, url). :param check_response_func: Function to validate the HTTP response. :rtype: :class:`Response` :returns: an object representing the server's response :raises: :exc:`gcloud.streaming.exceptions.RequestError` if no response could be parsed. """ connection_type = None # Handle overrides for connection types. This is used if the caller # wants control over the underlying connection for managing callbacks # or hash digestion. if getattr(http, 'connections', None): url_scheme = parse.urlsplit(http_request.url).scheme if url_scheme and url_scheme in http.connections: connection_type = http.connections[url_scheme] # Custom printing only at debuglevel 4 new_debuglevel = 4 if httplib2.debuglevel == 4 else 0 with _httplib2_debug_level(http_request, new_debuglevel, http=http): info, content = http.request( str(http_request.url), method=str(http_request.http_method), body=http_request.body, headers=http_request.headers, redirections=redirections, connection_type=connection_type) if info is None: raise RequestError() response = Response(info, content, http_request.url) check_response_func(response) return response def make_api_request(http, http_request, retries=7, max_retry_wait=60, redirections=5, check_response_func=_check_response, wo_retry_func=_make_api_request_no_retry): """Send an HTTP request via the given http, performing error/retry handling. :type http: :class:`httplib2.Http` :param http: an instance which impelements the `Http` API. :type http_request: :class:`Request` :param http_request: the request to send. :type retries: integer :param retries: Number of retries to attempt on retryable responses (such as 429 or 5XX). :type max_retry_wait: integer :param max_retry_wait: Maximum number of seconds to wait when retrying. :type redirections: integer :param redirections: Number of redirects to follow. :type check_response_func: function taking (response, content, url). :param check_response_func: Function to validate the HTTP response. :type wo_retry_func: function taking (http, request, redirections, check_response_func) :param wo_retry_func: Function to make HTTP request without retries. :rtype: :class:`Response` :returns: an object representing the server's response :raises: :exc:`gcloud.streaming.exceptions.RequestError` if no response could be parsed. """ retry = 0 while True: try: return wo_retry_func( http, http_request, redirections=redirections, check_response_func=check_response_func) except _RETRYABLE_EXCEPTIONS as exc: retry += 1 if retry >= retries: raise retry_after = getattr(exc, 'retry_after', None) if retry_after is None: retry_after = calculate_wait_for_retry(retry, max_retry_wait) _reset_http_connections(http) logging.debug('Retrying request to url %s after exception %s', http_request.url, type(exc).__name__) time.sleep(retry_after) _HTTP_FACTORIES = [] def _register_http_factory(factory): """Register a custom HTTP factory. :type factory: callable taking keyword arguments, returning an Http instance (or an instance implementing the same API); :param factory: the new factory (it may return ``None`` to defer to a later factory or the default). """ _HTTP_FACTORIES.append(factory) def get_http(**kwds): """Construct an Http instance. :type kwds: dict :param kwds: keyword arguments to pass to factories. :rtype: :class:`httplib2.Http` (or a workalike) """ for factory in _HTTP_FACTORIES: http = factory(**kwds) if http is not None: return http return httplib2.Http(**kwds)