Added delete option to database storage.

This commit is contained in:
Batuhan Berk Başoğlu 2020-10-12 12:10:01 -04:00
parent 308604a33c
commit 963b5bc68b
1868 changed files with 192402 additions and 13278 deletions

View file

@ -0,0 +1,97 @@
# Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transport - HTTP client library support.
:mod:`google.auth` is designed to work with various HTTP client libraries such
as urllib3 and requests. In order to work across these libraries with different
interfaces some abstraction is needed.
This module provides two interfaces that are implemented by transport adapters
to support HTTP libraries. :class:`Request` defines the interface expected by
:mod:`google.auth` to make requests. :class:`Response` defines the interface
for the return value of :class:`Request`.
"""
import abc
import six
from six.moves import http_client
DEFAULT_REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,)
"""Sequence[int]: Which HTTP status code indicate that credentials should be
refreshed and a request should be retried.
"""
DEFAULT_MAX_REFRESH_ATTEMPTS = 2
"""int: How many times to refresh the credentials and retry a request."""
@six.add_metaclass(abc.ABCMeta)
class Response(object):
"""HTTP Response data."""
@abc.abstractproperty
def status(self):
"""int: The HTTP status code."""
raise NotImplementedError("status must be implemented.")
@abc.abstractproperty
def headers(self):
"""Mapping[str, str]: The HTTP response headers."""
raise NotImplementedError("headers must be implemented.")
@abc.abstractproperty
def data(self):
"""bytes: The response body."""
raise NotImplementedError("data must be implemented.")
@six.add_metaclass(abc.ABCMeta)
class Request(object):
"""Interface for a callable that makes HTTP requests.
Specific transport implementations should provide an implementation of
this that adapts their specific request / response API.
.. automethod:: __call__
"""
@abc.abstractmethod
def __call__(
self, url, method="GET", body=None, headers=None, timeout=None, **kwargs
):
"""Make an HTTP request.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
transport-specific default timeout will be used.
kwargs: Additionally arguments passed on to the transport's
request method.
Returns:
Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
# pylint: disable=redundant-returns-doc, missing-raises-doc
# (pylint doesn't play well with abstract docstrings.)
raise NotImplementedError("__call__ must be implemented.")

View file

@ -0,0 +1,384 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transport adapter for Async HTTP (aiohttp).
NOTE: This async support is experimental and marked internal. This surface may
change in minor releases.
"""
from __future__ import absolute_import
import asyncio
import functools
import aiohttp
import six
import urllib3
from google.auth import exceptions
from google.auth import transport
from google.auth.transport import requests
# Timeout can be re-defined depending on async requirement. Currently made 60s more than
# sync timeout.
_DEFAULT_TIMEOUT = 180 # in seconds
class _CombinedResponse(transport.Response):
"""
In order to more closely resemble the `requests` interface, where a raw
and deflated content could be accessed at once, this class lazily reads the
stream in `transport.Response` so both return forms can be used.
The gzip and deflate transfer-encodings are automatically decoded for you
because the default parameter for autodecompress into the ClientSession is set
to False, and therefore we add this class to act as a wrapper for a user to be
able to access both the raw and decoded response bodies - mirroring the sync
implementation.
"""
def __init__(self, response):
self._response = response
self._raw_content = None
def _is_compressed(self):
headers = self._response.headers
return "Content-Encoding" in headers and (
headers["Content-Encoding"] == "gzip"
or headers["Content-Encoding"] == "deflate"
)
@property
def status(self):
return self._response.status
@property
def headers(self):
return self._response.headers
@property
def data(self):
return self._response.content
async def raw_content(self):
if self._raw_content is None:
self._raw_content = await self._response.content.read()
return self._raw_content
async def content(self):
# Load raw_content if necessary
await self.raw_content()
if self._is_compressed():
decoder = urllib3.response.MultiDecoder(
self._response.headers["Content-Encoding"]
)
decompressed = decoder.decompress(self._raw_content)
return decompressed
return self._raw_content
class _Response(transport.Response):
"""
Requests transport response adapter.
Args:
response (requests.Response): The raw Requests response.
"""
def __init__(self, response):
self._response = response
@property
def status(self):
return self._response.status
@property
def headers(self):
return self._response.headers
@property
def data(self):
return self._response.content
class Request(transport.Request):
"""Requests request adapter.
This class is used internally for making requests using asyncio transports
in a consistent way. If you use :class:`AuthorizedSession` you do not need
to construct or use this class directly.
This class can be useful if you want to manually refresh a
:class:`~google.auth.credentials.Credentials` instance::
import google.auth.transport.aiohttp_requests
request = google.auth.transport.aiohttp_requests.Request()
credentials.refresh(request)
Args:
session (aiohttp.ClientSession): An instance :class: aiohttp.ClientSession used
to make HTTP requests. If not specified, a session will be created.
.. automethod:: __call__
"""
def __init__(self, session=None):
self.session = None
async def __call__(
self,
url,
method="GET",
body=None,
headers=None,
timeout=_DEFAULT_TIMEOUT,
**kwargs,
):
"""
Make an HTTP request using aiohttp.
Args:
url (str): The URL to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
requests default timeout will be used.
kwargs: Additional arguments passed through to the underlying
requests :meth:`~requests.Session.request` method.
Returns:
google.auth.transport.Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
try:
if self.session is None: # pragma: NO COVER
self.session = aiohttp.ClientSession(
auto_decompress=False
) # pragma: NO COVER
requests._LOGGER.debug("Making request: %s %s", method, url)
response = await self.session.request(
method, url, data=body, headers=headers, timeout=timeout, **kwargs
)
return _CombinedResponse(response)
except aiohttp.ClientError as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
except asyncio.TimeoutError as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
class AuthorizedSession(aiohttp.ClientSession):
"""This is an async implementation of the Authorized Session class. We utilize an
aiohttp transport instance, and the interface mirrors the google.auth.transport.requests
Authorized Session class, except for the change in the transport used in the async use case.
A Requests Session class with credentials.
This class is used to perform requests to API endpoints that require
authorization::
from google.auth.transport import aiohttp_requests
async with aiohttp_requests.AuthorizedSession(credentials) as authed_session:
response = await authed_session.request(
'GET', 'https://www.googleapis.com/storage/v1/b')
The underlying :meth:`request` implementation handles adding the
credentials' headers to the request and refreshing credentials as needed.
Args:
credentials (google.auth._credentials_async.Credentials): The credentials to
add to the request.
refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
that credentials should be refreshed and the request should be
retried.
max_refresh_attempts (int): The maximum number of times to attempt to
refresh the credentials and retry the request.
refresh_timeout (Optional[int]): The timeout value in seconds for
credential refresh HTTP requests.
auth_request (google.auth.transport.aiohttp_requests.Request):
(Optional) An instance of
:class:`~google.auth.transport.aiohttp_requests.Request` used when
refreshing credentials. If not passed,
an instance of :class:`~google.auth.transport.aiohttp_requests.Request`
is created.
"""
def __init__(
self,
credentials,
refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
refresh_timeout=None,
auth_request=None,
auto_decompress=False,
):
super(AuthorizedSession, self).__init__()
self.credentials = credentials
self._refresh_status_codes = refresh_status_codes
self._max_refresh_attempts = max_refresh_attempts
self._refresh_timeout = refresh_timeout
self._is_mtls = False
self._auth_request = auth_request
self._auth_request_session = None
self._loop = asyncio.get_event_loop()
self._refresh_lock = asyncio.Lock()
self._auto_decompress = auto_decompress
async def request(
self,
method,
url,
data=None,
headers=None,
max_allowed_time=None,
timeout=_DEFAULT_TIMEOUT,
auto_decompress=False,
**kwargs,
):
"""Implementation of Authorized Session aiohttp request.
Args:
method: The http request method used (e.g. GET, PUT, DELETE)
url: The url at which the http request is sent.
data, headers: These fields parallel the associated data and headers
fields of a regular http request. Using the aiohttp client session to
send the http request allows us to use this parallel corresponding structure
in our Authorized Session class.
timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
The amount of time in seconds to wait for the server response
with each individual request.
Can also be passed as an `aiohttp.ClientTimeout` object.
max_allowed_time (Optional[float]):
If the method runs longer than this, a ``Timeout`` exception is
automatically raised. Unlike the ``timeout` parameter, this
value applies to the total method execution time, even if
multiple requests are made under the hood.
Mind that it is not guaranteed that the timeout error is raised
at ``max_allowed_time`. It might take longer, for example, if
an underlying request takes a lot of time, but the request
itself does not timeout, e.g. if a large file is being
transmitted. The timout error will be raised after such
request completes.
"""
# Headers come in as bytes which isn't expected behavior, the resumable
# media libraries in some cases expect a str type for the header values,
# but sometimes the operations return these in bytes types.
if headers:
for key in headers.keys():
if type(headers[key]) is bytes:
headers[key] = headers[key].decode("utf-8")
async with aiohttp.ClientSession(
auto_decompress=self._auto_decompress
) as self._auth_request_session:
auth_request = Request(self._auth_request_session)
self._auth_request = auth_request
# Use a kwarg for this instead of an attribute to maintain
# thread-safety.
_credential_refresh_attempt = kwargs.pop("_credential_refresh_attempt", 0)
# Make a copy of the headers. They will be modified by the credentials
# and we want to pass the original headers if we recurse.
request_headers = headers.copy() if headers is not None else {}
# Do not apply the timeout unconditionally in order to not override the
# _auth_request's default timeout.
auth_request = (
self._auth_request
if timeout is None
else functools.partial(self._auth_request, timeout=timeout)
)
remaining_time = max_allowed_time
with requests.TimeoutGuard(remaining_time, asyncio.TimeoutError) as guard:
await self.credentials.before_request(
auth_request, method, url, request_headers
)
with requests.TimeoutGuard(remaining_time, asyncio.TimeoutError) as guard:
response = await super(AuthorizedSession, self).request(
method,
url,
data=data,
headers=request_headers,
timeout=timeout,
**kwargs,
)
remaining_time = guard.remaining_timeout
if (
response.status in self._refresh_status_codes
and _credential_refresh_attempt < self._max_refresh_attempts
):
requests._LOGGER.info(
"Refreshing credentials due to a %s response. Attempt %s/%s.",
response.status,
_credential_refresh_attempt + 1,
self._max_refresh_attempts,
)
# Do not apply the timeout unconditionally in order to not override the
# _auth_request's default timeout.
auth_request = (
self._auth_request
if timeout is None
else functools.partial(self._auth_request, timeout=timeout)
)
with requests.TimeoutGuard(
remaining_time, asyncio.TimeoutError
) as guard:
async with self._refresh_lock:
await self._loop.run_in_executor(
None, self.credentials.refresh, auth_request
)
remaining_time = guard.remaining_timeout
return await self.request(
method,
url,
data=data,
headers=headers,
max_allowed_time=remaining_time,
timeout=timeout,
_credential_refresh_attempt=_credential_refresh_attempt + 1,
**kwargs,
)
return response

View file

@ -0,0 +1,115 @@
# Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transport adapter for http.client, for internal use only."""
import logging
import socket
import six
from six.moves import http_client
from six.moves import urllib
from google.auth import exceptions
from google.auth import transport
_LOGGER = logging.getLogger(__name__)
class Response(transport.Response):
"""http.client transport response adapter.
Args:
response (http.client.HTTPResponse): The raw http client response.
"""
def __init__(self, response):
self._status = response.status
self._headers = {key.lower(): value for key, value in response.getheaders()}
self._data = response.read()
@property
def status(self):
return self._status
@property
def headers(self):
return self._headers
@property
def data(self):
return self._data
class Request(transport.Request):
"""http.client transport request adapter."""
def __call__(
self, url, method="GET", body=None, headers=None, timeout=None, **kwargs
):
"""Make an HTTP request using http.client.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping): Request headers.
timeout (Optional(int)): The number of seconds to wait for a
response from the server. If not specified or if None, the
socket global default timeout will be used.
kwargs: Additional arguments passed throught to the underlying
:meth:`~http.client.HTTPConnection.request` method.
Returns:
Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
# socket._GLOBAL_DEFAULT_TIMEOUT is the default in http.client.
if timeout is None:
timeout = socket._GLOBAL_DEFAULT_TIMEOUT
# http.client doesn't allow None as the headers argument.
if headers is None:
headers = {}
# http.client needs the host and path parts specified separately.
parts = urllib.parse.urlsplit(url)
path = urllib.parse.urlunsplit(
("", "", parts.path, parts.query, parts.fragment)
)
if parts.scheme != "http":
raise exceptions.TransportError(
"http.client transport only supports the http scheme, {}"
"was specified".format(parts.scheme)
)
connection = http_client.HTTPConnection(parts.netloc, timeout=timeout)
try:
_LOGGER.debug("Making request: %s %s", method, url)
connection.request(method, path, body=body, headers=headers, **kwargs)
response = connection.getresponse()
return Response(response)
except (http_client.HTTPException, socket.error) as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
finally:
connection.close()

View file

@ -0,0 +1,250 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions for getting mTLS cert and key."""
import json
import logging
from os import path
import re
import subprocess
import six
from google.auth import exceptions
CONTEXT_AWARE_METADATA_PATH = "~/.secureConnect/context_aware_metadata.json"
_CERT_PROVIDER_COMMAND = "cert_provider_command"
_CERT_REGEX = re.compile(
b"-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\r?\n?", re.DOTALL
)
# support various format of key files, e.g.
# "-----BEGIN PRIVATE KEY-----...",
# "-----BEGIN EC PRIVATE KEY-----...",
# "-----BEGIN RSA PRIVATE KEY-----..."
# "-----BEGIN ENCRYPTED PRIVATE KEY-----"
_KEY_REGEX = re.compile(
b"-----BEGIN [A-Z ]*PRIVATE KEY-----.+-----END [A-Z ]*PRIVATE KEY-----\r?\n?",
re.DOTALL,
)
_LOGGER = logging.getLogger(__name__)
_PASSPHRASE_REGEX = re.compile(
b"-----BEGIN PASSPHRASE-----(.+)-----END PASSPHRASE-----", re.DOTALL
)
def _check_dca_metadata_path(metadata_path):
"""Checks for context aware metadata. If it exists, returns the absolute path;
otherwise returns None.
Args:
metadata_path (str): context aware metadata path.
Returns:
str: absolute path if exists and None otherwise.
"""
metadata_path = path.expanduser(metadata_path)
if not path.exists(metadata_path):
_LOGGER.debug("%s is not found, skip client SSL authentication.", metadata_path)
return None
return metadata_path
def _read_dca_metadata_file(metadata_path):
"""Loads context aware metadata from the given path.
Args:
metadata_path (str): context aware metadata path.
Returns:
Dict[str, str]: The metadata.
Raises:
google.auth.exceptions.ClientCertError: If failed to parse metadata as JSON.
"""
try:
with open(metadata_path) as f:
metadata = json.load(f)
except ValueError as caught_exc:
new_exc = exceptions.ClientCertError(caught_exc)
six.raise_from(new_exc, caught_exc)
return metadata
def _run_cert_provider_command(command, expect_encrypted_key=False):
"""Run the provided command, and return client side mTLS cert, key and
passphrase.
Args:
command (List[str]): cert provider command.
expect_encrypted_key (bool): If encrypted private key is expected.
Returns:
Tuple[bytes, bytes, bytes]: client certificate bytes in PEM format, key
bytes in PEM format and passphrase bytes.
Raises:
google.auth.exceptions.ClientCertError: if problems occurs when running
the cert provider command or generating cert, key and passphrase.
"""
try:
process = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout, stderr = process.communicate()
except OSError as caught_exc:
new_exc = exceptions.ClientCertError(caught_exc)
six.raise_from(new_exc, caught_exc)
# Check cert provider command execution error.
if process.returncode != 0:
raise exceptions.ClientCertError(
"Cert provider command returns non-zero status code %s" % process.returncode
)
# Extract certificate (chain), key and passphrase.
cert_match = re.findall(_CERT_REGEX, stdout)
if len(cert_match) != 1:
raise exceptions.ClientCertError("Client SSL certificate is missing or invalid")
key_match = re.findall(_KEY_REGEX, stdout)
if len(key_match) != 1:
raise exceptions.ClientCertError("Client SSL key is missing or invalid")
passphrase_match = re.findall(_PASSPHRASE_REGEX, stdout)
if expect_encrypted_key:
if len(passphrase_match) != 1:
raise exceptions.ClientCertError("Passphrase is missing or invalid")
if b"ENCRYPTED" not in key_match[0]:
raise exceptions.ClientCertError("Encrypted private key is expected")
return cert_match[0], key_match[0], passphrase_match[0].strip()
if b"ENCRYPTED" in key_match[0]:
raise exceptions.ClientCertError("Encrypted private key is not expected")
if len(passphrase_match) > 0:
raise exceptions.ClientCertError("Passphrase is not expected")
return cert_match[0], key_match[0], None
def get_client_ssl_credentials(generate_encrypted_key=False):
"""Returns the client side certificate, private key and passphrase.
Args:
generate_encrypted_key (bool): If set to True, encrypted private key
and passphrase will be generated; otherwise, unencrypted private key
will be generated and passphrase will be None.
Returns:
Tuple[bool, bytes, bytes, bytes]:
A boolean indicating if cert, key and passphrase are obtained, the
cert bytes and key bytes both in PEM format, and passphrase bytes.
Raises:
google.auth.exceptions.ClientCertError: if problems occurs when getting
the cert, key and passphrase.
"""
metadata_path = _check_dca_metadata_path(CONTEXT_AWARE_METADATA_PATH)
if metadata_path:
metadata_json = _read_dca_metadata_file(metadata_path)
if _CERT_PROVIDER_COMMAND not in metadata_json:
raise exceptions.ClientCertError("Cert provider command is not found")
command = metadata_json[_CERT_PROVIDER_COMMAND]
if generate_encrypted_key and "--with_passphrase" not in command:
command.append("--with_passphrase")
# Execute the command.
cert, key, passphrase = _run_cert_provider_command(
command, expect_encrypted_key=generate_encrypted_key
)
return True, cert, key, passphrase
return False, None, None, None
def get_client_cert_and_key(client_cert_callback=None):
"""Returns the client side certificate and private key. The function first
tries to get certificate and key from client_cert_callback; if the callback
is None or doesn't provide certificate and key, the function tries application
default SSL credentials.
Args:
client_cert_callback (Optional[Callable[[], (bytes, bytes)]]): An
optional callback which returns client certificate bytes and private
key bytes both in PEM format.
Returns:
Tuple[bool, bytes, bytes]:
A boolean indicating if cert and key are obtained, the cert bytes
and key bytes both in PEM format.
Raises:
google.auth.exceptions.ClientCertError: if problems occurs when getting
the cert and key.
"""
if client_cert_callback:
cert, key = client_cert_callback()
return True, cert, key
has_cert, cert, key, _ = get_client_ssl_credentials(generate_encrypted_key=False)
return has_cert, cert, key
def decrypt_private_key(key, passphrase):
"""A helper function to decrypt the private key with the given passphrase.
google-auth library doesn't support passphrase protected private key for
mutual TLS channel. This helper function can be used to decrypt the
passphrase protected private key in order to estalish mutual TLS channel.
For example, if you have a function which produces client cert, passphrase
protected private key and passphrase, you can convert it to a client cert
callback function accepted by google-auth::
from google.auth.transport import _mtls_helper
def your_client_cert_function():
return cert, encrypted_key, passphrase
# callback accepted by google-auth for mutual TLS channel.
def client_cert_callback():
cert, encrypted_key, passphrase = your_client_cert_function()
decrypted_key = _mtls_helper.decrypt_private_key(encrypted_key,
passphrase)
return cert, decrypted_key
Args:
key (bytes): The private key bytes in PEM format.
passphrase (bytes): The passphrase bytes.
Returns:
bytes: The decrypted private key in PEM format.
Raises:
ImportError: If pyOpenSSL is not installed.
OpenSSL.crypto.Error: If there is any problem decrypting the private key.
"""
from OpenSSL import crypto
# First convert encrypted_key_bytes to PKey object
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key, passphrase=passphrase)
# Then dump the decrypted key bytes
return crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)

View file

@ -0,0 +1,334 @@
# Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Authorization support for gRPC."""
from __future__ import absolute_import
import logging
import os
import six
from google.auth import environment_vars
from google.auth import exceptions
from google.auth.transport import _mtls_helper
try:
import grpc
except ImportError as caught_exc: # pragma: NO COVER
six.raise_from(
ImportError(
"gRPC is not installed, please install the grpcio package "
"to use the gRPC transport."
),
caught_exc,
)
_LOGGER = logging.getLogger(__name__)
class AuthMetadataPlugin(grpc.AuthMetadataPlugin):
"""A `gRPC AuthMetadataPlugin`_ that inserts the credentials into each
request.
.. _gRPC AuthMetadataPlugin:
http://www.grpc.io/grpc/python/grpc.html#grpc.AuthMetadataPlugin
Args:
credentials (google.auth.credentials.Credentials): The credentials to
add to requests.
request (google.auth.transport.Request): A HTTP transport request
object used to refresh credentials as needed.
"""
def __init__(self, credentials, request):
# pylint: disable=no-value-for-parameter
# pylint doesn't realize that the super method takes no arguments
# because this class is the same name as the superclass.
super(AuthMetadataPlugin, self).__init__()
self._credentials = credentials
self._request = request
def _get_authorization_headers(self, context):
"""Gets the authorization headers for a request.
Returns:
Sequence[Tuple[str, str]]: A list of request headers (key, value)
to add to the request.
"""
headers = {}
self._credentials.before_request(
self._request, context.method_name, context.service_url, headers
)
return list(six.iteritems(headers))
def __call__(self, context, callback):
"""Passes authorization metadata into the given callback.
Args:
context (grpc.AuthMetadataContext): The RPC context.
callback (grpc.AuthMetadataPluginCallback): The callback that will
be invoked to pass in the authorization metadata.
"""
callback(self._get_authorization_headers(context), None)
def secure_authorized_channel(
credentials,
request,
target,
ssl_credentials=None,
client_cert_callback=None,
**kwargs
):
"""Creates a secure authorized gRPC channel.
This creates a channel with SSL and :class:`AuthMetadataPlugin`. This
channel can be used to create a stub that can make authorized requests.
Users can configure client certificate or rely on device certificates to
establish a mutual TLS channel, if the `GOOGLE_API_USE_CLIENT_CERTIFICATE`
variable is explicitly set to `true`.
Example::
import google.auth
import google.auth.transport.grpc
import google.auth.transport.requests
from google.cloud.speech.v1 import cloud_speech_pb2
# Get credentials.
credentials, _ = google.auth.default()
# Get an HTTP request function to refresh credentials.
request = google.auth.transport.requests.Request()
# Create a channel.
channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, regular_endpoint, request,
ssl_credentials=grpc.ssl_channel_credentials())
# Use the channel to create a stub.
cloud_speech.create_Speech_stub(channel)
Usage:
There are actually a couple of options to create a channel, depending on if
you want to create a regular or mutual TLS channel.
First let's list the endpoints (regular vs mutual TLS) to choose from::
regular_endpoint = 'speech.googleapis.com:443'
mtls_endpoint = 'speech.mtls.googleapis.com:443'
Option 1: create a regular (non-mutual) TLS channel by explicitly setting
the ssl_credentials::
regular_ssl_credentials = grpc.ssl_channel_credentials()
channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, regular_endpoint, request,
ssl_credentials=regular_ssl_credentials)
Option 2: create a mutual TLS channel by calling a callback which returns
the client side certificate and the key (Note that
`GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable must be explicitly
set to `true`)::
def my_client_cert_callback():
code_to_load_client_cert_and_key()
if loaded:
return (pem_cert_bytes, pem_key_bytes)
raise MyClientCertFailureException()
try:
channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, mtls_endpoint, request,
client_cert_callback=my_client_cert_callback)
except MyClientCertFailureException:
# handle the exception
Option 3: use application default SSL credentials. It searches and uses
the command in a context aware metadata file, which is available on devices
with endpoint verification support (Note that
`GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable must be explicitly
set to `true`).
See https://cloud.google.com/endpoint-verification/docs/overview::
try:
default_ssl_credentials = SslCredentials()
except:
# Exception can be raised if the context aware metadata is malformed.
# See :class:`SslCredentials` for the possible exceptions.
# Choose the endpoint based on the SSL credentials type.
if default_ssl_credentials.is_mtls:
endpoint_to_use = mtls_endpoint
else:
endpoint_to_use = regular_endpoint
channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, endpoint_to_use, request,
ssl_credentials=default_ssl_credentials)
Option 4: not setting ssl_credentials and client_cert_callback. For devices
without endpoint verification support or `GOOGLE_API_USE_CLIENT_CERTIFICATE`
environment variable is not `true`, a regular TLS channel is created;
otherwise, a mutual TLS channel is created, however, the call should be
wrapped in a try/except block in case of malformed context aware metadata.
The following code uses regular_endpoint, it works the same no matter the
created channle is regular or mutual TLS. Regular endpoint ignores client
certificate and key::
channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, regular_endpoint, request)
The following code uses mtls_endpoint, if the created channle is regular,
and API mtls_endpoint is confgured to require client SSL credentials, API
calls using this channel will be rejected::
channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, mtls_endpoint, request)
Args:
credentials (google.auth.credentials.Credentials): The credentials to
add to requests.
request (google.auth.transport.Request): A HTTP transport request
object used to refresh credentials as needed. Even though gRPC
is a separate transport, there's no way to refresh the credentials
without using a standard http transport.
target (str): The host and port of the service.
ssl_credentials (grpc.ChannelCredentials): Optional SSL channel
credentials. This can be used to specify different certificates.
This argument is mutually exclusive with client_cert_callback;
providing both will raise an exception.
If ssl_credentials and client_cert_callback are None, application
default SSL credentials are used if `GOOGLE_API_USE_CLIENT_CERTIFICATE`
environment variable is explicitly set to `true`, otherwise one way TLS
SSL credentials are used.
client_cert_callback (Callable[[], (bytes, bytes)]): Optional
callback function to obtain client certicate and key for mutual TLS
connection. This argument is mutually exclusive with
ssl_credentials; providing both will raise an exception.
This argument does nothing unless `GOOGLE_API_USE_CLIENT_CERTIFICATE`
environment variable is explicitly set to `true`.
kwargs: Additional arguments to pass to :func:`grpc.secure_channel`.
Returns:
grpc.Channel: The created gRPC channel.
Raises:
google.auth.exceptions.MutualTLSChannelError: If mutual TLS channel
creation failed for any reason.
"""
# Create the metadata plugin for inserting the authorization header.
metadata_plugin = AuthMetadataPlugin(credentials, request)
# Create a set of grpc.CallCredentials using the metadata plugin.
google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin)
if ssl_credentials and client_cert_callback:
raise ValueError(
"Received both ssl_credentials and client_cert_callback; "
"these are mutually exclusive."
)
# If SSL credentials are not explicitly set, try client_cert_callback and ADC.
if not ssl_credentials:
use_client_cert = os.getenv(
environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"
)
if use_client_cert == "true" and client_cert_callback:
# Use the callback if provided.
cert, key = client_cert_callback()
ssl_credentials = grpc.ssl_channel_credentials(
certificate_chain=cert, private_key=key
)
elif use_client_cert == "true":
# Use application default SSL credentials.
adc_ssl_credentils = SslCredentials()
ssl_credentials = adc_ssl_credentils.ssl_credentials
else:
ssl_credentials = grpc.ssl_channel_credentials()
# Combine the ssl credentials and the authorization credentials.
composite_credentials = grpc.composite_channel_credentials(
ssl_credentials, google_auth_credentials
)
return grpc.secure_channel(target, composite_credentials, **kwargs)
class SslCredentials:
"""Class for application default SSL credentials.
The behavior is controlled by `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment
variable whose default value is `false`. Client certificate will not be used
unless the environment variable is explicitly set to `true`. See
https://google.aip.dev/auth/4114
If the environment variable is `true`, then for devices with endpoint verification
support, a device certificate will be automatically loaded and mutual TLS will
be established.
See https://cloud.google.com/endpoint-verification/docs/overview.
"""
def __init__(self):
use_client_cert = os.getenv(
environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"
)
if use_client_cert != "true":
self._is_mtls = False
else:
# Load client SSL credentials.
metadata_path = _mtls_helper._check_dca_metadata_path(
_mtls_helper.CONTEXT_AWARE_METADATA_PATH
)
self._is_mtls = metadata_path is not None
@property
def ssl_credentials(self):
"""Get the created SSL channel credentials.
For devices with endpoint verification support, if the device certificate
loading has any problems, corresponding exceptions will be raised. For
a device without endpoint verification support, no exceptions will be
raised.
Returns:
grpc.ChannelCredentials: The created grpc channel credentials.
Raises:
google.auth.exceptions.MutualTLSChannelError: If mutual TLS channel
creation failed for any reason.
"""
if self._is_mtls:
try:
_, cert, key, _ = _mtls_helper.get_client_ssl_credentials()
self._ssl_credentials = grpc.ssl_channel_credentials(
certificate_chain=cert, private_key=key
)
except exceptions.ClientCertError as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
else:
self._ssl_credentials = grpc.ssl_channel_credentials()
return self._ssl_credentials
@property
def is_mtls(self):
"""Indicates if the created SSL channel credentials is mutual TLS."""
return self._is_mtls

View file

@ -0,0 +1,105 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilites for mutual TLS."""
import six
from google.auth import exceptions
from google.auth.transport import _mtls_helper
def has_default_client_cert_source():
"""Check if default client SSL credentials exists on the device.
Returns:
bool: indicating if the default client cert source exists.
"""
metadata_path = _mtls_helper._check_dca_metadata_path(
_mtls_helper.CONTEXT_AWARE_METADATA_PATH
)
return metadata_path is not None
def default_client_cert_source():
"""Get a callback which returns the default client SSL credentials.
Returns:
Callable[[], [bytes, bytes]]: A callback which returns the default
client certificate bytes and private key bytes, both in PEM format.
Raises:
google.auth.exceptions.DefaultClientCertSourceError: If the default
client SSL credentials don't exist or are malformed.
"""
if not has_default_client_cert_source():
raise exceptions.MutualTLSChannelError(
"Default client cert source doesn't exist"
)
def callback():
try:
_, cert_bytes, key_bytes = _mtls_helper.get_client_cert_and_key()
except (OSError, RuntimeError, ValueError) as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
return cert_bytes, key_bytes
return callback
def default_client_encrypted_cert_source(cert_path, key_path):
"""Get a callback which returns the default encrpyted client SSL credentials.
Args:
cert_path (str): The cert file path. The default client certificate will
be written to this file when the returned callback is called.
key_path (str): The key file path. The default encrypted client key will
be written to this file when the returned callback is called.
Returns:
Callable[[], [str, str, bytes]]: A callback which generates the default
client certificate, encrpyted private key and passphrase. It writes
the certificate and private key into the cert_path and key_path, and
returns the cert_path, key_path and passphrase bytes.
Raises:
google.auth.exceptions.DefaultClientCertSourceError: If any problem
occurs when loading or saving the client certificate and key.
"""
if not has_default_client_cert_source():
raise exceptions.MutualTLSChannelError(
"Default client encrypted cert source doesn't exist"
)
def callback():
try:
(
_,
cert_bytes,
key_bytes,
passphrase_bytes,
) = _mtls_helper.get_client_ssl_credentials(generate_encrypted_key=True)
with open(cert_path, "wb") as cert_file:
cert_file.write(cert_bytes)
with open(key_path, "wb") as key_file:
key_file.write(key_bytes)
except (exceptions.ClientCertError, OSError) as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
return cert_path, key_path, passphrase_bytes
return callback

View file

@ -0,0 +1,521 @@
# Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transport adapter for Requests."""
from __future__ import absolute_import
import functools
import logging
import numbers
import os
import time
try:
import requests
except ImportError as caught_exc: # pragma: NO COVER
import six
six.raise_from(
ImportError(
"The requests library is not installed, please install the "
"requests package to use the requests transport."
),
caught_exc,
)
import requests.adapters # pylint: disable=ungrouped-imports
import requests.exceptions # pylint: disable=ungrouped-imports
from requests.packages.urllib3.util.ssl_ import (
create_urllib3_context,
) # pylint: disable=ungrouped-imports
import six # pylint: disable=ungrouped-imports
from google.auth import environment_vars
from google.auth import exceptions
from google.auth import transport
import google.auth.transport._mtls_helper
_LOGGER = logging.getLogger(__name__)
_DEFAULT_TIMEOUT = 120 # in seconds
class _Response(transport.Response):
"""Requests transport response adapter.
Args:
response (requests.Response): The raw Requests response.
"""
def __init__(self, response):
self._response = response
@property
def status(self):
return self._response.status_code
@property
def headers(self):
return self._response.headers
@property
def data(self):
return self._response.content
class TimeoutGuard(object):
"""A context manager raising an error if the suite execution took too long.
Args:
timeout ([Union[None, float, Tuple[float, float]]]):
The maximum number of seconds a suite can run without the context
manager raising a timeout exception on exit. If passed as a tuple,
the smaller of the values is taken as a timeout. If ``None``, a
timeout error is never raised.
timeout_error_type (Optional[Exception]):
The type of the error to raise on timeout. Defaults to
:class:`requests.exceptions.Timeout`.
"""
def __init__(self, timeout, timeout_error_type=requests.exceptions.Timeout):
self._timeout = timeout
self.remaining_timeout = timeout
self._timeout_error_type = timeout_error_type
def __enter__(self):
self._start = time.time()
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_value:
return # let the error bubble up automatically
if self._timeout is None:
return # nothing to do, the timeout was not specified
elapsed = time.time() - self._start
deadline_hit = False
if isinstance(self._timeout, numbers.Number):
self.remaining_timeout = self._timeout - elapsed
deadline_hit = self.remaining_timeout <= 0
else:
self.remaining_timeout = tuple(x - elapsed for x in self._timeout)
deadline_hit = min(self.remaining_timeout) <= 0
if deadline_hit:
raise self._timeout_error_type()
class Request(transport.Request):
"""Requests request adapter.
This class is used internally for making requests using various transports
in a consistent way. If you use :class:`AuthorizedSession` you do not need
to construct or use this class directly.
This class can be useful if you want to manually refresh a
:class:`~google.auth.credentials.Credentials` instance::
import google.auth.transport.requests
import requests
request = google.auth.transport.requests.Request()
credentials.refresh(request)
Args:
session (requests.Session): An instance :class:`requests.Session` used
to make HTTP requests. If not specified, a session will be created.
.. automethod:: __call__
"""
def __init__(self, session=None):
if not session:
session = requests.Session()
self.session = session
def __call__(
self,
url,
method="GET",
body=None,
headers=None,
timeout=_DEFAULT_TIMEOUT,
**kwargs
):
"""Make an HTTP request using requests.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
requests default timeout will be used.
kwargs: Additional arguments passed through to the underlying
requests :meth:`~requests.Session.request` method.
Returns:
google.auth.transport.Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
try:
_LOGGER.debug("Making request: %s %s", method, url)
response = self.session.request(
method, url, data=body, headers=headers, timeout=timeout, **kwargs
)
return _Response(response)
except requests.exceptions.RequestException as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
class _MutualTlsAdapter(requests.adapters.HTTPAdapter):
"""
A TransportAdapter that enables mutual TLS.
Args:
cert (bytes): client certificate in PEM format
key (bytes): client private key in PEM format
Raises:
ImportError: if certifi or pyOpenSSL is not installed
OpenSSL.crypto.Error: if client cert or key is invalid
"""
def __init__(self, cert, key):
import certifi
from OpenSSL import crypto
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert)
ctx_poolmanager = create_urllib3_context()
ctx_poolmanager.load_verify_locations(cafile=certifi.where())
ctx_poolmanager._ctx.use_certificate(x509)
ctx_poolmanager._ctx.use_privatekey(pkey)
self._ctx_poolmanager = ctx_poolmanager
ctx_proxymanager = create_urllib3_context()
ctx_proxymanager.load_verify_locations(cafile=certifi.where())
ctx_proxymanager._ctx.use_certificate(x509)
ctx_proxymanager._ctx.use_privatekey(pkey)
self._ctx_proxymanager = ctx_proxymanager
super(_MutualTlsAdapter, self).__init__()
def init_poolmanager(self, *args, **kwargs):
kwargs["ssl_context"] = self._ctx_poolmanager
super(_MutualTlsAdapter, self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
kwargs["ssl_context"] = self._ctx_proxymanager
return super(_MutualTlsAdapter, self).proxy_manager_for(*args, **kwargs)
class AuthorizedSession(requests.Session):
"""A Requests Session class with credentials.
This class is used to perform requests to API endpoints that require
authorization::
from google.auth.transport.requests import AuthorizedSession
authed_session = AuthorizedSession(credentials)
response = authed_session.request(
'GET', 'https://www.googleapis.com/storage/v1/b')
The underlying :meth:`request` implementation handles adding the
credentials' headers to the request and refreshing credentials as needed.
This class also supports mutual TLS via :meth:`configure_mtls_channel`
method. In order to use this method, the `GOOGLE_API_USE_CLIENT_CERTIFICATE`
environment variable must be explicitly set to `true`, otherwise it does
nothing. Assume the environment is set to `true`, the method behaves in the
following manner:
If client_cert_callback is provided, client certificate and private
key are loaded using the callback; if client_cert_callback is None,
application default SSL credentials will be used. Exceptions are raised if
there are problems with the certificate, private key, or the loading process,
so it should be called within a try/except block.
First we set the environment variable to `true`, then create an :class:`AuthorizedSession`
instance and specify the endpoints::
regular_endpoint = 'https://pubsub.googleapis.com/v1/projects/{my_project_id}/topics'
mtls_endpoint = 'https://pubsub.mtls.googleapis.com/v1/projects/{my_project_id}/topics'
authed_session = AuthorizedSession(credentials)
Now we can pass a callback to :meth:`configure_mtls_channel`::
def my_cert_callback():
# some code to load client cert bytes and private key bytes, both in
# PEM format.
some_code_to_load_client_cert_and_key()
if loaded:
return cert, key
raise MyClientCertFailureException()
# Always call configure_mtls_channel within a try/except block.
try:
authed_session.configure_mtls_channel(my_cert_callback)
except:
# handle exceptions.
if authed_session.is_mtls:
response = authed_session.request('GET', mtls_endpoint)
else:
response = authed_session.request('GET', regular_endpoint)
You can alternatively use application default SSL credentials like this::
try:
authed_session.configure_mtls_channel()
except:
# handle exceptions.
Args:
credentials (google.auth.credentials.Credentials): The credentials to
add to the request.
refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
that credentials should be refreshed and the request should be
retried.
max_refresh_attempts (int): The maximum number of times to attempt to
refresh the credentials and retry the request.
refresh_timeout (Optional[int]): The timeout value in seconds for
credential refresh HTTP requests.
auth_request (google.auth.transport.requests.Request):
(Optional) An instance of
:class:`~google.auth.transport.requests.Request` used when
refreshing credentials. If not passed,
an instance of :class:`~google.auth.transport.requests.Request`
is created.
"""
def __init__(
self,
credentials,
refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
refresh_timeout=None,
auth_request=None,
):
super(AuthorizedSession, self).__init__()
self.credentials = credentials
self._refresh_status_codes = refresh_status_codes
self._max_refresh_attempts = max_refresh_attempts
self._refresh_timeout = refresh_timeout
self._is_mtls = False
if auth_request is None:
auth_request_session = requests.Session()
# Using an adapter to make HTTP requests robust to network errors.
# This adapter retrys HTTP requests when network errors occur
# and the requests seems safely retryable.
retry_adapter = requests.adapters.HTTPAdapter(max_retries=3)
auth_request_session.mount("https://", retry_adapter)
# Do not pass `self` as the session here, as it can lead to
# infinite recursion.
auth_request = Request(auth_request_session)
# Request instance used by internal methods (for example,
# credentials.refresh).
self._auth_request = auth_request
def configure_mtls_channel(self, client_cert_callback=None):
"""Configure the client certificate and key for SSL connection.
The function does nothing unless `GOOGLE_API_USE_CLIENT_CERTIFICATE` is
explicitly set to `true`. In this case if client certificate and key are
successfully obtained (from the given client_cert_callback or from application
default SSL credentials), a :class:`_MutualTlsAdapter` instance will be mounted
to "https://" prefix.
Args:
client_cert_callback (Optional[Callable[[], (bytes, bytes)]]):
The optional callback returns the client certificate and private
key bytes both in PEM format.
If the callback is None, application default SSL credentials
will be used.
Raises:
google.auth.exceptions.MutualTLSChannelError: If mutual TLS channel
creation failed for any reason.
"""
use_client_cert = os.getenv(
environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"
)
if use_client_cert != "true":
self._is_mtls = False
return
try:
import OpenSSL
except ImportError as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
try:
(
self._is_mtls,
cert,
key,
) = google.auth.transport._mtls_helper.get_client_cert_and_key(
client_cert_callback
)
if self._is_mtls:
mtls_adapter = _MutualTlsAdapter(cert, key)
self.mount("https://", mtls_adapter)
except (
exceptions.ClientCertError,
ImportError,
OpenSSL.crypto.Error,
) as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
def request(
self,
method,
url,
data=None,
headers=None,
max_allowed_time=None,
timeout=_DEFAULT_TIMEOUT,
**kwargs
):
"""Implementation of Requests' request.
Args:
timeout (Optional[Union[float, Tuple[float, float]]]):
The amount of time in seconds to wait for the server response
with each individual request.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
max_allowed_time (Optional[float]):
If the method runs longer than this, a ``Timeout`` exception is
automatically raised. Unlike the ``timeout` parameter, this
value applies to the total method execution time, even if
multiple requests are made under the hood.
Mind that it is not guaranteed that the timeout error is raised
at ``max_allowed_time`. It might take longer, for example, if
an underlying request takes a lot of time, but the request
itself does not timeout, e.g. if a large file is being
transmitted. The timout error will be raised after such
request completes.
"""
# pylint: disable=arguments-differ
# Requests has a ton of arguments to request, but only two
# (method, url) are required. We pass through all of the other
# arguments to super, so no need to exhaustively list them here.
# Use a kwarg for this instead of an attribute to maintain
# thread-safety.
_credential_refresh_attempt = kwargs.pop("_credential_refresh_attempt", 0)
# Make a copy of the headers. They will be modified by the credentials
# and we want to pass the original headers if we recurse.
request_headers = headers.copy() if headers is not None else {}
# Do not apply the timeout unconditionally in order to not override the
# _auth_request's default timeout.
auth_request = (
self._auth_request
if timeout is None
else functools.partial(self._auth_request, timeout=timeout)
)
remaining_time = max_allowed_time
with TimeoutGuard(remaining_time) as guard:
self.credentials.before_request(auth_request, method, url, request_headers)
remaining_time = guard.remaining_timeout
with TimeoutGuard(remaining_time) as guard:
response = super(AuthorizedSession, self).request(
method,
url,
data=data,
headers=request_headers,
timeout=timeout,
**kwargs
)
remaining_time = guard.remaining_timeout
# If the response indicated that the credentials needed to be
# refreshed, then refresh the credentials and re-attempt the
# request.
# A stored token may expire between the time it is retrieved and
# the time the request is made, so we may need to try twice.
if (
response.status_code in self._refresh_status_codes
and _credential_refresh_attempt < self._max_refresh_attempts
):
_LOGGER.info(
"Refreshing credentials due to a %s response. Attempt %s/%s.",
response.status_code,
_credential_refresh_attempt + 1,
self._max_refresh_attempts,
)
# Do not apply the timeout unconditionally in order to not override the
# _auth_request's default timeout.
auth_request = (
self._auth_request
if timeout is None
else functools.partial(self._auth_request, timeout=timeout)
)
with TimeoutGuard(remaining_time) as guard:
self.credentials.refresh(auth_request)
remaining_time = guard.remaining_timeout
# Recurse. Pass in the original headers, not our modified set, but
# do pass the adjusted max allowed time (i.e. the remaining total time).
return self.request(
method,
url,
data=data,
headers=headers,
max_allowed_time=remaining_time,
timeout=timeout,
_credential_refresh_attempt=_credential_refresh_attempt + 1,
**kwargs
)
return response
@property
def is_mtls(self):
"""Indicates if the created SSL channel is mutual TLS."""
return self._is_mtls

View file

@ -0,0 +1,426 @@
# Copyright 2016 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transport adapter for urllib3."""
from __future__ import absolute_import
import logging
import os
import warnings
# Certifi is Mozilla's certificate bundle. Urllib3 needs a certificate bundle
# to verify HTTPS requests, and certifi is the recommended and most reliable
# way to get a root certificate bundle. See
# http://urllib3.readthedocs.io/en/latest/user-guide.html\
# #certificate-verification
# For more details.
try:
import certifi
except ImportError: # pragma: NO COVER
certifi = None
try:
import urllib3
except ImportError as caught_exc: # pragma: NO COVER
import six
six.raise_from(
ImportError(
"The urllib3 library is not installed, please install the "
"urllib3 package to use the urllib3 transport."
),
caught_exc,
)
import six
import urllib3.exceptions # pylint: disable=ungrouped-imports
from google.auth import environment_vars
from google.auth import exceptions
from google.auth import transport
_LOGGER = logging.getLogger(__name__)
class _Response(transport.Response):
"""urllib3 transport response adapter.
Args:
response (urllib3.response.HTTPResponse): The raw urllib3 response.
"""
def __init__(self, response):
self._response = response
@property
def status(self):
return self._response.status
@property
def headers(self):
return self._response.headers
@property
def data(self):
return self._response.data
class Request(transport.Request):
"""urllib3 request adapter.
This class is used internally for making requests using various transports
in a consistent way. If you use :class:`AuthorizedHttp` you do not need
to construct or use this class directly.
This class can be useful if you want to manually refresh a
:class:`~google.auth.credentials.Credentials` instance::
import google.auth.transport.urllib3
import urllib3
http = urllib3.PoolManager()
request = google.auth.transport.urllib3.Request(http)
credentials.refresh(request)
Args:
http (urllib3.request.RequestMethods): An instance of any urllib3
class that implements :class:`~urllib3.request.RequestMethods`,
usually :class:`urllib3.PoolManager`.
.. automethod:: __call__
"""
def __init__(self, http):
self.http = http
def __call__(
self, url, method="GET", body=None, headers=None, timeout=None, **kwargs
):
"""Make an HTTP request using urllib3.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
urllib3 default timeout will be used.
kwargs: Additional arguments passed throught to the underlying
urllib3 :meth:`urlopen` method.
Returns:
google.auth.transport.Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
# urllib3 uses a sentinel default value for timeout, so only set it if
# specified.
if timeout is not None:
kwargs["timeout"] = timeout
try:
_LOGGER.debug("Making request: %s %s", method, url)
response = self.http.request(
method, url, body=body, headers=headers, **kwargs
)
return _Response(response)
except urllib3.exceptions.HTTPError as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
def _make_default_http():
if certifi is not None:
return urllib3.PoolManager(cert_reqs="CERT_REQUIRED", ca_certs=certifi.where())
else:
return urllib3.PoolManager()
def _make_mutual_tls_http(cert, key):
"""Create a mutual TLS HTTP connection with the given client cert and key.
See https://github.com/urllib3/urllib3/issues/474#issuecomment-253168415
Args:
cert (bytes): client certificate in PEM format
key (bytes): client private key in PEM format
Returns:
urllib3.PoolManager: Mutual TLS HTTP connection.
Raises:
ImportError: If certifi or pyOpenSSL is not installed.
OpenSSL.crypto.Error: If the cert or key is invalid.
"""
import certifi
from OpenSSL import crypto
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()
ctx = urllib3.util.ssl_.create_urllib3_context()
ctx.load_verify_locations(cafile=certifi.where())
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert)
ctx._ctx.use_certificate(x509)
ctx._ctx.use_privatekey(pkey)
http = urllib3.PoolManager(ssl_context=ctx)
return http
class AuthorizedHttp(urllib3.request.RequestMethods):
"""A urllib3 HTTP class with credentials.
This class is used to perform requests to API endpoints that require
authorization::
from google.auth.transport.urllib3 import AuthorizedHttp
authed_http = AuthorizedHttp(credentials)
response = authed_http.request(
'GET', 'https://www.googleapis.com/storage/v1/b')
This class implements :class:`urllib3.request.RequestMethods` and can be
used just like any other :class:`urllib3.PoolManager`.
The underlying :meth:`urlopen` implementation handles adding the
credentials' headers to the request and refreshing credentials as needed.
This class also supports mutual TLS via :meth:`configure_mtls_channel`
method. In order to use this method, the `GOOGLE_API_USE_CLIENT_CERTIFICATE`
environment variable must be explicitly set to `true`, otherwise it does
nothing. Assume the environment is set to `true`, the method behaves in the
following manner:
If client_cert_callback is provided, client certificate and private
key are loaded using the callback; if client_cert_callback is None,
application default SSL credentials will be used. Exceptions are raised if
there are problems with the certificate, private key, or the loading process,
so it should be called within a try/except block.
First we set the environment variable to `true`, then create an :class:`AuthorizedHttp`
instance and specify the endpoints::
regular_endpoint = 'https://pubsub.googleapis.com/v1/projects/{my_project_id}/topics'
mtls_endpoint = 'https://pubsub.mtls.googleapis.com/v1/projects/{my_project_id}/topics'
authed_http = AuthorizedHttp(credentials)
Now we can pass a callback to :meth:`configure_mtls_channel`::
def my_cert_callback():
# some code to load client cert bytes and private key bytes, both in
# PEM format.
some_code_to_load_client_cert_and_key()
if loaded:
return cert, key
raise MyClientCertFailureException()
# Always call configure_mtls_channel within a try/except block.
try:
is_mtls = authed_http.configure_mtls_channel(my_cert_callback)
except:
# handle exceptions.
if is_mtls:
response = authed_http.request('GET', mtls_endpoint)
else:
response = authed_http.request('GET', regular_endpoint)
You can alternatively use application default SSL credentials like this::
try:
is_mtls = authed_http.configure_mtls_channel()
except:
# handle exceptions.
Args:
credentials (google.auth.credentials.Credentials): The credentials to
add to the request.
http (urllib3.PoolManager): The underlying HTTP object to
use to make requests. If not specified, a
:class:`urllib3.PoolManager` instance will be constructed with
sane defaults.
refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
that credentials should be refreshed and the request should be
retried.
max_refresh_attempts (int): The maximum number of times to attempt to
refresh the credentials and retry the request.
"""
def __init__(
self,
credentials,
http=None,
refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
):
if http is None:
self.http = _make_default_http()
self._has_user_provided_http = False
else:
self.http = http
self._has_user_provided_http = True
self.credentials = credentials
self._refresh_status_codes = refresh_status_codes
self._max_refresh_attempts = max_refresh_attempts
# Request instance used by internal methods (for example,
# credentials.refresh).
self._request = Request(self.http)
super(AuthorizedHttp, self).__init__()
def configure_mtls_channel(self, client_cert_callback=None):
"""Configures mutual TLS channel using the given client_cert_callback or
application default SSL credentials. The behavior is controlled by
`GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable.
(1) If the environment variable value is `true`, the function returns True
if the channel is mutual TLS and False otherwise. The `http` provided
in the constructor will be overwritten.
(2) If the environment variable is not set or `false`, the function does
nothing and it always return False.
Args:
client_cert_callback (Optional[Callable[[], (bytes, bytes)]]):
The optional callback returns the client certificate and private
key bytes both in PEM format.
If the callback is None, application default SSL credentials
will be used.
Returns:
True if the channel is mutual TLS and False otherwise.
Raises:
google.auth.exceptions.MutualTLSChannelError: If mutual TLS channel
creation failed for any reason.
"""
use_client_cert = os.getenv(
environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"
)
if use_client_cert != "true":
return False
try:
import OpenSSL
except ImportError as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
try:
found_cert_key, cert, key = transport._mtls_helper.get_client_cert_and_key(
client_cert_callback
)
if found_cert_key:
self.http = _make_mutual_tls_http(cert, key)
else:
self.http = _make_default_http()
except (
exceptions.ClientCertError,
ImportError,
OpenSSL.crypto.Error,
) as caught_exc:
new_exc = exceptions.MutualTLSChannelError(caught_exc)
six.raise_from(new_exc, caught_exc)
if self._has_user_provided_http:
self._has_user_provided_http = False
warnings.warn(
"`http` provided in the constructor is overwritten", UserWarning
)
return found_cert_key
def urlopen(self, method, url, body=None, headers=None, **kwargs):
"""Implementation of urllib3's urlopen."""
# pylint: disable=arguments-differ
# We use kwargs to collect additional args that we don't need to
# introspect here. However, we do explicitly collect the two
# positional arguments.
# Use a kwarg for this instead of an attribute to maintain
# thread-safety.
_credential_refresh_attempt = kwargs.pop("_credential_refresh_attempt", 0)
if headers is None:
headers = self.headers
# Make a copy of the headers. They will be modified by the credentials
# and we want to pass the original headers if we recurse.
request_headers = headers.copy()
self.credentials.before_request(self._request, method, url, request_headers)
response = self.http.urlopen(
method, url, body=body, headers=request_headers, **kwargs
)
# If the response indicated that the credentials needed to be
# refreshed, then refresh the credentials and re-attempt the
# request.
# A stored token may expire between the time it is retrieved and
# the time the request is made, so we may need to try twice.
# The reason urllib3's retries aren't used is because they
# don't allow you to modify the request headers. :/
if (
response.status in self._refresh_status_codes
and _credential_refresh_attempt < self._max_refresh_attempts
):
_LOGGER.info(
"Refreshing credentials due to a %s response. Attempt %s/%s.",
response.status,
_credential_refresh_attempt + 1,
self._max_refresh_attempts,
)
self.credentials.refresh(self._request)
# Recurse. Pass in the original headers, not our modified set.
return self.urlopen(
method,
url,
body=body,
headers=headers,
_credential_refresh_attempt=_credential_refresh_attempt + 1,
**kwargs
)
return response
# Proxy methods for compliance with the urllib3.PoolManager interface
def __enter__(self):
"""Proxy to ``self.http``."""
return self.http.__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
"""Proxy to ``self.http``."""
return self.http.__exit__(exc_type, exc_val, exc_tb)
@property
def headers(self):
"""Proxy to ``self.http``."""
return self.http.headers
@headers.setter
def headers(self, value):
"""Proxy to ``self.http``."""
self.http.headers = value