553 lines
17 KiB
Python
553 lines
17 KiB
Python
# Copyright 2014 Google Inc. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Thread-local resource stack.
|
|
|
|
This module is not part of the public API surface of `gcloud`.
|
|
"""
|
|
|
|
import calendar
|
|
import datetime
|
|
import json
|
|
import os
|
|
import re
|
|
import socket
|
|
import sys
|
|
from threading import local as Local
|
|
|
|
from google.protobuf import timestamp_pb2
|
|
import six
|
|
from six.moves.http_client import HTTPConnection
|
|
from six.moves import configparser
|
|
|
|
from gcloud.environment_vars import PROJECT
|
|
from gcloud.environment_vars import CREDENTIALS
|
|
|
|
try:
|
|
from google.appengine.api import app_identity
|
|
except ImportError:
|
|
app_identity = None
|
|
|
|
|
|
_NOW = datetime.datetime.utcnow # To be replaced by tests.
|
|
_RFC3339_MICROS = '%Y-%m-%dT%H:%M:%S.%fZ'
|
|
_RFC3339_NO_FRACTION = '%Y-%m-%dT%H:%M:%S'
|
|
# datetime.strptime cannot handle nanosecond precision: parse w/ regex
|
|
_RFC3339_NANOS = re.compile(r"""
|
|
(?P<no_fraction>
|
|
\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} # YYYY-MM-DDTHH:MM:SS
|
|
)
|
|
\. # decimal point
|
|
(?P<nanos>\d{1,9}) # nanoseconds, maybe truncated
|
|
Z # Zulu
|
|
""", re.VERBOSE)
|
|
DEFAULT_CONFIGURATION_PATH = '~/.config/gcloud/configurations/config_default'
|
|
|
|
|
|
class _LocalStack(Local):
|
|
"""Manage a thread-local LIFO stack of resources.
|
|
|
|
Intended for use in :class:`gcloud.datastore.batch.Batch.__enter__`,
|
|
:class:`gcloud.storage.batch.Batch.__enter__`, etc.
|
|
"""
|
|
def __init__(self):
|
|
super(_LocalStack, self).__init__()
|
|
self._stack = []
|
|
|
|
def __iter__(self):
|
|
"""Iterate the stack in LIFO order.
|
|
"""
|
|
return iter(reversed(self._stack))
|
|
|
|
def push(self, resource):
|
|
"""Push a resource onto our stack.
|
|
"""
|
|
self._stack.append(resource)
|
|
|
|
def pop(self):
|
|
"""Pop a resource from our stack.
|
|
|
|
:raises: IndexError if the stack is empty.
|
|
:returns: the top-most resource, after removing it.
|
|
"""
|
|
return self._stack.pop()
|
|
|
|
@property
|
|
def top(self):
|
|
"""Get the top-most resource
|
|
|
|
:returns: the top-most item, or None if the stack is empty.
|
|
"""
|
|
if len(self._stack) > 0:
|
|
return self._stack[-1]
|
|
|
|
|
|
class _UTC(datetime.tzinfo):
|
|
"""Basic UTC implementation.
|
|
|
|
Implementing a small surface area to avoid depending on ``pytz``.
|
|
"""
|
|
|
|
_dst = datetime.timedelta(0)
|
|
_tzname = 'UTC'
|
|
_utcoffset = _dst
|
|
|
|
def dst(self, dt): # pylint: disable=unused-argument
|
|
"""Daylight savings time offset."""
|
|
return self._dst
|
|
|
|
def fromutc(self, dt):
|
|
"""Convert a timestamp from (naive) UTC to this timezone."""
|
|
if dt.tzinfo is None:
|
|
return dt.replace(tzinfo=self)
|
|
return super(_UTC, self).fromutc(dt)
|
|
|
|
def tzname(self, dt): # pylint: disable=unused-argument
|
|
"""Get the name of this timezone."""
|
|
return self._tzname
|
|
|
|
def utcoffset(self, dt): # pylint: disable=unused-argument
|
|
"""UTC offset of this timezone."""
|
|
return self._utcoffset
|
|
|
|
def __repr__(self):
|
|
return '<%s>' % (self._tzname,)
|
|
|
|
def __str__(self):
|
|
return self._tzname
|
|
|
|
|
|
def _ensure_tuple_or_list(arg_name, tuple_or_list):
|
|
"""Ensures an input is a tuple or list.
|
|
|
|
This effectively reduces the iterable types allowed to a very short
|
|
whitelist: list and tuple.
|
|
|
|
:type arg_name: str
|
|
:param arg_name: Name of argument to use in error message.
|
|
|
|
:type tuple_or_list: sequence of str
|
|
:param tuple_or_list: Sequence to be verified.
|
|
|
|
:rtype: list of str
|
|
:returns: The ``tuple_or_list`` passed in cast to a ``list``.
|
|
:raises: class:`TypeError` if the ``tuple_or_list`` is not a tuple or
|
|
list.
|
|
"""
|
|
if not isinstance(tuple_or_list, (tuple, list)):
|
|
raise TypeError('Expected %s to be a tuple or list. '
|
|
'Received %r' % (arg_name, tuple_or_list))
|
|
return list(tuple_or_list)
|
|
|
|
|
|
def _app_engine_id():
|
|
"""Gets the App Engine application ID if it can be inferred.
|
|
|
|
:rtype: str or ``NoneType``
|
|
:returns: App Engine application ID if running in App Engine,
|
|
else ``None``.
|
|
"""
|
|
if app_identity is None:
|
|
return None
|
|
|
|
return app_identity.get_application_id()
|
|
|
|
|
|
def _file_project_id():
|
|
"""Gets the project id from the credentials file if one is available.
|
|
|
|
:rtype: str or ``NoneType``
|
|
:returns: Project-ID from JSON credentials file if value exists,
|
|
else ``None``.
|
|
"""
|
|
credentials_file_path = os.getenv(CREDENTIALS)
|
|
if credentials_file_path:
|
|
with open(credentials_file_path, 'rb') as credentials_file:
|
|
credentials_json = credentials_file.read()
|
|
credentials = json.loads(credentials_json.decode('utf-8'))
|
|
return credentials.get('project_id')
|
|
|
|
|
|
def _default_service_project_id():
|
|
"""Retrieves the project ID from the gcloud command line tool.
|
|
|
|
Files that cannot be opened with configparser are silently ignored; this is
|
|
designed so that you can specify a list of potential configuration file
|
|
locations.
|
|
|
|
:rtype: str or ``NoneType``
|
|
:returns: Project-ID from default configuration file else ``None``
|
|
"""
|
|
search_paths = []
|
|
# Workaround for GAE not supporting pwd which is used by expanduser.
|
|
try:
|
|
search_paths.append(os.path.expanduser(DEFAULT_CONFIGURATION_PATH))
|
|
except ImportError:
|
|
pass
|
|
win32_config_path = os.path.join(os.getenv('APPDATA', ''),
|
|
'gcloud', 'configurations',
|
|
'config_default')
|
|
search_paths.append(win32_config_path)
|
|
config = configparser.RawConfigParser()
|
|
config.read(search_paths)
|
|
|
|
if config.has_section('core'):
|
|
return config.get('core', 'project')
|
|
|
|
|
|
def _compute_engine_id():
|
|
"""Gets the Compute Engine project ID if it can be inferred.
|
|
|
|
Uses 169.254.169.254 for the metadata server to avoid request
|
|
latency from DNS lookup.
|
|
|
|
See https://cloud.google.com/compute/docs/metadata#metadataserver
|
|
for information about this IP address. (This IP is also used for
|
|
Amazon EC2 instances, so the metadata flavor is crucial.)
|
|
|
|
See https://github.com/google/oauth2client/issues/93 for context about
|
|
DNS latency.
|
|
|
|
:rtype: str or ``NoneType``
|
|
:returns: Compute Engine project ID if the metadata service is available,
|
|
else ``None``.
|
|
"""
|
|
host = '169.254.169.254'
|
|
uri_path = '/computeMetadata/v1/project/project-id'
|
|
headers = {'Metadata-Flavor': 'Google'}
|
|
connection = HTTPConnection(host, timeout=0.1)
|
|
|
|
try:
|
|
connection.request('GET', uri_path, headers=headers)
|
|
response = connection.getresponse()
|
|
if response.status == 200:
|
|
return response.read()
|
|
except socket.error: # socket.timeout or socket.error(64, 'Host is down')
|
|
pass
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def _get_production_project():
|
|
"""Gets the production project if it can be inferred."""
|
|
return os.getenv(PROJECT)
|
|
|
|
|
|
def _determine_default_project(project=None):
|
|
"""Determine default project ID explicitly or implicitly as fall-back.
|
|
|
|
In implicit case, supports three environments. In order of precedence, the
|
|
implicit environments are:
|
|
|
|
* GCLOUD_PROJECT environment variable
|
|
* GOOGLE_APPLICATION_CREDENTIALS JSON file
|
|
* Get default service project from
|
|
``$ gcloud beta auth application-default login``
|
|
* Google App Engine application ID
|
|
* Google Compute Engine project ID (from metadata server)
|
|
|
|
:type project: str
|
|
:param project: Optional. The project name to use as default.
|
|
|
|
:rtype: str or ``NoneType``
|
|
:returns: Default project if it can be determined.
|
|
"""
|
|
if project is None:
|
|
project = _get_production_project()
|
|
|
|
if project is None:
|
|
project = _file_project_id()
|
|
|
|
if project is None:
|
|
project = _default_service_project_id()
|
|
|
|
if project is None:
|
|
project = _app_engine_id()
|
|
|
|
if project is None:
|
|
project = _compute_engine_id()
|
|
|
|
return project
|
|
|
|
|
|
def _millis(when):
|
|
"""Convert a zone-aware datetime to integer milliseconds.
|
|
|
|
:type when: :class:`datetime.datetime`
|
|
:param when: the datetime to convert
|
|
|
|
:rtype: int
|
|
:returns: milliseconds since epoch for ``when``
|
|
"""
|
|
micros = _microseconds_from_datetime(when)
|
|
return micros // 1000
|
|
|
|
|
|
def _datetime_from_microseconds(value):
|
|
"""Convert timestamp to datetime, assuming UTC.
|
|
|
|
:type value: float
|
|
:param value: The timestamp to convert
|
|
|
|
:rtype: :class:`datetime.datetime`
|
|
:returns: The datetime object created from the value.
|
|
"""
|
|
return _EPOCH + datetime.timedelta(microseconds=value)
|
|
|
|
|
|
def _microseconds_from_datetime(value):
|
|
"""Convert non-none datetime to microseconds.
|
|
|
|
:type value: :class:`datetime.datetime`
|
|
:param value: The timestamp to convert.
|
|
|
|
:rtype: int
|
|
:returns: The timestamp, in microseconds.
|
|
"""
|
|
if not value.tzinfo:
|
|
value = value.replace(tzinfo=UTC)
|
|
# Regardless of what timezone is on the value, convert it to UTC.
|
|
value = value.astimezone(UTC)
|
|
# Convert the datetime to a microsecond timestamp.
|
|
return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond
|
|
|
|
|
|
def _millis_from_datetime(value):
|
|
"""Convert non-none datetime to timestamp, assuming UTC.
|
|
|
|
:type value: :class:`datetime.datetime`, or None
|
|
:param value: the timestamp
|
|
|
|
:rtype: int, or ``NoneType``
|
|
:returns: the timestamp, in milliseconds, or None
|
|
"""
|
|
if value is not None:
|
|
return _millis(value)
|
|
|
|
|
|
def _total_seconds_backport(offset):
|
|
"""Backport of timedelta.total_seconds() from python 2.7+.
|
|
|
|
:type offset: :class:`datetime.timedelta`
|
|
:param offset: A timedelta object.
|
|
|
|
:rtype: int
|
|
:returns: The total seconds (including microseconds) in the
|
|
duration.
|
|
"""
|
|
seconds = offset.days * 24 * 60 * 60 + offset.seconds
|
|
return seconds + offset.microseconds * 1e-6
|
|
|
|
|
|
def _total_seconds(offset):
|
|
"""Version independent total seconds for a time delta.
|
|
|
|
:type offset: :class:`datetime.timedelta`
|
|
:param offset: A timedelta object.
|
|
|
|
:rtype: int
|
|
:returns: The total seconds (including microseconds) in the
|
|
duration.
|
|
"""
|
|
if sys.version_info[:2] < (2, 7): # pragma: NO COVER Python 2.6
|
|
return _total_seconds_backport(offset)
|
|
else:
|
|
return offset.total_seconds()
|
|
|
|
|
|
def _rfc3339_to_datetime(dt_str):
|
|
"""Convert a microsecond-precision timetamp to a native datetime.
|
|
|
|
:type dt_str: str
|
|
:param dt_str: The string to convert.
|
|
|
|
:rtype: :class:`datetime.datetime`
|
|
:returns: The datetime object created from the string.
|
|
"""
|
|
return datetime.datetime.strptime(
|
|
dt_str, _RFC3339_MICROS).replace(tzinfo=UTC)
|
|
|
|
|
|
def _rfc3339_nanos_to_datetime(dt_str):
|
|
"""Convert a nanosecond-precision timestamp to a native datetime.
|
|
|
|
.. note::
|
|
|
|
Python datetimes do not support nanosecond precision; this function
|
|
therefore truncates such values to microseconds.
|
|
|
|
:type dt_str: str
|
|
:param dt_str: The string to convert.
|
|
|
|
:rtype: :class:`datetime.datetime`
|
|
:returns: The datetime object created from the string.
|
|
"""
|
|
with_nanos = _RFC3339_NANOS.match(dt_str)
|
|
if with_nanos is None:
|
|
raise ValueError(
|
|
'Timestamp: %r, does not match pattern: %r' % (
|
|
dt_str, _RFC3339_NANOS.pattern))
|
|
bare_seconds = datetime.datetime.strptime(
|
|
with_nanos.group('no_fraction'), _RFC3339_NO_FRACTION)
|
|
fraction = with_nanos.group('nanos')
|
|
scale = 9 - len(fraction)
|
|
nanos = int(fraction) * (10 ** scale)
|
|
micros = nanos // 1000
|
|
return bare_seconds.replace(microsecond=micros, tzinfo=UTC)
|
|
|
|
|
|
def _datetime_to_rfc3339(value):
|
|
"""Convert a native timestamp to a string.
|
|
|
|
:type value: :class:`datetime.datetime`
|
|
:param value: The datetime object to be converted to a string.
|
|
|
|
:rtype: str
|
|
:returns: The string representing the datetime stamp.
|
|
"""
|
|
return value.strftime(_RFC3339_MICROS)
|
|
|
|
|
|
def _to_bytes(value, encoding='ascii'):
|
|
"""Converts a string value to bytes, if necessary.
|
|
|
|
Unfortunately, ``six.b`` is insufficient for this task since in
|
|
Python2 it does not modify ``unicode`` objects.
|
|
|
|
:type value: str / bytes or unicode
|
|
:param value: The string/bytes value to be converted.
|
|
|
|
:type encoding: str
|
|
:param encoding: The encoding to use to convert unicode to bytes. Defaults
|
|
to "ascii", which will not allow any characters from
|
|
ordinals larger than 127. Other useful values are
|
|
"latin-1", which which will only allows byte ordinals
|
|
(up to 255) and "utf-8", which will encode any unicode
|
|
that needs to be.
|
|
|
|
:rtype: str / bytes
|
|
:returns: The original value converted to bytes (if unicode) or as passed
|
|
in if it started out as bytes.
|
|
:raises: :class:`TypeError <exceptions.TypeError>` if the value
|
|
could not be converted to bytes.
|
|
"""
|
|
result = (value.encode(encoding)
|
|
if isinstance(value, six.text_type) else value)
|
|
if isinstance(result, six.binary_type):
|
|
return result
|
|
else:
|
|
raise TypeError('%r could not be converted to bytes' % (value,))
|
|
|
|
|
|
def _bytes_to_unicode(value):
|
|
"""Converts bytes to a unicode value, if necessary.
|
|
|
|
:type value: bytes
|
|
:param value: bytes value to attempt string conversion on.
|
|
|
|
:rtype: str
|
|
:returns: The original value converted to unicode (if bytes) or as passed
|
|
in if it started out as unicode.
|
|
|
|
:raises: :class:`ValueError` if the value could not be converted to
|
|
unicode.
|
|
"""
|
|
result = (value.decode('utf-8')
|
|
if isinstance(value, six.binary_type) else value)
|
|
if isinstance(result, six.text_type):
|
|
return result
|
|
else:
|
|
raise ValueError('%r could not be converted to unicode' % (value,))
|
|
|
|
|
|
def _pb_timestamp_to_datetime(timestamp):
|
|
"""Convert a Timestamp protobuf to a datetime object.
|
|
|
|
:type timestamp: :class:`google.protobuf.timestamp_pb2.Timestamp`
|
|
:param timestamp: A Google returned timestamp protobuf.
|
|
|
|
:rtype: :class:`datetime.datetime`
|
|
:returns: A UTC datetime object converted from a protobuf timestamp.
|
|
"""
|
|
return (
|
|
_EPOCH +
|
|
datetime.timedelta(
|
|
seconds=timestamp.seconds,
|
|
microseconds=(timestamp.nanos / 1000.0),
|
|
)
|
|
)
|
|
|
|
|
|
def _datetime_to_pb_timestamp(when):
|
|
"""Convert a datetime object to a Timestamp protobuf.
|
|
|
|
:type when: :class:`datetime.datetime`
|
|
:param when: the datetime to convert
|
|
|
|
:rtype: :class:`google.protobuf.timestamp_pb2.Timestamp`
|
|
:returns: A timestamp protobuf corresponding to the object.
|
|
"""
|
|
ms_value = _microseconds_from_datetime(when)
|
|
seconds, micros = divmod(ms_value, 10**6)
|
|
nanos = micros * 10**3
|
|
return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
|
|
|
|
|
|
def _name_from_project_path(path, project, template):
|
|
"""Validate a URI path and get the leaf object's name.
|
|
|
|
:type path: str
|
|
:param path: URI path containing the name.
|
|
|
|
:type project: str or NoneType
|
|
:param project: The project associated with the request. It is
|
|
included for validation purposes. If passed as None,
|
|
disables validation.
|
|
|
|
:type template: str
|
|
:param template: Template regex describing the expected form of the path.
|
|
The regex must have two named groups, 'project' and
|
|
'name'.
|
|
|
|
:rtype: str
|
|
:returns: Name parsed from ``path``.
|
|
:raises: :class:`ValueError` if the ``path`` is ill-formed or if
|
|
the project from the ``path`` does not agree with the
|
|
``project`` passed in.
|
|
"""
|
|
if isinstance(template, str):
|
|
template = re.compile(template)
|
|
|
|
match = template.match(path)
|
|
|
|
if not match:
|
|
raise ValueError('path "%s" did not match expected pattern "%s"' % (
|
|
path, template.pattern,))
|
|
|
|
if project is not None:
|
|
found_project = match.group('project')
|
|
if found_project != project:
|
|
raise ValueError(
|
|
'Project from client (%s) should agree with '
|
|
'project from resource(%s).' % (project, found_project))
|
|
|
|
return match.group('name')
|
|
|
|
|
|
try:
|
|
from pytz import UTC # pylint: disable=unused-import,wrong-import-order
|
|
except ImportError:
|
|
UTC = _UTC() # Singleton instance to be used throughout.
|
|
|
|
# Need to define _EPOCH at the end of module since it relies on UTC.
|
|
_EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=UTC)
|