455 lines
16 KiB
Python
455 lines
16 KiB
Python
|
# Copyright 2014 Google Inc. All rights reserved.
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
"""Convenience wrapper for invoking APIs/factories w/ a project."""
|
||
|
|
|
||
|
|
import os
|
||
|
|
|
||
|
|
from gcloud._helpers import _LocalStack
|
||
|
|
from gcloud._helpers import _determine_default_project as _base_default_project
|
||
|
|
from gcloud.client import _ClientProjectMixin
|
||
|
|
from gcloud.client import Client as _BaseClient
|
||
|
|
from gcloud.datastore import helpers
|
||
|
|
from gcloud.datastore.connection import Connection
|
||
|
|
from gcloud.datastore.batch import Batch
|
||
|
|
from gcloud.datastore.entity import Entity
|
||
|
|
from gcloud.datastore.key import Key
|
||
|
|
from gcloud.datastore.query import Query
|
||
|
|
from gcloud.datastore.transaction import Transaction
|
||
|
|
from gcloud.environment_vars import GCD_DATASET
|
||
|
|
|
||
|
|
|
||
|
|
_MAX_LOOPS = 128
|
||
|
|
"""Maximum number of iterations to wait for deferred keys."""
|
||
|
|
|
||
|
|
|
||
|
|
def _get_gcd_project():
|
||
|
|
"""Gets the GCD application ID if it can be inferred."""
|
||
|
|
return os.getenv(GCD_DATASET)
|
||
|
|
|
||
|
|
|
||
|
|
def _determine_default_project(project=None):
|
||
|
|
"""Determine default project explicitly or implicitly as fall-back.
|
||
|
|
|
||
|
|
In implicit case, supports four environments. In order of precedence, the
|
||
|
|
implicit environments are:
|
||
|
|
|
||
|
|
* DATASTORE_DATASET environment variable (for ``gcd`` / emulator testing)
|
||
|
|
* GCLOUD_PROJECT environment variable
|
||
|
|
* Google App Engine application ID
|
||
|
|
* Google Compute Engine project ID (from metadata server)
|
||
|
|
|
||
|
|
:type project: string
|
||
|
|
:param project: Optional. The project to use as default.
|
||
|
|
|
||
|
|
:rtype: string or ``NoneType``
|
||
|
|
:returns: Default project if it can be determined.
|
||
|
|
"""
|
||
|
|
if project is None:
|
||
|
|
project = _get_gcd_project()
|
||
|
|
|
||
|
|
if project is None:
|
||
|
|
project = _base_default_project(project=project)
|
||
|
|
|
||
|
|
return project
|
||
|
|
|
||
|
|
|
||
|
|
def _extended_lookup(connection, project, key_pbs,
|
||
|
|
missing=None, deferred=None,
|
||
|
|
eventual=False, transaction_id=None):
|
||
|
|
"""Repeat lookup until all keys found (unless stop requested).
|
||
|
|
|
||
|
|
Helper function for :meth:`Client.get_multi`.
|
||
|
|
|
||
|
|
:type connection: :class:`gcloud.datastore.connection.Connection`
|
||
|
|
:param connection: The connection used to connect to datastore.
|
||
|
|
|
||
|
|
:type project: string
|
||
|
|
:param project: The project to make the request for.
|
||
|
|
|
||
|
|
:type key_pbs: list of :class:`gcloud.datastore._generated.entity_pb2.Key`
|
||
|
|
:param key_pbs: The keys to retrieve from the datastore.
|
||
|
|
|
||
|
|
:type missing: list
|
||
|
|
:param missing: (Optional) If a list is passed, the key-only entity
|
||
|
|
protobufs returned by the backend as "missing" will be
|
||
|
|
copied into it.
|
||
|
|
|
||
|
|
:type deferred: list
|
||
|
|
:param deferred: (Optional) If a list is passed, the key protobufs returned
|
||
|
|
by the backend as "deferred" will be copied into it.
|
||
|
|
|
||
|
|
:type eventual: bool
|
||
|
|
:param eventual: If False (the default), request ``STRONG`` read
|
||
|
|
consistency. If True, request ``EVENTUAL`` read
|
||
|
|
consistency.
|
||
|
|
|
||
|
|
:type transaction_id: string
|
||
|
|
:param transaction_id: If passed, make the request in the scope of
|
||
|
|
the given transaction. Incompatible with
|
||
|
|
``eventual==True``.
|
||
|
|
|
||
|
|
:rtype: list of :class:`gcloud.datastore._generated.entity_pb2.Entity`
|
||
|
|
:returns: The requested entities.
|
||
|
|
:raises: :class:`ValueError` if missing / deferred are not null or
|
||
|
|
empty list.
|
||
|
|
"""
|
||
|
|
if missing is not None and missing != []:
|
||
|
|
raise ValueError('missing must be None or an empty list')
|
||
|
|
|
||
|
|
if deferred is not None and deferred != []:
|
||
|
|
raise ValueError('deferred must be None or an empty list')
|
||
|
|
|
||
|
|
results = []
|
||
|
|
|
||
|
|
loop_num = 0
|
||
|
|
while loop_num < _MAX_LOOPS: # loop against possible deferred.
|
||
|
|
loop_num += 1
|
||
|
|
|
||
|
|
results_found, missing_found, deferred_found = connection.lookup(
|
||
|
|
project=project,
|
||
|
|
key_pbs=key_pbs,
|
||
|
|
eventual=eventual,
|
||
|
|
transaction_id=transaction_id,
|
||
|
|
)
|
||
|
|
|
||
|
|
results.extend(results_found)
|
||
|
|
|
||
|
|
if missing is not None:
|
||
|
|
missing.extend(missing_found)
|
||
|
|
|
||
|
|
if deferred is not None:
|
||
|
|
deferred.extend(deferred_found)
|
||
|
|
break
|
||
|
|
|
||
|
|
if len(deferred_found) == 0:
|
||
|
|
break
|
||
|
|
|
||
|
|
# We have deferred keys, and the user didn't ask to know about
|
||
|
|
# them, so retry (but only with the deferred ones).
|
||
|
|
key_pbs = deferred_found
|
||
|
|
|
||
|
|
return results
|
||
|
|
|
||
|
|
|
||
|
|
class Client(_BaseClient, _ClientProjectMixin):
|
||
|
|
"""Convenience wrapper for invoking APIs/factories w/ a project.
|
||
|
|
|
||
|
|
:type project: string
|
||
|
|
:param project: (optional) The project to pass to proxied API methods.
|
||
|
|
|
||
|
|
:type namespace: string
|
||
|
|
:param namespace: (optional) namespace to pass to proxied API methods.
|
||
|
|
|
||
|
|
:type credentials: :class:`oauth2client.client.OAuth2Credentials` or
|
||
|
|
:class:`NoneType`
|
||
|
|
:param credentials: The OAuth2 Credentials to use for the connection
|
||
|
|
owned by this client. If not passed (and if no ``http``
|
||
|
|
object is passed), falls back to the default inferred
|
||
|
|
from the environment.
|
||
|
|
|
||
|
|
:type http: :class:`httplib2.Http` or class that defines ``request()``.
|
||
|
|
:param http: An optional HTTP object to make requests. If not passed, an
|
||
|
|
``http`` object is created that is bound to the
|
||
|
|
``credentials`` for the current object.
|
||
|
|
"""
|
||
|
|
_connection_class = Connection
|
||
|
|
|
||
|
|
def __init__(self, project=None, namespace=None,
|
||
|
|
credentials=None, http=None):
|
||
|
|
_ClientProjectMixin.__init__(self, project=project)
|
||
|
|
self.namespace = namespace
|
||
|
|
self._batch_stack = _LocalStack()
|
||
|
|
super(Client, self).__init__(credentials, http)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _determine_default(project):
|
||
|
|
"""Helper: override default project detection."""
|
||
|
|
return _determine_default_project(project)
|
||
|
|
|
||
|
|
def _push_batch(self, batch):
|
||
|
|
"""Push a batch/transaction onto our stack.
|
||
|
|
|
||
|
|
"Protected", intended for use by batch / transaction context mgrs.
|
||
|
|
|
||
|
|
:type batch: :class:`gcloud.datastore.batch.Batch`, or an object
|
||
|
|
implementing its API.
|
||
|
|
:param batch: newly-active batch/transaction.
|
||
|
|
"""
|
||
|
|
self._batch_stack.push(batch)
|
||
|
|
|
||
|
|
def _pop_batch(self):
|
||
|
|
"""Pop a batch/transaction from our stack.
|
||
|
|
|
||
|
|
"Protected", intended for use by batch / transaction context mgrs.
|
||
|
|
|
||
|
|
:raises: IndexError if the stack is empty.
|
||
|
|
:rtype: :class:`gcloud.datastore.batch.Batch`, or an object
|
||
|
|
implementing its API.
|
||
|
|
:returns: the top-most batch/transaction, after removing it.
|
||
|
|
"""
|
||
|
|
return self._batch_stack.pop()
|
||
|
|
|
||
|
|
@property
|
||
|
|
def current_batch(self):
|
||
|
|
"""Currently-active batch.
|
||
|
|
|
||
|
|
:rtype: :class:`gcloud.datastore.batch.Batch`, or an object
|
||
|
|
implementing its API, or ``NoneType`` (if no batch is active).
|
||
|
|
:returns: The batch/transaction at the top of the batch stack.
|
||
|
|
"""
|
||
|
|
return self._batch_stack.top
|
||
|
|
|
||
|
|
@property
|
||
|
|
def current_transaction(self):
|
||
|
|
"""Currently-active transaction.
|
||
|
|
|
||
|
|
:rtype: :class:`gcloud.datastore.transaction.Transaction`, or an object
|
||
|
|
implementing its API, or ``NoneType`` (if no transaction is
|
||
|
|
active).
|
||
|
|
:returns: The transaction at the top of the batch stack.
|
||
|
|
"""
|
||
|
|
transaction = self.current_batch
|
||
|
|
if isinstance(transaction, Transaction):
|
||
|
|
return transaction
|
||
|
|
|
||
|
|
def get(self, key, missing=None, deferred=None, transaction=None):
|
||
|
|
"""Retrieve an entity from a single key (if it exists).
|
||
|
|
|
||
|
|
.. note::
|
||
|
|
|
||
|
|
This is just a thin wrapper over :meth:`get_multi`.
|
||
|
|
The backend API does not make a distinction between a single key or
|
||
|
|
multiple keys in a lookup request.
|
||
|
|
|
||
|
|
:type key: :class:`gcloud.datastore.key.Key`
|
||
|
|
:param key: The key to be retrieved from the datastore.
|
||
|
|
|
||
|
|
:type missing: list
|
||
|
|
:param missing: (Optional) If a list is passed, the key-only entities
|
||
|
|
returned by the backend as "missing" will be copied
|
||
|
|
into it.
|
||
|
|
|
||
|
|
:type deferred: list
|
||
|
|
:param deferred: (Optional) If a list is passed, the keys returned
|
||
|
|
by the backend as "deferred" will be copied into it.
|
||
|
|
|
||
|
|
:type transaction: :class:`gcloud.datastore.transaction.Transaction`
|
||
|
|
:param transaction: (Optional) Transaction to use for read consistency.
|
||
|
|
If not passed, uses current transaction, if set.
|
||
|
|
|
||
|
|
:rtype: :class:`gcloud.datastore.entity.Entity` or ``NoneType``
|
||
|
|
:returns: The requested entity if it exists.
|
||
|
|
"""
|
||
|
|
entities = self.get_multi(keys=[key], missing=missing,
|
||
|
|
deferred=deferred, transaction=transaction)
|
||
|
|
if entities:
|
||
|
|
return entities[0]
|
||
|
|
|
||
|
|
def get_multi(self, keys, missing=None, deferred=None, transaction=None):
|
||
|
|
"""Retrieve entities, along with their attributes.
|
||
|
|
|
||
|
|
:type keys: list of :class:`gcloud.datastore.key.Key`
|
||
|
|
:param keys: The keys to be retrieved from the datastore.
|
||
|
|
|
||
|
|
:type missing: list
|
||
|
|
:param missing: (Optional) If a list is passed, the key-only entities
|
||
|
|
returned by the backend as "missing" will be copied
|
||
|
|
into it. If the list is not empty, an error will occur.
|
||
|
|
|
||
|
|
:type deferred: list
|
||
|
|
:param deferred: (Optional) If a list is passed, the keys returned
|
||
|
|
by the backend as "deferred" will be copied into it.
|
||
|
|
If the list is not empty, an error will occur.
|
||
|
|
|
||
|
|
:type transaction: :class:`gcloud.datastore.transaction.Transaction`
|
||
|
|
:param transaction: (Optional) Transaction to use for read consistency.
|
||
|
|
If not passed, uses current transaction, if set.
|
||
|
|
|
||
|
|
:rtype: list of :class:`gcloud.datastore.entity.Entity`
|
||
|
|
:returns: The requested entities.
|
||
|
|
:raises: :class:`ValueError` if one or more of ``keys`` has a project
|
||
|
|
which does not match our project.
|
||
|
|
"""
|
||
|
|
if not keys:
|
||
|
|
return []
|
||
|
|
|
||
|
|
ids = set(key.project for key in keys)
|
||
|
|
for current_id in ids:
|
||
|
|
if current_id != self.project:
|
||
|
|
raise ValueError('Keys do not match project')
|
||
|
|
|
||
|
|
if transaction is None:
|
||
|
|
transaction = self.current_transaction
|
||
|
|
|
||
|
|
entity_pbs = _extended_lookup(
|
||
|
|
connection=self.connection,
|
||
|
|
project=self.project,
|
||
|
|
key_pbs=[k.to_protobuf() for k in keys],
|
||
|
|
missing=missing,
|
||
|
|
deferred=deferred,
|
||
|
|
transaction_id=transaction and transaction.id,
|
||
|
|
)
|
||
|
|
|
||
|
|
if missing is not None:
|
||
|
|
missing[:] = [
|
||
|
|
helpers.entity_from_protobuf(missed_pb)
|
||
|
|
for missed_pb in missing]
|
||
|
|
|
||
|
|
if deferred is not None:
|
||
|
|
deferred[:] = [
|
||
|
|
helpers.key_from_protobuf(deferred_pb)
|
||
|
|
for deferred_pb in deferred]
|
||
|
|
|
||
|
|
return [helpers.entity_from_protobuf(entity_pb)
|
||
|
|
for entity_pb in entity_pbs]
|
||
|
|
|
||
|
|
def put(self, entity):
|
||
|
|
"""Save an entity in the Cloud Datastore.
|
||
|
|
|
||
|
|
.. note::
|
||
|
|
|
||
|
|
This is just a thin wrapper over :meth:`put_multi`.
|
||
|
|
The backend API does not make a distinction between a single
|
||
|
|
entity or multiple entities in a commit request.
|
||
|
|
|
||
|
|
:type entity: :class:`gcloud.datastore.entity.Entity`
|
||
|
|
:param entity: The entity to be saved to the datastore.
|
||
|
|
"""
|
||
|
|
self.put_multi(entities=[entity])
|
||
|
|
|
||
|
|
def put_multi(self, entities):
|
||
|
|
"""Save entities in the Cloud Datastore.
|
||
|
|
|
||
|
|
:type entities: list of :class:`gcloud.datastore.entity.Entity`
|
||
|
|
:param entities: The entities to be saved to the datastore.
|
||
|
|
|
||
|
|
:raises: :class:`ValueError` if ``entities`` is a single entity.
|
||
|
|
"""
|
||
|
|
if isinstance(entities, Entity):
|
||
|
|
raise ValueError("Pass a sequence of entities")
|
||
|
|
|
||
|
|
if not entities:
|
||
|
|
return
|
||
|
|
|
||
|
|
current = self.current_batch
|
||
|
|
in_batch = current is not None
|
||
|
|
|
||
|
|
if not in_batch:
|
||
|
|
current = self.batch()
|
||
|
|
|
||
|
|
for entity in entities:
|
||
|
|
current.put(entity)
|
||
|
|
|
||
|
|
if not in_batch:
|
||
|
|
current.commit()
|
||
|
|
|
||
|
|
def delete(self, key):
|
||
|
|
"""Delete the key in the Cloud Datastore.
|
||
|
|
|
||
|
|
.. note::
|
||
|
|
|
||
|
|
This is just a thin wrapper over :meth:`delete_multi`.
|
||
|
|
The backend API does not make a distinction between a single key or
|
||
|
|
multiple keys in a commit request.
|
||
|
|
|
||
|
|
:type key: :class:`gcloud.datastore.key.Key`
|
||
|
|
:param key: The key to be deleted from the datastore.
|
||
|
|
"""
|
||
|
|
return self.delete_multi(keys=[key])
|
||
|
|
|
||
|
|
def delete_multi(self, keys):
|
||
|
|
"""Delete keys from the Cloud Datastore.
|
||
|
|
|
||
|
|
:type keys: list of :class:`gcloud.datastore.key.Key`
|
||
|
|
:param keys: The keys to be deleted from the datastore.
|
||
|
|
"""
|
||
|
|
if not keys:
|
||
|
|
return
|
||
|
|
|
||
|
|
# We allow partial keys to attempt a delete, the backend will fail.
|
||
|
|
current = self.current_batch
|
||
|
|
in_batch = current is not None
|
||
|
|
|
||
|
|
if not in_batch:
|
||
|
|
current = self.batch()
|
||
|
|
|
||
|
|
for key in keys:
|
||
|
|
current.delete(key)
|
||
|
|
|
||
|
|
if not in_batch:
|
||
|
|
current.commit()
|
||
|
|
|
||
|
|
def allocate_ids(self, incomplete_key, num_ids):
|
||
|
|
"""Allocate a list of IDs from a partial key.
|
||
|
|
|
||
|
|
:type incomplete_key: :class:`gcloud.datastore.key.Key`
|
||
|
|
:param incomplete_key: Partial key to use as base for allocated IDs.
|
||
|
|
|
||
|
|
:type num_ids: int
|
||
|
|
:param num_ids: The number of IDs to allocate.
|
||
|
|
|
||
|
|
:rtype: list of :class:`gcloud.datastore.key.Key`
|
||
|
|
:returns: The (complete) keys allocated with ``incomplete_key`` as
|
||
|
|
root.
|
||
|
|
:raises: :class:`ValueError` if ``incomplete_key`` is not a
|
||
|
|
partial key.
|
||
|
|
"""
|
||
|
|
if not incomplete_key.is_partial:
|
||
|
|
raise ValueError(('Key is not partial.', incomplete_key))
|
||
|
|
|
||
|
|
incomplete_key_pb = incomplete_key.to_protobuf()
|
||
|
|
incomplete_key_pbs = [incomplete_key_pb] * num_ids
|
||
|
|
|
||
|
|
conn = self.connection
|
||
|
|
allocated_key_pbs = conn.allocate_ids(incomplete_key.project,
|
||
|
|
incomplete_key_pbs)
|
||
|
|
allocated_ids = [allocated_key_pb.path[-1].id
|
||
|
|
for allocated_key_pb in allocated_key_pbs]
|
||
|
|
return [incomplete_key.completed_key(allocated_id)
|
||
|
|
for allocated_id in allocated_ids]
|
||
|
|
|
||
|
|
def key(self, *path_args, **kwargs):
|
||
|
|
"""Proxy to :class:`gcloud.datastore.key.Key`.
|
||
|
|
|
||
|
|
Passes our ``project``.
|
||
|
|
"""
|
||
|
|
if 'project' in kwargs:
|
||
|
|
raise TypeError('Cannot pass project')
|
||
|
|
kwargs['project'] = self.project
|
||
|
|
if 'namespace' not in kwargs:
|
||
|
|
kwargs['namespace'] = self.namespace
|
||
|
|
return Key(*path_args, **kwargs)
|
||
|
|
|
||
|
|
def batch(self):
|
||
|
|
"""Proxy to :class:`gcloud.datastore.batch.Batch`."""
|
||
|
|
return Batch(self)
|
||
|
|
|
||
|
|
def transaction(self):
|
||
|
|
"""Proxy to :class:`gcloud.datastore.transaction.Transaction`."""
|
||
|
|
return Transaction(self)
|
||
|
|
|
||
|
|
def query(self, **kwargs):
|
||
|
|
"""Proxy to :class:`gcloud.datastore.query.Query`.
|
||
|
|
|
||
|
|
Passes our ``project``.
|
||
|
|
"""
|
||
|
|
if 'client' in kwargs:
|
||
|
|
raise TypeError('Cannot pass client')
|
||
|
|
if 'project' in kwargs:
|
||
|
|
raise TypeError('Cannot pass project')
|
||
|
|
kwargs['project'] = self.project
|
||
|
|
if 'namespace' not in kwargs:
|
||
|
|
kwargs['namespace'] = self.namespace
|
||
|
|
return Query(self, **kwargs)
|