# Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """User friendly container for Google Cloud Bigtable Instance.""" import re from google.longrunning import operations_pb2 from gcloud._helpers import _pb_timestamp_to_datetime from gcloud.bigtable._generated_v2 import ( instance_pb2 as data_v2_pb2) from gcloud.bigtable._generated_v2 import ( bigtable_instance_admin_pb2 as messages_v2_pb2) from gcloud.bigtable._generated_v2 import ( bigtable_table_admin_pb2 as table_messages_v2_pb2) from gcloud.bigtable.cluster import Cluster from gcloud.bigtable.cluster import DEFAULT_SERVE_NODES from gcloud.bigtable.table import Table _EXISTING_INSTANCE_LOCATION_ID = 'see-existing-cluster' _INSTANCE_NAME_RE = re.compile(r'^projects/(?P[^/]+)/' r'instances/(?P[a-z][-a-z0-9]*)$') _OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/' r'instances/([a-z][-a-z0-9]*)/' r'locations/(?P[a-z][-a-z0-9]*)/' r'operations/(?P\d+)$') _TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' _ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.v2.' _INSTANCE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateInstanceMetadata' _TYPE_URL_MAP = { _INSTANCE_CREATE_METADATA: messages_v2_pb2.CreateInstanceMetadata, } def _prepare_create_request(instance): """Creates a protobuf request for a CreateInstance request. :type instance: :class:`Instance` :param instance: The instance to be created. :rtype: :class:`.messages_v2_pb2.CreateInstanceRequest` :returns: The CreateInstance request object containing the instance info. """ parent_name = ('projects/' + instance._client.project) message = messages_v2_pb2.CreateInstanceRequest( parent=parent_name, instance_id=instance.instance_id, instance=data_v2_pb2.Instance( display_name=instance.display_name, ), ) cluster = message.clusters[instance.instance_id] cluster.name = instance.name + '/clusters/' + instance.instance_id cluster.location = ( parent_name + '/locations/' + instance._cluster_location_id) cluster.serve_nodes = instance._cluster_serve_nodes return message def _parse_pb_any_to_native(any_val, expected_type=None): """Convert a serialized "google.protobuf.Any" value to actual type. :type any_val: :class:`google.protobuf.any_pb2.Any` :param any_val: A serialized protobuf value container. :type expected_type: str :param expected_type: (Optional) The type URL we expect ``any_val`` to have. :rtype: object :returns: The de-serialized object. :raises: :class:`ValueError ` if the ``expected_type`` does not match the ``type_url`` on the input. """ if expected_type is not None and expected_type != any_val.type_url: raise ValueError('Expected type: %s, Received: %s' % ( expected_type, any_val.type_url)) container_class = _TYPE_URL_MAP[any_val.type_url] return container_class.FromString(any_val.value) def _process_operation(operation_pb): """Processes a create protobuf response. :type operation_pb: :class:`google.longrunning.operations_pb2.Operation` :param operation_pb: The long-running operation response from a Create/Update/Undelete instance request. :rtype: (int, str, datetime) :returns: (operation_id, location_id, operation_begin). :raises: :class:`ValueError ` if the operation name doesn't match the :data:`_OPERATION_NAME_RE` regex. """ match = _OPERATION_NAME_RE.match(operation_pb.name) if match is None: raise ValueError('Operation name was not in the expected ' 'format after instance creation.', operation_pb.name) location_id = match.group('location_id') operation_id = int(match.group('operation_id')) request_metadata = _parse_pb_any_to_native(operation_pb.metadata) operation_begin = _pb_timestamp_to_datetime( request_metadata.request_time) return operation_id, location_id, operation_begin class Operation(object): """Representation of a Google API Long-Running Operation. In particular, these will be the result of operations on instances using the Cloud Bigtable API. :type op_type: str :param op_type: The type of operation being performed. Expect ``create``, ``update`` or ``undelete``. :type op_id: int :param op_id: The ID of the operation. :type begin: :class:`datetime.datetime` :param begin: The time when the operation was started. :type location_id: str :param location_id: ID of the location in which the operation is running :type instance: :class:`Instance` :param instance: The instance that created the operation. """ def __init__(self, op_type, op_id, begin, location_id, instance=None): self.op_type = op_type self.op_id = op_id self.begin = begin self.location_id = location_id self._instance = instance self._complete = False def __eq__(self, other): if not isinstance(other, self.__class__): return False return (other.op_type == self.op_type and other.op_id == self.op_id and other.begin == self.begin and other.location_id == self.location_id and other._instance == self._instance and other._complete == self._complete) def __ne__(self, other): return not self.__eq__(other) def finished(self): """Check if the operation has finished. :rtype: bool :returns: A boolean indicating if the current operation has completed. :raises: :class:`ValueError ` if the operation has already completed. """ if self._complete: raise ValueError('The operation has completed.') operation_name = ( 'operations/%s/locations/%s/operations/%d' % (self._instance.name, self.location_id, self.op_id)) request_pb = operations_pb2.GetOperationRequest(name=operation_name) # We expect a `google.longrunning.operations_pb2.Operation`. operation_pb = self._instance._client._operations_stub.GetOperation( request_pb, self._instance._client.timeout_seconds) if operation_pb.done: self._complete = True return True else: return False class Instance(object): """Representation of a Google Cloud Bigtable Instance. We can use a :class:`Instance` to: * :meth:`reload` itself * :meth:`create` itself * :meth:`update` itself * :meth:`delete` itself * :meth:`undelete` itself .. note:: For now, we leave out the ``default_storage_type`` (an enum) which if not sent will end up as :data:`.data_v2_pb2.STORAGE_SSD`. :type instance_id: str :param instance_id: The ID of the instance. :type client: :class:`Client ` :param client: The client that owns the instance. Provides authorization and a project ID. :type location_id: str :param location_id: ID of the location in which the instance will be created. Required for instances which do not yet exist. :type display_name: str :param display_name: (Optional) The display name for the instance in the Cloud Console UI. (Must be between 4 and 30 characters.) If this value is not set in the constructor, will fall back to the instance ID. :type serve_nodes: int :param serve_nodes: (Optional) The number of nodes in the instance's cluster; used to set up the instance's cluster. """ def __init__(self, instance_id, client, location_id=_EXISTING_INSTANCE_LOCATION_ID, display_name=None, serve_nodes=DEFAULT_SERVE_NODES): self.instance_id = instance_id self.display_name = display_name or instance_id self._cluster_location_id = location_id self._cluster_serve_nodes = serve_nodes self._client = client def _update_from_pb(self, instance_pb): """Refresh self from the server-provided protobuf. Helper for :meth:`from_pb` and :meth:`reload`. """ if not instance_pb.display_name: # Simple field (string) raise ValueError('Instance protobuf does not contain display_name') self.display_name = instance_pb.display_name @classmethod def from_pb(cls, instance_pb, client): """Creates a instance instance from a protobuf. :type instance_pb: :class:`instance_pb2.Instance` :param instance_pb: A instance protobuf object. :type client: :class:`Client ` :param client: The client that owns the instance. :rtype: :class:`Instance` :returns: The instance parsed from the protobuf response. :raises: :class:`ValueError ` if the instance name does not match ``projects/{project}/instances/{instance_id}`` or if the parsed project ID does not match the project ID on the client. """ match = _INSTANCE_NAME_RE.match(instance_pb.name) if match is None: raise ValueError('Instance protobuf name was not in the ' 'expected format.', instance_pb.name) if match.group('project') != client.project: raise ValueError('Project ID on instance does not match the ' 'project ID on the client') instance_id = match.group('instance_id') result = cls(instance_id, client, _EXISTING_INSTANCE_LOCATION_ID) result._update_from_pb(instance_pb) return result def copy(self): """Make a copy of this instance. Copies the local data stored as simple types and copies the client attached to this instance. :rtype: :class:`.Instance` :returns: A copy of the current instance. """ new_client = self._client.copy() return self.__class__(self.instance_id, new_client, self._cluster_location_id, display_name=self.display_name) @property def name(self): """Instance name used in requests. .. note:: This property will not change if ``instance_id`` does not, but the return value is not cached. The instance name is of the form ``"projects/{project}/instances/{instance_id}"`` :rtype: str :returns: The instance name. """ return self._client.project_name + '/instances/' + self.instance_id def __eq__(self, other): if not isinstance(other, self.__class__): return False # NOTE: This does not compare the configuration values, such as # the display_name. Instead, it only compares # identifying values instance ID and client. This is # intentional, since the same instance can be in different states # if not synchronized. Instances with similar instance # settings but different clients can't be used in the same way. return (other.instance_id == self.instance_id and other._client == self._client) def __ne__(self, other): return not self.__eq__(other) def reload(self): """Reload the metadata for this instance.""" request_pb = messages_v2_pb2.GetInstanceRequest(name=self.name) # We expect `data_v2_pb2.Instance`. instance_pb = self._client._instance_stub.GetInstance( request_pb, self._client.timeout_seconds) # NOTE: _update_from_pb does not check that the project and # instance ID on the response match the request. self._update_from_pb(instance_pb) def create(self): """Create this instance. .. note:: Uses the ``project`` and ``instance_id`` on the current :class:`Instance` in addition to the ``display_name``. To change them before creating, reset the values via .. code:: python instance.display_name = 'New display name' instance.instance_id = 'i-changed-my-mind' before calling :meth:`create`. :rtype: :class:`Operation` :returns: The long-running operation corresponding to the create operation. """ request_pb = _prepare_create_request(self) # We expect a `google.longrunning.operations_pb2.Operation`. operation_pb = self._client._instance_stub.CreateInstance( request_pb, self._client.timeout_seconds) op_id, loc_id, op_begin = _process_operation(operation_pb) return Operation('create', op_id, op_begin, loc_id, instance=self) def update(self): """Update this instance. .. note:: Updates the ``display_name``. To change that value before updating, reset its values via .. code:: python instance.display_name = 'New display name' before calling :meth:`update`. """ request_pb = data_v2_pb2.Instance( name=self.name, display_name=self.display_name, ) # Ignore the expected `data_v2_pb2.Instance`. self._client._instance_stub.UpdateInstance( request_pb, self._client.timeout_seconds) def delete(self): """Delete this instance. Marks a instance and all of its tables for permanent deletion in 7 days. Immediately upon completion of the request: * Billing will cease for all of the instance's reserved resources. * The instance's ``delete_time`` field will be set 7 days in the future. Soon afterward: * All tables within the instance will become unavailable. Prior to the instance's ``delete_time``: * The instance can be recovered with a call to ``UndeleteInstance``. * All other attempts to modify or delete the instance will be rejected. At the instance's ``delete_time``: * The instance and **all of its tables** will immediately and irrevocably disappear from the API, and their data will be permanently deleted. """ request_pb = messages_v2_pb2.DeleteInstanceRequest(name=self.name) # We expect a `google.protobuf.empty_pb2.Empty` self._client._instance_stub.DeleteInstance( request_pb, self._client.timeout_seconds) def cluster(self, cluster_id, serve_nodes=3): """Factory to create a cluster associated with this client. :type cluster_id: str :param cluster_id: The ID of the cluster. :type serve_nodes: int :param serve_nodes: (Optional) The number of nodes in the cluster. Defaults to 3. :rtype: :class:`.Cluster` :returns: The cluster owned by this client. """ return Cluster(cluster_id, self, serve_nodes=serve_nodes) def list_clusters(self): """Lists clusters in this instance. :rtype: tuple :returns: A pair of results, the first is a list of :class:`.Cluster` s returned and the second is a list of strings (the failed locations in the request). """ request_pb = messages_v2_pb2.ListClustersRequest(parent=self.name) # We expect a `.cluster_messages_v1_pb2.ListClustersResponse` list_clusters_response = self._client._instance_stub.ListClusters( request_pb, self._client.timeout_seconds) failed_locations = [ location for location in list_clusters_response.failed_locations] clusters = [Cluster.from_pb(cluster_pb, self) for cluster_pb in list_clusters_response.clusters] return clusters, failed_locations def table(self, table_id): """Factory to create a table associated with this instance. :type table_id: str :param table_id: The ID of the table. :rtype: :class:`Table ` :returns: The table owned by this instance. """ return Table(table_id, self) def list_tables(self): """List the tables in this instance. :rtype: list of :class:`Table ` :returns: The list of tables owned by the instance. :raises: :class:`ValueError ` if one of the returned tables has a name that is not of the expected format. """ request_pb = table_messages_v2_pb2.ListTablesRequest(parent=self.name) # We expect a `table_messages_v2_pb2.ListTablesResponse` table_list_pb = self._client._table_stub.ListTables( request_pb, self._client.timeout_seconds) result = [] for table_pb in table_list_pb.tables: table_prefix = self.name + '/tables/' if not table_pb.name.startswith(table_prefix): raise ValueError('Table name %s not of expected format' % ( table_pb.name,)) table_id = table_pb.name[len(table_prefix):] result.append(self.table(table_id)) return result