# Copyright 2016 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """GAX wrapper for Logging API requests.""" import json # pylint: disable=import-error from google.gax import CallOptions from google.gax import INITIAL_PAGE from google.gax.errors import GaxError from google.gax.grpc import exc_to_code from google.logging.type.log_severity_pb2 import LogSeverity from google.logging.v2.logging_config_pb2 import LogSink from google.logging.v2.logging_metrics_pb2 import LogMetric from google.logging.v2.log_entry_pb2 import LogEntry from google.protobuf.json_format import Parse from grpc.beta.interfaces import StatusCode # pylint: enable=import-error from gcloud.exceptions import Conflict from gcloud.exceptions import NotFound from gcloud._helpers import _datetime_to_pb_timestamp from gcloud._helpers import _datetime_to_rfc3339 from gcloud._helpers import _pb_timestamp_to_datetime class _LoggingAPI(object): """Helper mapping logging-related APIs. :type gax_api: :class:`google.logging.v2.logging_service_v2_api.LoggingServiceV2Api` :param gax_api: API object used to make GAX requests. """ def __init__(self, gax_api): self._gax_api = gax_api def list_entries(self, projects, filter_='', order_by='', page_size=0, page_token=None): """Return a page of log entry resources. :type projects: list of strings :param projects: project IDs to include. If not passed, defaults to the project bound to the API's client. :type filter_: str :param filter_: a filter expression. See: https://cloud.google.com/logging/docs/view/advanced_filters :type order_by: str :param order_by: One of :data:`gcloud.logging.ASCENDING` or :data:`gcloud.logging.DESCENDING`. :type page_size: int :param page_size: maximum number of entries to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: opaque marker for the next "page" of entries. If not passed, the API will return the first page of entries. :rtype: tuple, (list, str) :returns: list of mappings, plus a "next page token" string: if not None, indicates that more entries can be retrieved with another call (pass that value as ``page_token``). """ options = _build_paging_options(page_token) page_iter = self._gax_api.list_log_entries( projects, filter_, order_by, page_size, options) entries = [_log_entry_pb_to_mapping(entry_pb) for entry_pb in page_iter.next()] token = page_iter.page_token or None return entries, token def write_entries(self, entries, logger_name=None, resource=None, labels=None): """API call: log an entry resource via a POST request :type entries: sequence of mapping :param entries: the log entry resources to log. :type logger_name: string :param logger_name: name of default logger to which to log the entries; individual entries may override. :type resource: mapping :param resource: default resource to associate with entries; individual entries may override. :type labels: mapping :param labels: default labels to associate with entries; individual entries may override. """ options = None partial_success = False entry_pbs = [_log_entry_mapping_to_pb(entry) for entry in entries] self._gax_api.write_log_entries(entry_pbs, logger_name, resource, labels, partial_success, options) def logger_delete(self, project, logger_name): """API call: delete all entries in a logger via a DELETE request :type project: string :param project: ID of project containing the log entries to delete :type logger_name: string :param logger_name: name of logger containing the log entries to delete """ options = None path = 'projects/%s/logs/%s' % (project, logger_name) self._gax_api.delete_log(path, options) class _SinksAPI(object): """Helper mapping sink-related APIs. :type gax_api: :class:`google.logging.v2.config_service_v2_api.ConfigServiceV2Api` :param gax_api: API object used to make GAX requests. """ def __init__(self, gax_api): self._gax_api = gax_api def list_sinks(self, project, page_size=0, page_token=None): """List sinks for the project associated with this client. :type project: string :param project: ID of the project whose sinks are to be listed. :type page_size: int :param page_size: maximum number of sinks to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: opaque marker for the next "page" of sinks. If not passed, the API will return the first page of sinks. :rtype: tuple, (list, str) :returns: list of mappings, plus a "next page token" string: if not None, indicates that more sinks can be retrieved with another call (pass that value as ``page_token``). """ options = _build_paging_options(page_token) page_iter = self._gax_api.list_sinks(project, page_size, options) sinks = [_log_sink_pb_to_mapping(log_sink_pb) for log_sink_pb in page_iter.next()] token = page_iter.page_token or None return sinks, token def sink_create(self, project, sink_name, filter_, destination): """API call: create a sink resource. See: https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/projects.sinks/create :type project: string :param project: ID of the project in which to create the sink. :type sink_name: string :param sink_name: the name of the sink :type filter_: string :param filter_: the advanced logs filter expression defining the entries exported by the sink. :type destination: string :param destination: destination URI for the entries exported by the sink. """ options = None parent = 'projects/%s' % (project,) sink_pb = LogSink(name=sink_name, filter=filter_, destination=destination) try: self._gax_api.create_sink(parent, sink_pb, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.FAILED_PRECONDITION: path = 'projects/%s/sinks/%s' % (project, sink_name) raise Conflict(path) raise def sink_get(self, project, sink_name): """API call: retrieve a sink resource. :type project: string :param project: ID of the project containing the sink. :type sink_name: string :param sink_name: the name of the sink """ options = None path = 'projects/%s/sinks/%s' % (project, sink_name) try: sink_pb = self._gax_api.get_sink(path, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.NOT_FOUND: raise NotFound(path) raise return _log_sink_pb_to_mapping(sink_pb) def sink_update(self, project, sink_name, filter_, destination): """API call: update a sink resource. :type project: string :param project: ID of the project containing the sink. :type sink_name: string :param sink_name: the name of the sink :type filter_: string :param filter_: the advanced logs filter expression defining the entries exported by the sink. :type destination: string :param destination: destination URI for the entries exported by the sink. """ options = None path = 'projects/%s/sinks/%s' % (project, sink_name) sink_pb = LogSink(name=path, filter=filter_, destination=destination) try: self._gax_api.update_sink(path, sink_pb, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.NOT_FOUND: raise NotFound(path) raise return _log_sink_pb_to_mapping(sink_pb) def sink_delete(self, project, sink_name): """API call: delete a sink resource. :type project: string :param project: ID of the project containing the sink. :type sink_name: string :param sink_name: the name of the sink """ options = None path = 'projects/%s/sinks/%s' % (project, sink_name) try: self._gax_api.delete_sink(path, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.NOT_FOUND: raise NotFound(path) raise class _MetricsAPI(object): """Helper mapping sink-related APIs. :type gax_api: :class:`google.logging.v2.metrics_service_v2_api.MetricsServiceV2Api` :param gax_api: API object used to make GAX requests. """ def __init__(self, gax_api): self._gax_api = gax_api def list_metrics(self, project, page_size=0, page_token=None): """List metrics for the project associated with this client. :type project: string :param project: ID of the project whose metrics are to be listed. :type page_size: int :param page_size: maximum number of metrics to return, If not passed, defaults to a value set by the API. :type page_token: str :param page_token: opaque marker for the next "page" of metrics. If not passed, the API will return the first page of metrics. :rtype: tuple, (list, str) :returns: list of mappings, plus a "next page token" string: if not None, indicates that more metrics can be retrieved with another call (pass that value as ``page_token``). """ options = _build_paging_options(page_token) page_iter = self._gax_api.list_log_metrics(project, page_size, options) metrics = [_log_metric_pb_to_mapping(log_metric_pb) for log_metric_pb in page_iter.next()] token = page_iter.page_token or None return metrics, token def metric_create(self, project, metric_name, filter_, description): """API call: create a metric resource. See: https://cloud.google.com/logging/docs/api/ref_v2beta1/rest/v2beta1/projects.metrics/create :type project: string :param project: ID of the project in which to create the metric. :type metric_name: string :param metric_name: the name of the metric :type filter_: string :param filter_: the advanced logs filter expression defining the entries exported by the metric. :type description: string :param description: description of the metric. """ options = None parent = 'projects/%s' % (project,) metric_pb = LogMetric(name=metric_name, filter=filter_, description=description) try: self._gax_api.create_log_metric(parent, metric_pb, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.FAILED_PRECONDITION: path = 'projects/%s/metrics/%s' % (project, metric_name) raise Conflict(path) raise def metric_get(self, project, metric_name): """API call: retrieve a metric resource. :type project: string :param project: ID of the project containing the metric. :type metric_name: string :param metric_name: the name of the metric """ options = None path = 'projects/%s/metrics/%s' % (project, metric_name) try: metric_pb = self._gax_api.get_log_metric(path, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.NOT_FOUND: raise NotFound(path) raise return _log_metric_pb_to_mapping(metric_pb) def metric_update(self, project, metric_name, filter_, description): """API call: update a metric resource. :type project: string :param project: ID of the project containing the metric. :type metric_name: string :param metric_name: the name of the metric :type filter_: string :param filter_: the advanced logs filter expression defining the entries exported by the metric. :type description: string :param description: description of the metric. """ options = None path = 'projects/%s/metrics/%s' % (project, metric_name) metric_pb = LogMetric(name=path, filter=filter_, description=description) try: self._gax_api.update_log_metric(path, metric_pb, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.NOT_FOUND: raise NotFound(path) raise return _log_metric_pb_to_mapping(metric_pb) def metric_delete(self, project, metric_name): """API call: delete a metric resource. :type project: string :param project: ID of the project containing the metric. :type metric_name: string :param metric_name: the name of the metric """ options = None path = 'projects/%s/metrics/%s' % (project, metric_name) try: self._gax_api.delete_log_metric(path, options) except GaxError as exc: if exc_to_code(exc.cause) == StatusCode.NOT_FOUND: raise NotFound(path) raise def _build_paging_options(page_token=None): """Helper for :meth:'_PublisherAPI.list_topics' et aliae.""" if page_token is None: page_token = INITIAL_PAGE options = {'page_token': page_token} return CallOptions(**options) def _mon_resource_pb_to_mapping(resource_pb): """Helper for :func:_log_entry_pb_to_mapping""" mapping = { 'type': resource_pb.type, } if resource_pb.labels: mapping['labels'] = resource_pb.labels return mapping def _pb_timestamp_to_rfc3339(timestamp_pb): """Helper for :func:_log_entry_pb_to_mapping""" timestamp = _pb_timestamp_to_datetime(timestamp_pb) return _datetime_to_rfc3339(timestamp) def _log_entry_pb_to_mapping(entry_pb): """Helper for :meth:`list_entries`, et aliae Ideally, would use a function from :mod:`protobuf.json_format`, but the right one isn't public. See: https://github.com/google/protobuf/issues/1351 """ mapping = { 'logName': entry_pb.log_name, 'resource': _mon_resource_pb_to_mapping(entry_pb.resource), 'severity': entry_pb.severity, 'insertId': entry_pb.insert_id, 'timestamp': _pb_timestamp_to_rfc3339(entry_pb.timestamp), 'labels': entry_pb.labels, 'textPayload': entry_pb.text_payload, 'jsonPayload': entry_pb.json_payload, 'protoPayload': entry_pb.proto_payload, } if entry_pb.http_request: request = entry_pb.http_request mapping['httpRequest'] = { 'request_method': request.request_method, 'request_url': request.request_url, 'status': request.status, 'referer': request.referer, 'user_agent': request.user_agent, 'cache_hit': request.cache_hit, 'request_size': request.request_size, 'response_size': request.response_size, 'remote_ip': request.remote_ip, } if entry_pb.operation: operation = entry_pb.operation mapping['operation'] = { 'producer': operation.producer, 'id': operation.id, 'first': operation.first, 'last': operation.last, } return mapping def _http_request_mapping_to_pb(info, request): """Helper for _log_entry_mapping_to_pb""" optional_request_keys = { 'requestMethod': 'request_method', 'requestUrl': 'request_url', 'status': 'status', 'referer': 'referer', 'userAgent': 'user_agent', 'cacheHit': 'cache_hit', 'requestSize': 'request_size', 'responseSize': 'response_size', 'remoteIp': 'remote_ip', } for key, pb_name in optional_request_keys.items(): if key in info: setattr(request, pb_name, info[key]) def _log_operation_mapping_to_pb(info, operation): """Helper for _log_entry_mapping_to_pb""" operation.producer = info['producer'] operation.id = info['id'] if 'first' in info: operation.first = info['first'] if 'last' in info: operation.last = info['last'] def _log_entry_mapping_to_pb(mapping): """Helper for :meth:`write_entries`, et aliae Ideally, would use a function from :mod:`protobuf.json_format`, but the right one isn't public. See: https://github.com/google/protobuf/issues/1351 """ # pylint: disable=too-many-branches entry_pb = LogEntry() optional_scalar_keys = { 'logName': 'log_name', 'insertId': 'insert_id', 'textPayload': 'text_payload', } for key, pb_name in optional_scalar_keys.items(): if key in mapping: setattr(entry_pb, pb_name, mapping[key]) if 'resource' in mapping: entry_pb.resource.type = mapping['resource']['type'] if 'severity' in mapping: severity = mapping['severity'] if isinstance(severity, str): severity = LogSeverity.Value(severity) entry_pb.severity = severity if 'timestamp' in mapping: timestamp = _datetime_to_pb_timestamp(mapping['timestamp']) entry_pb.timestamp.CopyFrom(timestamp) if 'labels' in mapping: for key, value in mapping['labels'].items(): entry_pb.labels[key] = value if 'jsonPayload' in mapping: for key, value in mapping['jsonPayload'].items(): entry_pb.json_payload[key] = value if 'protoPayload' in mapping: Parse(json.dumps(mapping['protoPayload']), entry_pb.proto_payload) if 'httpRequest' in mapping: _http_request_mapping_to_pb( mapping['httpRequest'], entry_pb.http_request) if 'operation' in mapping: _log_operation_mapping_to_pb( mapping['operation'], entry_pb.operation) return entry_pb # pylint: enable=too-many-branches def _log_sink_pb_to_mapping(sink_pb): """Helper for :meth:`list_sinks`, et aliae Ideally, would use a function from :mod:`protobuf.json_format`, but the right one isn't public. See: https://github.com/google/protobuf/issues/1351 """ return { 'name': sink_pb.name, 'destination': sink_pb.destination, 'filter': sink_pb.filter, } def _log_metric_pb_to_mapping(metric_pb): """Helper for :meth:`list_metrics`, et aliae Ideally, would use a function from :mod:`protobuf.json_format`, but the right one isn't public. See: https://github.com/google/protobuf/issues/1351 """ return { 'name': metric_pb.name, 'description': metric_pb.description, 'filter': metric_pb.filter, }