187 lines
6.5 KiB
Python
187 lines
6.5 KiB
Python
# Copyright 2015 Google Inc. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Iterators for paging through API responses.
|
|
|
|
These iterators simplify the process of paging through API responses
|
|
where the response is a list of results with a ``nextPageToken``.
|
|
|
|
To make an iterator work, just override the ``get_items_from_response``
|
|
method so that given a response (containing a page of results) it parses
|
|
those results into an iterable of the actual objects you want::
|
|
|
|
class MyIterator(Iterator):
|
|
def get_items_from_response(self, response):
|
|
items = response.get('items', [])
|
|
for item in items:
|
|
my_item = MyItemClass(other_arg=True)
|
|
my_item._set_properties(item)
|
|
yield my_item
|
|
|
|
You then can use this to get **all** the results from a resource::
|
|
|
|
>>> iterator = MyIterator(...)
|
|
>>> list(iterator) # Convert to a list (consumes all values).
|
|
|
|
Or you can walk your way through items and call off the search early if
|
|
you find what you're looking for (resulting in possibly fewer
|
|
requests)::
|
|
|
|
>>> for item in MyIterator(...):
|
|
>>> print item.name
|
|
>>> if not item.is_valid:
|
|
>>> break
|
|
"""
|
|
|
|
|
|
class Iterator(object):
|
|
"""A generic class for iterating through Cloud JSON APIs list responses.
|
|
|
|
:type client: :class:`gcloud.client.Client`
|
|
:param client: The client, which owns a connection to make requests.
|
|
|
|
:type path: string
|
|
:param path: The path to query for the list of items.
|
|
|
|
:type extra_params: dict or None
|
|
:param extra_params: Extra query string parameters for the API call.
|
|
"""
|
|
|
|
PAGE_TOKEN = 'pageToken'
|
|
RESERVED_PARAMS = frozenset([PAGE_TOKEN])
|
|
|
|
def __init__(self, client, path, extra_params=None):
|
|
self.client = client
|
|
self.path = path
|
|
self.page_number = 0
|
|
self.next_page_token = None
|
|
self.extra_params = extra_params or {}
|
|
reserved_in_use = self.RESERVED_PARAMS.intersection(
|
|
self.extra_params)
|
|
if reserved_in_use:
|
|
raise ValueError(('Using a reserved parameter',
|
|
reserved_in_use))
|
|
|
|
def __iter__(self):
|
|
"""Iterate through the list of items."""
|
|
while self.has_next_page():
|
|
response = self.get_next_page_response()
|
|
for item in self.get_items_from_response(response):
|
|
yield item
|
|
|
|
def has_next_page(self):
|
|
"""Determines whether or not this iterator has more pages.
|
|
|
|
:rtype: boolean
|
|
:returns: Whether the iterator has more pages or not.
|
|
"""
|
|
if self.page_number == 0:
|
|
return True
|
|
|
|
return self.next_page_token is not None
|
|
|
|
def get_query_params(self):
|
|
"""Getter for query parameters for the next request.
|
|
|
|
:rtype: dict
|
|
:returns: A dictionary of query parameters.
|
|
"""
|
|
result = ({self.PAGE_TOKEN: self.next_page_token}
|
|
if self.next_page_token else {})
|
|
result.update(self.extra_params)
|
|
return result
|
|
|
|
def get_next_page_response(self):
|
|
"""Requests the next page from the path provided.
|
|
|
|
:rtype: dict
|
|
:returns: The parsed JSON response of the next page's contents.
|
|
"""
|
|
if not self.has_next_page():
|
|
raise RuntimeError('No more pages. Try resetting the iterator.')
|
|
|
|
response = self.client.connection.api_request(
|
|
method='GET', path=self.path, query_params=self.get_query_params())
|
|
|
|
self.page_number += 1
|
|
self.next_page_token = response.get('nextPageToken')
|
|
|
|
return response
|
|
|
|
def reset(self):
|
|
"""Resets the iterator to the beginning."""
|
|
self.page_number = 0
|
|
self.next_page_token = None
|
|
|
|
def get_items_from_response(self, response):
|
|
"""Factory method called while iterating. This should be overriden.
|
|
|
|
This method should be overridden by a subclass. It should
|
|
accept the API response of a request for the next page of items,
|
|
and return a list (or other iterable) of items.
|
|
|
|
Typically this method will construct a Bucket or a Blob from the
|
|
page of results in the response.
|
|
|
|
:type response: dict
|
|
:param response: The response of asking for the next page of items.
|
|
|
|
:rtype: iterable
|
|
:returns: Items that the iterator should yield.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
class MethodIterator(object):
|
|
"""Method-based iterator iterating through Cloud JSON APIs list responses.
|
|
|
|
:type method: instance method
|
|
:param method: ``list_foo`` method of a domain object, taking as arguments
|
|
``page_token``, ``page_size``, and optional additional
|
|
keyword arguments.
|
|
|
|
:type page_token: string or ``NoneType``
|
|
:param page_token: Initial page token to pass. if ``None``, fetch the
|
|
first page from the ``method`` API call.
|
|
|
|
:type page_size: integer or ``NoneType``
|
|
:param page_size: Maximum number of items to return from the ``method``
|
|
API call; if ``None``, uses the default for the API.
|
|
|
|
:type max_calls: integer or ``NoneType``
|
|
:param max_calls: Maximum number of times to make the ``method``
|
|
API call; if ``None``, applies no limit.
|
|
|
|
:type kw: dict
|
|
:param kw: optional keyword argments to be passed to ``method``.
|
|
"""
|
|
def __init__(self, method, page_token=None, page_size=None,
|
|
max_calls=None, **kw):
|
|
self._method = method
|
|
self._token = page_token
|
|
self._page_size = page_size
|
|
self._kw = kw
|
|
self._max_calls = max_calls
|
|
self._page_num = 0
|
|
|
|
def __iter__(self):
|
|
while self._max_calls is None or self._page_num < self._max_calls:
|
|
items, new_token = self._method(
|
|
page_token=self._token, page_size=self._page_size, **self._kw)
|
|
for item in items:
|
|
yield item
|
|
if new_token is None:
|
|
return
|
|
self._page_num += 1
|
|
self._token = new_token
|