# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Define API Datasets."""
import datetime
import json
import os
import six
from gcloud._helpers import _datetime_from_microseconds
from gcloud._helpers import _microseconds_from_datetime
from gcloud._helpers import _millis_from_datetime
from gcloud.exceptions import NotFound
from gcloud.streaming.http_wrapper import Request
from gcloud.streaming.http_wrapper import make_api_request
from gcloud.streaming.transfer import RESUMABLE_UPLOAD
from gcloud.streaming.transfer import Upload
from gcloud.bigquery._helpers import _rows_from_json
_MARKER = object()
class SchemaField(object):
"""Describe a single field within a table schema.
:type name: str
:param name: the name of the field
:type field_type: str
:param field_type: the type of the field (one of 'STRING', 'INTEGER',
'FLOAT', 'BOOLEAN', 'TIMESTAMP' or 'RECORD')
:type mode: str
:param mode: the type of the field (one of 'NULLABLE', 'REQUIRED',
or 'REPEATED')
:type description: str
:param description: optional description for the field
:type fields: list of :class:`SchemaField`, or None
:param fields: subfields (requires ``field_type`` of 'RECORD').
"""
def __init__(self, name, field_type, mode='NULLABLE', description=None,
fields=None):
self.name = name
self.field_type = field_type
self.mode = mode
self.description = description
self.fields = fields
def __eq__(self, other):
return (
self.name == other.name and
self.field_type.lower() == other.field_type.lower() and
self.mode == other.mode and
self.description == other.description and
self.fields == other.fields)
class Table(object):
"""Tables represent a set of rows whose values correspond to a schema.
See:
https://cloud.google.com/bigquery/docs/reference/v2/tables
:type name: str
:param name: the name of the table
:type dataset: :class:`gcloud.bigquery.dataset.Dataset`
:param dataset: The dataset which contains the table.
:type schema: list of :class:`SchemaField`
:param schema: The table's schema
"""
_schema = None
def __init__(self, name, dataset, schema=()):
self.name = name
self._dataset = dataset
self._properties = {}
# Let the @property do validation.
self.schema = schema
@property
def project(self):
"""Project bound to the table.
:rtype: str
:returns: the project (derived from the dataset).
"""
return self._dataset.project
@property
def dataset_name(self):
"""Name of dataset containing the table.
:rtype: str
:returns: the ID (derived from the dataset).
"""
return self._dataset.name
@property
def path(self):
"""URL path for the table's APIs.
:rtype: str
:returns: the path based on project and dataste name.
"""
return '%s/tables/%s' % (self._dataset.path, self.name)
@property
def schema(self):
"""Table's schema.
:rtype: list of :class:`SchemaField`
:returns: fields describing the schema
"""
return list(self._schema)
@schema.setter
def schema(self, value):
"""Update table's schema
:type value: list of :class:`SchemaField`
:param value: fields describing the schema
:raises: TypeError if 'value' is not a sequence, or ValueError if
any item in the sequence is not a SchemaField
"""
if not all(isinstance(field, SchemaField) for field in value):
raise ValueError('Schema items must be fields')
self._schema = tuple(value)
@property
def created(self):
"""Datetime at which the table was created.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the creation time (None until set from the server).
"""
creation_time = self._properties.get('creationTime')
if creation_time is not None:
# creation_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * creation_time)
@property
def etag(self):
"""ETag for the table resource.
:rtype: str, or ``NoneType``
:returns: the ETag (None until set from the server).
"""
return self._properties.get('etag')
@property
def modified(self):
"""Datetime at which the table was last modified.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the modification time (None until set from the server).
"""
modified_time = self._properties.get('lastModifiedTime')
if modified_time is not None:
# modified_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * modified_time)
@property
def num_bytes(self):
"""The size of the table in bytes.
:rtype: integer, or ``NoneType``
:returns: the byte count (None until set from the server).
"""
num_bytes_as_str = self._properties.get('numBytes')
if num_bytes_as_str is not None:
return int(num_bytes_as_str)
@property
def num_rows(self):
"""The number of rows in the table.
:rtype: integer, or ``NoneType``
:returns: the row count (None until set from the server).
"""
num_rows_as_str = self._properties.get('numRows')
if num_rows_as_str is not None:
return int(num_rows_as_str)
@property
def self_link(self):
"""URL for the table resource.
:rtype: str, or ``NoneType``
:returns: the URL (None until set from the server).
"""
return self._properties.get('selfLink')
@property
def table_id(self):
"""ID for the table resource.
:rtype: str, or ``NoneType``
:returns: the ID (None until set from the server).
"""
return self._properties.get('id')
@property
def table_type(self):
"""The type of the table.
Possible values are "TABLE" or "VIEW".
:rtype: str, or ``NoneType``
:returns: the URL (None until set from the server).
"""
return self._properties.get('type')
@property
def description(self):
"""Description of the table.
:rtype: str, or ``NoneType``
:returns: The description as set by the user, or None (the default).
"""
return self._properties.get('description')
@description.setter
def description(self, value):
"""Update description of the table.
:type value: str, or ``NoneType``
:param value: new description
:raises: ValueError for invalid value types.
"""
if not isinstance(value, six.string_types) and value is not None:
raise ValueError("Pass a string, or None")
self._properties['description'] = value
@property
def expires(self):
"""Datetime at which the table will be removed.
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the expiration time, or None
"""
expiration_time = self._properties.get('expirationTime')
if expiration_time is not None:
# expiration_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * expiration_time)
@expires.setter
def expires(self, value):
"""Update datetime at which the table will be removed.
:type value: ``datetime.datetime``, or ``NoneType``
:param value: the new expiration time, or None
"""
if not isinstance(value, datetime.datetime) and value is not None:
raise ValueError("Pass a datetime, or None")
self._properties['expirationTime'] = _millis_from_datetime(value)
@property
def friendly_name(self):
"""Title of the table.
:rtype: str, or ``NoneType``
:returns: The name as set by the user, or None (the default).
"""
return self._properties.get('friendlyName')
@friendly_name.setter
def friendly_name(self, value):
"""Update title of the table.
:type value: str, or ``NoneType``
:param value: new title
:raises: ValueError for invalid value types.
"""
if not isinstance(value, six.string_types) and value is not None:
raise ValueError("Pass a string, or None")
self._properties['friendlyName'] = value
@property
def location(self):
"""Location in which the table is hosted.
:rtype: str, or ``NoneType``
:returns: The location as set by the user, or None (the default).
"""
return self._properties.get('location')
@location.setter
def location(self, value):
"""Update location in which the table is hosted.
:type value: str, or ``NoneType``
:param value: new location
:raises: ValueError for invalid value types.
"""
if not isinstance(value, six.string_types) and value is not None:
raise ValueError("Pass a string, or None")
self._properties['location'] = value
@property
def view_query(self):
"""SQL query defining the table as a view.
:rtype: str, or ``NoneType``
:returns: The query as set by the user, or None (the default).
"""
view = self._properties.get('view')
if view is not None:
return view.get('query')
@view_query.setter
def view_query(self, value):
"""Update SQL query defining the table as a view.
:type value: str
:param value: new query
:raises: ValueError for invalid value types.
"""
if not isinstance(value, six.string_types):
raise ValueError("Pass a string")
self._properties['view'] = {'query': value}
@view_query.deleter
def view_query(self):
"""Delete SQL query defining the table as a view."""
self._properties.pop('view', None)
@classmethod
def from_api_repr(cls, resource, dataset):
"""Factory: construct a table given its API representation
:type resource: dict
:param resource: table resource representation returned from the API
:type dataset: :class:`gcloud.bigquery.dataset.Dataset`
:param dataset: The dataset containing the table.
:rtype: :class:`gcloud.bigquery.table.Table`
:returns: Table parsed from ``resource``.
"""
if ('tableReference' not in resource or
'tableId' not in resource['tableReference']):
raise KeyError('Resource lacks required identity information:'
'["tableReference"]["tableId"]')
table_name = resource['tableReference']['tableId']
table = cls(table_name, dataset=dataset)
table._set_properties(resource)
return table
def _require_client(self, client):
"""Check client or verify over-ride.
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
:rtype: :class:`gcloud.bigquery.client.Client`
:returns: The client passed in or the currently bound client.
"""
if client is None:
client = self._dataset._client
return client
def _set_properties(self, api_response):
"""Update properties from resource in body of ``api_response``
:type api_response: httplib2.Response
:param api_response: response returned from an API call
"""
self._properties.clear()
cleaned = api_response.copy()
schema = cleaned.pop('schema', {'fields': ()})
self.schema = _parse_schema_resource(schema)
if 'creationTime' in cleaned:
cleaned['creationTime'] = float(cleaned['creationTime'])
if 'lastModifiedTime' in cleaned:
cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime'])
if 'expirationTime' in cleaned:
cleaned['expirationTime'] = float(cleaned['expirationTime'])
self._properties.update(cleaned)
def _build_resource(self):
"""Generate a resource for ``create`` or ``update``."""
resource = {
'tableReference': {
'projectId': self._dataset.project,
'datasetId': self._dataset.name,
'tableId': self.name},
}
if self.description is not None:
resource['description'] = self.description
if self.expires is not None:
value = _millis_from_datetime(self.expires)
resource['expirationTime'] = value
if self.friendly_name is not None:
resource['friendlyName'] = self.friendly_name
if self.location is not None:
resource['location'] = self.location
if self.view_query is not None:
view = resource['view'] = {}
view['query'] = self.view_query
elif self._schema:
resource['schema'] = {
'fields': _build_schema_resource(self._schema)
}
else:
raise ValueError("Set either 'view_query' or 'schema'.")
return resource
def create(self, client=None):
"""API call: create the dataset via a PUT request
See:
https://cloud.google.com/bigquery/docs/reference/v2/tables/insert
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
"""
client = self._require_client(client)
path = '/projects/%s/datasets/%s/tables' % (
self._dataset.project, self._dataset.name)
api_response = client.connection.api_request(
method='POST', path=path, data=self._build_resource())
self._set_properties(api_response)
def exists(self, client=None):
"""API call: test for the existence of the table via a GET request
See
https://cloud.google.com/bigquery/docs/reference/v2/tables/get
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
"""
client = self._require_client(client)
try:
client.connection.api_request(method='GET', path=self.path,
query_params={'fields': 'id'})
except NotFound:
return False
else:
return True
def reload(self, client=None):
"""API call: refresh table properties via a GET request
See
https://cloud.google.com/bigquery/docs/reference/v2/tables/get
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
"""
client = self._require_client(client)
api_response = client.connection.api_request(
method='GET', path=self.path)
self._set_properties(api_response)
def patch(self,
client=None,
friendly_name=_MARKER,
description=_MARKER,
location=_MARKER,
expires=_MARKER,
view_query=_MARKER,
schema=_MARKER):
"""API call: update individual table properties via a PATCH request
See
https://cloud.google.com/bigquery/docs/reference/v2/tables/patch
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
:type friendly_name: str or ``NoneType``
:param friendly_name: point in time at which the table expires.
:type description: str or ``NoneType``
:param description: point in time at which the table expires.
:type location: str or ``NoneType``
:param location: point in time at which the table expires.
:type expires: :class:`datetime.datetime` or ``NoneType``
:param expires: point in time at which the table expires.
:type view_query: str
:param view_query: SQL query defining the table as a view
:type schema: list of :class:`SchemaField`
:param schema: fields describing the schema
:raises: ValueError for invalid value types.
"""
client = self._require_client(client)
partial = {}
if expires is not _MARKER:
if (not isinstance(expires, datetime.datetime) and
expires is not None):
raise ValueError("Pass a datetime, or None")
partial['expirationTime'] = _millis_from_datetime(expires)
if description is not _MARKER:
partial['description'] = description
if friendly_name is not _MARKER:
partial['friendlyName'] = friendly_name
if location is not _MARKER:
partial['location'] = location
if view_query is not _MARKER:
if view_query is None:
partial['view'] = None
else:
partial['view'] = {'query': view_query}
if schema is not _MARKER:
if schema is None:
partial['schema'] = None
else:
partial['schema'] = {
'fields': _build_schema_resource(schema)}
api_response = client.connection.api_request(
method='PATCH', path=self.path, data=partial)
self._set_properties(api_response)
def update(self, client=None):
"""API call: update table properties via a PUT request
See
https://cloud.google.com/bigquery/docs/reference/v2/tables/update
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
"""
client = self._require_client(client)
api_response = client.connection.api_request(
method='PUT', path=self.path, data=self._build_resource())
self._set_properties(api_response)
def delete(self, client=None):
"""API call: delete the table via a DELETE request
See:
https://cloud.google.com/bigquery/docs/reference/v2/tables/delete
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
"""
client = self._require_client(client)
client.connection.api_request(method='DELETE', path=self.path)
def fetch_data(self, max_results=None, page_token=None, client=None):
"""API call: fetch the table data via a GET request
See:
https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list
.. note::
This method assumes that its instance's ``schema`` attribute is
up-to-date with the schema as defined on the back-end: if the
two schemas are not identical, the values returned may be
incomplete. To ensure that the local copy of the schema is
up-to-date, call the table's ``reload`` method.
:type max_results: integer or ``NoneType``
:param max_results: maximum number of rows to return.
:type page_token: str or ``NoneType``
:param page_token: token representing a cursor into the table's rows.
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
:rtype: tuple
:returns: ``(row_data, total_rows, page_token)``, where ``row_data``
is a list of tuples, one per result row, containing only
the values; ``total_rows`` is a count of the total number
of rows in the table; and ``page_token`` is an opaque
string which can be used to fetch the next batch of rows
(``None`` if no further batches can be fetched).
"""
client = self._require_client(client)
params = {}
if max_results is not None:
params['maxResults'] = max_results
if page_token is not None:
params['pageToken'] = page_token
response = client.connection.api_request(method='GET',
path='%s/data' % self.path,
query_params=params)
total_rows = response.get('totalRows')
if total_rows is not None:
total_rows = int(total_rows)
page_token = response.get('pageToken')
rows_data = _rows_from_json(response.get('rows', ()), self._schema)
return rows_data, total_rows, page_token
def insert_data(self,
rows,
row_ids=None,
skip_invalid_rows=None,
ignore_unknown_values=None,
template_suffix=None,
client=None):
"""API call: insert table data via a POST request
See:
https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll
:type rows: list of tuples
:param rows: Row data to be inserted. Each tuple should contain data
for each schema field on the current table and in the
same order as the schema fields.
:type row_ids: list of string
:param row_ids: Unique ids, one per row being inserted. If not
passed, no de-duplication occurs.
:type skip_invalid_rows: boolean or ``NoneType``
:param skip_invalid_rows: skip rows w/ invalid data?
:type ignore_unknown_values: boolean or ``NoneType``
:param ignore_unknown_values: ignore columns beyond schema?
:type template_suffix: str or ``NoneType``
:param template_suffix: treat ``name`` as a template table and provide
a suffix. BigQuery will create the table
``<name> + <template_suffix>`` based on the
schema of the template table. See:
https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables
:type client: :class:`gcloud.bigquery.client.Client` or ``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current dataset.
:rtype: list of mappings
:returns: One mapping per row with insert errors: the "index" key
identifies the row, and the "errors" key contains a list
of the mappings describing one or more problems with the
row.
"""
client = self._require_client(client)
rows_info = []
data = {'rows': rows_info}
for index, row in enumerate(rows):
row_info = {}
for field, value in zip(self._schema, row):
if field.field_type == 'TIMESTAMP' and value is not None:
# BigQuery stores TIMESTAMP data internally as a
# UNIX timestamp with microsecond precision.
# Specifies the number of seconds since the epoch.
value = _microseconds_from_datetime(value) * 1e-6
row_info[field.name] = value
info = {'json': row_info}
if row_ids is not None:
info['insertId'] = row_ids[index]
rows_info.append(info)
if skip_invalid_rows is not None:
data['skipInvalidRows'] = skip_invalid_rows
if ignore_unknown_values is not None:
data['ignoreUnknownValues'] = ignore_unknown_values
if template_suffix is not None:
data['templateSuffix'] = template_suffix
response = client.connection.api_request(
method='POST',
path='%s/insertAll' % self.path,
data=data)
errors = []
for error in response.get('insertErrors', ()):
errors.append({'index': int(error['index']),
'errors': error['errors']})
return errors
# pylint: disable=too-many-arguments,too-many-locals
def upload_from_file(self,
file_obj,
source_format,
rewind=False,
size=None,
num_retries=6,
allow_jagged_rows=None,
allow_quoted_newlines=None,
create_disposition=None,
encoding=None,
field_delimiter=None,
ignore_unknown_values=None,
max_bad_records=None,
quote_character=None,
skip_leading_rows=None,
write_disposition=None,
client=None):
"""Upload the contents of this table from a file-like object.
The content type of the upload will either be
- The value passed in to the function (if any)
- ``text/csv``.
:type file_obj: file
:param file_obj: A file handle opened in binary mode for reading.
:type source_format: str
:param source_format: one of 'CSV' or 'NEWLINE_DELIMITED_JSON'.
job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type rewind: boolean
:param rewind: If True, seek to the beginning of the file handle before
writing the file to Cloud Storage.
:type size: int
:param size: The number of bytes to read from the file handle.
If not provided, we'll try to guess the size using
:func:`os.fstat`. (If the file handle is not from the
filesystem this won't be possible.)
:type num_retries: integer
:param num_retries: Number of upload retries. Defaults to 6.
:type allow_jagged_rows: boolean
:param allow_jagged_rows: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type allow_quoted_newlines: boolean
:param allow_quoted_newlines: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type create_disposition: str
:param create_disposition: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type encoding: str
:param encoding: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type field_delimiter: str
:param field_delimiter: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type ignore_unknown_values: boolean
:param ignore_unknown_values: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type max_bad_records: integer
:param max_bad_records: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type quote_character: str
:param quote_character: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type skip_leading_rows: integer
:param skip_leading_rows: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type write_disposition: str
:param write_disposition: job configuration option; see
:meth:`gcloud.bigquery.job.LoadJob`
:type client: :class:`gcloud.storage.client.Client` or ``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the current dataset.
:rtype: :class:`gcloud.bigquery.jobs.LoadTableFromStorageJob`
:returns: the job instance used to load the data (e.g., for
querying status)
:raises: :class:`ValueError` if ``size`` is not passed in and can not
be determined, or if the ``file_obj`` can be detected to be
a file opened in text mode.
"""
client = self._require_client(client)
connection = client.connection
content_type = 'application/octet-stream'
# Rewind the file if desired.
if rewind:
file_obj.seek(0, os.SEEK_SET)
mode = getattr(file_obj, 'mode', None)
if mode is not None and mode != 'rb':
raise ValueError(
"Cannot upload files opened in text mode: use "
"open(filename, mode='rb')")
# Get the basic stats about the file.
total_bytes = size
if total_bytes is None:
if hasattr(file_obj, 'fileno'):
total_bytes = os.fstat(file_obj.fileno()).st_size
else:
raise ValueError('total bytes could not be determined. Please '
'pass an explicit size.')
headers = {
'Accept': 'application/json',
'Accept-Encoding': 'gzip, deflate',
'User-Agent': connection.USER_AGENT,
'content-type': 'application/json',
}
metadata = {
'configuration': {
'load': {
'sourceFormat': source_format,
'schema': {
'fields': _build_schema_resource(self._schema),
},
'destinationTable': {
'projectId': self._dataset.project,
'datasetId': self._dataset.name,
'tableId': self.name,
}
}
}
}
_configure_job_metadata(metadata, allow_jagged_rows,
allow_quoted_newlines, create_disposition,
encoding, field_delimiter,
ignore_unknown_values, max_bad_records,
quote_character, skip_leading_rows,
write_disposition)
upload = Upload(file_obj, content_type, total_bytes,
auto_transfer=False)
url_builder = _UrlBuilder()
upload_config = _UploadConfig()
# Base URL may change once we know simple vs. resumable.
base_url = connection.API_BASE_URL + '/upload'
path = '/projects/%s/jobs' % (self._dataset.project,)
upload_url = connection.build_api_url(api_base_url=base_url, path=path)
# Use apitools 'Upload' facility.
request = Request(upload_url, 'POST', headers,
body=json.dumps(metadata))
upload.configure_request(upload_config, request, url_builder)
query_params = url_builder.query_params
base_url = connection.API_BASE_URL + '/upload'
request.url = connection.build_api_url(api_base_url=base_url,
path=path,
query_params=query_params)
upload.initialize_upload(request, connection.http)
if upload.strategy == RESUMABLE_UPLOAD:
http_response = upload.stream_file(use_chunks=True)
else:
http_response = make_api_request(connection.http, request,
retries=num_retries)
response_content = http_response.content
if not isinstance(response_content,
six.string_types): # pragma: NO COVER Python3
response_content = response_content.decode('utf-8')
return client.job_from_resource(json.loads(response_content))
# pylint: enable=too-many-arguments,too-many-locals
def _configure_job_metadata(metadata, # pylint: disable=too-many-arguments
allow_jagged_rows,
allow_quoted_newlines,
create_disposition,
encoding,
field_delimiter,
ignore_unknown_values,
max_bad_records,
quote_character,
skip_leading_rows,
write_disposition):
"""Helper for :meth:`Table.upload_from_file`."""
load_config = metadata['configuration']['load']
if allow_jagged_rows is not None:
load_config['allowJaggedRows'] = allow_jagged_rows
if allow_quoted_newlines is not None:
load_config['allowQuotedNewlines'] = allow_quoted_newlines
if create_disposition is not None:
load_config['createDisposition'] = create_disposition
if encoding is not None:
load_config['encoding'] = encoding
if field_delimiter is not None:
load_config['fieldDelimiter'] = field_delimiter
if ignore_unknown_values is not None:
load_config['ignoreUnknownValues'] = ignore_unknown_values
if max_bad_records is not None:
load_config['maxBadRecords'] = max_bad_records
if quote_character is not None:
load_config['quote'] = quote_character
if skip_leading_rows is not None:
load_config['skipLeadingRows'] = skip_leading_rows
if write_disposition is not None:
load_config['writeDisposition'] = write_disposition
def _parse_schema_resource(info):
"""Parse a resource fragment into a schema field.
:type info: mapping
:param info: should contain a "fields" key to be parsed
:rtype: list of :class:`SchemaField`, or ``NoneType``
:returns: a list of parsed fields, or ``None`` if no "fields" key is
present in ``info``.
"""
if 'fields' not in info:
return None
schema = []
for r_field in info['fields']:
name = r_field['name']
field_type = r_field['type']
mode = r_field.get('mode', 'NULLABLE')
description = r_field.get('description')
sub_fields = _parse_schema_resource(r_field)
schema.append(
SchemaField(name, field_type, mode, description, sub_fields))
return schema
def _build_schema_resource(fields):
"""Generate a resource fragment for a schema.
:type fields: sequence of :class:`SchemaField`
:param fields: schema to be dumped
:rtype: mapping
:returns: a mapping describing the schema of the supplied fields.
"""
infos = []
for field in fields:
info = {'name': field.name,
'type': field.field_type,
'mode': field.mode}
if field.description is not None:
info['description'] = field.description
if field.fields is not None:
info['fields'] = _build_schema_resource(field.fields)
infos.append(info)
return infos
class _UploadConfig(object):
"""Faux message FBO apitools' 'configure_request'."""
accept = ['*/*']
max_size = None
resumable_multipart = True
resumable_path = u'/upload/bigquery/v2/projects/{project}/jobs'
simple_multipart = True
simple_path = u'/upload/bigquery/v2/projects/{project}/jobs'
class _UrlBuilder(object):
"""Faux builder FBO apitools' 'configure_request'"""
def __init__(self):
self.query_params = {}
self._relative_path = ''