Vehicle-Anti-Theft-Face-Rec.../venv/Lib/site-packages/gcloud/bigquery/test_table.py

1742 lines
68 KiB
Python
Raw Normal View History

# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest2
class TestSchemaField(unittest2.TestCase):
def _getTargetClass(self):
from gcloud.bigquery.table import SchemaField
return SchemaField
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
def test_ctor_defaults(self):
field = self._makeOne('test', 'STRING')
self.assertEqual(field.name, 'test')
self.assertEqual(field.field_type, 'STRING')
self.assertEqual(field.mode, 'NULLABLE')
self.assertEqual(field.description, None)
self.assertEqual(field.fields, None)
def test_ctor_explicit(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
self.assertEqual(field.name, 'test')
self.assertEqual(field.field_type, 'STRING')
self.assertEqual(field.mode, 'REQUIRED')
self.assertEqual(field.description, 'Testing')
self.assertEqual(field.fields, None)
def test_ctor_subfields(self):
field = self._makeOne('phone_number', 'RECORD',
fields=[self._makeOne('area_code', 'STRING'),
self._makeOne('local_number', 'STRING')])
self.assertEqual(field.name, 'phone_number')
self.assertEqual(field.field_type, 'RECORD')
self.assertEqual(field.mode, 'NULLABLE')
self.assertEqual(field.description, None)
self.assertEqual(len(field.fields), 2)
self.assertEqual(field.fields[0].name, 'area_code')
self.assertEqual(field.fields[0].field_type, 'STRING')
self.assertEqual(field.fields[0].mode, 'NULLABLE')
self.assertEqual(field.fields[0].description, None)
self.assertEqual(field.fields[0].fields, None)
self.assertEqual(field.fields[1].name, 'local_number')
self.assertEqual(field.fields[1].field_type, 'STRING')
self.assertEqual(field.fields[1].mode, 'NULLABLE')
self.assertEqual(field.fields[1].description, None)
self.assertEqual(field.fields[1].fields, None)
def test___eq___name_mismatch(self):
field = self._makeOne('test', 'STRING')
other = self._makeOne('other', 'STRING')
self.assertNotEqual(field, other)
def test___eq___field_type_mismatch(self):
field = self._makeOne('test', 'STRING')
other = self._makeOne('test', 'INTEGER')
self.assertNotEqual(field, other)
def test___eq___mode_mismatch(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED')
other = self._makeOne('test', 'STRING', mode='NULLABLE')
self.assertNotEqual(field, other)
def test___eq___description_mismatch(self):
field = self._makeOne('test', 'STRING', description='Testing')
other = self._makeOne('test', 'STRING', description='Other')
self.assertNotEqual(field, other)
def test___eq___fields_mismatch(self):
sub1 = self._makeOne('sub1', 'STRING')
sub2 = self._makeOne('sub2', 'STRING')
field = self._makeOne('test', 'RECORD', fields=[sub1])
other = self._makeOne('test', 'RECORD', fields=[sub2])
self.assertNotEqual(field, other)
def test___eq___hit(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
other = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
self.assertEqual(field, other)
def test___eq___hit_case_diff_on_type(self):
field = self._makeOne('test', 'STRING', mode='REQUIRED',
description='Testing')
other = self._makeOne('test', 'string', mode='REQUIRED',
description='Testing')
self.assertEqual(field, other)
def test___eq___hit_w_fields(self):
sub1 = self._makeOne('sub1', 'STRING')
sub2 = self._makeOne('sub2', 'STRING')
field = self._makeOne('test', 'RECORD', fields=[sub1, sub2])
other = self._makeOne('test', 'RECORD', fields=[sub1, sub2])
self.assertEqual(field, other)
class _SchemaBase(object):
def _verify_field(self, field, r_field):
self.assertEqual(field.name, r_field['name'])
self.assertEqual(field.field_type, r_field['type'])
self.assertEqual(field.mode, r_field.get('mode', 'NULLABLE'))
def _verifySchema(self, schema, resource):
r_fields = resource['schema']['fields']
self.assertEqual(len(schema), len(r_fields))
for field, r_field in zip(schema, r_fields):
self._verify_field(field, r_field)
class TestTable(unittest2.TestCase, _SchemaBase):
PROJECT = 'project'
DS_NAME = 'dataset-name'
TABLE_NAME = 'table-name'
def _getTargetClass(self):
from gcloud.bigquery.table import Table
return Table
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
def _setUpConstants(self):
import datetime
from gcloud._helpers import UTC
self.WHEN_TS = 1437767599.006
self.WHEN = datetime.datetime.utcfromtimestamp(self.WHEN_TS).replace(
tzinfo=UTC)
self.ETAG = 'ETAG'
self.TABLE_ID = '%s:%s:%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
self.RESOURCE_URL = 'http://example.com/path/to/resource'
self.NUM_BYTES = 12345
self.NUM_ROWS = 67
def _makeResource(self):
self._setUpConstants()
return {
'creationTime': self.WHEN_TS * 1000,
'tableReference':
{'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME},
'schema': {'fields': [
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]},
'etag': 'ETAG',
'id': self.TABLE_ID,
'lastModifiedTime': self.WHEN_TS * 1000,
'location': 'US',
'selfLink': self.RESOURCE_URL,
'numRows': self.NUM_ROWS,
'numBytes': self.NUM_BYTES,
'type': 'TABLE',
}
def _verifyReadonlyResourceProperties(self, table, resource):
if 'creationTime' in resource:
self.assertEqual(table.created, self.WHEN)
else:
self.assertEqual(table.created, None)
if 'etag' in resource:
self.assertEqual(table.etag, self.ETAG)
else:
self.assertEqual(table.etag, None)
if 'numRows' in resource:
self.assertEqual(table.num_rows, self.NUM_ROWS)
else:
self.assertEqual(table.num_rows, None)
if 'numBytes' in resource:
self.assertEqual(table.num_bytes, self.NUM_BYTES)
else:
self.assertEqual(table.num_bytes, None)
if 'selfLink' in resource:
self.assertEqual(table.self_link, self.RESOURCE_URL)
else:
self.assertEqual(table.self_link, None)
self.assertEqual(table.table_id, self.TABLE_ID)
self.assertEqual(table.table_type,
'TABLE' if 'view' not in resource else 'VIEW')
def _verifyResourceProperties(self, table, resource):
self._verifyReadonlyResourceProperties(table, resource)
if 'expirationTime' in resource:
self.assertEqual(table.expires, self.EXP_TIME)
else:
self.assertEqual(table.expires, None)
self.assertEqual(table.description, resource.get('description'))
self.assertEqual(table.friendly_name, resource.get('friendlyName'))
self.assertEqual(table.location, resource.get('location'))
if 'view' in resource:
self.assertEqual(table.view_query, resource['view']['query'])
else:
self.assertEqual(table.view_query, None)
if 'schema' in resource:
self._verifySchema(table.schema, resource)
else:
self.assertEqual(table.schema, [])
def test_ctor(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
self.assertEqual(table.name, self.TABLE_NAME)
self.assertTrue(table._dataset is dataset)
self.assertEqual(table.project, self.PROJECT)
self.assertEqual(table.dataset_name, self.DS_NAME)
self.assertEqual(
table.path,
'/projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME))
self.assertEqual(table.schema, [])
self.assertEqual(table.created, None)
self.assertEqual(table.etag, None)
self.assertEqual(table.modified, None)
self.assertEqual(table.num_bytes, None)
self.assertEqual(table.num_rows, None)
self.assertEqual(table.self_link, None)
self.assertEqual(table.table_id, None)
self.assertEqual(table.table_type, None)
self.assertEqual(table.description, None)
self.assertEqual(table.expires, None)
self.assertEqual(table.friendly_name, None)
self.assertEqual(table.location, None)
self.assertEqual(table.view_query, None)
def test_ctor_w_schema(self):
from gcloud.bigquery.table import SchemaField
client = _Client(self.PROJECT)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
table = self._makeOne(self.TABLE_NAME, dataset,
schema=[full_name, age])
self.assertEqual(table.schema, [full_name, age])
def test_num_bytes_getter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
# Check with no value set.
self.assertEqual(table.num_bytes, None)
num_bytes = 1337
# Check with integer value set.
table._properties = {'numBytes': num_bytes}
self.assertEqual(table.num_bytes, num_bytes)
# Check with a string value set.
table._properties = {'numBytes': str(num_bytes)}
self.assertEqual(table.num_bytes, num_bytes)
# Check with invalid int value.
table._properties = {'numBytes': 'x'}
with self.assertRaises(ValueError):
getattr(table, 'num_bytes')
def test_num_rows_getter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
# Check with no value set.
self.assertEqual(table.num_rows, None)
num_rows = 42
# Check with integer value set.
table._properties = {'numRows': num_rows}
self.assertEqual(table.num_rows, num_rows)
# Check with a string value set.
table._properties = {'numRows': str(num_rows)}
self.assertEqual(table.num_rows, num_rows)
# Check with invalid int value.
table._properties = {'numRows': 'x'}
with self.assertRaises(ValueError):
getattr(table, 'num_rows')
def test_schema_setter_non_list(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(TypeError):
table.schema = object()
def test_schema_setter_invalid_field(self):
from gcloud.bigquery.table import SchemaField
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
with self.assertRaises(ValueError):
table.schema = [full_name, object()]
def test_schema_setter(self):
from gcloud.bigquery.table import SchemaField
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
table.schema = [full_name, age]
self.assertEqual(table.schema, [full_name, age])
def test_props_set_by_server(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _millis
CREATED = datetime.datetime(2015, 7, 29, 12, 13, 22, tzinfo=UTC)
MODIFIED = datetime.datetime(2015, 7, 29, 14, 47, 15, tzinfo=UTC)
TABLE_ID = '%s:%s:%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
URL = 'http://example.com/projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table._properties['creationTime'] = _millis(CREATED)
table._properties['etag'] = 'ETAG'
table._properties['lastModifiedTime'] = _millis(MODIFIED)
table._properties['numBytes'] = 12345
table._properties['numRows'] = 66
table._properties['selfLink'] = URL
table._properties['id'] = TABLE_ID
table._properties['type'] = 'TABLE'
self.assertEqual(table.created, CREATED)
self.assertEqual(table.etag, 'ETAG')
self.assertEqual(table.modified, MODIFIED)
self.assertEqual(table.num_bytes, 12345)
self.assertEqual(table.num_rows, 66)
self.assertEqual(table.self_link, URL)
self.assertEqual(table.table_id, TABLE_ID)
self.assertEqual(table.table_type, 'TABLE')
def test_description_setter_bad_value(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(ValueError):
table.description = 12345
def test_description_setter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table.description = 'DESCRIPTION'
self.assertEqual(table.description, 'DESCRIPTION')
def test_expires_setter_bad_value(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(ValueError):
table.expires = object()
def test_expires_setter(self):
import datetime
from gcloud._helpers import UTC
WHEN = datetime.datetime(2015, 7, 28, 16, 39, tzinfo=UTC)
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table.expires = WHEN
self.assertEqual(table.expires, WHEN)
def test_friendly_name_setter_bad_value(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(ValueError):
table.friendly_name = 12345
def test_friendly_name_setter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table.friendly_name = 'FRIENDLY'
self.assertEqual(table.friendly_name, 'FRIENDLY')
def test_location_setter_bad_value(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(ValueError):
table.location = 12345
def test_location_setter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table.location = 'LOCATION'
self.assertEqual(table.location, 'LOCATION')
def test_view_query_setter_bad_value(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(ValueError):
table.view_query = 12345
def test_view_query_setter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table.view_query = 'select * from foo'
self.assertEqual(table.view_query, 'select * from foo')
def test_view_query_deleter(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
table.view_query = 'select * from foo'
del table.view_query
self.assertEqual(table.view_query, None)
def test_from_api_repr_missing_identity(self):
self._setUpConstants()
client = _Client(self.PROJECT)
dataset = _Dataset(client)
RESOURCE = {}
klass = self._getTargetClass()
with self.assertRaises(KeyError):
klass.from_api_repr(RESOURCE, dataset)
def test_from_api_repr_bare(self):
self._setUpConstants()
client = _Client(self.PROJECT)
dataset = _Dataset(client)
RESOURCE = {
'id': '%s:%s:%s' % (self.PROJECT, self.DS_NAME, self.TABLE_NAME),
'tableReference': {
'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME,
},
'type': 'TABLE',
}
klass = self._getTargetClass()
table = klass.from_api_repr(RESOURCE, dataset)
self.assertEqual(table.name, self.TABLE_NAME)
self.assertTrue(table._dataset is dataset)
self._verifyResourceProperties(table, RESOURCE)
def test_from_api_repr_w_properties(self):
client = _Client(self.PROJECT)
dataset = _Dataset(client)
RESOURCE = self._makeResource()
klass = self._getTargetClass()
table = klass.from_api_repr(RESOURCE, dataset)
self.assertTrue(table._dataset._client is client)
self._verifyResourceProperties(table, RESOURCE)
def test_create_no_view_query_no_schema(self):
conn = _Connection()
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset)
with self.assertRaises(ValueError):
table.create()
def test_create_w_bound_client(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME)
RESOURCE = self._makeResource()
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
table = self._makeOne(self.TABLE_NAME, dataset,
schema=[full_name, age])
table.create()
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
SENT = {
'tableReference': {
'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME},
'schema': {'fields': [
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]},
}
self.assertEqual(req['data'], SENT)
self._verifyResourceProperties(table, RESOURCE)
def test_create_w_alternate_client(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _millis
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME)
DESCRIPTION = 'DESCRIPTION'
TITLE = 'TITLE'
QUERY = 'select fullname, age from person_ages'
RESOURCE = self._makeResource()
RESOURCE['description'] = DESCRIPTION
RESOURCE['friendlyName'] = TITLE
self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59,
tzinfo=UTC)
RESOURCE['expirationTime'] = _millis(self.EXP_TIME)
RESOURCE['view'] = {}
RESOURCE['view']['query'] = QUERY
RESOURCE['type'] = 'VIEW'
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection(RESOURCE)
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client=client1)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age])
table.friendly_name = TITLE
table.description = DESCRIPTION
table.view_query = QUERY
table.create(client=client2)
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
SENT = {
'tableReference': {
'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME},
'description': DESCRIPTION,
'friendlyName': TITLE,
'view': {'query': QUERY},
}
self.assertEqual(req['data'], SENT)
self._verifyResourceProperties(table, RESOURCE)
def test_create_w_missing_output_properties(self):
# In the wild, the resource returned from 'dataset.create' sometimes
# lacks 'creationTime' / 'lastModifiedTime'
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME)
RESOURCE = self._makeResource()
del RESOURCE['creationTime']
del RESOURCE['lastModifiedTime']
self.WHEN = None
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
table = self._makeOne(self.TABLE_NAME, dataset,
schema=[full_name, age])
table.create()
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
SENT = {
'tableReference': {
'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME},
'schema': {'fields': [
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]},
}
self.assertEqual(req['data'], SENT)
self._verifyResourceProperties(table, RESOURCE)
def test_exists_miss_w_bound_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn = _Connection()
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
self.assertFalse(table.exists())
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['query_params'], {'fields': 'id'})
def test_exists_hit_w_alternate_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection({})
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
self.assertTrue(table.exists(client=client2))
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['query_params'], {'fields': 'id'})
def test_reload_w_bound_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
RESOURCE = self._makeResource()
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.reload()
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
self._verifyResourceProperties(table, RESOURCE)
def test_reload_w_alternate_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
RESOURCE = self._makeResource()
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection(RESOURCE)
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.reload(client=client2)
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
self._verifyResourceProperties(table, RESOURCE)
def test_patch_w_invalid_expiration(self):
RESOURCE = self._makeResource()
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
with self.assertRaises(ValueError):
table.patch(expires='BOGUS')
def test_patch_w_bound_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
DESCRIPTION = 'DESCRIPTION'
TITLE = 'TITLE'
RESOURCE = self._makeResource()
RESOURCE['description'] = DESCRIPTION
RESOURCE['friendlyName'] = TITLE
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.patch(description=DESCRIPTION,
friendly_name=TITLE,
view_query=None)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'PATCH')
SENT = {
'description': DESCRIPTION,
'friendlyName': TITLE,
'view': None,
}
self.assertEqual(req['data'], SENT)
self.assertEqual(req['path'], '/%s' % PATH)
self._verifyResourceProperties(table, RESOURCE)
def test_patch_w_alternate_client(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _millis
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
QUERY = 'select fullname, age from person_ages'
LOCATION = 'EU'
RESOURCE = self._makeResource()
RESOURCE['view'] = {'query': QUERY}
RESOURCE['type'] = 'VIEW'
RESOURCE['location'] = LOCATION
self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59,
tzinfo=UTC)
RESOURCE['expirationTime'] = _millis(self.EXP_TIME)
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection(RESOURCE)
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='NULLABLE')
table.patch(client=client2, view_query=QUERY, location=LOCATION,
expires=self.EXP_TIME, schema=[full_name, age])
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'PATCH')
self.assertEqual(req['path'], '/%s' % PATH)
SENT = {
'view': {'query': QUERY},
'location': LOCATION,
'expirationTime': _millis(self.EXP_TIME),
'schema': {'fields': [
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'age', 'type': 'INTEGER', 'mode': 'NULLABLE'}]},
}
self.assertEqual(req['data'], SENT)
self._verifyResourceProperties(table, RESOURCE)
def test_patch_w_schema_None(self):
# Simulate deleting schema: not sure if back-end will actually
# allow this operation, but the spec says it is optional.
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
DESCRIPTION = 'DESCRIPTION'
TITLE = 'TITLE'
RESOURCE = self._makeResource()
RESOURCE['description'] = DESCRIPTION
RESOURCE['friendlyName'] = TITLE
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.patch(schema=None)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'PATCH')
SENT = {'schema': None}
self.assertEqual(req['data'], SENT)
self.assertEqual(req['path'], '/%s' % PATH)
self._verifyResourceProperties(table, RESOURCE)
def test_update_w_bound_client(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
DESCRIPTION = 'DESCRIPTION'
TITLE = 'TITLE'
RESOURCE = self._makeResource()
RESOURCE['description'] = DESCRIPTION
RESOURCE['friendlyName'] = TITLE
conn = _Connection(RESOURCE)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age])
table.description = DESCRIPTION
table.friendly_name = TITLE
table.update()
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'PUT')
SENT = {
'tableReference':
{'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME},
'schema': {'fields': [
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]},
'description': DESCRIPTION,
'friendlyName': TITLE,
}
self.assertEqual(req['data'], SENT)
self.assertEqual(req['path'], '/%s' % PATH)
self._verifyResourceProperties(table, RESOURCE)
def test_update_w_alternate_client(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _millis
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
DEF_TABLE_EXP = 12345
LOCATION = 'EU'
QUERY = 'select fullname, age from person_ages'
RESOURCE = self._makeResource()
RESOURCE['defaultTableExpirationMs'] = 12345
RESOURCE['location'] = LOCATION
self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59,
tzinfo=UTC)
RESOURCE['expirationTime'] = _millis(self.EXP_TIME)
RESOURCE['view'] = {'query': QUERY}
RESOURCE['type'] = 'VIEW'
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection(RESOURCE)
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.default_table_expiration_ms = DEF_TABLE_EXP
table.location = LOCATION
table.expires = self.EXP_TIME
table.view_query = QUERY
table.update(client=client2)
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'PUT')
self.assertEqual(req['path'], '/%s' % PATH)
SENT = {
'tableReference':
{'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME},
'expirationTime': _millis(self.EXP_TIME),
'location': 'EU',
'view': {'query': QUERY},
}
self.assertEqual(req['data'], SENT)
self._verifyResourceProperties(table, RESOURCE)
def test_delete_w_bound_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn = _Connection({})
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.delete()
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'DELETE')
self.assertEqual(req['path'], '/%s' % PATH)
def test_delete_w_alternate_client(self):
PATH = 'projects/%s/datasets/%s/tables/%s' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection({})
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
table.delete(client=client2)
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'DELETE')
self.assertEqual(req['path'], '/%s' % PATH)
def test_fetch_data_w_bound_client(self):
import datetime
from gcloud._helpers import UTC
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/data' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
WHEN_TS = 1437767599.006
WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace(
tzinfo=UTC)
WHEN_1 = WHEN + datetime.timedelta(seconds=1)
WHEN_2 = WHEN + datetime.timedelta(seconds=2)
ROWS = 1234
TOKEN = 'TOKEN'
def _bigquery_timestamp_float_repr(ts_float):
# Preserve microsecond precision for E+09 timestamps
return '%0.15E' % (ts_float,)
DATA = {
'totalRows': str(ROWS),
'pageToken': TOKEN,
'rows': [
{'f': [
{'v': 'Phred Phlyntstone'},
{'v': '32'},
{'v': _bigquery_timestamp_float_repr(WHEN_TS)},
]},
{'f': [
{'v': 'Bharney Rhubble'},
{'v': '33'},
{'v': _bigquery_timestamp_float_repr(WHEN_TS + 1)},
]},
{'f': [
{'v': 'Wylma Phlyntstone'},
{'v': '29'},
{'v': _bigquery_timestamp_float_repr(WHEN_TS + 2)},
]},
{'f': [
{'v': 'Bhettye Rhubble'},
{'v': None},
{'v': None},
]},
]
}
conn = _Connection(DATA)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='NULLABLE')
joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age, joined])
rows, total_rows, page_token = table.fetch_data()
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0], ('Phred Phlyntstone', 32, WHEN))
self.assertEqual(rows[1], ('Bharney Rhubble', 33, WHEN_1))
self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, WHEN_2))
self.assertEqual(rows[3], ('Bhettye Rhubble', None, None))
self.assertEqual(total_rows, ROWS)
self.assertEqual(page_token, TOKEN)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
def test_fetch_data_w_alternate_client(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/data' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
MAX = 10
TOKEN = 'TOKEN'
DATA = {
'rows': [
{'f': [
{'v': 'Phred Phlyntstone'},
{'v': '32'},
{'v': 'true'},
{'v': '3.1415926'},
]},
{'f': [
{'v': 'Bharney Rhubble'},
{'v': '33'},
{'v': 'false'},
{'v': '1.414'},
]},
{'f': [
{'v': 'Wylma Phlyntstone'},
{'v': '29'},
{'v': 'true'},
{'v': '2.71828'},
]},
{'f': [
{'v': 'Bhettye Rhubble'},
{'v': '27'},
{'v': None},
{'v': None},
]},
]
}
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection(DATA)
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
voter = SchemaField('voter', 'BOOLEAN', mode='NULLABLE')
score = SchemaField('score', 'FLOAT', mode='NULLABLE')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age, voter, score])
rows, total_rows, page_token = table.fetch_data(client=client2,
max_results=MAX,
page_token=TOKEN)
self.assertEqual(len(rows), 4)
self.assertEqual(rows[0], ('Phred Phlyntstone', 32, True, 3.1415926))
self.assertEqual(rows[1], ('Bharney Rhubble', 33, False, 1.414))
self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, True, 2.71828))
self.assertEqual(rows[3], ('Bhettye Rhubble', 27, None, None))
self.assertEqual(total_rows, None)
self.assertEqual(page_token, None)
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['query_params'],
{'maxResults': MAX, 'pageToken': TOKEN})
def test_fetch_data_w_repeated_fields(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/data' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
ROWS = 1234
TOKEN = 'TOKEN'
DATA = {
'totalRows': ROWS,
'pageToken': TOKEN,
'rows': [
{'f': [
{'v': ['red', 'green']},
{'v': [{'f': [{'v': ['1', '2']},
{'v': ['3.1415', '1.414']}]}]},
]},
]
}
conn = _Connection(DATA)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('color', 'STRING', mode='REPEATED')
index = SchemaField('index', 'INTEGER', 'REPEATED')
score = SchemaField('score', 'FLOAT', 'REPEATED')
struct = SchemaField('struct', 'RECORD', mode='REPEATED',
fields=[index, score])
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, struct])
rows, total_rows, page_token = table.fetch_data()
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0][0], ['red', 'green'])
self.assertEqual(rows[0][1], [{'index': [1, 2],
'score': [3.1415, 1.414]}])
self.assertEqual(total_rows, ROWS)
self.assertEqual(page_token, TOKEN)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
def test_fetch_data_w_record_schema(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/data' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
ROWS = 1234
TOKEN = 'TOKEN'
DATA = {
'totalRows': ROWS,
'pageToken': TOKEN,
'rows': [
{'f': [
{'v': 'Phred Phlyntstone'},
{'v': {'f': [{'v': '800'}, {'v': '555-1212'}, {'v': 1}]}},
]},
{'f': [
{'v': 'Bharney Rhubble'},
{'v': {'f': [{'v': '877'}, {'v': '768-5309'}, {'v': 2}]}},
]},
{'f': [
{'v': 'Wylma Phlyntstone'},
{'v': None},
]},
]
}
conn = _Connection(DATA)
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
area_code = SchemaField('area_code', 'STRING', 'REQUIRED')
local_number = SchemaField('local_number', 'STRING', 'REQUIRED')
rank = SchemaField('rank', 'INTEGER', 'REQUIRED')
phone = SchemaField('phone', 'RECORD', mode='NULLABLE',
fields=[area_code, local_number, rank])
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, phone])
rows, total_rows, page_token = table.fetch_data()
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0][0], 'Phred Phlyntstone')
self.assertEqual(rows[0][1], {'area_code': '800',
'local_number': '555-1212',
'rank': 1})
self.assertEqual(rows[1][0], 'Bharney Rhubble')
self.assertEqual(rows[1][1], {'area_code': '877',
'local_number': '768-5309',
'rank': 2})
self.assertEqual(rows[2][0], 'Wylma Phlyntstone')
self.assertEqual(rows[2][1], None)
self.assertEqual(total_rows, ROWS)
self.assertEqual(page_token, TOKEN)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'GET')
self.assertEqual(req['path'], '/%s' % PATH)
def test_insert_data_w_bound_client(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _microseconds_from_datetime
from gcloud.bigquery.table import SchemaField
WHEN_TS = 1437767599.006
WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace(
tzinfo=UTC)
PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn = _Connection({})
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age, joined])
ROWS = [
('Phred Phlyntstone', 32, WHEN),
('Bharney Rhubble', 33, WHEN + datetime.timedelta(seconds=1)),
('Wylma Phlyntstone', 29, WHEN + datetime.timedelta(seconds=2)),
('Bhettye Rhubble', 27, None),
]
def _row_data(row):
joined = None
if row[2] is not None:
joined = _microseconds_from_datetime(row[2]) * 1e-6
return {'full_name': row[0],
'age': row[1],
'joined': joined}
SENT = {
'rows': [{'json': _row_data(row)} for row in ROWS],
}
errors = table.insert_data(ROWS)
self.assertEqual(len(errors), 0)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['data'], SENT)
def test_insert_data_w_alternate_client(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
RESPONSE = {
'insertErrors': [
{'index': 1,
'errors': [
{'reason': 'REASON',
'location': 'LOCATION',
'debugInfo': 'INFO',
'message': 'MESSAGE'}
]},
]}
conn1 = _Connection()
client1 = _Client(project=self.PROJECT, connection=conn1)
conn2 = _Connection(RESPONSE)
client2 = _Client(project=self.PROJECT, connection=conn2)
dataset = _Dataset(client1)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
voter = SchemaField('voter', 'BOOLEAN', mode='NULLABLE')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age, voter])
ROWS = [
('Phred Phlyntstone', 32, True),
('Bharney Rhubble', 33, False),
('Wylma Phlyntstone', 29, True),
('Bhettye Rhubble', 27, True),
]
def _row_data(row):
return {'full_name': row[0], 'age': row[1], 'voter': row[2]}
SENT = {
'skipInvalidRows': True,
'ignoreUnknownValues': True,
'templateSuffix': '20160303',
'rows': [{'insertId': index, 'json': _row_data(row)}
for index, row in enumerate(ROWS)],
}
errors = table.insert_data(
client=client2,
rows=ROWS,
row_ids=[index for index, _ in enumerate(ROWS)],
skip_invalid_rows=True,
ignore_unknown_values=True,
template_suffix='20160303',
)
self.assertEqual(len(errors), 1)
self.assertEqual(errors[0]['index'], 1)
self.assertEqual(len(errors[0]['errors']), 1)
self.assertEqual(errors[0]['errors'][0],
RESPONSE['insertErrors'][0]['errors'][0])
self.assertEqual(len(conn1._requested), 0)
self.assertEqual(len(conn2._requested), 1)
req = conn2._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['data'], SENT)
def test_insert_data_w_repeated_fields(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn = _Connection({})
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('color', 'STRING', mode='REPEATED')
index = SchemaField('index', 'INTEGER', 'REPEATED')
score = SchemaField('score', 'FLOAT', 'REPEATED')
struct = SchemaField('struct', 'RECORD', mode='REPEATED',
fields=[index, score])
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, struct])
ROWS = [
(['red', 'green'], [{'index': [1, 2], 'score': [3.1415, 1.414]}]),
]
def _row_data(row):
return {'color': row[0],
'struct': row[1]}
SENT = {
'rows': [{'json': _row_data(row)} for row in ROWS],
}
errors = table.insert_data(ROWS)
self.assertEqual(len(errors), 0)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['data'], SENT)
def test_insert_data_w_record_schema(self):
from gcloud.bigquery.table import SchemaField
PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % (
self.PROJECT, self.DS_NAME, self.TABLE_NAME)
conn = _Connection({})
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
area_code = SchemaField('area_code', 'STRING', 'REQUIRED')
local_number = SchemaField('local_number', 'STRING', 'REQUIRED')
rank = SchemaField('rank', 'INTEGER', 'REQUIRED')
phone = SchemaField('phone', 'RECORD', mode='NULLABLE',
fields=[area_code, local_number, rank])
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, phone])
ROWS = [
('Phred Phlyntstone', {'area_code': '800',
'local_number': '555-1212',
'rank': 1}),
('Bharney Rhubble', {'area_code': '877',
'local_number': '768-5309',
'rank': 2}),
('Wylma Phlyntstone', None),
]
def _row_data(row):
return {'full_name': row[0],
'phone': row[1]}
SENT = {
'rows': [{'json': _row_data(row)} for row in ROWS],
}
errors = table.insert_data(ROWS)
self.assertEqual(len(errors), 0)
self.assertEqual(len(conn._requested), 1)
req = conn._requested[0]
self.assertEqual(req['method'], 'POST')
self.assertEqual(req['path'], '/%s' % PATH)
self.assertEqual(req['data'], SENT)
def test_upload_from_file_text_mode_file_failure(self):
class TextModeFile(object):
mode = 'r'
conn = _Connection()
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
file_obj = TextModeFile()
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
with self.assertRaises(ValueError):
table.upload_from_file(file_obj, 'CSV', size=1234)
def test_upload_from_file_size_failure(self):
conn = _Connection()
client = _Client(project=self.PROJECT, connection=conn)
dataset = _Dataset(client)
file_obj = object()
table = self._makeOne(self.TABLE_NAME, dataset=dataset)
with self.assertRaises(ValueError):
table.upload_from_file(file_obj, 'CSV', size=None)
def _upload_from_file_helper(self, **kw):
import csv
import datetime
from six.moves.http_client import OK
from gcloud._helpers import UTC
from gcloud._testing import _NamedTemporaryFile
from gcloud.bigquery.table import SchemaField
WHEN_TS = 1437767599.006
WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace(
tzinfo=UTC)
PATH = 'projects/%s/jobs' % (self.PROJECT,)
response = {'status': OK}
conn = _Connection(
(response, b'{}'),
)
client = _Client(project=self.PROJECT, connection=conn)
expected_job = object()
if 'client' in kw:
kw['client']._job = expected_job
else:
client._job = expected_job
dataset = _Dataset(client)
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE')
table = self._makeOne(self.TABLE_NAME, dataset=dataset,
schema=[full_name, age, joined])
ROWS = [
('Phred Phlyntstone', 32, WHEN),
('Bharney Rhubble', 33, WHEN + datetime.timedelta(seconds=1)),
('Wylma Phlyntstone', 29, WHEN + datetime.timedelta(seconds=2)),
('Bhettye Rhubble', 27, None),
]
with _NamedTemporaryFile() as temp:
with open(temp.name, 'w') as file_obj:
writer = csv.writer(file_obj)
writer.writerow(('full_name', 'age', 'joined'))
writer.writerows(ROWS)
with open(temp.name, 'rb') as file_obj:
BODY = file_obj.read()
explicit_size = kw.pop('_explicit_size', False)
if explicit_size:
kw['size'] = len(BODY)
job = table.upload_from_file(
file_obj, 'CSV', rewind=True, **kw)
self.assertTrue(job is expected_job)
return conn.http._requested, PATH, BODY
def test_upload_from_file_w_bound_client_multipart(self):
import json
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._helpers import _to_bytes
from gcloud.streaming.test_transfer import _email_chunk_parser
requested, PATH, BODY = self._upload_from_file_helper()
parse_chunk = _email_chunk_parser()
self.assertEqual(len(requested), 1)
req = requested[0]
self.assertEqual(req['method'], 'POST')
uri = req['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/%s' % PATH)
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'multipart'})
ctype, boundary = [x.strip()
for x in req['headers']['content-type'].split(';')]
self.assertEqual(ctype, 'multipart/related')
self.assertTrue(boundary.startswith('boundary="=='))
self.assertTrue(boundary.endswith('=="'))
divider = b'--' + _to_bytes(boundary[len('boundary="'):-1])
chunks = req['body'].split(divider)[1:-1] # discard prolog / epilog
self.assertEqual(len(chunks), 2)
text_msg = parse_chunk(chunks[0].strip())
self.assertEqual(dict(text_msg._headers),
{'Content-Type': 'application/json',
'MIME-Version': '1.0'})
metadata = json.loads(text_msg._payload)
load_config = metadata['configuration']['load']
DESTINATION_TABLE = {
'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME,
}
self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE)
self.assertEqual(load_config['sourceFormat'], 'CSV')
app_msg = parse_chunk(chunks[1].strip())
self.assertEqual(dict(app_msg._headers),
{'Content-Type': 'application/octet-stream',
'Content-Transfer-Encoding': 'binary',
'MIME-Version': '1.0'})
body = BODY.decode('ascii').rstrip()
body_lines = [line.strip() for line in body.splitlines()]
payload_lines = app_msg._payload.rstrip().splitlines()
self.assertEqual(payload_lines, body_lines)
# pylint: disable=too-many-statements
def test_upload_from_file_w_explicit_client_resumable(self):
import json
from six.moves.http_client import OK
from six.moves.urllib.parse import parse_qsl
from six.moves.urllib.parse import urlsplit
from gcloud._testing import _Monkey
from gcloud.bigquery import table as MUT
UPLOAD_PATH = 'https://example.com/upload/test'
initial_response = {'status': OK, 'location': UPLOAD_PATH}
upload_response = {'status': OK}
conn = _Connection(
(initial_response, b'{}'),
(upload_response, b'{}'),
)
client = _Client(project=self.PROJECT, connection=conn)
class _UploadConfig(object):
accept = ['*/*']
max_size = None
resumable_multipart = True
resumable_path = u'/upload/bigquery/v2/projects/{project}/jobs'
simple_multipart = True
simple_path = u'' # force resumable
with _Monkey(MUT, _UploadConfig=_UploadConfig):
orig_requested, PATH, BODY = self._upload_from_file_helper(
allow_jagged_rows=False,
allow_quoted_newlines=False,
create_disposition='CREATE_IF_NEEDED',
encoding='utf8',
field_delimiter=',',
ignore_unknown_values=False,
max_bad_records=0,
quote_character='"',
skip_leading_rows=1,
write_disposition='WRITE_APPEND',
client=client,
_explicit_size=True)
self.assertEqual(len(orig_requested), 0)
requested = conn.http._requested
self.assertEqual(len(requested), 2)
req = requested[0]
self.assertEqual(req['method'], 'POST')
uri = req['uri']
scheme, netloc, path, qs, _ = urlsplit(uri)
self.assertEqual(scheme, 'http')
self.assertEqual(netloc, 'example.com')
self.assertEqual(path, '/%s' % PATH)
self.assertEqual(dict(parse_qsl(qs)),
{'uploadType': 'resumable'})
self.assertEqual(req['headers']['content-type'], 'application/json')
metadata = json.loads(req['body'])
load_config = metadata['configuration']['load']
DESTINATION_TABLE = {
'projectId': self.PROJECT,
'datasetId': self.DS_NAME,
'tableId': self.TABLE_NAME,
}
self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE)
self.assertEqual(load_config['sourceFormat'], 'CSV')
self.assertEqual(load_config['allowJaggedRows'], False)
self.assertEqual(load_config['allowQuotedNewlines'], False)
self.assertEqual(load_config['createDisposition'], 'CREATE_IF_NEEDED')
self.assertEqual(load_config['encoding'], 'utf8')
self.assertEqual(load_config['fieldDelimiter'], ',')
self.assertEqual(load_config['ignoreUnknownValues'], False)
self.assertEqual(load_config['maxBadRecords'], 0)
self.assertEqual(load_config['quote'], '"')
self.assertEqual(load_config['skipLeadingRows'], 1)
self.assertEqual(load_config['writeDisposition'], 'WRITE_APPEND')
req = requested[1]
self.assertEqual(req['method'], 'PUT')
self.assertEqual(req['uri'], UPLOAD_PATH)
headers = req['headers']
length = len(BODY)
self.assertEqual(headers['Content-Type'], 'application/octet-stream')
self.assertEqual(headers['Content-Range'],
'bytes 0-%d/%d' % (length - 1, length))
self.assertEqual(headers['content-length'], '%d' % (length,))
self.assertEqual(req['body'], BODY)
# pylint: enable=too-many-statements
class Test_parse_schema_resource(unittest2.TestCase, _SchemaBase):
def _callFUT(self, resource):
from gcloud.bigquery.table import _parse_schema_resource
return _parse_schema_resource(resource)
def _makeResource(self):
return {
'schema': {'fields': [
{'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'},
]},
}
def test__parse_schema_resource_defaults(self):
RESOURCE = self._makeResource()
schema = self._callFUT(RESOURCE['schema'])
self._verifySchema(schema, RESOURCE)
def test__parse_schema_resource_subfields(self):
RESOURCE = self._makeResource()
RESOURCE['schema']['fields'].append(
{'name': 'phone',
'type': 'RECORD',
'mode': 'REPEATABLE',
'fields': [{'name': 'type',
'type': 'STRING',
'mode': 'REQUIRED'},
{'name': 'number',
'type': 'STRING',
'mode': 'REQUIRED'}]})
schema = self._callFUT(RESOURCE['schema'])
self._verifySchema(schema, RESOURCE)
def test__parse_schema_resource_fields_without_mode(self):
RESOURCE = self._makeResource()
RESOURCE['schema']['fields'].append(
{'name': 'phone',
'type': 'STRING'})
schema = self._callFUT(RESOURCE['schema'])
self._verifySchema(schema, RESOURCE)
class Test_build_schema_resource(unittest2.TestCase, _SchemaBase):
def _callFUT(self, resource):
from gcloud.bigquery.table import _build_schema_resource
return _build_schema_resource(resource)
def test_defaults(self):
from gcloud.bigquery.table import SchemaField
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
resource = self._callFUT([full_name, age])
self.assertEqual(len(resource), 2)
self.assertEqual(resource[0],
{'name': 'full_name',
'type': 'STRING',
'mode': 'REQUIRED'})
self.assertEqual(resource[1],
{'name': 'age',
'type': 'INTEGER',
'mode': 'REQUIRED'})
def test_w_description(self):
from gcloud.bigquery.table import SchemaField
DESCRIPTION = 'DESCRIPTION'
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED',
description=DESCRIPTION)
age = SchemaField('age', 'INTEGER', mode='REQUIRED')
resource = self._callFUT([full_name, age])
self.assertEqual(len(resource), 2)
self.assertEqual(resource[0],
{'name': 'full_name',
'type': 'STRING',
'mode': 'REQUIRED',
'description': DESCRIPTION})
self.assertEqual(resource[1],
{'name': 'age',
'type': 'INTEGER',
'mode': 'REQUIRED'})
def test_w_subfields(self):
from gcloud.bigquery.table import SchemaField
full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
ph_type = SchemaField('type', 'STRING', 'REQUIRED')
ph_num = SchemaField('number', 'STRING', 'REQUIRED')
phone = SchemaField('phone', 'RECORD', mode='REPEATABLE',
fields=[ph_type, ph_num])
resource = self._callFUT([full_name, phone])
self.assertEqual(len(resource), 2)
self.assertEqual(resource[0],
{'name': 'full_name',
'type': 'STRING',
'mode': 'REQUIRED'})
self.assertEqual(resource[1],
{'name': 'phone',
'type': 'RECORD',
'mode': 'REPEATABLE',
'fields': [{'name': 'type',
'type': 'STRING',
'mode': 'REQUIRED'},
{'name': 'number',
'type': 'STRING',
'mode': 'REQUIRED'}]})
class _Client(object):
def __init__(self, project='project', connection=None):
self.project = project
self.connection = connection
def job_from_resource(self, resource): # pylint: disable=unused-argument
return self._job
class _Dataset(object):
def __init__(self, client, name=TestTable.DS_NAME):
self._client = client
self.name = name
@property
def path(self):
return '/projects/%s/datasets/%s' % (
self._client.project, self.name)
@property
def project(self):
return self._client.project
class _Responder(object):
def __init__(self, *responses):
self._responses = responses[:]
self._requested = []
def _respond(self, **kw):
self._requested.append(kw)
response, self._responses = self._responses[0], self._responses[1:]
return response
class _HTTP(_Responder):
connections = {} # For google-apitools debugging.
def request(self, uri, method, headers, body, **kw):
if hasattr(body, 'read'):
body = body.read()
return self._respond(uri=uri, method=method, headers=headers,
body=body, **kw)
class _Connection(_Responder):
API_BASE_URL = 'http://example.com'
USER_AGENT = 'testing 1.2.3'
def __init__(self, *responses):
super(_Connection, self).__init__(*responses)
self.http = _HTTP(*responses)
def api_request(self, **kw):
from gcloud.exceptions import NotFound
self._requested.append(kw)
try:
response, self._responses = self._responses[0], self._responses[1:]
except:
raise NotFound('miss')
else:
return response
def build_api_url(self, path, query_params=None,
api_base_url=API_BASE_URL):
from six.moves.urllib.parse import urlencode
from six.moves.urllib.parse import urlsplit
from six.moves.urllib.parse import urlunsplit
# Mimic the build_api_url interface.
qs = urlencode(query_params or {})
scheme, netloc, _, _, _ = urlsplit(api_base_url)
return urlunsplit((scheme, netloc, path, qs, ''))