# Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest2 class TestSchemaField(unittest2.TestCase): def _getTargetClass(self): from gcloud.bigquery.table import SchemaField return SchemaField def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) def test_ctor_defaults(self): field = self._makeOne('test', 'STRING') self.assertEqual(field.name, 'test') self.assertEqual(field.field_type, 'STRING') self.assertEqual(field.mode, 'NULLABLE') self.assertEqual(field.description, None) self.assertEqual(field.fields, None) def test_ctor_explicit(self): field = self._makeOne('test', 'STRING', mode='REQUIRED', description='Testing') self.assertEqual(field.name, 'test') self.assertEqual(field.field_type, 'STRING') self.assertEqual(field.mode, 'REQUIRED') self.assertEqual(field.description, 'Testing') self.assertEqual(field.fields, None) def test_ctor_subfields(self): field = self._makeOne('phone_number', 'RECORD', fields=[self._makeOne('area_code', 'STRING'), self._makeOne('local_number', 'STRING')]) self.assertEqual(field.name, 'phone_number') self.assertEqual(field.field_type, 'RECORD') self.assertEqual(field.mode, 'NULLABLE') self.assertEqual(field.description, None) self.assertEqual(len(field.fields), 2) self.assertEqual(field.fields[0].name, 'area_code') self.assertEqual(field.fields[0].field_type, 'STRING') self.assertEqual(field.fields[0].mode, 'NULLABLE') self.assertEqual(field.fields[0].description, None) self.assertEqual(field.fields[0].fields, None) self.assertEqual(field.fields[1].name, 'local_number') self.assertEqual(field.fields[1].field_type, 'STRING') self.assertEqual(field.fields[1].mode, 'NULLABLE') self.assertEqual(field.fields[1].description, None) self.assertEqual(field.fields[1].fields, None) def test___eq___name_mismatch(self): field = self._makeOne('test', 'STRING') other = self._makeOne('other', 'STRING') self.assertNotEqual(field, other) def test___eq___field_type_mismatch(self): field = self._makeOne('test', 'STRING') other = self._makeOne('test', 'INTEGER') self.assertNotEqual(field, other) def test___eq___mode_mismatch(self): field = self._makeOne('test', 'STRING', mode='REQUIRED') other = self._makeOne('test', 'STRING', mode='NULLABLE') self.assertNotEqual(field, other) def test___eq___description_mismatch(self): field = self._makeOne('test', 'STRING', description='Testing') other = self._makeOne('test', 'STRING', description='Other') self.assertNotEqual(field, other) def test___eq___fields_mismatch(self): sub1 = self._makeOne('sub1', 'STRING') sub2 = self._makeOne('sub2', 'STRING') field = self._makeOne('test', 'RECORD', fields=[sub1]) other = self._makeOne('test', 'RECORD', fields=[sub2]) self.assertNotEqual(field, other) def test___eq___hit(self): field = self._makeOne('test', 'STRING', mode='REQUIRED', description='Testing') other = self._makeOne('test', 'STRING', mode='REQUIRED', description='Testing') self.assertEqual(field, other) def test___eq___hit_case_diff_on_type(self): field = self._makeOne('test', 'STRING', mode='REQUIRED', description='Testing') other = self._makeOne('test', 'string', mode='REQUIRED', description='Testing') self.assertEqual(field, other) def test___eq___hit_w_fields(self): sub1 = self._makeOne('sub1', 'STRING') sub2 = self._makeOne('sub2', 'STRING') field = self._makeOne('test', 'RECORD', fields=[sub1, sub2]) other = self._makeOne('test', 'RECORD', fields=[sub1, sub2]) self.assertEqual(field, other) class _SchemaBase(object): def _verify_field(self, field, r_field): self.assertEqual(field.name, r_field['name']) self.assertEqual(field.field_type, r_field['type']) self.assertEqual(field.mode, r_field.get('mode', 'NULLABLE')) def _verifySchema(self, schema, resource): r_fields = resource['schema']['fields'] self.assertEqual(len(schema), len(r_fields)) for field, r_field in zip(schema, r_fields): self._verify_field(field, r_field) class TestTable(unittest2.TestCase, _SchemaBase): PROJECT = 'project' DS_NAME = 'dataset-name' TABLE_NAME = 'table-name' def _getTargetClass(self): from gcloud.bigquery.table import Table return Table def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) def _setUpConstants(self): import datetime from gcloud._helpers import UTC self.WHEN_TS = 1437767599.006 self.WHEN = datetime.datetime.utcfromtimestamp(self.WHEN_TS).replace( tzinfo=UTC) self.ETAG = 'ETAG' self.TABLE_ID = '%s:%s:%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) self.RESOURCE_URL = 'http://example.com/path/to/resource' self.NUM_BYTES = 12345 self.NUM_ROWS = 67 def _makeResource(self): self._setUpConstants() return { 'creationTime': self.WHEN_TS * 1000, 'tableReference': {'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME}, 'schema': {'fields': [ {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, 'etag': 'ETAG', 'id': self.TABLE_ID, 'lastModifiedTime': self.WHEN_TS * 1000, 'location': 'US', 'selfLink': self.RESOURCE_URL, 'numRows': self.NUM_ROWS, 'numBytes': self.NUM_BYTES, 'type': 'TABLE', } def _verifyReadonlyResourceProperties(self, table, resource): if 'creationTime' in resource: self.assertEqual(table.created, self.WHEN) else: self.assertEqual(table.created, None) if 'etag' in resource: self.assertEqual(table.etag, self.ETAG) else: self.assertEqual(table.etag, None) if 'numRows' in resource: self.assertEqual(table.num_rows, self.NUM_ROWS) else: self.assertEqual(table.num_rows, None) if 'numBytes' in resource: self.assertEqual(table.num_bytes, self.NUM_BYTES) else: self.assertEqual(table.num_bytes, None) if 'selfLink' in resource: self.assertEqual(table.self_link, self.RESOURCE_URL) else: self.assertEqual(table.self_link, None) self.assertEqual(table.table_id, self.TABLE_ID) self.assertEqual(table.table_type, 'TABLE' if 'view' not in resource else 'VIEW') def _verifyResourceProperties(self, table, resource): self._verifyReadonlyResourceProperties(table, resource) if 'expirationTime' in resource: self.assertEqual(table.expires, self.EXP_TIME) else: self.assertEqual(table.expires, None) self.assertEqual(table.description, resource.get('description')) self.assertEqual(table.friendly_name, resource.get('friendlyName')) self.assertEqual(table.location, resource.get('location')) if 'view' in resource: self.assertEqual(table.view_query, resource['view']['query']) else: self.assertEqual(table.view_query, None) if 'schema' in resource: self._verifySchema(table.schema, resource) else: self.assertEqual(table.schema, []) def test_ctor(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) self.assertEqual(table.name, self.TABLE_NAME) self.assertTrue(table._dataset is dataset) self.assertEqual(table.project, self.PROJECT) self.assertEqual(table.dataset_name, self.DS_NAME) self.assertEqual( table.path, '/projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME)) self.assertEqual(table.schema, []) self.assertEqual(table.created, None) self.assertEqual(table.etag, None) self.assertEqual(table.modified, None) self.assertEqual(table.num_bytes, None) self.assertEqual(table.num_rows, None) self.assertEqual(table.self_link, None) self.assertEqual(table.table_id, None) self.assertEqual(table.table_type, None) self.assertEqual(table.description, None) self.assertEqual(table.expires, None) self.assertEqual(table.friendly_name, None) self.assertEqual(table.location, None) self.assertEqual(table.view_query, None) def test_ctor_w_schema(self): from gcloud.bigquery.table import SchemaField client = _Client(self.PROJECT) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table = self._makeOne(self.TABLE_NAME, dataset, schema=[full_name, age]) self.assertEqual(table.schema, [full_name, age]) def test_num_bytes_getter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) # Check with no value set. self.assertEqual(table.num_bytes, None) num_bytes = 1337 # Check with integer value set. table._properties = {'numBytes': num_bytes} self.assertEqual(table.num_bytes, num_bytes) # Check with a string value set. table._properties = {'numBytes': str(num_bytes)} self.assertEqual(table.num_bytes, num_bytes) # Check with invalid int value. table._properties = {'numBytes': 'x'} with self.assertRaises(ValueError): getattr(table, 'num_bytes') def test_num_rows_getter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) # Check with no value set. self.assertEqual(table.num_rows, None) num_rows = 42 # Check with integer value set. table._properties = {'numRows': num_rows} self.assertEqual(table.num_rows, num_rows) # Check with a string value set. table._properties = {'numRows': str(num_rows)} self.assertEqual(table.num_rows, num_rows) # Check with invalid int value. table._properties = {'numRows': 'x'} with self.assertRaises(ValueError): getattr(table, 'num_rows') def test_schema_setter_non_list(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(TypeError): table.schema = object() def test_schema_setter_invalid_field(self): from gcloud.bigquery.table import SchemaField client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') with self.assertRaises(ValueError): table.schema = [full_name, object()] def test_schema_setter(self): from gcloud.bigquery.table import SchemaField client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table.schema = [full_name, age] self.assertEqual(table.schema, [full_name, age]) def test_props_set_by_server(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _millis CREATED = datetime.datetime(2015, 7, 29, 12, 13, 22, tzinfo=UTC) MODIFIED = datetime.datetime(2015, 7, 29, 14, 47, 15, tzinfo=UTC) TABLE_ID = '%s:%s:%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) URL = 'http://example.com/projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table._properties['creationTime'] = _millis(CREATED) table._properties['etag'] = 'ETAG' table._properties['lastModifiedTime'] = _millis(MODIFIED) table._properties['numBytes'] = 12345 table._properties['numRows'] = 66 table._properties['selfLink'] = URL table._properties['id'] = TABLE_ID table._properties['type'] = 'TABLE' self.assertEqual(table.created, CREATED) self.assertEqual(table.etag, 'ETAG') self.assertEqual(table.modified, MODIFIED) self.assertEqual(table.num_bytes, 12345) self.assertEqual(table.num_rows, 66) self.assertEqual(table.self_link, URL) self.assertEqual(table.table_id, TABLE_ID) self.assertEqual(table.table_type, 'TABLE') def test_description_setter_bad_value(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(ValueError): table.description = 12345 def test_description_setter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table.description = 'DESCRIPTION' self.assertEqual(table.description, 'DESCRIPTION') def test_expires_setter_bad_value(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(ValueError): table.expires = object() def test_expires_setter(self): import datetime from gcloud._helpers import UTC WHEN = datetime.datetime(2015, 7, 28, 16, 39, tzinfo=UTC) client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table.expires = WHEN self.assertEqual(table.expires, WHEN) def test_friendly_name_setter_bad_value(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(ValueError): table.friendly_name = 12345 def test_friendly_name_setter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table.friendly_name = 'FRIENDLY' self.assertEqual(table.friendly_name, 'FRIENDLY') def test_location_setter_bad_value(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(ValueError): table.location = 12345 def test_location_setter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table.location = 'LOCATION' self.assertEqual(table.location, 'LOCATION') def test_view_query_setter_bad_value(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(ValueError): table.view_query = 12345 def test_view_query_setter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table.view_query = 'select * from foo' self.assertEqual(table.view_query, 'select * from foo') def test_view_query_deleter(self): client = _Client(self.PROJECT) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) table.view_query = 'select * from foo' del table.view_query self.assertEqual(table.view_query, None) def test_from_api_repr_missing_identity(self): self._setUpConstants() client = _Client(self.PROJECT) dataset = _Dataset(client) RESOURCE = {} klass = self._getTargetClass() with self.assertRaises(KeyError): klass.from_api_repr(RESOURCE, dataset) def test_from_api_repr_bare(self): self._setUpConstants() client = _Client(self.PROJECT) dataset = _Dataset(client) RESOURCE = { 'id': '%s:%s:%s' % (self.PROJECT, self.DS_NAME, self.TABLE_NAME), 'tableReference': { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME, }, 'type': 'TABLE', } klass = self._getTargetClass() table = klass.from_api_repr(RESOURCE, dataset) self.assertEqual(table.name, self.TABLE_NAME) self.assertTrue(table._dataset is dataset) self._verifyResourceProperties(table, RESOURCE) def test_from_api_repr_w_properties(self): client = _Client(self.PROJECT) dataset = _Dataset(client) RESOURCE = self._makeResource() klass = self._getTargetClass() table = klass.from_api_repr(RESOURCE, dataset) self.assertTrue(table._dataset._client is client) self._verifyResourceProperties(table, RESOURCE) def test_create_no_view_query_no_schema(self): conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset) with self.assertRaises(ValueError): table.create() def test_create_w_bound_client(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) RESOURCE = self._makeResource() conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table = self._makeOne(self.TABLE_NAME, dataset, schema=[full_name, age]) table.create() self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) SENT = { 'tableReference': { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME}, 'schema': {'fields': [ {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, } self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) def test_create_w_alternate_client(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _millis from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) DESCRIPTION = 'DESCRIPTION' TITLE = 'TITLE' QUERY = 'select fullname, age from person_ages' RESOURCE = self._makeResource() RESOURCE['description'] = DESCRIPTION RESOURCE['friendlyName'] = TITLE self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, tzinfo=UTC) RESOURCE['expirationTime'] = _millis(self.EXP_TIME) RESOURCE['view'] = {} RESOURCE['view']['query'] = QUERY RESOURCE['type'] = 'VIEW' conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection(RESOURCE) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client=client1) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age]) table.friendly_name = TITLE table.description = DESCRIPTION table.view_query = QUERY table.create(client=client2) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) SENT = { 'tableReference': { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME}, 'description': DESCRIPTION, 'friendlyName': TITLE, 'view': {'query': QUERY}, } self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) def test_create_w_missing_output_properties(self): # In the wild, the resource returned from 'dataset.create' sometimes # lacks 'creationTime' / 'lastModifiedTime' from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) RESOURCE = self._makeResource() del RESOURCE['creationTime'] del RESOURCE['lastModifiedTime'] self.WHEN = None conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table = self._makeOne(self.TABLE_NAME, dataset, schema=[full_name, age]) table.create() self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) SENT = { 'tableReference': { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME}, 'schema': {'fields': [ {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, } self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) def test_exists_miss_w_bound_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) self.assertFalse(table.exists()) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_exists_hit_w_alternate_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection({}) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) table = self._makeOne(self.TABLE_NAME, dataset=dataset) self.assertTrue(table.exists(client=client2)) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['query_params'], {'fields': 'id'}) def test_reload_w_bound_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) RESOURCE = self._makeResource() conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.reload() self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) self._verifyResourceProperties(table, RESOURCE) def test_reload_w_alternate_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) RESOURCE = self._makeResource() conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection(RESOURCE) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.reload(client=client2) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) self._verifyResourceProperties(table, RESOURCE) def test_patch_w_invalid_expiration(self): RESOURCE = self._makeResource() conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) with self.assertRaises(ValueError): table.patch(expires='BOGUS') def test_patch_w_bound_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) DESCRIPTION = 'DESCRIPTION' TITLE = 'TITLE' RESOURCE = self._makeResource() RESOURCE['description'] = DESCRIPTION RESOURCE['friendlyName'] = TITLE conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.patch(description=DESCRIPTION, friendly_name=TITLE, view_query=None) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'PATCH') SENT = { 'description': DESCRIPTION, 'friendlyName': TITLE, 'view': None, } self.assertEqual(req['data'], SENT) self.assertEqual(req['path'], '/%s' % PATH) self._verifyResourceProperties(table, RESOURCE) def test_patch_w_alternate_client(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _millis from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) QUERY = 'select fullname, age from person_ages' LOCATION = 'EU' RESOURCE = self._makeResource() RESOURCE['view'] = {'query': QUERY} RESOURCE['type'] = 'VIEW' RESOURCE['location'] = LOCATION self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, tzinfo=UTC) RESOURCE['expirationTime'] = _millis(self.EXP_TIME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection(RESOURCE) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) table = self._makeOne(self.TABLE_NAME, dataset=dataset) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='NULLABLE') table.patch(client=client2, view_query=QUERY, location=LOCATION, expires=self.EXP_TIME, schema=[full_name, age]) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'PATCH') self.assertEqual(req['path'], '/%s' % PATH) SENT = { 'view': {'query': QUERY}, 'location': LOCATION, 'expirationTime': _millis(self.EXP_TIME), 'schema': {'fields': [ {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'age', 'type': 'INTEGER', 'mode': 'NULLABLE'}]}, } self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) def test_patch_w_schema_None(self): # Simulate deleting schema: not sure if back-end will actually # allow this operation, but the spec says it is optional. PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) DESCRIPTION = 'DESCRIPTION' TITLE = 'TITLE' RESOURCE = self._makeResource() RESOURCE['description'] = DESCRIPTION RESOURCE['friendlyName'] = TITLE conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.patch(schema=None) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'PATCH') SENT = {'schema': None} self.assertEqual(req['data'], SENT) self.assertEqual(req['path'], '/%s' % PATH) self._verifyResourceProperties(table, RESOURCE) def test_update_w_bound_client(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) DESCRIPTION = 'DESCRIPTION' TITLE = 'TITLE' RESOURCE = self._makeResource() RESOURCE['description'] = DESCRIPTION RESOURCE['friendlyName'] = TITLE conn = _Connection(RESOURCE) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age]) table.description = DESCRIPTION table.friendly_name = TITLE table.update() self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'PUT') SENT = { 'tableReference': {'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME}, 'schema': {'fields': [ {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, 'description': DESCRIPTION, 'friendlyName': TITLE, } self.assertEqual(req['data'], SENT) self.assertEqual(req['path'], '/%s' % PATH) self._verifyResourceProperties(table, RESOURCE) def test_update_w_alternate_client(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _millis from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) DEF_TABLE_EXP = 12345 LOCATION = 'EU' QUERY = 'select fullname, age from person_ages' RESOURCE = self._makeResource() RESOURCE['defaultTableExpirationMs'] = 12345 RESOURCE['location'] = LOCATION self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, tzinfo=UTC) RESOURCE['expirationTime'] = _millis(self.EXP_TIME) RESOURCE['view'] = {'query': QUERY} RESOURCE['type'] = 'VIEW' conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection(RESOURCE) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.default_table_expiration_ms = DEF_TABLE_EXP table.location = LOCATION table.expires = self.EXP_TIME table.view_query = QUERY table.update(client=client2) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'PUT') self.assertEqual(req['path'], '/%s' % PATH) SENT = { 'tableReference': {'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME}, 'expirationTime': _millis(self.EXP_TIME), 'location': 'EU', 'view': {'query': QUERY}, } self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) def test_delete_w_bound_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn = _Connection({}) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.delete() self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'DELETE') self.assertEqual(req['path'], '/%s' % PATH) def test_delete_w_alternate_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection({}) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) table = self._makeOne(self.TABLE_NAME, dataset=dataset) table.delete(client=client2) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'DELETE') self.assertEqual(req['path'], '/%s' % PATH) def test_fetch_data_w_bound_client(self): import datetime from gcloud._helpers import UTC from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) WHEN_TS = 1437767599.006 WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace( tzinfo=UTC) WHEN_1 = WHEN + datetime.timedelta(seconds=1) WHEN_2 = WHEN + datetime.timedelta(seconds=2) ROWS = 1234 TOKEN = 'TOKEN' def _bigquery_timestamp_float_repr(ts_float): # Preserve microsecond precision for E+09 timestamps return '%0.15E' % (ts_float,) DATA = { 'totalRows': str(ROWS), 'pageToken': TOKEN, 'rows': [ {'f': [ {'v': 'Phred Phlyntstone'}, {'v': '32'}, {'v': _bigquery_timestamp_float_repr(WHEN_TS)}, ]}, {'f': [ {'v': 'Bharney Rhubble'}, {'v': '33'}, {'v': _bigquery_timestamp_float_repr(WHEN_TS + 1)}, ]}, {'f': [ {'v': 'Wylma Phlyntstone'}, {'v': '29'}, {'v': _bigquery_timestamp_float_repr(WHEN_TS + 2)}, ]}, {'f': [ {'v': 'Bhettye Rhubble'}, {'v': None}, {'v': None}, ]}, ] } conn = _Connection(DATA) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='NULLABLE') joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age, joined]) rows, total_rows, page_token = table.fetch_data() self.assertEqual(len(rows), 4) self.assertEqual(rows[0], ('Phred Phlyntstone', 32, WHEN)) self.assertEqual(rows[1], ('Bharney Rhubble', 33, WHEN_1)) self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, WHEN_2)) self.assertEqual(rows[3], ('Bhettye Rhubble', None, None)) self.assertEqual(total_rows, ROWS) self.assertEqual(page_token, TOKEN) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) def test_fetch_data_w_alternate_client(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) MAX = 10 TOKEN = 'TOKEN' DATA = { 'rows': [ {'f': [ {'v': 'Phred Phlyntstone'}, {'v': '32'}, {'v': 'true'}, {'v': '3.1415926'}, ]}, {'f': [ {'v': 'Bharney Rhubble'}, {'v': '33'}, {'v': 'false'}, {'v': '1.414'}, ]}, {'f': [ {'v': 'Wylma Phlyntstone'}, {'v': '29'}, {'v': 'true'}, {'v': '2.71828'}, ]}, {'f': [ {'v': 'Bhettye Rhubble'}, {'v': '27'}, {'v': None}, {'v': None}, ]}, ] } conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection(DATA) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') voter = SchemaField('voter', 'BOOLEAN', mode='NULLABLE') score = SchemaField('score', 'FLOAT', mode='NULLABLE') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age, voter, score]) rows, total_rows, page_token = table.fetch_data(client=client2, max_results=MAX, page_token=TOKEN) self.assertEqual(len(rows), 4) self.assertEqual(rows[0], ('Phred Phlyntstone', 32, True, 3.1415926)) self.assertEqual(rows[1], ('Bharney Rhubble', 33, False, 1.414)) self.assertEqual(rows[2], ('Wylma Phlyntstone', 29, True, 2.71828)) self.assertEqual(rows[3], ('Bhettye Rhubble', 27, None, None)) self.assertEqual(total_rows, None) self.assertEqual(page_token, None) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['query_params'], {'maxResults': MAX, 'pageToken': TOKEN}) def test_fetch_data_w_repeated_fields(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) ROWS = 1234 TOKEN = 'TOKEN' DATA = { 'totalRows': ROWS, 'pageToken': TOKEN, 'rows': [ {'f': [ {'v': ['red', 'green']}, {'v': [{'f': [{'v': ['1', '2']}, {'v': ['3.1415', '1.414']}]}]}, ]}, ] } conn = _Connection(DATA) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('color', 'STRING', mode='REPEATED') index = SchemaField('index', 'INTEGER', 'REPEATED') score = SchemaField('score', 'FLOAT', 'REPEATED') struct = SchemaField('struct', 'RECORD', mode='REPEATED', fields=[index, score]) table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, struct]) rows, total_rows, page_token = table.fetch_data() self.assertEqual(len(rows), 1) self.assertEqual(rows[0][0], ['red', 'green']) self.assertEqual(rows[0][1], [{'index': [1, 2], 'score': [3.1415, 1.414]}]) self.assertEqual(total_rows, ROWS) self.assertEqual(page_token, TOKEN) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) def test_fetch_data_w_record_schema(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/data' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) ROWS = 1234 TOKEN = 'TOKEN' DATA = { 'totalRows': ROWS, 'pageToken': TOKEN, 'rows': [ {'f': [ {'v': 'Phred Phlyntstone'}, {'v': {'f': [{'v': '800'}, {'v': '555-1212'}, {'v': 1}]}}, ]}, {'f': [ {'v': 'Bharney Rhubble'}, {'v': {'f': [{'v': '877'}, {'v': '768-5309'}, {'v': 2}]}}, ]}, {'f': [ {'v': 'Wylma Phlyntstone'}, {'v': None}, ]}, ] } conn = _Connection(DATA) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') area_code = SchemaField('area_code', 'STRING', 'REQUIRED') local_number = SchemaField('local_number', 'STRING', 'REQUIRED') rank = SchemaField('rank', 'INTEGER', 'REQUIRED') phone = SchemaField('phone', 'RECORD', mode='NULLABLE', fields=[area_code, local_number, rank]) table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, phone]) rows, total_rows, page_token = table.fetch_data() self.assertEqual(len(rows), 3) self.assertEqual(rows[0][0], 'Phred Phlyntstone') self.assertEqual(rows[0][1], {'area_code': '800', 'local_number': '555-1212', 'rank': 1}) self.assertEqual(rows[1][0], 'Bharney Rhubble') self.assertEqual(rows[1][1], {'area_code': '877', 'local_number': '768-5309', 'rank': 2}) self.assertEqual(rows[2][0], 'Wylma Phlyntstone') self.assertEqual(rows[2][1], None) self.assertEqual(total_rows, ROWS) self.assertEqual(page_token, TOKEN) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'GET') self.assertEqual(req['path'], '/%s' % PATH) def test_insert_data_w_bound_client(self): import datetime from gcloud._helpers import UTC from gcloud._helpers import _microseconds_from_datetime from gcloud.bigquery.table import SchemaField WHEN_TS = 1437767599.006 WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace( tzinfo=UTC) PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn = _Connection({}) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age, joined]) ROWS = [ ('Phred Phlyntstone', 32, WHEN), ('Bharney Rhubble', 33, WHEN + datetime.timedelta(seconds=1)), ('Wylma Phlyntstone', 29, WHEN + datetime.timedelta(seconds=2)), ('Bhettye Rhubble', 27, None), ] def _row_data(row): joined = None if row[2] is not None: joined = _microseconds_from_datetime(row[2]) * 1e-6 return {'full_name': row[0], 'age': row[1], 'joined': joined} SENT = { 'rows': [{'json': _row_data(row)} for row in ROWS], } errors = table.insert_data(ROWS) self.assertEqual(len(errors), 0) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['data'], SENT) def test_insert_data_w_alternate_client(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) RESPONSE = { 'insertErrors': [ {'index': 1, 'errors': [ {'reason': 'REASON', 'location': 'LOCATION', 'debugInfo': 'INFO', 'message': 'MESSAGE'} ]}, ]} conn1 = _Connection() client1 = _Client(project=self.PROJECT, connection=conn1) conn2 = _Connection(RESPONSE) client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') voter = SchemaField('voter', 'BOOLEAN', mode='NULLABLE') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age, voter]) ROWS = [ ('Phred Phlyntstone', 32, True), ('Bharney Rhubble', 33, False), ('Wylma Phlyntstone', 29, True), ('Bhettye Rhubble', 27, True), ] def _row_data(row): return {'full_name': row[0], 'age': row[1], 'voter': row[2]} SENT = { 'skipInvalidRows': True, 'ignoreUnknownValues': True, 'templateSuffix': '20160303', 'rows': [{'insertId': index, 'json': _row_data(row)} for index, row in enumerate(ROWS)], } errors = table.insert_data( client=client2, rows=ROWS, row_ids=[index for index, _ in enumerate(ROWS)], skip_invalid_rows=True, ignore_unknown_values=True, template_suffix='20160303', ) self.assertEqual(len(errors), 1) self.assertEqual(errors[0]['index'], 1) self.assertEqual(len(errors[0]['errors']), 1) self.assertEqual(errors[0]['errors'][0], RESPONSE['insertErrors'][0]['errors'][0]) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) req = conn2._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['data'], SENT) def test_insert_data_w_repeated_fields(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn = _Connection({}) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('color', 'STRING', mode='REPEATED') index = SchemaField('index', 'INTEGER', 'REPEATED') score = SchemaField('score', 'FLOAT', 'REPEATED') struct = SchemaField('struct', 'RECORD', mode='REPEATED', fields=[index, score]) table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, struct]) ROWS = [ (['red', 'green'], [{'index': [1, 2], 'score': [3.1415, 1.414]}]), ] def _row_data(row): return {'color': row[0], 'struct': row[1]} SENT = { 'rows': [{'json': _row_data(row)} for row in ROWS], } errors = table.insert_data(ROWS) self.assertEqual(len(errors), 0) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['data'], SENT) def test_insert_data_w_record_schema(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s/insertAll' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) conn = _Connection({}) client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') area_code = SchemaField('area_code', 'STRING', 'REQUIRED') local_number = SchemaField('local_number', 'STRING', 'REQUIRED') rank = SchemaField('rank', 'INTEGER', 'REQUIRED') phone = SchemaField('phone', 'RECORD', mode='NULLABLE', fields=[area_code, local_number, rank]) table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, phone]) ROWS = [ ('Phred Phlyntstone', {'area_code': '800', 'local_number': '555-1212', 'rank': 1}), ('Bharney Rhubble', {'area_code': '877', 'local_number': '768-5309', 'rank': 2}), ('Wylma Phlyntstone', None), ] def _row_data(row): return {'full_name': row[0], 'phone': row[1]} SENT = { 'rows': [{'json': _row_data(row)} for row in ROWS], } errors = table.insert_data(ROWS) self.assertEqual(len(errors), 0) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] self.assertEqual(req['method'], 'POST') self.assertEqual(req['path'], '/%s' % PATH) self.assertEqual(req['data'], SENT) def test_upload_from_file_text_mode_file_failure(self): class TextModeFile(object): mode = 'r' conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) file_obj = TextModeFile() table = self._makeOne(self.TABLE_NAME, dataset=dataset) with self.assertRaises(ValueError): table.upload_from_file(file_obj, 'CSV', size=1234) def test_upload_from_file_size_failure(self): conn = _Connection() client = _Client(project=self.PROJECT, connection=conn) dataset = _Dataset(client) file_obj = object() table = self._makeOne(self.TABLE_NAME, dataset=dataset) with self.assertRaises(ValueError): table.upload_from_file(file_obj, 'CSV', size=None) def _upload_from_file_helper(self, **kw): import csv import datetime from six.moves.http_client import OK from gcloud._helpers import UTC from gcloud._testing import _NamedTemporaryFile from gcloud.bigquery.table import SchemaField WHEN_TS = 1437767599.006 WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace( tzinfo=UTC) PATH = 'projects/%s/jobs' % (self.PROJECT,) response = {'status': OK} conn = _Connection( (response, b'{}'), ) client = _Client(project=self.PROJECT, connection=conn) expected_job = object() if 'client' in kw: kw['client']._job = expected_job else: client._job = expected_job dataset = _Dataset(client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE') table = self._makeOne(self.TABLE_NAME, dataset=dataset, schema=[full_name, age, joined]) ROWS = [ ('Phred Phlyntstone', 32, WHEN), ('Bharney Rhubble', 33, WHEN + datetime.timedelta(seconds=1)), ('Wylma Phlyntstone', 29, WHEN + datetime.timedelta(seconds=2)), ('Bhettye Rhubble', 27, None), ] with _NamedTemporaryFile() as temp: with open(temp.name, 'w') as file_obj: writer = csv.writer(file_obj) writer.writerow(('full_name', 'age', 'joined')) writer.writerows(ROWS) with open(temp.name, 'rb') as file_obj: BODY = file_obj.read() explicit_size = kw.pop('_explicit_size', False) if explicit_size: kw['size'] = len(BODY) job = table.upload_from_file( file_obj, 'CSV', rewind=True, **kw) self.assertTrue(job is expected_job) return conn.http._requested, PATH, BODY def test_upload_from_file_w_bound_client_multipart(self): import json from six.moves.urllib.parse import parse_qsl from six.moves.urllib.parse import urlsplit from gcloud._helpers import _to_bytes from gcloud.streaming.test_transfer import _email_chunk_parser requested, PATH, BODY = self._upload_from_file_helper() parse_chunk = _email_chunk_parser() self.assertEqual(len(requested), 1) req = requested[0] self.assertEqual(req['method'], 'POST') uri = req['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') self.assertEqual(path, '/%s' % PATH) self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'multipart'}) ctype, boundary = [x.strip() for x in req['headers']['content-type'].split(';')] self.assertEqual(ctype, 'multipart/related') self.assertTrue(boundary.startswith('boundary="==')) self.assertTrue(boundary.endswith('=="')) divider = b'--' + _to_bytes(boundary[len('boundary="'):-1]) chunks = req['body'].split(divider)[1:-1] # discard prolog / epilog self.assertEqual(len(chunks), 2) text_msg = parse_chunk(chunks[0].strip()) self.assertEqual(dict(text_msg._headers), {'Content-Type': 'application/json', 'MIME-Version': '1.0'}) metadata = json.loads(text_msg._payload) load_config = metadata['configuration']['load'] DESTINATION_TABLE = { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME, } self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE) self.assertEqual(load_config['sourceFormat'], 'CSV') app_msg = parse_chunk(chunks[1].strip()) self.assertEqual(dict(app_msg._headers), {'Content-Type': 'application/octet-stream', 'Content-Transfer-Encoding': 'binary', 'MIME-Version': '1.0'}) body = BODY.decode('ascii').rstrip() body_lines = [line.strip() for line in body.splitlines()] payload_lines = app_msg._payload.rstrip().splitlines() self.assertEqual(payload_lines, body_lines) # pylint: disable=too-many-statements def test_upload_from_file_w_explicit_client_resumable(self): import json from six.moves.http_client import OK from six.moves.urllib.parse import parse_qsl from six.moves.urllib.parse import urlsplit from gcloud._testing import _Monkey from gcloud.bigquery import table as MUT UPLOAD_PATH = 'https://example.com/upload/test' initial_response = {'status': OK, 'location': UPLOAD_PATH} upload_response = {'status': OK} conn = _Connection( (initial_response, b'{}'), (upload_response, b'{}'), ) client = _Client(project=self.PROJECT, connection=conn) class _UploadConfig(object): accept = ['*/*'] max_size = None resumable_multipart = True resumable_path = u'/upload/bigquery/v2/projects/{project}/jobs' simple_multipart = True simple_path = u'' # force resumable with _Monkey(MUT, _UploadConfig=_UploadConfig): orig_requested, PATH, BODY = self._upload_from_file_helper( allow_jagged_rows=False, allow_quoted_newlines=False, create_disposition='CREATE_IF_NEEDED', encoding='utf8', field_delimiter=',', ignore_unknown_values=False, max_bad_records=0, quote_character='"', skip_leading_rows=1, write_disposition='WRITE_APPEND', client=client, _explicit_size=True) self.assertEqual(len(orig_requested), 0) requested = conn.http._requested self.assertEqual(len(requested), 2) req = requested[0] self.assertEqual(req['method'], 'POST') uri = req['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') self.assertEqual(path, '/%s' % PATH) self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'resumable'}) self.assertEqual(req['headers']['content-type'], 'application/json') metadata = json.loads(req['body']) load_config = metadata['configuration']['load'] DESTINATION_TABLE = { 'projectId': self.PROJECT, 'datasetId': self.DS_NAME, 'tableId': self.TABLE_NAME, } self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE) self.assertEqual(load_config['sourceFormat'], 'CSV') self.assertEqual(load_config['allowJaggedRows'], False) self.assertEqual(load_config['allowQuotedNewlines'], False) self.assertEqual(load_config['createDisposition'], 'CREATE_IF_NEEDED') self.assertEqual(load_config['encoding'], 'utf8') self.assertEqual(load_config['fieldDelimiter'], ',') self.assertEqual(load_config['ignoreUnknownValues'], False) self.assertEqual(load_config['maxBadRecords'], 0) self.assertEqual(load_config['quote'], '"') self.assertEqual(load_config['skipLeadingRows'], 1) self.assertEqual(load_config['writeDisposition'], 'WRITE_APPEND') req = requested[1] self.assertEqual(req['method'], 'PUT') self.assertEqual(req['uri'], UPLOAD_PATH) headers = req['headers'] length = len(BODY) self.assertEqual(headers['Content-Type'], 'application/octet-stream') self.assertEqual(headers['Content-Range'], 'bytes 0-%d/%d' % (length - 1, length)) self.assertEqual(headers['content-length'], '%d' % (length,)) self.assertEqual(req['body'], BODY) # pylint: enable=too-many-statements class Test_parse_schema_resource(unittest2.TestCase, _SchemaBase): def _callFUT(self, resource): from gcloud.bigquery.table import _parse_schema_resource return _parse_schema_resource(resource) def _makeResource(self): return { 'schema': {'fields': [ {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}, ]}, } def test__parse_schema_resource_defaults(self): RESOURCE = self._makeResource() schema = self._callFUT(RESOURCE['schema']) self._verifySchema(schema, RESOURCE) def test__parse_schema_resource_subfields(self): RESOURCE = self._makeResource() RESOURCE['schema']['fields'].append( {'name': 'phone', 'type': 'RECORD', 'mode': 'REPEATABLE', 'fields': [{'name': 'type', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'number', 'type': 'STRING', 'mode': 'REQUIRED'}]}) schema = self._callFUT(RESOURCE['schema']) self._verifySchema(schema, RESOURCE) def test__parse_schema_resource_fields_without_mode(self): RESOURCE = self._makeResource() RESOURCE['schema']['fields'].append( {'name': 'phone', 'type': 'STRING'}) schema = self._callFUT(RESOURCE['schema']) self._verifySchema(schema, RESOURCE) class Test_build_schema_resource(unittest2.TestCase, _SchemaBase): def _callFUT(self, resource): from gcloud.bigquery.table import _build_schema_resource return _build_schema_resource(resource) def test_defaults(self): from gcloud.bigquery.table import SchemaField full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') resource = self._callFUT([full_name, age]) self.assertEqual(len(resource), 2) self.assertEqual(resource[0], {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}) self.assertEqual(resource[1], {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}) def test_w_description(self): from gcloud.bigquery.table import SchemaField DESCRIPTION = 'DESCRIPTION' full_name = SchemaField('full_name', 'STRING', mode='REQUIRED', description=DESCRIPTION) age = SchemaField('age', 'INTEGER', mode='REQUIRED') resource = self._callFUT([full_name, age]) self.assertEqual(len(resource), 2) self.assertEqual(resource[0], {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED', 'description': DESCRIPTION}) self.assertEqual(resource[1], {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}) def test_w_subfields(self): from gcloud.bigquery.table import SchemaField full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') ph_type = SchemaField('type', 'STRING', 'REQUIRED') ph_num = SchemaField('number', 'STRING', 'REQUIRED') phone = SchemaField('phone', 'RECORD', mode='REPEATABLE', fields=[ph_type, ph_num]) resource = self._callFUT([full_name, phone]) self.assertEqual(len(resource), 2) self.assertEqual(resource[0], {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}) self.assertEqual(resource[1], {'name': 'phone', 'type': 'RECORD', 'mode': 'REPEATABLE', 'fields': [{'name': 'type', 'type': 'STRING', 'mode': 'REQUIRED'}, {'name': 'number', 'type': 'STRING', 'mode': 'REQUIRED'}]}) class _Client(object): def __init__(self, project='project', connection=None): self.project = project self.connection = connection def job_from_resource(self, resource): # pylint: disable=unused-argument return self._job class _Dataset(object): def __init__(self, client, name=TestTable.DS_NAME): self._client = client self.name = name @property def path(self): return '/projects/%s/datasets/%s' % ( self._client.project, self.name) @property def project(self): return self._client.project class _Responder(object): def __init__(self, *responses): self._responses = responses[:] self._requested = [] def _respond(self, **kw): self._requested.append(kw) response, self._responses = self._responses[0], self._responses[1:] return response class _HTTP(_Responder): connections = {} # For google-apitools debugging. def request(self, uri, method, headers, body, **kw): if hasattr(body, 'read'): body = body.read() return self._respond(uri=uri, method=method, headers=headers, body=body, **kw) class _Connection(_Responder): API_BASE_URL = 'http://example.com' USER_AGENT = 'testing 1.2.3' def __init__(self, *responses): super(_Connection, self).__init__(*responses) self.http = _HTTP(*responses) def api_request(self, **kw): from gcloud.exceptions import NotFound self._requested.append(kw) try: response, self._responses = self._responses[0], self._responses[1:] except: raise NotFound('miss') else: return response def build_api_url(self, path, query_params=None, api_base_url=API_BASE_URL): from six.moves.urllib.parse import urlencode from six.moves.urllib.parse import urlsplit from six.moves.urllib.parse import urlunsplit # Mimic the build_api_url interface. qs = urlencode(query_params or {}) scheme, netloc, _, _, _ = urlsplit(api_base_url) return urlunsplit((scheme, netloc, path, qs, ''))