我正在尝试将Apache光束输出写入Bigquery Table,但它的下载失败,如下所示 . 请帮助 .
代码:
import logging
import apache_beam as beam
from mongodbio import ReadFromMongo
TABLE_SCHEMA = ('clause_type:STRING, count:INTEGER')
def transform_doc(document):
data={str(document['clause_type']):int(document['count'])}
return data
def run():
connection_string = 'mongodb://localhost:27017'
with beam.Pipeline(runner='DirectRunner') as pipeline:
(pipeline
| 'read' >> ReadFromMongo(connection_string, 'new', 'Data',
query={}, fields=['count','clause_type'])
| 'transform' >>
beam.Map(transform_doc).with_output_types(str)
| 'save' >> beam.io.WriteToBigQuery('<table_name>',schema=TABLE_SCHEMA,create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND))
if __name__ == '__main__':
logging.getLogger().setLevel(logging.DEBUG)
run()
错误:
index: <index number>>, <InsertErrorsValueListEntry
errors: [<ErrorProto
debugInfo: u'generic::not_found: no such field.'
location: u'fiscal year'
message: u'no such field.'
reason: u'invalid'>]