Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
E
edx-analytics-pipeline
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
edx-analytics-pipeline
Commits
95865b2f
Commit
95865b2f
authored
May 31, 2017
by
Hassan Javeed
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix table_updates entry.
parent
ba890d35
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
7 additions
and
3 deletions
+7
-3
edx/analytics/tasks/common/bigquery_load.py
+7
-3
No files found.
edx/analytics/tasks/common/bigquery_load.py
View file @
95865b2f
...
@@ -60,17 +60,21 @@ class BigQueryTarget(luigi.Target):
...
@@ -60,17 +60,21 @@ class BigQueryTarget(luigi.Target):
# Use a tempfile for loading data into table_updates
# Use a tempfile for loading data into table_updates
# We deliberately don't use table.insert_data as it we cannot use delete on
# We deliberately don't use table.insert_data as it we cannot use delete on
# a bigquery table with streaming inserts.
# a bigquery table with streaming inserts.
tmp
=
tempfile
.
NamedTemporaryFile
(
bufsize
=
0
)
tmp
=
tempfile
.
NamedTemporaryFile
(
delete
=
False
)
table_update_row
=
(
self
.
update_id
,
"{dataset}.{table}"
.
format
(
dataset
=
self
.
dataset_id
,
table
=
self
.
table
))
table_update_row
=
(
self
.
update_id
,
"{dataset}.{table}"
.
format
(
dataset
=
self
.
dataset_id
,
table
=
self
.
table
))
tmp
.
write
(
','
.
join
(
table_update_row
))
tmp
.
write
(
','
.
join
(
table_update_row
))
tmp
.
close
()
tmp
.
seek
(
0
)
job
=
table
.
upload_from_file
(
tmp
,
source_format
=
'text/csv'
)
# table.upload_from_file requires the file to be opened in 'rb' mode
with
open
(
tmp
.
name
,
'rb'
)
as
source_file
:
job
=
table
.
upload_from_file
(
source_file
,
source_format
=
'text/csv'
)
try
:
try
:
wait_for_job
(
job
)
wait_for_job
(
job
)
finally
:
finally
:
tmp
.
close
()
tmp
.
close
()
os
.
unlink
(
tmp
.
name
)
def
create_marker_table
(
self
):
def
create_marker_table
(
self
):
marker_table_schema
=
[
marker_table_schema
=
[
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment