Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
E
edx-video-pipeline
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
edx-video-pipeline
Commits
921a3e2b
Commit
921a3e2b
authored
Dec 06, 2017
by
Gregory Martin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Deployment Bugfix
parent
f579279e
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
5 additions
and
1 deletions
+5
-1
control/veda_file_discovery.py
+1
-0
youtube_callback/sftp_id_retrieve.py
+4
-1
No files found.
control/veda_file_discovery.py
View file @
921a3e2b
...
@@ -249,6 +249,7 @@ class FileDiscovery(object):
...
@@ -249,6 +249,7 @@ class FileDiscovery(object):
connection
=
boto
.
connect_s3
()
connection
=
boto
.
connect_s3
()
self
.
bucket
=
connection
.
get_bucket
(
self
.
auth_dict
[
'edx_s3_ingest_bucket'
])
self
.
bucket
=
connection
.
get_bucket
(
self
.
auth_dict
[
'edx_s3_ingest_bucket'
])
for
video_s3_key
in
self
.
bucket
.
list
(
self
.
auth_dict
[
'edx_s3_ingest_prefix'
],
'/'
):
for
video_s3_key
in
self
.
bucket
.
list
(
self
.
auth_dict
[
'edx_s3_ingest_prefix'
],
'/'
):
if
video_s3_key
.
name
!=
'prod-edx/unprocessed/'
:
self
.
validate_metadata_and_feed_to_ingest
(
video_s3_key
=
self
.
bucket
.
get_key
(
video_s3_key
.
name
))
self
.
validate_metadata_and_feed_to_ingest
(
video_s3_key
=
self
.
bucket
.
get_key
(
video_s3_key
.
name
))
except
S3ResponseError
:
except
S3ResponseError
:
ErrorObject
.
print_error
(
message
=
'[File Ingest] S3 Ingest Connection Failure'
)
ErrorObject
.
print_error
(
message
=
'[File Ingest] S3 Ingest Connection Failure'
)
...
...
youtube_callback/sftp_id_retrieve.py
View file @
921a3e2b
...
@@ -10,6 +10,7 @@ import sys
...
@@ -10,6 +10,7 @@ import sys
import
xml.etree.ElementTree
as
ET
import
xml.etree.ElementTree
as
ET
from
datetime
import
timedelta
from
datetime
import
timedelta
from
os.path
import
expanduser
from
os.path
import
expanduser
from
paramiko.ssh_exception
import
AuthenticationException
import
django
import
django
import
pysftp
import
pysftp
...
@@ -73,7 +74,7 @@ def xml_downloader(course):
...
@@ -73,7 +74,7 @@ def xml_downloader(course):
cnopts
=
pysftp
.
CnOpts
()
cnopts
=
pysftp
.
CnOpts
()
cnopts
.
hostkeys
=
None
cnopts
.
hostkeys
=
None
try
:
with
pysftp
.
Connection
(
with
pysftp
.
Connection
(
'partnerupload.google.com'
,
'partnerupload.google.com'
,
username
=
course
.
yt_logon
,
username
=
course
.
yt_logon
,
...
@@ -84,6 +85,8 @@ def xml_downloader(course):
...
@@ -84,6 +85,8 @@ def xml_downloader(course):
s1
.
timeout
=
60.0
s1
.
timeout
=
60.0
for
d
in
s1
.
listdir_attr
():
for
d
in
s1
.
listdir_attr
():
crawl_sftp
(
d
=
d
,
s1
=
s1
)
crawl_sftp
(
d
=
d
,
s1
=
s1
)
except
AuthenticationException
:
print
"{inst}{clss} : Authentication Failed"
.
format
(
inst
=
course
.
institution
,
clss
=
course
.
edx_classid
)
def
crawl_sftp
(
d
,
s1
):
def
crawl_sftp
(
d
,
s1
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment