Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
E
edx-analytics-data-api
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
edx-analytics-data-api
Commits
80d99fb7
Commit
80d99fb7
authored
Jun 01, 2014
by
Brian Wilson
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix bug in generate_s3_sources when root is empty.
Change-Id: I0297e92c22496a1443a3e4e9ea05acd7d7af95fd
parent
7832943b
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
19 additions
and
6 deletions
+19
-6
edx/analytics/tasks/s3_util.py
+1
-1
edx/analytics/tasks/tests/test_s3_util.py
+18
-5
No files found.
edx/analytics/tasks/s3_util.py
View file @
80d99fb7
...
...
@@ -79,7 +79,7 @@ def generate_s3_sources(s3_conn, source, patterns):
# Make sure that the listing is done on a "folder" boundary,
# since list() just looks for matching prefixes.
root_with_slash
=
root
+
'/'
if
root
[
-
1
]
!=
'/'
else
root
root_with_slash
=
root
if
len
(
root
)
==
0
or
root
.
endswith
(
'/'
)
else
root
+
'/'
# Skip keys that have zero size. This allows directories
# to be skipped, but also skips legitimate files that are
...
...
edx/analytics/tasks/tests/test_s3_util.py
View file @
80d99fb7
...
...
@@ -31,6 +31,14 @@ class GenerateS3SourcesTestCase(unittest.TestCase):
output
=
list
(
generator
)
return
output
def
_run_without_filtering
(
self
,
bucket_name
,
root
,
path_info
):
"""Runs generator and checks output."""
patterns
=
[
'*'
]
output
=
self
.
_make_s3_generator
(
bucket_name
,
root
,
path_info
,
patterns
)
self
.
assertEquals
(
len
(
output
),
len
(
path_info
))
expected
=
[(
bucket_name
,
root
,
key
)
for
key
in
path_info
]
self
.
assertEquals
(
set
(
output
),
set
(
expected
))
def
test_normal_generate
(
self
):
bucket_name
=
"bucket_name"
root
=
"root1/root2"
...
...
@@ -38,11 +46,16 @@ class GenerateS3SourcesTestCase(unittest.TestCase):
"subdir1/path1"
:
1000
,
"path2"
:
2000
,
}
patterns
=
[
'*'
]
output
=
self
.
_make_s3_generator
(
bucket_name
,
root
,
path_info
,
patterns
)
self
.
assertEquals
(
len
(
output
),
2
)
self
.
assertEquals
(
set
(
output
),
set
([(
bucket_name
,
root
,
"subdir1/path1"
),
(
bucket_name
,
root
,
"path2"
)]))
self
.
_run_without_filtering
(
bucket_name
,
root
,
path_info
)
def
test_generate_with_empty_root
(
self
):
bucket_name
=
"bucket_name"
root
=
""
path_info
=
{
"subdir1/path1"
:
1000
,
"path2"
:
2000
,
}
self
.
_run_without_filtering
(
bucket_name
,
root
,
path_info
)
def
test_generate_with_pattern_filtering
(
self
):
bucket_name
=
"bucket_name"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment