Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
E
edx-platform
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
edx-platform
Commits
f94530ba
Commit
f94530ba
authored
Jul 15, 2016
by
Calen Pennington
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[EV-62] Add the ability to log timing information during paver tasks
parent
4fca1d9d
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
334 additions
and
1 deletions
+334
-1
pavelib/assets.py
+5
-0
pavelib/bok_choy.py
+8
-0
pavelib/docs.py
+3
-0
pavelib/i18n.py
+17
-0
pavelib/js_test.py
+4
-0
pavelib/paver_tests/test_timer.py
+188
-0
pavelib/prereqs.py
+5
-0
pavelib/quality.py
+9
-0
pavelib/servers.py
+3
-0
pavelib/tests.py
+8
-1
pavelib/utils/test/utils.py
+4
-0
pavelib/utils/timer.py
+80
-0
No files found.
pavelib/assets.py
View file @
f94530ba
...
@@ -17,6 +17,7 @@ from watchdog.events import PatternMatchingEventHandler
...
@@ -17,6 +17,7 @@ from watchdog.events import PatternMatchingEventHandler
from
.utils.envs
import
Env
from
.utils.envs
import
Env
from
.utils.cmd
import
cmd
,
django_cmd
from
.utils.cmd
import
cmd
,
django_cmd
from
.utils.timer
import
timed
from
openedx.core.djangoapps.theming.paver_helpers
import
get_theme_paths
from
openedx.core.djangoapps.theming.paver_helpers
import
get_theme_paths
...
@@ -383,6 +384,7 @@ def coffeescript_files():
...
@@ -383,6 +384,7 @@ def coffeescript_files():
@task
@task
@no_help
@no_help
@timed
def
compile_coffeescript
(
*
files
):
def
compile_coffeescript
(
*
files
):
"""
"""
Compile CoffeeScript to JavaScript.
Compile CoffeeScript to JavaScript.
...
@@ -403,6 +405,7 @@ def compile_coffeescript(*files):
...
@@ -403,6 +405,7 @@ def compile_coffeescript(*files):
(
'debug'
,
'd'
,
'Debug mode'
),
(
'debug'
,
'd'
,
'Debug mode'
),
(
'force'
,
''
,
'Force full compilation'
),
(
'force'
,
''
,
'Force full compilation'
),
])
])
@timed
def
compile_sass
(
options
):
def
compile_sass
(
options
):
"""
"""
Compile Sass to CSS. If command is called without any arguments, it will
Compile Sass to CSS. If command is called without any arguments, it will
...
@@ -682,6 +685,7 @@ def execute_compile_sass(args):
...
@@ -682,6 +685,7 @@ def execute_compile_sass(args):
(
'theme-dirs='
,
'-td'
,
'The themes dir containing all themes (defaults to None)'
),
(
'theme-dirs='
,
'-td'
,
'The themes dir containing all themes (defaults to None)'
),
(
'themes='
,
'-t'
,
'The themes to add sass watchers for (defaults to None)'
),
(
'themes='
,
'-t'
,
'The themes to add sass watchers for (defaults to None)'
),
])
])
@timed
def
watch_assets
(
options
):
def
watch_assets
(
options
):
"""
"""
Watch for changes to asset files, and regenerate js/css
Watch for changes to asset files, and regenerate js/css
...
@@ -730,6 +734,7 @@ def watch_assets(options):
...
@@ -730,6 +734,7 @@ def watch_assets(options):
'pavelib.prereqs.install_node_prereqs'
,
'pavelib.prereqs.install_node_prereqs'
,
)
)
@consume_args
@consume_args
@timed
def
update_assets
(
args
):
def
update_assets
(
args
):
"""
"""
Compile CoffeeScript and Sass, then collect static assets.
Compile CoffeeScript and Sass, then collect static assets.
...
...
pavelib/bok_choy.py
View file @
f94530ba
...
@@ -7,6 +7,7 @@ from pavelib.utils.test.suites.bokchoy_suite import BokChoyTestSuite, Pa11yCrawl
...
@@ -7,6 +7,7 @@ from pavelib.utils.test.suites.bokchoy_suite import BokChoyTestSuite, Pa11yCrawl
from
pavelib.utils.envs
import
Env
from
pavelib.utils.envs
import
Env
from
pavelib.utils.test.utils
import
check_firefox_version
from
pavelib.utils.test.utils
import
check_firefox_version
from
pavelib.utils.passthrough_opts
import
PassthroughTask
from
pavelib.utils.passthrough_opts
import
PassthroughTask
from
pavelib.utils.timer
import
timed
from
optparse
import
make_option
from
optparse
import
make_option
import
os
import
os
...
@@ -86,6 +87,7 @@ def parse_bokchoy_opts(options, passthrough_options=None):
...
@@ -86,6 +87,7 @@ def parse_bokchoy_opts(options, passthrough_options=None):
@needs
(
'pavelib.prereqs.install_prereqs'
)
@needs
(
'pavelib.prereqs.install_prereqs'
)
@cmdopts
(
BOKCHOY_OPTS
)
@cmdopts
(
BOKCHOY_OPTS
)
@PassthroughTask
@PassthroughTask
@timed
def
test_bokchoy
(
options
,
passthrough_options
):
def
test_bokchoy
(
options
,
passthrough_options
):
"""
"""
Run acceptance tests that use the bok-choy framework.
Run acceptance tests that use the bok-choy framework.
...
@@ -116,6 +118,7 @@ def test_bokchoy(options, passthrough_options):
...
@@ -116,6 +118,7 @@ def test_bokchoy(options, passthrough_options):
@needs
(
'pavelib.prereqs.install_prereqs'
)
@needs
(
'pavelib.prereqs.install_prereqs'
)
@cmdopts
(
BOKCHOY_OPTS
)
@cmdopts
(
BOKCHOY_OPTS
)
@PassthroughTask
@PassthroughTask
@timed
def
test_a11y
(
options
,
passthrough_options
):
def
test_a11y
(
options
,
passthrough_options
):
"""
"""
Run accessibility tests that use the bok-choy framework.
Run accessibility tests that use the bok-choy framework.
...
@@ -142,6 +145,7 @@ def test_a11y(options, passthrough_options):
...
@@ -142,6 +145,7 @@ def test_a11y(options, passthrough_options):
@needs
(
'pavelib.prereqs.install_prereqs'
)
@needs
(
'pavelib.prereqs.install_prereqs'
)
@cmdopts
(
BOKCHOY_OPTS
)
@cmdopts
(
BOKCHOY_OPTS
)
@PassthroughTask
@PassthroughTask
@timed
def
perf_report_bokchoy
(
options
,
passthrough_options
):
def
perf_report_bokchoy
(
options
,
passthrough_options
):
"""
"""
Generates a har file for with page performance info.
Generates a har file for with page performance info.
...
@@ -164,6 +168,7 @@ def perf_report_bokchoy(options, passthrough_options):
...
@@ -164,6 +168,7 @@ def perf_report_bokchoy(options, passthrough_options):
),
),
])
])
@PassthroughTask
@PassthroughTask
@timed
def
pa11ycrawler
(
options
,
passthrough_options
):
def
pa11ycrawler
(
options
,
passthrough_options
):
"""
"""
Runs pa11ycrawler against the demo-test-course to generates accessibility
Runs pa11ycrawler against the demo-test-course to generates accessibility
...
@@ -220,6 +225,7 @@ def parse_coverage(report_dir, coveragerc):
...
@@ -220,6 +225,7 @@ def parse_coverage(report_dir, coveragerc):
@task
@task
@timed
def
bokchoy_coverage
():
def
bokchoy_coverage
():
"""
"""
Generate coverage reports for bok-choy tests
Generate coverage reports for bok-choy tests
...
@@ -231,6 +237,7 @@ def bokchoy_coverage():
...
@@ -231,6 +237,7 @@ def bokchoy_coverage():
@task
@task
@timed
def
a11y_coverage
():
def
a11y_coverage
():
"""
"""
Generate coverage reports for a11y tests. Note that this coverage report
Generate coverage reports for a11y tests. Note that this coverage report
...
@@ -246,6 +253,7 @@ def a11y_coverage():
...
@@ -246,6 +253,7 @@ def a11y_coverage():
@task
@task
@timed
def
pa11ycrawler_coverage
():
def
pa11ycrawler_coverage
():
"""
"""
Generate coverage reports for bok-choy tests
Generate coverage reports for bok-choy tests
...
...
pavelib/docs.py
View file @
f94530ba
...
@@ -7,6 +7,8 @@ import sys
...
@@ -7,6 +7,8 @@ import sys
from
paver.easy
import
cmdopts
,
needs
,
sh
,
task
from
paver.easy
import
cmdopts
,
needs
,
sh
,
task
from
.utils.timer
import
timed
DOC_PATHS
=
{
DOC_PATHS
=
{
"dev"
:
"docs/en_us/developers"
,
"dev"
:
"docs/en_us/developers"
,
...
@@ -64,6 +66,7 @@ def doc_path(options, allow_default=True):
...
@@ -64,6 +66,7 @@ def doc_path(options, allow_default=True):
(
"type="
,
"t"
,
"Type of docs to compile"
),
(
"type="
,
"t"
,
"Type of docs to compile"
),
(
"verbose"
,
"v"
,
"Display verbose output"
),
(
"verbose"
,
"v"
,
"Display verbose output"
),
])
])
@timed
def
build_docs
(
options
):
def
build_docs
(
options
):
"""
"""
Invoke sphinx 'make build' to generate docs.
Invoke sphinx 'make build' to generate docs.
...
...
pavelib/i18n.py
View file @
f94530ba
...
@@ -10,6 +10,7 @@ from path import Path as path
...
@@ -10,6 +10,7 @@ from path import Path as path
from
paver.easy
import
task
,
cmdopts
,
needs
,
sh
from
paver.easy
import
task
,
cmdopts
,
needs
,
sh
from
.utils.cmd
import
django_cmd
from
.utils.cmd
import
django_cmd
from
.utils.timer
import
timed
try
:
try
:
from
pygments.console
import
colorize
from
pygments.console
import
colorize
...
@@ -28,6 +29,7 @@ DEFAULT_SETTINGS = 'devstack'
...
@@ -28,6 +29,7 @@ DEFAULT_SETTINGS = 'devstack'
@cmdopts
([
@cmdopts
([
(
"verbose"
,
"v"
,
"Sets 'verbose' to True"
),
(
"verbose"
,
"v"
,
"Sets 'verbose' to True"
),
])
])
@timed
def
i18n_extract
(
options
):
def
i18n_extract
(
options
):
"""
"""
Extract localizable strings from sources
Extract localizable strings from sources
...
@@ -42,6 +44,7 @@ def i18n_extract(options):
...
@@ -42,6 +44,7 @@ def i18n_extract(options):
@task
@task
@timed
def
i18n_fastgenerate
():
def
i18n_fastgenerate
():
"""
"""
Compile localizable strings from sources without re-extracting strings first.
Compile localizable strings from sources without re-extracting strings first.
...
@@ -51,6 +54,7 @@ def i18n_fastgenerate():
...
@@ -51,6 +54,7 @@ def i18n_fastgenerate():
@task
@task
@needs
(
"pavelib.i18n.i18n_extract"
)
@needs
(
"pavelib.i18n.i18n_extract"
)
@timed
def
i18n_generate
():
def
i18n_generate
():
"""
"""
Compile localizable strings from sources, extracting strings first.
Compile localizable strings from sources, extracting strings first.
...
@@ -60,6 +64,7 @@ def i18n_generate():
...
@@ -60,6 +64,7 @@ def i18n_generate():
@task
@task
@needs
(
"pavelib.i18n.i18n_extract"
)
@needs
(
"pavelib.i18n.i18n_extract"
)
@timed
def
i18n_generate_strict
():
def
i18n_generate_strict
():
"""
"""
Compile localizable strings from sources, extracting strings first.
Compile localizable strings from sources, extracting strings first.
...
@@ -70,6 +75,7 @@ def i18n_generate_strict():
...
@@ -70,6 +75,7 @@ def i18n_generate_strict():
@task
@task
@needs
(
"pavelib.i18n.i18n_extract"
)
@needs
(
"pavelib.i18n.i18n_extract"
)
@timed
def
i18n_dummy
():
def
i18n_dummy
():
"""
"""
Simulate international translation by generating dummy strings
Simulate international translation by generating dummy strings
...
@@ -85,6 +91,7 @@ def i18n_dummy():
...
@@ -85,6 +91,7 @@ def i18n_dummy():
@task
@task
@timed
def
i18n_validate_gettext
():
def
i18n_validate_gettext
():
"""
"""
Make sure GNU gettext utilities are available
Make sure GNU gettext utilities are available
...
@@ -107,6 +114,7 @@ def i18n_validate_gettext():
...
@@ -107,6 +114,7 @@ def i18n_validate_gettext():
@task
@task
@timed
def
i18n_validate_transifex_config
():
def
i18n_validate_transifex_config
():
"""
"""
Make sure config file with username/password exists
Make sure config file with username/password exists
...
@@ -130,6 +138,7 @@ def i18n_validate_transifex_config():
...
@@ -130,6 +138,7 @@ def i18n_validate_transifex_config():
@task
@task
@needs
(
"pavelib.i18n.i18n_validate_transifex_config"
)
@needs
(
"pavelib.i18n.i18n_validate_transifex_config"
)
@timed
def
i18n_transifex_push
():
def
i18n_transifex_push
():
"""
"""
Push source strings to Transifex for translation
Push source strings to Transifex for translation
...
@@ -139,6 +148,7 @@ def i18n_transifex_push():
...
@@ -139,6 +148,7 @@ def i18n_transifex_push():
@task
@task
@needs
(
"pavelib.i18n.i18n_validate_transifex_config"
)
@needs
(
"pavelib.i18n.i18n_validate_transifex_config"
)
@timed
def
i18n_transifex_pull
():
def
i18n_transifex_pull
():
"""
"""
Pull translated strings from Transifex
Pull translated strings from Transifex
...
@@ -147,6 +157,7 @@ def i18n_transifex_pull():
...
@@ -147,6 +157,7 @@ def i18n_transifex_pull():
@task
@task
@timed
def
i18n_rtl
():
def
i18n_rtl
():
"""
"""
Pull all RTL translations (reviewed AND unreviewed) from Transifex
Pull all RTL translations (reviewed AND unreviewed) from Transifex
...
@@ -164,6 +175,7 @@ def i18n_rtl():
...
@@ -164,6 +175,7 @@ def i18n_rtl():
@task
@task
@timed
def
i18n_ltr
():
def
i18n_ltr
():
"""
"""
Pull all LTR translations (reviewed AND unreviewed) from Transifex
Pull all LTR translations (reviewed AND unreviewed) from Transifex
...
@@ -188,6 +200,7 @@ def i18n_ltr():
...
@@ -188,6 +200,7 @@ def i18n_ltr():
"pavelib.i18n.i18n_dummy"
,
"pavelib.i18n.i18n_dummy"
,
"pavelib.i18n.i18n_generate_strict"
,
"pavelib.i18n.i18n_generate_strict"
,
)
)
@timed
def
i18n_robot_pull
():
def
i18n_robot_pull
():
"""
"""
Pull source strings, generate po and mo files, and validate
Pull source strings, generate po and mo files, and validate
...
@@ -215,6 +228,7 @@ def i18n_robot_pull():
...
@@ -215,6 +228,7 @@ def i18n_robot_pull():
@task
@task
@timed
def
i18n_clean
():
def
i18n_clean
():
"""
"""
Clean the i18n directory of artifacts
Clean the i18n directory of artifacts
...
@@ -227,6 +241,7 @@ def i18n_clean():
...
@@ -227,6 +241,7 @@ def i18n_clean():
"pavelib.i18n.i18n_extract"
,
"pavelib.i18n.i18n_extract"
,
"pavelib.i18n.i18n_transifex_push"
,
"pavelib.i18n.i18n_transifex_push"
,
)
)
@timed
def
i18n_robot_push
():
def
i18n_robot_push
():
"""
"""
Extract new strings, and push to transifex
Extract new strings, and push to transifex
...
@@ -239,6 +254,7 @@ def i18n_robot_push():
...
@@ -239,6 +254,7 @@ def i18n_robot_push():
"pavelib.i18n.i18n_validate_transifex_config"
,
"pavelib.i18n.i18n_validate_transifex_config"
,
"pavelib.i18n.i18n_generate"
,
"pavelib.i18n.i18n_generate"
,
)
)
@timed
def
i18n_release_push
():
def
i18n_release_push
():
"""
"""
Push release-specific resources to Transifex.
Push release-specific resources to Transifex.
...
@@ -251,6 +267,7 @@ def i18n_release_push():
...
@@ -251,6 +267,7 @@ def i18n_release_push():
@needs
(
@needs
(
"pavelib.i18n.i18n_validate_transifex_config"
,
"pavelib.i18n.i18n_validate_transifex_config"
,
)
)
@timed
def
i18n_release_pull
():
def
i18n_release_pull
():
"""
"""
Pull release-specific translations from Transifex.
Pull release-specific translations from Transifex.
...
...
pavelib/js_test.py
View file @
f94530ba
...
@@ -5,6 +5,7 @@ import sys
...
@@ -5,6 +5,7 @@ import sys
from
paver.easy
import
task
,
cmdopts
,
needs
from
paver.easy
import
task
,
cmdopts
,
needs
from
pavelib.utils.test.suites
import
JsTestSuite
from
pavelib.utils.test.suites
import
JsTestSuite
from
pavelib.utils.envs
import
Env
from
pavelib.utils.envs
import
Env
from
pavelib.utils.timer
import
timed
__test__
=
False
# do not collect
__test__
=
False
# do not collect
...
@@ -22,6 +23,7 @@ __test__ = False # do not collect
...
@@ -22,6 +23,7 @@ __test__ = False # do not collect
(
'skip-clean'
,
'C'
,
'skip cleaning repository before running tests'
),
(
'skip-clean'
,
'C'
,
'skip cleaning repository before running tests'
),
(
'skip_clean'
,
None
,
'deprecated in favor of skip-clean'
),
(
'skip_clean'
,
None
,
'deprecated in favor of skip-clean'
),
],
share_with
=
[
"pavelib.utils.tests.utils.clean_reports_dir"
])
],
share_with
=
[
"pavelib.utils.tests.utils.clean_reports_dir"
])
@timed
def
test_js
(
options
):
def
test_js
(
options
):
"""
"""
Run the JavaScript tests
Run the JavaScript tests
...
@@ -58,6 +60,7 @@ def test_js(options):
...
@@ -58,6 +60,7 @@ def test_js(options):
(
"suite="
,
"s"
,
"Test suite to run"
),
(
"suite="
,
"s"
,
"Test suite to run"
),
(
"coverage"
,
"c"
,
"Run test under coverage"
),
(
"coverage"
,
"c"
,
"Run test under coverage"
),
])
])
@timed
def
test_js_run
(
options
):
def
test_js_run
(
options
):
"""
"""
Run the JavaScript tests and print results to the console
Run the JavaScript tests and print results to the console
...
@@ -71,6 +74,7 @@ def test_js_run(options):
...
@@ -71,6 +74,7 @@ def test_js_run(options):
(
"suite="
,
"s"
,
"Test suite to run"
),
(
"suite="
,
"s"
,
"Test suite to run"
),
(
"port="
,
"p"
,
"Port to run test server on"
),
(
"port="
,
"p"
,
"Port to run test server on"
),
])
])
@timed
def
test_js_dev
(
options
):
def
test_js_dev
(
options
):
"""
"""
Run the JavaScript tests in your default browsers
Run the JavaScript tests in your default browsers
...
...
pavelib/paver_tests/test_timer.py
0 → 100644
View file @
f94530ba
"""
Tests of the pavelib.utils.timer module.
"""
from
datetime
import
datetime
,
timedelta
from
mock
import
patch
,
MagicMock
from
unittest
import
TestCase
from
pavelib.utils
import
timer
@timer.timed
def
identity
(
*
args
,
**
kwargs
):
"""
An identity function used as a default task to test the timing of.
"""
return
args
,
kwargs
MOCK_OPEN
=
MagicMock
(
spec
=
open
)
@patch.dict
(
'pavelib.utils.timer.__builtins__'
,
open
=
MOCK_OPEN
)
class
TimedDecoratorTests
(
TestCase
):
"""
Tests of the pavelib.utils.timer:timed decorator.
"""
def
setUp
(
self
):
super
(
TimedDecoratorTests
,
self
)
.
setUp
()
patch_dumps
=
patch
.
object
(
timer
.
json
,
'dump'
,
autospec
=
True
)
self
.
mock_dump
=
patch_dumps
.
start
()
self
.
addCleanup
(
patch_dumps
.
stop
)
patch_makedirs
=
patch
.
object
(
timer
.
os
,
'makedirs'
,
autospec
=
True
)
self
.
mock_makedirs
=
patch_makedirs
.
start
()
self
.
addCleanup
(
patch_makedirs
.
stop
)
patch_datetime
=
patch
.
object
(
timer
,
'datetime'
,
autospec
=
True
)
self
.
mock_datetime
=
patch_datetime
.
start
()
self
.
addCleanup
(
patch_datetime
.
stop
)
patch_exists
=
patch
.
object
(
timer
,
'exists'
,
autospec
=
True
)
self
.
mock_exists
=
patch_exists
.
start
()
self
.
addCleanup
(
patch_exists
.
stop
)
MOCK_OPEN
.
reset_mock
()
def
get_log_messages
(
self
,
task
=
identity
,
args
=
None
,
kwargs
=
None
,
raises
=
None
):
"""
Return all timing messages recorded during the execution of ``task``.
"""
if
args
is
None
:
args
=
[]
if
kwargs
is
None
:
kwargs
=
{}
if
raises
is
None
:
task
(
*
args
,
**
kwargs
)
else
:
self
.
assertRaises
(
raises
,
task
,
*
args
,
**
kwargs
)
return
[
call
[
0
][
0
]
# log_message
for
call
in
self
.
mock_dump
.
call_args_list
]
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
'/tmp/some-log'
)
def
test_times
(
self
):
start
=
datetime
(
2016
,
7
,
20
,
10
,
56
,
19
)
end
=
start
+
timedelta
(
seconds
=
35.6
)
self
.
mock_datetime
.
utcnow
.
side_effect
=
[
start
,
end
]
messages
=
self
.
get_log_messages
()
self
.
assertEquals
(
len
(
messages
),
1
)
# I'm not using assertDictContainsSubset because it is
# removed in python 3.2 (because the arguments were backwards)
# and it wasn't ever replaced by anything *headdesk*
self
.
assertIn
(
'duration'
,
messages
[
0
])
self
.
assertEquals
(
35.6
,
messages
[
0
][
'duration'
])
self
.
assertIn
(
'started_at'
,
messages
[
0
])
self
.
assertEquals
(
start
.
isoformat
(
' '
),
messages
[
0
][
'started_at'
])
self
.
assertIn
(
'ended_at'
,
messages
[
0
])
self
.
assertEquals
(
end
.
isoformat
(
' '
),
messages
[
0
][
'ended_at'
])
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
None
)
def
test_no_logs
(
self
):
messages
=
self
.
get_log_messages
()
self
.
assertEquals
(
len
(
messages
),
0
)
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
'/tmp/some-log'
)
def
test_arguments
(
self
):
messages
=
self
.
get_log_messages
(
args
=
(
1
,
'foo'
),
kwargs
=
dict
(
bar
=
'baz'
))
self
.
assertEquals
(
len
(
messages
),
1
)
# I'm not using assertDictContainsSubset because it is
# removed in python 3.2 (because the arguments were backwards)
# and it wasn't ever replaced by anything *headdesk*
self
.
assertIn
(
'args'
,
messages
[
0
])
self
.
assertEquals
([
repr
(
1
),
repr
(
'foo'
)],
messages
[
0
][
'args'
])
self
.
assertIn
(
'kwargs'
,
messages
[
0
])
self
.
assertEquals
({
'bar'
:
repr
(
'baz'
)},
messages
[
0
][
'kwargs'
])
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
'/tmp/some-log'
)
def
test_task_name
(
self
):
messages
=
self
.
get_log_messages
()
self
.
assertEquals
(
len
(
messages
),
1
)
# I'm not using assertDictContainsSubset because it is
# removed in python 3.2 (because the arguments were backwards)
# and it wasn't ever replaced by anything *headdesk*
self
.
assertIn
(
'task'
,
messages
[
0
])
self
.
assertEquals
(
'pavelib.paver_tests.test_timer.identity'
,
messages
[
0
][
'task'
])
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
'/tmp/some-log'
)
def
test_exceptions
(
self
):
@timer.timed
def
raises
():
"""
A task used for testing exception handling of the timed decorator.
"""
raise
Exception
(
'The Message!'
)
messages
=
self
.
get_log_messages
(
task
=
raises
,
raises
=
Exception
)
self
.
assertEquals
(
len
(
messages
),
1
)
# I'm not using assertDictContainsSubset because it is
# removed in python 3.2 (because the arguments were backwards)
# and it wasn't ever replaced by anything *headdesk*
self
.
assertIn
(
'exception'
,
messages
[
0
])
self
.
assertEquals
(
"Exception: The Message!"
,
messages
[
0
][
'exception'
])
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
'/tmp/some-log-
%
Y-
%
m-
%
d-
%
H-
%
M-
%
S.log'
)
def
test_date_formatting
(
self
):
start
=
datetime
(
2016
,
7
,
20
,
10
,
56
,
19
)
end
=
start
+
timedelta
(
seconds
=
35.6
)
self
.
mock_datetime
.
utcnow
.
side_effect
=
[
start
,
end
]
messages
=
self
.
get_log_messages
()
self
.
assertEquals
(
len
(
messages
),
1
)
MOCK_OPEN
.
assert_called_once_with
(
'/tmp/some-log-2016-07-20-10-56-19.log'
,
'a'
)
@patch.object
(
timer
,
'PAVER_TIMER_LOG'
,
'/tmp/some-log'
)
def
test_nested_tasks
(
self
):
@timer.timed
def
parent
():
"""
A timed task that calls another task
"""
identity
()
parent_start
=
datetime
(
2016
,
7
,
20
,
10
,
56
,
19
)
parent_end
=
parent_start
+
timedelta
(
seconds
=
60
)
child_start
=
parent_start
+
timedelta
(
seconds
=
10
)
child_end
=
parent_end
-
timedelta
(
seconds
=
10
)
self
.
mock_datetime
.
utcnow
.
side_effect
=
[
parent_start
,
child_start
,
child_end
,
parent_end
]
messages
=
self
.
get_log_messages
(
task
=
parent
)
self
.
assertEquals
(
len
(
messages
),
2
)
# Child messages first
self
.
assertIn
(
'duration'
,
messages
[
0
])
self
.
assertEquals
(
40
,
messages
[
0
][
'duration'
])
self
.
assertIn
(
'started_at'
,
messages
[
0
])
self
.
assertEquals
(
child_start
.
isoformat
(
' '
),
messages
[
0
][
'started_at'
])
self
.
assertIn
(
'ended_at'
,
messages
[
0
])
self
.
assertEquals
(
child_end
.
isoformat
(
' '
),
messages
[
0
][
'ended_at'
])
# Parent messages after
self
.
assertIn
(
'duration'
,
messages
[
1
])
self
.
assertEquals
(
60
,
messages
[
1
][
'duration'
])
self
.
assertIn
(
'started_at'
,
messages
[
1
])
self
.
assertEquals
(
parent_start
.
isoformat
(
' '
),
messages
[
1
][
'started_at'
])
self
.
assertIn
(
'ended_at'
,
messages
[
1
])
self
.
assertEquals
(
parent_end
.
isoformat
(
' '
),
messages
[
1
][
'ended_at'
])
pavelib/prereqs.py
View file @
f94530ba
...
@@ -11,6 +11,7 @@ import sys
...
@@ -11,6 +11,7 @@ import sys
from
paver.easy
import
sh
,
task
from
paver.easy
import
sh
,
task
from
.utils.envs
import
Env
from
.utils.envs
import
Env
from
.utils.timer
import
timed
PREREQS_STATE_DIR
=
os
.
getenv
(
'PREREQ_CACHE_DIR'
,
Env
.
REPO_ROOT
/
'.prereqs_cache'
)
PREREQS_STATE_DIR
=
os
.
getenv
(
'PREREQ_CACHE_DIR'
,
Env
.
REPO_ROOT
/
'.prereqs_cache'
)
...
@@ -145,6 +146,7 @@ def python_prereqs_installation():
...
@@ -145,6 +146,7 @@ def python_prereqs_installation():
@task
@task
@timed
def
install_node_prereqs
():
def
install_node_prereqs
():
"""
"""
Installs Node prerequisites
Installs Node prerequisites
...
@@ -168,6 +170,7 @@ PACKAGES_TO_UNINSTALL = [
...
@@ -168,6 +170,7 @@ PACKAGES_TO_UNINSTALL = [
@task
@task
@timed
def
uninstall_python_packages
():
def
uninstall_python_packages
():
"""
"""
Uninstall Python packages that need explicit uninstallation.
Uninstall Python packages that need explicit uninstallation.
...
@@ -235,6 +238,7 @@ def package_in_frozen(package_name, frozen_output):
...
@@ -235,6 +238,7 @@ def package_in_frozen(package_name, frozen_output):
@task
@task
@timed
def
install_python_prereqs
():
def
install_python_prereqs
():
"""
"""
Installs Python prerequisites.
Installs Python prerequisites.
...
@@ -268,6 +272,7 @@ def install_python_prereqs():
...
@@ -268,6 +272,7 @@ def install_python_prereqs():
@task
@task
@timed
def
install_prereqs
():
def
install_prereqs
():
"""
"""
Installs Node and Python prerequisites
Installs Node and Python prerequisites
...
...
pavelib/quality.py
View file @
f94530ba
...
@@ -9,6 +9,7 @@ import re
...
@@ -9,6 +9,7 @@ import re
from
openedx.core.djangolib.markup
import
HTML
from
openedx.core.djangolib.markup
import
HTML
from
.utils.envs
import
Env
from
.utils.envs
import
Env
from
.utils.timer
import
timed
ALL_SYSTEMS
=
'lms,cms,common,openedx,pavelib'
ALL_SYSTEMS
=
'lms,cms,common,openedx,pavelib'
...
@@ -38,6 +39,7 @@ def top_python_dirs(dirname):
...
@@ -38,6 +39,7 @@ def top_python_dirs(dirname):
@cmdopts
([
@cmdopts
([
(
"system="
,
"s"
,
"System to act on"
),
(
"system="
,
"s"
,
"System to act on"
),
])
])
@timed
def
find_fixme
(
options
):
def
find_fixme
(
options
):
"""
"""
Run pylint on system code, only looking for fixme items.
Run pylint on system code, only looking for fixme items.
...
@@ -82,6 +84,7 @@ def find_fixme(options):
...
@@ -82,6 +84,7 @@ def find_fixme(options):
(
"errors"
,
"e"
,
"Check for errors only"
),
(
"errors"
,
"e"
,
"Check for errors only"
),
(
"limit="
,
"l"
,
"limit for number of acceptable violations"
),
(
"limit="
,
"l"
,
"limit for number of acceptable violations"
),
])
])
@timed
def
run_pylint
(
options
):
def
run_pylint
(
options
):
"""
"""
Run pylint on system code. When violations limit is passed in,
Run pylint on system code. When violations limit is passed in,
...
@@ -197,6 +200,7 @@ def _pep8_violations(report_file):
...
@@ -197,6 +200,7 @@ def _pep8_violations(report_file):
@cmdopts
([
@cmdopts
([
(
"system="
,
"s"
,
"System to act on"
),
(
"system="
,
"s"
,
"System to act on"
),
])
])
@timed
def
run_pep8
(
options
):
# pylint: disable=unused-argument
def
run_pep8
(
options
):
# pylint: disable=unused-argument
"""
"""
Run pep8 on system code.
Run pep8 on system code.
...
@@ -224,6 +228,7 @@ def run_pep8(options): # pylint: disable=unused-argument
...
@@ -224,6 +228,7 @@ def run_pep8(options): # pylint: disable=unused-argument
@task
@task
@needs
(
'pavelib.prereqs.install_python_prereqs'
)
@needs
(
'pavelib.prereqs.install_python_prereqs'
)
@timed
def
run_complexity
():
def
run_complexity
():
"""
"""
Uses radon to examine cyclomatic complexity.
Uses radon to examine cyclomatic complexity.
...
@@ -262,6 +267,7 @@ def run_complexity():
...
@@ -262,6 +267,7 @@ def run_complexity():
@cmdopts
([
@cmdopts
([
(
"limit="
,
"l"
,
"limit for number of acceptable violations"
),
(
"limit="
,
"l"
,
"limit for number of acceptable violations"
),
])
])
@timed
def
run_jshint
(
options
):
def
run_jshint
(
options
):
"""
"""
Runs jshint on static asset directories
Runs jshint on static asset directories
...
@@ -306,6 +312,7 @@ def run_jshint(options):
...
@@ -306,6 +312,7 @@ def run_jshint(options):
@cmdopts
([
@cmdopts
([
(
"thresholds="
,
"t"
,
"json containing limit for number of acceptable violations per rule"
),
(
"thresholds="
,
"t"
,
"json containing limit for number of acceptable violations per rule"
),
])
])
@timed
def
run_safelint
(
options
):
def
run_safelint
(
options
):
"""
"""
Runs safe_template_linter.py on the codebase
Runs safe_template_linter.py on the codebase
...
@@ -407,6 +414,7 @@ def run_safelint(options):
...
@@ -407,6 +414,7 @@ def run_safelint(options):
@task
@task
@needs
(
'pavelib.prereqs.install_python_prereqs'
)
@needs
(
'pavelib.prereqs.install_python_prereqs'
)
@timed
def
run_safecommit_report
():
def
run_safecommit_report
():
"""
"""
Runs safe-commit-linter.sh on the current branch.
Runs safe-commit-linter.sh on the current branch.
...
@@ -580,6 +588,7 @@ def _get_safecommit_count(filename):
...
@@ -580,6 +588,7 @@ def _get_safecommit_count(filename):
(
"compare-branch="
,
"b"
,
"Branch to compare against, defaults to origin/master"
),
(
"compare-branch="
,
"b"
,
"Branch to compare against, defaults to origin/master"
),
(
"percentage="
,
"p"
,
"fail if diff-quality is below this percentage"
),
(
"percentage="
,
"p"
,
"fail if diff-quality is below this percentage"
),
])
])
@timed
def
run_quality
(
options
):
def
run_quality
(
options
):
"""
"""
Build the html diff quality reports, and print the reports to the console.
Build the html diff quality reports, and print the reports to the console.
...
...
pavelib/servers.py
View file @
f94530ba
...
@@ -10,6 +10,7 @@ from paver.easy import call_task, cmdopts, consume_args, needs, sh, task
...
@@ -10,6 +10,7 @@ from paver.easy import call_task, cmdopts, consume_args, needs, sh, task
from
.assets
import
collect_assets
from
.assets
import
collect_assets
from
.utils.cmd
import
django_cmd
from
.utils.cmd
import
django_cmd
from
.utils.process
import
run_process
,
run_multi_processes
from
.utils.process
import
run_process
,
run_multi_processes
from
.utils.timer
import
timed
DEFAULT_PORT
=
{
"lms"
:
8000
,
"studio"
:
8001
}
DEFAULT_PORT
=
{
"lms"
:
8000
,
"studio"
:
8001
}
...
@@ -244,6 +245,7 @@ def run_all_servers(options):
...
@@ -244,6 +245,7 @@ def run_all_servers(options):
(
"settings="
,
"s"
,
"Django settings"
),
(
"settings="
,
"s"
,
"Django settings"
),
(
"fake-initial"
,
None
,
"Fake the initial migrations"
),
(
"fake-initial"
,
None
,
"Fake the initial migrations"
),
])
])
@timed
def
update_db
(
options
):
def
update_db
(
options
):
"""
"""
Migrates the lms and cms across all databases
Migrates the lms and cms across all databases
...
@@ -261,6 +263,7 @@ def update_db(options):
...
@@ -261,6 +263,7 @@ def update_db(options):
@task
@task
@needs
(
'pavelib.prereqs.install_prereqs'
)
@needs
(
'pavelib.prereqs.install_prereqs'
)
@consume_args
@consume_args
@timed
def
check_settings
(
args
):
def
check_settings
(
args
):
"""
"""
Checks settings files.
Checks settings files.
...
...
pavelib/tests.py
View file @
f94530ba
...
@@ -7,6 +7,7 @@ import sys
...
@@ -7,6 +7,7 @@ import sys
from
paver.easy
import
sh
,
task
,
cmdopts
,
needs
,
call_task
from
paver.easy
import
sh
,
task
,
cmdopts
,
needs
,
call_task
from
pavelib.utils.test
import
suites
from
pavelib.utils.test
import
suites
from
pavelib.utils.envs
import
Env
from
pavelib.utils.envs
import
Env
from
pavelib.utils.timer
import
timed
from
pavelib.utils.passthrough_opts
import
PassthroughTask
from
pavelib.utils.passthrough_opts
import
PassthroughTask
from
optparse
import
make_option
from
optparse
import
make_option
...
@@ -55,6 +56,7 @@ __test__ = False # do not collect
...
@@ -55,6 +56,7 @@ __test__ = False # do not collect
(
'skip_clean'
,
None
,
'deprecated in favor of skip-clean'
),
(
'skip_clean'
,
None
,
'deprecated in favor of skip-clean'
),
],
share_with
=
[
'pavelib.utils.test.utils.clean_reports_dir'
])
],
share_with
=
[
'pavelib.utils.test.utils.clean_reports_dir'
])
@PassthroughTask
@PassthroughTask
@timed
def
test_system
(
options
,
passthrough_options
):
def
test_system
(
options
,
passthrough_options
):
"""
"""
Run tests on our djangoapps for lms and cms
Run tests on our djangoapps for lms and cms
...
@@ -120,6 +122,7 @@ def test_system(options, passthrough_options):
...
@@ -120,6 +122,7 @@ def test_system(options, passthrough_options):
(
"test_id="
,
None
,
"deprecated in favor of test-id"
),
(
"test_id="
,
None
,
"deprecated in favor of test-id"
),
],
share_with
=
[
'pavelib.utils.test.utils.clean_reports_dir'
])
],
share_with
=
[
'pavelib.utils.test.utils.clean_reports_dir'
])
@PassthroughTask
@PassthroughTask
@timed
def
test_lib
(
options
,
passthrough_options
):
def
test_lib
(
options
,
passthrough_options
):
"""
"""
Run tests for common/lib/ and pavelib/ (paver-tests)
Run tests for common/lib/ and pavelib/ (paver-tests)
...
@@ -184,6 +187,7 @@ def test_lib(options, passthrough_options):
...
@@ -184,6 +187,7 @@ def test_lib(options, passthrough_options):
(
"fail_fast"
,
None
,
"deprecated in favor of fail-fast"
),
(
"fail_fast"
,
None
,
"deprecated in favor of fail-fast"
),
])
])
@PassthroughTask
@PassthroughTask
@timed
def
test_python
(
options
,
passthrough_options
):
def
test_python
(
options
,
passthrough_options
):
"""
"""
Run all python tests
Run all python tests
...
@@ -216,6 +220,7 @@ def test_python(options, passthrough_options):
...
@@ -216,6 +220,7 @@ def test_python(options, passthrough_options):
),
),
])
])
@PassthroughTask
@PassthroughTask
@timed
def
test
(
options
,
passthrough_options
):
def
test
(
options
,
passthrough_options
):
"""
"""
Run all tests
Run all tests
...
@@ -239,7 +244,8 @@ def test(options, passthrough_options):
...
@@ -239,7 +244,8 @@ def test(options, passthrough_options):
(
"compare-branch="
,
"b"
,
"Branch to compare against, defaults to origin/master"
),
(
"compare-branch="
,
"b"
,
"Branch to compare against, defaults to origin/master"
),
(
"compare_branch="
,
None
,
"deprecated in favor of compare-branch"
),
(
"compare_branch="
,
None
,
"deprecated in favor of compare-branch"
),
])
])
def
coverage
(
options
):
@timed
def
coverage
():
"""
"""
Build the html, xml, and diff coverage reports
Build the html, xml, and diff coverage reports
"""
"""
...
@@ -276,6 +282,7 @@ def coverage(options):
...
@@ -276,6 +282,7 @@ def coverage(options):
(
"compare-branch="
,
"b"
,
"Branch to compare against, defaults to origin/master"
),
(
"compare-branch="
,
"b"
,
"Branch to compare against, defaults to origin/master"
),
(
"compare_branch="
,
None
,
"deprecated in favor of compare-branch"
),
(
"compare_branch="
,
None
,
"deprecated in favor of compare-branch"
),
],
share_with
=
[
'coverage'
])
],
share_with
=
[
'coverage'
])
@timed
def
diff_coverage
(
options
):
def
diff_coverage
(
options
):
"""
"""
Build the diff coverage reports
Build the diff coverage reports
...
...
pavelib/utils/test/utils.py
View file @
f94530ba
...
@@ -3,6 +3,7 @@ Helper functions for test tasks
...
@@ -3,6 +3,7 @@ Helper functions for test tasks
"""
"""
from
paver.easy
import
sh
,
task
,
cmdopts
from
paver.easy
import
sh
,
task
,
cmdopts
from
pavelib.utils.envs
import
Env
from
pavelib.utils.envs
import
Env
from
pavelib.utils.timer
import
timed
import
os
import
os
import
re
import
re
import
subprocess
import
subprocess
...
@@ -15,6 +16,7 @@ __test__ = False # do not collect
...
@@ -15,6 +16,7 @@ __test__ = False # do not collect
@task
@task
@timed
def
clean_test_files
():
def
clean_test_files
():
"""
"""
Clean fixture files used by tests and .pyc files
Clean fixture files used by tests and .pyc files
...
@@ -42,6 +44,7 @@ def clean_dir(directory):
...
@@ -42,6 +44,7 @@ def clean_dir(directory):
(
'skip-clean'
,
'C'
,
'skip cleaning repository before running tests'
),
(
'skip-clean'
,
'C'
,
'skip cleaning repository before running tests'
),
(
'skip_clean'
,
None
,
'deprecated in favor of skip-clean'
),
(
'skip_clean'
,
None
,
'deprecated in favor of skip-clean'
),
])
])
@timed
def
clean_reports_dir
(
options
):
def
clean_reports_dir
(
options
):
"""
"""
Clean coverage files, to ensure that we don't use stale data to generate reports.
Clean coverage files, to ensure that we don't use stale data to generate reports.
...
@@ -57,6 +60,7 @@ def clean_reports_dir(options):
...
@@ -57,6 +60,7 @@ def clean_reports_dir(options):
@task
@task
@timed
def
clean_mongo
():
def
clean_mongo
():
"""
"""
Clean mongo test databases
Clean mongo test databases
...
...
pavelib/utils/timer.py
0 → 100644
View file @
f94530ba
"""
Tools for timing paver tasks
"""
from
datetime
import
datetime
import
json
import
logging
import
os
from
os.path
import
dirname
,
exists
import
sys
import
traceback
import
wrapt
LOGGER
=
logging
.
getLogger
(
__file__
)
PAVER_TIMER_LOG
=
os
.
environ
.
get
(
'PAVER_TIMER_LOG'
)
@wrapt.decorator
def
timed
(
wrapped
,
instance
,
args
,
kwargs
):
# pylint: disable=unused-argument
"""
Log execution time for a function to a log file.
Logging is only actually executed if the PAVER_TIMER_LOG environment variable
is set. That variable is expanded for the current user and current
environment variables. It also can have :meth:`~Datetime.strftime` format
identifiers which are substituted using the time when the task started.
For example, ``PAVER_TIMER_LOG='~/.paver.logs/
%
Y-
%
d-
%
m.log'`` will create a new
log file every day containing reconds for paver tasks run that day, and
will put those log files in the ``.paver.logs`` directory inside the users
home.
Must be earlier in the decorator stack than the paver task declaration.
"""
start
=
datetime
.
utcnow
()
exception_info
=
{}
try
:
return
wrapped
(
*
args
,
**
kwargs
)
except
Exception
as
exc
:
# pylint: disable=broad-except
exception_info
=
{
'exception'
:
""
.
join
(
traceback
.
format_exception_only
(
type
(
exc
),
exc
))
.
strip
()
}
raise
finally
:
end
=
datetime
.
utcnow
()
# N.B. This is intended to provide a consistent interface and message format
# across all of Open edX tooling, so it deliberately eschews standard
# python logging infrastructure.
if
PAVER_TIMER_LOG
is
not
None
:
log_path
=
start
.
strftime
(
PAVER_TIMER_LOG
)
log_message
=
{
'python_version'
:
sys
.
version
,
'task'
:
"{}.{}"
.
format
(
wrapped
.
__module__
,
wrapped
.
__name__
),
'args'
:
[
repr
(
arg
)
for
arg
in
args
],
'kwargs'
:
{
key
:
repr
(
value
)
for
key
,
value
in
kwargs
.
items
()},
'started_at'
:
start
.
isoformat
(
' '
),
'ended_at'
:
end
.
isoformat
(
' '
),
'duration'
:
(
end
-
start
)
.
total_seconds
(),
}
log_message
.
update
(
exception_info
)
try
:
if
not
exists
(
dirname
(
log_path
)):
os
.
makedirs
(
dirname
(
log_path
))
with
open
(
log_path
,
'a'
)
as
outfile
:
json
.
dump
(
log_message
,
outfile
,
separators
=
(
','
,
':'
),
sort_keys
=
True
,
)
outfile
.
write
(
'
\n
'
)
except
OSError
:
# Squelch OSErrors, because we expect them and they shouldn't
# interrupt the rest of the process.
LOGGER
.
exception
(
"Unable to write timing logs"
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment