Skip to content

A
ansible
Commit fe2d00d9 by Michael DeHaan
Options

WIP on refactoring the module formatter code that we use to build the doc site with.

parent 31d0060d
Showing with 126 additions and 414 deletions
Makefile
...@@ -64,9 +64,6 @@ all: clean python ...@@ -64,9 +64,6 @@ all: clean python
tests: tests:
PYTHONPATH=./lib ANSIBLE_LIBRARY=./library $(NOSETESTS) -d -v PYTHONPATH=./lib ANSIBLE_LIBRARY=./library $(NOSETESTS) -d -v
# To force a rebuild of the docs run 'touch VERSION && make docs'
docs: $(MANPAGES) modulepages
authors: authors:
sh hacking/authors.sh sh hacking/authors.sh
...@@ -172,11 +169,6 @@ deb: debian ...@@ -172,11 +169,6 @@ deb: debian
# for arch or gentoo, read instructions in the appropriate 'packaging' subdirectory directory # for arch or gentoo, read instructions in the appropriate 'packaging' subdirectory directory
modulepages: webdocs: $(MANPAGES)
PYTHONPATH=./lib $(PYTHON) hacking/module_formatter.py -A $(VERSION) -t man -o docs/man/man3/ --module-dir=library --template-dir=hacking/templates # --verbose
# because this requires Sphinx it is not run as part of every build, those building the RPM and so on can ignore this
webdocs:
(cd docsite/; make docs) (cd docsite/; make docs)
hacking/module_formatter.py
...@@ -24,16 +24,20 @@ import yaml ...@@ -24,16 +24,20 @@ import yaml
import codecs import codecs
import json import json
import ast import ast
from jinja2 import Environment, FileSystemLoader
import re import re
import optparse import optparse
import time import time
import datetime import datetime
import subprocess import subprocess
import cgi import cgi
from jinja2 import Environment, FileSystemLoader
import ansible.utils import ansible.utils
import ansible.utils.module_docs as module_docs import ansible.utils.module_docs as module_docs
#####################################################################################
# constants and paths
# if a module is added in a version of Ansible older than this, don't print the version added information # if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already. # in the module documentation because everyone is assumed to be running something newer than this already.
TO_OLD_TO_BE_NOTABLE = 1.0 TO_OLD_TO_BE_NOTABLE = 1.0
...@@ -48,65 +52,16 @@ EXAMPLE_YAML=os.path.abspath(os.path.join( ...@@ -48,65 +52,16 @@ EXAMPLE_YAML=os.path.abspath(os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml' os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
)) ))
# There is a better way of doing this!
# TODO: somebody add U(text, http://foo.bar/) as described by Tim in #991
_ITALIC = re.compile(r"I\(([^)]+)\)") _ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD = re.compile(r"B\(([^)]+)\)") _BOLD = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)") _MODULE = re.compile(r"M\(([^)]+)\)")
_URL = re.compile(r"U\(([^)]+)\)") _URL = re.compile(r"U\(([^)]+)\)")
_CONST = re.compile(r"C\(([^)]+)\)") _CONST = re.compile(r"C\(([^)]+)\)")
def latex_ify(text): #####################################################################################
t = _ITALIC.sub("\\I{" + r"\1" + "}", text)
t = _BOLD.sub("\\B{" + r"\1" + "}", t)
t = _MODULE.sub("\\M{" + r"\1" + "}", t)
t = _URL.sub("\\url{" + r"\1" + "}", t)
t = _CONST.sub("\\C{" + r"\1" + "}", t)
return t
def html_ify(text):
#print "DEBUG: text=%s" % text
t = cgi.escape(text)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def json_ify(text):
t = _ITALIC.sub("<em>" + r"\1" + "</em>", text)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def js_ify(text):
return text
def man_ify(text):
t = _ITALIC.sub(r'\\fI' + r"\1" + r"\\fR", text)
t = _BOLD.sub(r'\\fB' + r"\1" + r"\\fR", t)
t = _MODULE.sub(r'\\fI' + r"\1" + r"\\fR", t)
t = _URL.sub(r'\\fI' + r"\1" + r"\\fR", t)
t = _CONST.sub(r'\\fC' + r"\1" + r"\\fR", t)
return t
def rst_ify(text): def rst_ify(text):
''' convert symbols like I(this is in italics) to valid restructured text '''
t = _ITALIC.sub(r'*' + r"\1" + r"*", text) t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
t = _BOLD.sub(r'**' + r"\1" + r"**", t) t = _BOLD.sub(r'**' + r"\1" + r"**", t)
...@@ -116,31 +71,40 @@ def rst_ify(text): ...@@ -116,31 +71,40 @@ def rst_ify(text):
return t return t
_MARKDOWN = re.compile(r"[*_`]") #####################################################################################
def markdown_ify(text): def html_ify(text):
''' convert symbols like I(this is in italics) to valid HTML '''
t = cgi.escape(text) t = cgi.escape(text)
t = _MARKDOWN.sub(r"\\\g<0>", t) t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _ITALIC.sub("_" + r"\1" + "_", t) t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _BOLD.sub("**" + r"\1" + "**", t) t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _MODULE.sub("*" + r"\1" + "*", t) t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _URL.sub("[" + r"\1" + "](" + r"\1" + ")", t) t = _CONST.sub("<code>" + r"\1" + "</code>", t)
t = _CONST.sub("`" + r"\1" + "`", t)
return t return t
# Helper for Jinja2 (format() doesn't work here...)
#####################################################################################
def rst_fmt(text, fmt): def rst_fmt(text, fmt):
''' helper for Jinja2 to do format strings '''
return fmt % (text) return fmt % (text)
#####################################################################################
def rst_xline(width, char="="): def rst_xline(width, char="="):
''' return a restructured text line of a given length '''
return char * width return char * width
def load_examples_section(text): #####################################################################################
return text.split('***BREAK***')
def return_data(text, options, outputname, module): def return_data(text, options, outputname, module):
''' dumps module output to a file or the screen, as requested '''
if options.output_dir is not None: if options.output_dir is not None:
f = open(os.path.join(options.output_dir, outputname % module), 'w') f = open(os.path.join(options.output_dir, outputname % module), 'w')
f.write(text.encode('utf-8')) f.write(text.encode('utf-8'))
...@@ -148,15 +112,29 @@ def return_data(text, options, outputname, module): ...@@ -148,15 +112,29 @@ def return_data(text, options, outputname, module):
else: else:
print text print text
#####################################################################################
def boilerplate(): def boilerplate():
''' prints the boilerplate for module docs '''
if not os.path.exists(EXAMPLE_YAML): if not os.path.exists(EXAMPLE_YAML):
print >>sys.stderr, "Missing example boiler plate: %s" % EXAMPLE_YAML print >>sys.stderr, "Missing example boiler plate: %s" % EXAMPLE_YAML
print "DOCUMENTATION = '''" print "DOCUMENTATION = '''"
print file(EXAMPLE_YAML).read() print file(EXAMPLE_YAML).read()
print "'''" print "'''"
print "" print ""
print ""
print "EXAMPLES = '''"
print "# example of doing ___ from a playbook"
print "your_module: some_arg=1 other_arg=2"
print "'''"
print ""
#####################################################################################
def list_modules(module_dir): def list_modules(module_dir):
''' returns a hash of categories, each category being a hash of module names to file paths '''
categories = {} categories = {}
files = glob.glob("%s/*" % module_dir) files = glob.glob("%s/*" % module_dir)
for d in files: for d in files:
...@@ -171,202 +149,72 @@ def list_modules(module_dir): ...@@ -171,202 +149,72 @@ def list_modules(module_dir):
categories[category][module] = f categories[category][module] = f
return categories return categories
def main(): #####################################################################################
def generate_parser():
''' generate an optparse parser '''
p = optparse.OptionParser( p = optparse.OptionParser(
version='%prog 1.0', version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2', usage='usage: %prog [options] arg1 arg2',
description='Convert Ansible module DOCUMENTATION strings to other formats', description='Generate module documentation from metadata',
) )
p.add_option("-A", "--ansible-version", p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
action="store", p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
dest="ansible_version", p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
default="unknown", p.add_option("-t", "--type", action='store', dest='type', choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'], default='latex', help="Document type")
help="Ansible version number") p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
p.add_option("-M", "--module-dir", p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
action="store", p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
dest="module_dir", p.add_option("-G", "--generate", action="store_true", dest="do_boilerplate", default=False, help="generate boilerplate docs to stdout")
default=MODULEDIR,
help="Ansible modules/ directory")
p.add_option("-T", "--template-dir",
action="store",
dest="template_dir",
default="hacking/templates",
help="directory containing Jinja2 templates")
p.add_option("-t", "--type",
action='store',
dest='type',
choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'],
default='latex',
help="Output type")
p.add_option("-m", "--module",
action='append',
default=[],
dest='module_list',
help="Add modules to process in module_dir")
p.add_option("-v", "--verbose",
action='store_true',
default=False,
help="Verbose")
p.add_option("-o", "--output-dir",
action="store",
dest="output_dir",
default=None,
help="Output directory for module files")
p.add_option("-I", "--includes-file",
action="store",
dest="includes_file",
default=None,
help="Create a file containing list of processed modules")
p.add_option("-G", "--generate",
action="store_true",
dest="do_boilerplate",
default=False,
help="generate boilerplate DOCUMENTATION to stdout")
p.add_option('-V', action='version', help='Show version number and exit') p.add_option('-V', action='version', help='Show version number and exit')
return p
(options, args) = p.parse_args() #####################################################################################
# print "M: %s" % options.module_dir
# print "t: %s" % options.type
# print "m: %s" % options.module_list
# print "v: %s" % options.verbose
if options.do_boilerplate:
boilerplate()
print ""
print "EXAMPLES = '''"
print "# example of doing ___ from a playbook"
print "your_module: some_arg=1 other_arg=2"
print "'''"
print ""
sys.exit(0)
if not options.module_dir:
print "Need module_dir"
sys.exit(1)
if not os.path.exists(options.module_dir):
print >>sys.stderr, "Module directory does not exist: %s" % options.module_dir
sys.exit(1)
if not options.template_dir: def jinja2_environment(template_dir, typ):
print "Need template_dir"
sys.exit(1)
env = Environment(loader=FileSystemLoader(options.template_dir), env = Environment(loader=FileSystemLoader(template_dir),
variable_start_string="@{", variable_start_string="@{",
variable_end_string="}@", variable_end_string="}@",
trim_blocks=True, trim_blocks=True,
) )
env.globals['xline'] = rst_xline env.globals['xline'] = rst_xline
if options.type == 'latex': if typ == 'rst':
env.filters['convert_symbols_to_format'] = latex_ify
template = env.get_template('latex.j2')
outputname = "%s.tex"
includecmt = ""
includefmt = "%s\n"
if options.type == 'html':
env.filters['convert_symbols_to_format'] = html_ify
template = env.get_template('html.j2')
outputname = "%s.html"
includecmt = ""
includefmt = ""
if options.type == 'man':
env.filters['convert_symbols_to_format'] = man_ify
template = env.get_template('man.j2')
outputname = "ansible.%s.3"
includecmt = ""
includefmt = ""
if options.type == 'rst':
env.filters['convert_symbols_to_format'] = rst_ify env.filters['convert_symbols_to_format'] = rst_ify
env.filters['html_ify'] = html_ify env.filters['html_ify'] = html_ify
env.filters['fmt'] = rst_fmt env.filters['fmt'] = rst_fmt
env.filters['xline'] = rst_xline env.filters['xline'] = rst_xline
template = env.get_template('rst.j2') template = env.get_template('rst.j2')
outputname = "%s.rst" outputname = "%s.rst"
includecmt = ".. Generated by module_formatter\n" else:
includefmt = ".. include:: modules/%s.rst\n" raise Exception("unknown module format type: %s" % typ)
if options.type == 'json':
env.filters['convert_symbols_to_format'] = json_ify
outputname = "%s.json"
includecmt = ""
includefmt = ""
if options.type == 'js':
env.filters['convert_symbols_to_format'] = js_ify
template = env.get_template('js.j2')
outputname = "%s.js"
if options.type == 'markdown':
env.filters['convert_symbols_to_format'] = markdown_ify
env.filters['html_ify'] = html_ify
template = env.get_template('markdown.j2')
outputname = "%s.md"
includecmt = ""
includefmt = ""
if options.includes_file is not None and includefmt != "":
incfile = open(options.includes_file, "w")
incfile.write(includecmt)
# Temporary variable required to genrate aggregated content in 'js' format.
js_data = []
categories = list_modules(options.module_dir)
last_category = None
category_names = categories.keys()
category_names.sort()
for category in category_names:
module_map = categories[category] return env, template, outputname
category = category.replace("_"," ") #####################################################################################
category = category.title()
modules = module_map.keys() def process_module(module, options, env, template, outputname, module_map):
modules.sort()
for module in modules:
print "rendering: %s" % module print "rendering: %s" % module
fname = module_map[module] fname = module_map[module]
if len(options.module_list): # ignore files with extensions
if not module in options.module_list: if os.path.basename(fname).find(".") != -1:
continue return
# fname = os.path.join(options.module_dir, module)
extra = os.path.join("inc", "%s.tex" % module)
# probably could just throw out everything with extensions
if fname.endswith(".swp") or fname.endswith(".orig") or fname.endswith(".rej"):
continue
# print " processing module source ---> %s" % fname
if options.type == 'js':
if fname.endswith(".json"):
f = open(fname)
j = json.load(f)
f.close()
js_data.append(j)
continue
# use ansible core library to parse out doc metadata YAML and plaintext examples
doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose) doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose)
# crash if module is missing documentation and not explicitly hidden from docs index
if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES: if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES:
print " while processing module source ---> %s" % fname sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s ***\n" % module) sys.exit(1)
#sys.exit(1) if doc is None:
return
if not doc is None:
all_keys = [] all_keys = []
...@@ -399,43 +247,67 @@ def main(): ...@@ -399,43 +247,67 @@ def main():
doc['ansible_version'] = options.ansible_version doc['ansible_version'] = options.ansible_version
doc['plainexamples'] = examples #plain text doc['plainexamples'] = examples #plain text
# BOOKMARK: here is where we build the table of contents... # here is where we build the table of contents...
text = template.render(doc)
return_data(text, options, outputname, module)
#####################################################################################
if options.includes_file is not None and includefmt != "": def process_category(category, categories, options, env, template, outputname):
if last_category != category: module_map = categories[category]
incfile.write("\n\n")
incfile.write(category)
incfile.write("\n")
incfile.write('`' * len(category))
incfile.write("\n\n")
last_category = category
incfile.write(includefmt % module) # TODO: start a new category file
if options.verbose: category = category.replace("_"," ")
print json.dumps(doc, indent=4) category = category.title()
modules = module_map.keys()
modules.sort()
if options.type == 'latex': for module in modules:
if os.path.exists(extra): process_module(module, options, env, template, outputname, module_map)
f = open(extra)
extradata = f.read()
f.close()
doc['extradata'] = extradata
if options.type == 'json': # TODO: end a new category file
text = json.dumps(doc, indent=2)
else:
text = template.render(doc)
return_data(text, options, outputname, module) #####################################################################################
def validate_options(options):
''' validate option parser options '''
if options.type == 'js': if options.do_boilerplate:
docs = {} boilerplate()
docs['json'] = json.dumps(js_data, indent=2) sys.exit(0)
text = template.render(docs)
return_data(text, options, outputname, 'modules') if not options.module_dir:
print >>sys.stderr, "--module-dir is required"
sys.exit(1)
if not os.path.exists(options.module_dir):
print >>sys.stderr, "--module-dir does not exist: %s" % options.module_dir
sys.exit(1)
if not options.template_dir:
print "--template-dir must be specified"
sys.exit(1)
#####################################################################################
def main():
p = generate_parser()
(options, args) = p.parse_args()
validate_options(options)
env, template, outputname = jinja2_environment(options.template_dir, options.type)
categories = list_modules(options.module_dir)
last_category = None
category_names = categories.keys()
category_names.sort()
for category in category_names:
process_category(category, categories, options, env, template, outputname)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
hacking/templates/html.j2 deleted 100644 → 0
<!-- @{ module | upper }@ -->
<h2>@{module}@</h2>
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor %}
hacking/templates/js.j2 deleted 100644 → 0
function AnsibleModules($scope) {
$scope.modules = @{ json }@;
$scope.orderProp = "module";
}
hacking/templates/latex.j2 deleted 100644 → 0
{# -------------------------------------------------------------------
template for module_formatter.py for LaTeX output (Ansible Booklet)
by Jan-Piet Mens.
Note: nodes & code examples are omitted on purpose.
-------------------------------------------------------------------- #}
%--- @{ module | upper }@ ---- from @{ filename }@ ---
%: -- module header
\mods{@{module}@}{@{docuri}@}{
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor -%}
{% if version_added is defined -%}
(\I{* new in version @{ version_added }@})
{% endif -%}
}
%: -- module options
{% if options %}
\begin{xlist}{abcdefghijklmno}
{% for (opt,v) in options.iteritems() %}
{% if v['required'] %}
\item[\man\,\C{@{ opt }@}]
{% else %}
\item[\opt\,\C{@{ opt }@}]
{% endif %}
{# -------- option description ----------#}
{% for desc in v.description %}
@{ desc | convert_symbols_to_format }@
{% endfor %}
{% if v['choices'] %}
\B{Choices}:\,
{% for choice in v['choices'] %}\C{@{ choice }@}{% if not loop.last %},{% else %}.{% endif %}
{% endfor %}
{% endif %}
{% if v['default'] %}
(default \C{@{ v['default'] }@})
{% endif %}
{% if v['version_added'] is defined %}
(\I{* version @{ v['version_added'] }@})
{% endif %}
{% endfor %}
\end{xlist}
{% endif %}
{# ---------------------------------------
{% if notes %}
{% for note in notes %}
\I{@{ note | convert_symbols_to_format }@}
{% endfor %}
{% endif %}
----------------------------- #}
{#-------------------------------------------
{% if examples is defined -%}
{% for e in examples %}
\begin{extymeta}
@{ e['code'] }@
\end{extymeta}
{% endfor %}
{% endif %}
----------------------------------- #}
{% if extradata is defined %}
%--- BEGIN-EXTRADATA
\begin{extymeta}
@{ extradata }@
\end{extymeta}
%----- END-EXTRADATA
{% endif %}
hacking/templates/markdown.j2 deleted 100644 → 0
## @{ module | convert_symbols_to_format }@
{# ------------------------------------------
#
# This is Github-flavored Markdown
#
--------------------------------------------#}
{% if version_added is defined -%}
New in version @{ version_added }@.
{% endif %}
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor %}
{% if options -%}
<table>
<tr>
<th class="head">parameter</th>
<th class="head">required</th>
<th class="head">default</th>
<th class="head">choices</th>
<th class="head">comments</th>
</tr>
{% for (k,v) in options.iteritems() %}
<tr>
<td>@{ k }@</td>
<td>{% if v.get('required', False) %}yes{% else %}no{% endif %}</td>
<td>{% if v['default'] %}@{ v['default'] }@{% endif %}</td>
<td><ul>{% for choice in v.get('choices',[]) -%}<li>@{ choice }@</li>{% endfor -%}</ul></td>
<td>{% for desc in v.description -%}@{ desc | html_ify }@{% endfor -%}{% if v['version_added'] %} (added in Ansible @{v['version_added']}@){% endif %}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% if examples or plainexamples %}
#### Examples
{% endif %}
{% for example in examples %}
{% if example['description'] %}
* @{ example['description'] | convert_symbols_to_format }@
{% endif %}
```
@{ example['code'] }@
```
{% endfor %}
{% if plainexamples -%}
```
@{ plainexamples }@
```
{% endif %}
{% if notes %}
#### Notes
{% for note in notes %}
@{ note | convert_symbols_to_format }@
{% endfor %}
{% endif %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment