Skip to content

A
ansible
Commit fe2d00d9 by Michael DeHaan
Options

WIP on refactoring the module formatter code that we use to build the doc site with.

parent 31d0060d
Showing with 126 additions and 414 deletions
Makefile
......@@ -64,9 +64,6 @@ all: clean python
tests:
PYTHONPATH=./lib ANSIBLE_LIBRARY=./library $(NOSETESTS) -d -v
# To force a rebuild of the docs run 'touch VERSION && make docs'
docs: $(MANPAGES) modulepages
authors:
sh hacking/authors.sh
......@@ -172,11 +169,6 @@ deb: debian
# for arch or gentoo, read instructions in the appropriate 'packaging' subdirectory directory
modulepages:
PYTHONPATH=./lib $(PYTHON) hacking/module_formatter.py -A $(VERSION) -t man -o docs/man/man3/ --module-dir=library --template-dir=hacking/templates # --verbose
# because this requires Sphinx it is not run as part of every build, those building the RPM and so on can ignore this
webdocs:
webdocs: $(MANPAGES)
(cd docsite/; make docs)
hacking/module_formatter.py
......@@ -24,16 +24,20 @@ import yaml
import codecs
import json
import ast
from jinja2 import Environment, FileSystemLoader
import re
import optparse
import time
import datetime
import subprocess
import cgi
from jinja2 import Environment, FileSystemLoader
import ansible.utils
import ansible.utils.module_docs as module_docs
#####################################################################################
# constants and paths
# if a module is added in a version of Ansible older than this, don't print the version added information
# in the module documentation because everyone is assumed to be running something newer than this already.
TO_OLD_TO_BE_NOTABLE = 1.0
......@@ -48,65 +52,16 @@ EXAMPLE_YAML=os.path.abspath(os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.pardir, 'examples', 'DOCUMENTATION.yml'
))
# There is a better way of doing this!
# TODO: somebody add U(text, http://foo.bar/) as described by Tim in #991
_ITALIC = re.compile(r"I\(([^)]+)\)")
_BOLD = re.compile(r"B\(([^)]+)\)")
_MODULE = re.compile(r"M\(([^)]+)\)")
_URL = re.compile(r"U\(([^)]+)\)")
_CONST = re.compile(r"C\(([^)]+)\)")
def latex_ify(text):
t = _ITALIC.sub("\\I{" + r"\1" + "}", text)
t = _BOLD.sub("\\B{" + r"\1" + "}", t)
t = _MODULE.sub("\\M{" + r"\1" + "}", t)
t = _URL.sub("\\url{" + r"\1" + "}", t)
t = _CONST.sub("\\C{" + r"\1" + "}", t)
return t
def html_ify(text):
#print "DEBUG: text=%s" % text
t = cgi.escape(text)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def json_ify(text):
t = _ITALIC.sub("<em>" + r"\1" + "</em>", text)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
def js_ify(text):
return text
def man_ify(text):
t = _ITALIC.sub(r'\\fI' + r"\1" + r"\\fR", text)
t = _BOLD.sub(r'\\fB' + r"\1" + r"\\fR", t)
t = _MODULE.sub(r'\\fI' + r"\1" + r"\\fR", t)
t = _URL.sub(r'\\fI' + r"\1" + r"\\fR", t)
t = _CONST.sub(r'\\fC' + r"\1" + r"\\fR", t)
return t
#####################################################################################
def rst_ify(text):
''' convert symbols like I(this is in italics) to valid restructured text '''
t = _ITALIC.sub(r'*' + r"\1" + r"*", text)
t = _BOLD.sub(r'**' + r"\1" + r"**", t)
......@@ -116,31 +71,40 @@ def rst_ify(text):
return t
_MARKDOWN = re.compile(r"[*_`]")
#####################################################################################
def markdown_ify(text):
def html_ify(text):
''' convert symbols like I(this is in italics) to valid HTML '''
t = cgi.escape(text)
t = _MARKDOWN.sub(r"\\\g<0>", t)
t = _ITALIC.sub("_" + r"\1" + "_", t)
t = _BOLD.sub("**" + r"\1" + "**", t)
t = _MODULE.sub("*" + r"\1" + "*", t)
t = _URL.sub("[" + r"\1" + "](" + r"\1" + ")", t)
t = _CONST.sub("`" + r"\1" + "`", t)
t = _ITALIC.sub("<em>" + r"\1" + "</em>", t)
t = _BOLD.sub("<b>" + r"\1" + "</b>", t)
t = _MODULE.sub("<span class='module'>" + r"\1" + "</span>", t)
t = _URL.sub("<a href='" + r"\1" + "'>" + r"\1" + "</a>", t)
t = _CONST.sub("<code>" + r"\1" + "</code>", t)
return t
# Helper for Jinja2 (format() doesn't work here...)
#####################################################################################
def rst_fmt(text, fmt):
''' helper for Jinja2 to do format strings '''
return fmt % (text)
#####################################################################################
def rst_xline(width, char="="):
''' return a restructured text line of a given length '''
return char * width
def load_examples_section(text):
return text.split('***BREAK***')
#####################################################################################
def return_data(text, options, outputname, module):
''' dumps module output to a file or the screen, as requested '''
if options.output_dir is not None:
f = open(os.path.join(options.output_dir, outputname % module), 'w')
f.write(text.encode('utf-8'))
......@@ -148,15 +112,29 @@ def return_data(text, options, outputname, module):
else:
print text
#####################################################################################
def boilerplate():
''' prints the boilerplate for module docs '''
if not os.path.exists(EXAMPLE_YAML):
print >>sys.stderr, "Missing example boiler plate: %s" % EXAMPLE_YAML
print "DOCUMENTATION = '''"
print file(EXAMPLE_YAML).read()
print "'''"
print ""
print ""
print "EXAMPLES = '''"
print "# example of doing ___ from a playbook"
print "your_module: some_arg=1 other_arg=2"
print "'''"
print ""
#####################################################################################
def list_modules(module_dir):
''' returns a hash of categories, each category being a hash of module names to file paths '''
categories = {}
files = glob.glob("%s/*" % module_dir)
for d in files:
......@@ -171,202 +149,72 @@ def list_modules(module_dir):
categories[category][module] = f
return categories
def main():
#####################################################################################
def generate_parser():
''' generate an optparse parser '''
p = optparse.OptionParser(
version='%prog 1.0',
usage='usage: %prog [options] arg1 arg2',
description='Convert Ansible module DOCUMENTATION strings to other formats',
description='Generate module documentation from metadata',
)
p.add_option("-A", "--ansible-version",
action="store",
dest="ansible_version",
default="unknown",
help="Ansible version number")
p.add_option("-M", "--module-dir",
action="store",
dest="module_dir",
default=MODULEDIR,
help="Ansible modules/ directory")
p.add_option("-T", "--template-dir",
action="store",
dest="template_dir",
default="hacking/templates",
help="directory containing Jinja2 templates")
p.add_option("-t", "--type",
action='store',
dest='type',
choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'],
default='latex',
help="Output type")
p.add_option("-m", "--module",
action='append',
default=[],
dest='module_list',
help="Add modules to process in module_dir")
p.add_option("-v", "--verbose",
action='store_true',
default=False,
help="Verbose")
p.add_option("-o", "--output-dir",
action="store",
dest="output_dir",
default=None,
help="Output directory for module files")
p.add_option("-I", "--includes-file",
action="store",
dest="includes_file",
default=None,
help="Create a file containing list of processed modules")
p.add_option("-G", "--generate",
action="store_true",
dest="do_boilerplate",
default=False,
help="generate boilerplate DOCUMENTATION to stdout")
p.add_option("-A", "--ansible-version", action="store", dest="ansible_version", default="unknown", help="Ansible version number")
p.add_option("-M", "--module-dir", action="store", dest="module_dir", default=MODULEDIR, help="Ansible library path")
p.add_option("-T", "--template-dir", action="store", dest="template_dir", default="hacking/templates", help="directory containing Jinja2 templates")
p.add_option("-t", "--type", action='store', dest='type', choices=['html', 'latex', 'man', 'rst', 'json', 'markdown', 'js'], default='latex', help="Document type")
p.add_option("-v", "--verbose", action='store_true', default=False, help="Verbose")
p.add_option("-o", "--output-dir", action="store", dest="output_dir", default=None, help="Output directory for module files")
p.add_option("-I", "--includes-file", action="store", dest="includes_file", default=None, help="Create a file containing list of processed modules")
p.add_option("-G", "--generate", action="store_true", dest="do_boilerplate", default=False, help="generate boilerplate docs to stdout")
p.add_option('-V', action='version', help='Show version number and exit')
return p
(options, args) = p.parse_args()
# print "M: %s" % options.module_dir
# print "t: %s" % options.type
# print "m: %s" % options.module_list
# print "v: %s" % options.verbose
if options.do_boilerplate:
boilerplate()
print ""
print "EXAMPLES = '''"
print "# example of doing ___ from a playbook"
print "your_module: some_arg=1 other_arg=2"
print "'''"
print ""
sys.exit(0)
if not options.module_dir:
print "Need module_dir"
sys.exit(1)
if not os.path.exists(options.module_dir):
print >>sys.stderr, "Module directory does not exist: %s" % options.module_dir
sys.exit(1)
#####################################################################################
if not options.template_dir:
print "Need template_dir"
sys.exit(1)
def jinja2_environment(template_dir, typ):
env = Environment(loader=FileSystemLoader(options.template_dir),
env = Environment(loader=FileSystemLoader(template_dir),
variable_start_string="@{",
variable_end_string="}@",
trim_blocks=True,
)
env.globals['xline'] = rst_xline
if options.type == 'latex':
env.filters['convert_symbols_to_format'] = latex_ify
template = env.get_template('latex.j2')
outputname = "%s.tex"
includecmt = ""
includefmt = "%s\n"
if options.type == 'html':
env.filters['convert_symbols_to_format'] = html_ify
template = env.get_template('html.j2')
outputname = "%s.html"
includecmt = ""
includefmt = ""
if options.type == 'man':
env.filters['convert_symbols_to_format'] = man_ify
template = env.get_template('man.j2')
outputname = "ansible.%s.3"
includecmt = ""
includefmt = ""
if options.type == 'rst':
if typ == 'rst':
env.filters['convert_symbols_to_format'] = rst_ify
env.filters['html_ify'] = html_ify
env.filters['fmt'] = rst_fmt
env.filters['xline'] = rst_xline
template = env.get_template('rst.j2')
outputname = "%s.rst"
includecmt = ".. Generated by module_formatter\n"
includefmt = ".. include:: modules/%s.rst\n"
if options.type == 'json':
env.filters['convert_symbols_to_format'] = json_ify
outputname = "%s.json"
includecmt = ""
includefmt = ""
if options.type == 'js':
env.filters['convert_symbols_to_format'] = js_ify
template = env.get_template('js.j2')
outputname = "%s.js"
if options.type == 'markdown':
env.filters['convert_symbols_to_format'] = markdown_ify
env.filters['html_ify'] = html_ify
template = env.get_template('markdown.j2')
outputname = "%s.md"
includecmt = ""
includefmt = ""
if options.includes_file is not None and includefmt != "":
incfile = open(options.includes_file, "w")
incfile.write(includecmt)
# Temporary variable required to genrate aggregated content in 'js' format.
js_data = []
categories = list_modules(options.module_dir)
last_category = None
category_names = categories.keys()
category_names.sort()
for category in category_names:
else:
raise Exception("unknown module format type: %s" % typ)
module_map = categories[category]
return env, template, outputname
category = category.replace("_"," ")
category = category.title()
#####################################################################################
modules = module_map.keys()
modules.sort()
for module in modules:
def process_module(module, options, env, template, outputname, module_map):
print "rendering: %s" % module
fname = module_map[module]
if len(options.module_list):
if not module in options.module_list:
continue
# fname = os.path.join(options.module_dir, module)
extra = os.path.join("inc", "%s.tex" % module)
# probably could just throw out everything with extensions
if fname.endswith(".swp") or fname.endswith(".orig") or fname.endswith(".rej"):
continue
# print " processing module source ---> %s" % fname
if options.type == 'js':
if fname.endswith(".json"):
f = open(fname)
j = json.load(f)
f.close()
js_data.append(j)
continue
# ignore files with extensions
if os.path.basename(fname).find(".") != -1:
return
# use ansible core library to parse out doc metadata YAML and plaintext examples
doc, examples = ansible.utils.module_docs.get_docstring(fname, verbose=options.verbose)
# crash if module is missing documentation and not explicitly hidden from docs index
if doc is None and module not in ansible.utils.module_docs.BLACKLIST_MODULES:
print " while processing module source ---> %s" % fname
sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s ***\n" % module)
#sys.exit(1)
if not doc is None:
sys.stderr.write("*** ERROR: CORE MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
sys.exit(1)
if doc is None:
return
all_keys = []
......@@ -399,43 +247,67 @@ def main():
doc['ansible_version'] = options.ansible_version
doc['plainexamples'] = examples #plain text
# BOOKMARK: here is where we build the table of contents...
# here is where we build the table of contents...
text = template.render(doc)
return_data(text, options, outputname, module)
#####################################################################################
if options.includes_file is not None and includefmt != "":
def process_category(category, categories, options, env, template, outputname):
if last_category != category:
incfile.write("\n\n")
incfile.write(category)
incfile.write("\n")
incfile.write('`' * len(category))
incfile.write("\n\n")
last_category = category
module_map = categories[category]
incfile.write(includefmt % module)
# TODO: start a new category file
if options.verbose:
print json.dumps(doc, indent=4)
category = category.replace("_"," ")
category = category.title()
modules = module_map.keys()
modules.sort()
if options.type == 'latex':
if os.path.exists(extra):
f = open(extra)
extradata = f.read()
f.close()
doc['extradata'] = extradata
for module in modules:
process_module(module, options, env, template, outputname, module_map)
if options.type == 'json':
text = json.dumps(doc, indent=2)
else:
text = template.render(doc)
# TODO: end a new category file
return_data(text, options, outputname, module)
#####################################################################################
def validate_options(options):
''' validate option parser options '''
if options.type == 'js':
docs = {}
docs['json'] = json.dumps(js_data, indent=2)
text = template.render(docs)
return_data(text, options, outputname, 'modules')
if options.do_boilerplate:
boilerplate()
sys.exit(0)
if not options.module_dir:
print >>sys.stderr, "--module-dir is required"
sys.exit(1)
if not os.path.exists(options.module_dir):
print >>sys.stderr, "--module-dir does not exist: %s" % options.module_dir
sys.exit(1)
if not options.template_dir:
print "--template-dir must be specified"
sys.exit(1)
#####################################################################################
def main():
p = generate_parser()
(options, args) = p.parse_args()
validate_options(options)
env, template, outputname = jinja2_environment(options.template_dir, options.type)
categories = list_modules(options.module_dir)
last_category = None
category_names = categories.keys()
category_names.sort()
for category in category_names:
process_category(category, categories, options, env, template, outputname)
if __name__ == '__main__':
main()
hacking/templates/html.j2 deleted 100644 → 0
<!-- @{ module | upper }@ -->
<h2>@{module}@</h2>
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor %}
hacking/templates/js.j2 deleted 100644 → 0
function AnsibleModules($scope) {
$scope.modules = @{ json }@;
$scope.orderProp = "module";
}
hacking/templates/latex.j2 deleted 100644 → 0
{# -------------------------------------------------------------------
template for module_formatter.py for LaTeX output (Ansible Booklet)
by Jan-Piet Mens.
Note: nodes & code examples are omitted on purpose.
-------------------------------------------------------------------- #}
%--- @{ module | upper }@ ---- from @{ filename }@ ---
%: -- module header
\mods{@{module}@}{@{docuri}@}{
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor -%}
{% if version_added is defined -%}
(\I{* new in version @{ version_added }@})
{% endif -%}
}
%: -- module options
{% if options %}
\begin{xlist}{abcdefghijklmno}
{% for (opt,v) in options.iteritems() %}
{% if v['required'] %}
\item[\man\,\C{@{ opt }@}]
{% else %}
\item[\opt\,\C{@{ opt }@}]
{% endif %}
{# -------- option description ----------#}
{% for desc in v.description %}
@{ desc | convert_symbols_to_format }@
{% endfor %}
{% if v['choices'] %}
\B{Choices}:\,
{% for choice in v['choices'] %}\C{@{ choice }@}{% if not loop.last %},{% else %}.{% endif %}
{% endfor %}
{% endif %}
{% if v['default'] %}
(default \C{@{ v['default'] }@})
{% endif %}
{% if v['version_added'] is defined %}
(\I{* version @{ v['version_added'] }@})
{% endif %}
{% endfor %}
\end{xlist}
{% endif %}
{# ---------------------------------------
{% if notes %}
{% for note in notes %}
\I{@{ note | convert_symbols_to_format }@}
{% endfor %}
{% endif %}
----------------------------- #}
{#-------------------------------------------
{% if examples is defined -%}
{% for e in examples %}
\begin{extymeta}
@{ e['code'] }@
\end{extymeta}
{% endfor %}
{% endif %}
----------------------------------- #}
{% if extradata is defined %}
%--- BEGIN-EXTRADATA
\begin{extymeta}
@{ extradata }@
\end{extymeta}
%----- END-EXTRADATA
{% endif %}
hacking/templates/markdown.j2 deleted 100644 → 0
## @{ module | convert_symbols_to_format }@
{# ------------------------------------------
#
# This is Github-flavored Markdown
#
--------------------------------------------#}
{% if version_added is defined -%}
New in version @{ version_added }@.
{% endif %}
{% for desc in description -%}
@{ desc | convert_symbols_to_format }@
{% endfor %}
{% if options -%}
<table>
<tr>
<th class="head">parameter</th>
<th class="head">required</th>
<th class="head">default</th>
<th class="head">choices</th>
<th class="head">comments</th>
</tr>
{% for (k,v) in options.iteritems() %}
<tr>
<td>@{ k }@</td>
<td>{% if v.get('required', False) %}yes{% else %}no{% endif %}</td>
<td>{% if v['default'] %}@{ v['default'] }@{% endif %}</td>
<td><ul>{% for choice in v.get('choices',[]) -%}<li>@{ choice }@</li>{% endfor -%}</ul></td>
<td>{% for desc in v.description -%}@{ desc | html_ify }@{% endfor -%}{% if v['version_added'] %} (added in Ansible @{v['version_added']}@){% endif %}</td>
</tr>
{% endfor %}
</table>
{% endif %}
{% if examples or plainexamples %}
#### Examples
{% endif %}
{% for example in examples %}
{% if example['description'] %}
* @{ example['description'] | convert_symbols_to_format }@
{% endif %}
```
@{ example['code'] }@
```
{% endfor %}
{% if plainexamples -%}
```
@{ plainexamples }@
```
{% endif %}
{% if notes %}
#### Notes
{% for note in notes %}
@{ note | convert_symbols_to_format }@
{% endfor %}
{% endif %}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment