[examples] Implement Python-only doc generator using pyparsing

examples/convert_code_to_doc.py

+#!/usr/bin/env python3
+
+"""
+A simple documentation generator
+generating Markdown-documented examples
+from annotated source code
+"""
+from pyparsing import *
+
+# tell pyparsing to never ignore white space characters
+ParserElement.setDefaultWhitespaceChars('')
+
+def taggedContent(keyname, action, open="[[", close="]]", endTag="/"):
+    """
+    Match content between [[keyname]] and [[/keyname]] and apply the action on it
+    """
+    start = LineStart() + ZeroOrMore(" ") + Literal("//") + ZeroOrMore(" ") + Literal(open + str(keyname) + close)
+    end = LineStart() + ZeroOrMore(" ") + Literal("//") + ZeroOrMore(" ") + Literal(open + endTag + str(keyname) + close)
+    return start.suppress() + SkipTo(end).setParseAction(action) + end.suppress()
+
+def createMarkdownCode(markDownToken):
+    """
+    Put code into Markdown syntax for code blocks with syntax highlighting
+    """
+    def action(token):
+        # only print code snippets with content
+        if not token[0].rstrip():
+            return ""
+        else:
+            return "```" + markDownToken + "\n" + token[0].rstrip() + "\n```\n"
+    return action
+
+def cppRules():
+    """
+    Define a list of rules to apply for cpp source code
+    """
+    header = Suppress(Combine(Literal("// -*-") + SkipTo(Literal("*******/") + LineEnd()) + Literal("*******/")))
+    headerGuard = Suppress(Literal("#ifndef") + Optional(restOfLine) + LineEnd() + Literal("#define") + Optional(restOfLine))
+    endHeaderGuard = Suppress(Literal("#endif") + Optional(restOfLine))
+
+    # exclude stuff between [[exclude]] and [[/exclude]]
+    exclude = taggedContent("exclude", action=replaceWith(""))
+
+    # make a code block (possibly containing comments) between [[codeblock]] and [[/codeblock]]
+    action = createMarkdownCode("cpp")
+    codeblock = taggedContent("codeblock", action=action)
+
+    # treat doc and code line
+    doc = LineStart() + Suppress(ZeroOrMore(" ") + Literal("//") + ZeroOrMore(" ")) + Optional(restOfLine)
+    code = LineStart() + ~(ZeroOrMore(" ") + Literal("//")) + (SkipTo(doc) | SkipTo(StringEnd()))
+    code.setParseAction(action)
+    docTransforms = codeblock | doc | code
+
+    return [header, headerGuard, endHeaderGuard, exclude, docTransforms]
+
+def transformCode(code, rules):
+    for transform in rules:
+        code = transform.transformString(code)
+    return code

examples/generate_example_docs.py

 #!/usr/bin/env python3
+
 import os
 import argparse
-import subprocess
 
-parser = argparse.ArgumentParser()
-parser.add_argument("-d", "--directory", help="The folder to look for examples", default=".")
-parser.add_argument("-m", "--markdowngenerator", help="The markdown generator script taking a list of files", default="../bin/doc/cpp_to_md.sh")
-args = vars(parser.parse_args())
+from convert_code_to_doc import *
 
 def convertToMarkdownAndMerge(dir, includeList):
-    script = os.path.join(os.path.abspath(args["markdowngenerator"]))
     with open(os.path.join(dir, "README.md"), "w") as readme:
-        for include in includeList:
-            if os.path.splitext(include)[1] == ".md":
-                with open(include, "r") as markdown:
+        for include, description in includeList:
+            fileExtension = os.path.splitext(include)[1]
+            if fileExtension == ".md":
+                with open(os.path.join(dir, include), "r") as markdown:
                     readme.write(markdown.read())
+            elif fileExtension == ".hh" or fileExtension == ".cc":
+                title = description if description else "The file "
+                readme.write("\n\n## " + title + "`{}`\n\n\n".format(os.path.split(include)[1]))
+                with open(os.path.join(dir, include), "r") as cppCode:
+                    readme.write(transformCode(cppCode.read(), cppRules()) + "\n")
             else:
-                readme.write("\n\n## The file `{}`\n\n".format(os.path.split(include)[1]))
-                markdown = subprocess.check_output(["bash", script, include], encoding="utf-8")
-                readme.write(markdown + "\n")
+                raise IOError("Unsupported or unknown file extension *{}".format(fileExtension))
 
 def generateReadme(dir):
     includeList = None
+    # look for .doc_config, if not found we pass
     try:
         configname = os.path.join(dir, ".doc_config")
         with open(configname, 'r') as config:
-            includeList = [os.path.join(dir, include) for include in config.read().splitlines()]
+            def readline(line):
+                line = line.split()
+                if len(line) == 1:
+                    return [line[0], ""]
+                else:
+                    return [line[0], line[1]]
+            includeList = [readline(line) for line in config.read().splitlines()]
     except FileNotFoundError:
-        print("Error: The example directory {} does not contain a .doc_config file! Could not generate README.md!".format(dir))
-        raise
+        pass
     if includeList is not None:
         convertToMarkdownAndMerge(dir, includeList)
 
-for path in os.listdir(args["directory"]):
-    abspath = os.path.join(os.path.abspath(args["directory"]), path)
-    if os.path.isdir(abspath):
-        generateReadme(abspath)
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--directory", help="The folder to look for examples", default=".")
+    args = vars(parser.parse_args())
+
+    for path in os.listdir(args["directory"]):
+        abspath = os.path.join(os.path.abspath(args["directory"]), path)
+        if os.path.isdir(abspath):
+            generateReadme(abspath)