Automatically include sample code into user guide

This change add scripts to generate `user_guide.md` automatically from special
markers that are now visible in the source file `user_guide.md.in`. This allows
us to easily keep the source code up-to-date in the docs without having to
manually copy-paste the code, and to ensure that the code we test (in the
example files) is exactly the code that ends up in the documentation.

Closes #54

.gitignore

@@ -0,0 +1,5 @@
+# C compiled objects
+*.o
+
+# Python generated files
+*.pyc

docs/Makefile

@@ -0,0 +1,31 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+VERB = @
+ifeq ($(VERBOSE),1)
+	VERB =
+endif
+
+gen: user_guide.md
+
+%.md: %.md.in doc_gen.py 
+	$(VERB) ./doc_gen.py $< > $@
+
+diff:
+	$(VERB) ./doc_gen_diff.sh
+
+test:
+	$(VERB) python -m unittest discover -p '*_test.py'
+
+all-tests: diff test

docs/doc_gen.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+#
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generates user_guide.md from user_guide.md.in ."""
+
+import json
+import os
+import re
+import sys
+
+
+def ReadFileRaw(filename):
+    with open(filename, 'r') as source:
+        for line in source:
+            yield line
+
+
+def ReadFileContentsWithMarker(filename, marker):
+    begin_comment_c = re.compile(r'^/\* BEGIN: (\w+) \*/$')
+    end_comment_c = re.compile(r'^/\* END: (\w+) \*/$')
+    begin_comment_cpp = re.compile(r'^// BEGIN: (\w+)$')
+    end_comment_cpp = re.compile(r'^// END: (\w+)$')
+
+    def Matches(matcherA, matcherB, content):
+        return (matcherA.match(content), matcherB.match(content))
+
+    def Valid(matches):
+        return matches[0] or matches[1]
+
+    def Group1(matches):
+        if matches[0]:
+            return matches[0].group(1)
+        else:
+            return matches[1].group(1)
+
+    output = False
+    for line in ReadFileRaw(filename):
+        begin_matches = Matches(begin_comment_c, begin_comment_cpp, line)
+        if Valid(begin_matches):
+            begin_marker = Group1(begin_matches)
+            if begin_marker == marker:
+                yield '~~~c\n'  # Markdown C formatting header
+                output = True
+                continue        # avoid outputting our own begin line
+
+        end_matches = Matches(end_comment_c, end_comment_cpp, line)
+        if Valid(end_matches):
+            end_marker = Group1(end_matches)
+            if end_marker == marker:
+                yield '~~~\n'  # Markdown formatting end block
+                return         # we are done with this region
+
+        if output:
+            yield line  # enables outputting nested region markers
+
+
+def ProcessFile(source):
+    pattern = re.compile(r'^{({.*})}$')
+    for line in source:
+        match = pattern.match(line)
+        if not match:
+            yield line
+            continue
+
+        # Handle special include
+        params = json.loads(match.group(1))
+        full_path = params['source']
+        base_name = os.path.basename(full_path)
+        yield '[`%s`](%s)\n' % (base_name, full_path)
+        yield '\n'
+        marker = params['marker']
+        for item in ReadFileContentsWithMarker(full_path, marker):
+            yield item
+
+
+def main(argv):
+    if len(argv) < 2:
+        sys.stderr.write('Syntax: %s [filename]\n' % argv[0])
+        sys.exit(1)
+
+    filename = argv[1]
+    with open(filename, 'r') as source:
+        sys.stdout.write('''\
+<!-- This file was auto-generated by `%s %s`.
+     Do not modify manually; changes will be overwritten. -->
+
+''' % (argv[0], argv[1]))
+        results = ProcessFile(source.readlines())
+        for line in results:
+            sys.stdout.write('%s' % line)
+
+
+if __name__ == '__main__':
+    main(sys.argv)

docs/doc_gen_diff.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -eu
+#
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO(mbrukman): figure out how to encode this in Makefile syntax for
+# simplicity. In particular, the diff command with subcommand diff does not seem
+# to work and it's unclear what the correct syntax is for this, so keeping this
+# in a shell script for now.
+for md in *.md.in ; do
+  diff -u <(./doc_gen.py "${md}") "${md%.in}"
+done

docs/doc_gen_test.py

@@ -0,0 +1,119 @@
+#!/usr/bin/python
+#
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the `doc_gen` module."""
+
+import unittest
+import doc_gen
+
+
+class ProcessFileTest(unittest.TestCase):
+
+    def setUp(self):
+        self.docgen_readfile = None
+
+
+    def tearDown(self):
+        if self.docgen_readfile is not None:
+            doc_gen.ReadFileRaw = self.docgen_readfile
+
+
+    def _setCustomReadFile(self, filepath, contents):
+        self.docgen_readfile = doc_gen.ReadFileRaw
+        def readFileCustom(filename):
+            if filename == filepath:
+                return contents
+            else:
+                raise Exception('Expected file: "%s", received: "%s"' %
+                        (filepath, filename))
+
+        doc_gen.ReadFileRaw = readFileCustom
+
+
+    def testPreC99Comments(self):
+        source_file = 'file.c'
+        source_file_fullpath = '../src/examples/%s' % source_file
+        marker = 'docs'
+        source_file_contents = [
+            '/* license header comment */\n',
+            '\n',
+            '/* BEGIN: %s */\n' % marker,
+            '#include <stdio.h>\n',
+            '/* END: %s */\n' % marker,
+            'int main(int argc, char** argv) {\n',
+            '  return 0;\n',
+            '}\n',
+        ]
+
+        md_file_contents = [
+            'text before code\n',
+            '{{"source": "%s", "marker": "%s"}}\n' % (source_file_fullpath, marker),
+            'text after code\n',
+        ]
+
+        expected_output = [
+            'text before code\n',
+            '[`%s`](%s)\n' % (source_file, source_file_fullpath),
+            '\n',
+            '~~~c\n',
+            '#include <stdio.h>\n',
+            '~~~\n',
+            'text after code\n',
+        ]
+
+        self._setCustomReadFile(source_file_fullpath, source_file_contents)
+        actual_output = list(doc_gen.ProcessFile(md_file_contents))
+        self.assertEquals(expected_output, actual_output)
+
+
+    def testC99Comments(self):
+        source_file = 'file.c'
+        source_file_fullpath = '../src/examples/%s' % source_file
+        marker = 'docs'
+        source_file_contents = [
+            '/* license header comment */\n',
+            '\n',
+            '// BEGIN: %s\n' % marker,
+            '#include <stdio.h>\n',
+            '// END: %s\n' % marker,
+            'int main(int argc, char** argv) {\n',
+            '  return 0;\n',
+            '}\n',
+        ]
+
+        md_file_contents = [
+            'text before code\n',
+            '{{"source": "%s", "marker": "%s"}}\n' % (source_file_fullpath, marker),
+            'text after code\n',
+        ]
+
+        expected_output = [
+            'text before code\n',
+            '[`%s`](%s)\n' % (source_file, source_file_fullpath),
+            '\n',
+            '~~~c\n',
+            '#include <stdio.h>\n',
+            '~~~\n',
+            'text after code\n',
+        ]
+
+        self._setCustomReadFile(source_file_fullpath, source_file_contents)
+        actual_output = list(doc_gen.ProcessFile(md_file_contents))
+        self.assertEquals(expected_output, actual_output)
+
+
+if __name__ == '__main__':
+    unittest.main()