fix: Tools for reformatting code. (GIT8266O-364)

.gitignore

@@ -14,3 +14,6 @@ examples/**/build
 tools/unit-test-app/sdkconfig
 tools/unit-test-app/sdkconfig.old
 tools/unit-test-app/build
+
+# Python
+venv

README.md

@@ -170,3 +170,50 @@ git pull
 ```
 
 The ``git pull`` command is fetching and merging changes from ESP8266_RTOS_SDK repository on GitHub.
+
+# Developers
+
+*This section is for developers of ESP8266_RTOS_SDK itself*
+
+## Code style
+
+We use [astyle](http://astyle.sourceforge.net/) to format the code.
+The formatting settings can be seen in
+[tools/format.sh](tools/format.sh).
+
+## The code-style tool
+
+`tools/code-style` is a tool to expand tabs and run clang-format on the
+entire repository.
+
+Usage: `tools/code-style [-e] [-c]`
+
+* `-t` shows all the files that will be tagged.
+* `-e` runs untabbing (replace tab characters with 4 spaces)
+* `-a` runs astyle.
+* `-n` enabled dry-run mode. In this mode no files will be changed,
+  but it will let you know which files that would have been changed.
+
+It uses `tools/astyle` to find files in the repository. The
+tool reads `tools/code-style.ini` (see the current version
+[here](tools/code-style.ini)) and tags each file it finds with attributes
+explaining what should be done with the file.
+
+The tools are written in bash and Python.
+
+## Installing dependencies for Python
+
+`tag-files` requires the following packages:
+
+* pathspec
+* configparser (build-in in python3)
+
+If these are not available through your OSes package manager you can
+use virtualenv to install them locally:
+
+    virtualenv venv
+    venv/bin/pip install -r tools/requirements.txt
+
+After that you can add `venv/bin` to path and run `tools/tag-files` like this:
+
+    PATH=$(pwd)/venv/bin:$PATH

tools/code-style

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -e -o pipefail
+cd $(dirname $0)/..
+
+function fix() {
+    local f=$1; shift
+    orig=$(md5sum "$f" | cut -f 1 -d ' ')
+    "$@" < "$f" > "$f".tmp
+    new=$(md5sum "$f.tmp" | cut -f 1 -d ' ')
+
+    if [[ $orig != $new ]]
+    then
+        echo Fixed "$f"
+
+        if [[ $dry_run -ne 0 ]]
+        then
+            mv "$f.tmp" "$f"
+        fi
+    fi
+
+    rm -f "$f.tmp"
+}
+
+function usage() {
+    echo "$0: usage: [OPTION]" >/dev/stderr
+    echo " -n: dry run" >/dev/stderr
+    echo " -t: show all files and tags" >/dev/stderr
+    echo " -e: run tab expansion" >/dev/stderr
+    echo " -a: run astyle" >/dev/stderr
+    exit 1
+}
+
+dry_run=0
+run_tag=0
+run_expand_tabs=0
+run_astyle=0
+
+while getopts "ntaeh?" opt; do
+    case $opt in
+    n)
+        dry_run=1
+        ;;
+    t)
+        run_tag=1
+        ;;
+    e)
+        run_expand_tabs=1
+        ;;
+    a)
+        run_astyle=1
+        ;;
+    h|?)
+        usage
+        ;;
+    esac
+done
+
+tf=(tools/tag-files --config tools/code-style.ini)
+
+if [[ $run_tag -ne 0 ]]
+then
+    "${tf[@]}"
+fi
+
+if [[ $run_expand_tabs -ne 0 ]]
+then
+    echo "Replacing tabs with spaces..."
+    "${tf[@]}" |\
+        grep expand-tabs=yes |\
+    while IFS=$'\t' read f params
+    do
+        fix "$f" sed 's,\t,    ,g'
+    done
+
+    echo "To undo run"
+    echo "tools/tag-files | grep -v ignore=True | grep no_expand_tab=False | cut -f 1 -d $'\t' | xargs git checkout"
+fi
+
+if [[ $run_astyle -ne 0 ]]
+then
+    echo "Running astyle..."
+    "${tf[@]}" |\
+        grep astyle=yes |\
+    while IFS=$'\t' read f params
+    do
+        fix "$f" tools/format.sh -n
+    done
+
+    echo "To undo run"
+    echo "tools/tag-files | grep -v ignore=True | grep clang_format=True | cut -f 1 -d $'\t' | xargs git checkout"
+fi

tools/code-style.ini

@@ -0,0 +1,45 @@
+# Configuration for tag-files. The 'tree' set controls which files that
+# are tagged, the other sets control are tags applied to files found in the
+# 'tree' set.
+
+[tree:include]
+*
+
+# These will never be printed
+[tree:exclude]
+.git
+venv
+tools/cmake/third_party
+tools/kconfig
+tools/kconfig_new
+*.bin
+*.o
+*.a
+*.key
+*.rar
+sdkconfig
+sdkconfig.old
+
+# Third party code
+components/cjson
+components/esp8266/include/xtensa
+components/esptool_py
+components/freertos/freertos
+components/freertos/include
+components/lwip
+components/newlib/newlib/include
+components/spiffs
+components/mqtt/paho
+components/ssl/axtls
+components/ssl/mbedtls
+components/ssl/wolfssl
+
+[expand-tabs:exclude]
+*.mk
+Makefile*
+makefile*
+
+[astyle:include]
+*.h
+*.c
+*.cpp

tools/format.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Runs astyle with the full set of formatting options
+exec astyle \
+	--style=otbs \
+	--indent=spaces=4 \
+	--convert-tabs \
+	--align-pointer=name \
+	--align-reference=name \
+	--keep-one-line-statements \
+	--pad-header \
+	--pad-oper \
+	"$@"

tools/requirements.txt

@@ -0,0 +1,2 @@
+pathspec
+configparser

tools/tag-files

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8
+# Written by Trygve Laugstøl <[email protected]>
+
+from __future__ import print_function
+from pathspec import PathSpec
+import argparse
+import configparser
+import os
+import sys
+
+def err_print(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+parser = argparse.ArgumentParser(description="Tag files")
+parser.add_argument("--basedir", action="store", help="Where to start scanning")
+parser.add_argument("--config", action="store", help="Config file to use. Default: tag-files.ini")
+
+args = parser.parse_args()
+if help in args:
+    sys.exit(0)
+
+basedir = args.basedir if args.basedir else os.getcwd()
+ini_file = args.config if args.config else "tag-files.ini"
+
+err_print("basedir={}".format(basedir))
+err_print("ini_file={}".format(ini_file))
+
+config = configparser.ConfigParser(allow_no_value=True)
+config.optionxform = str
+try:
+    with open(ini_file, "r") as f:
+        config.read_file(f)
+except IOError as e:
+    err_print("Could not open ini file: {}".format(ini_file))
+    sys.exit(1)
+
+keys = set(k[:k.rindex(":")] for k in config if
+        (k.endswith(":include") or k.endswith(":exclude"))
+        and not k.startswith("tree:"))
+
+def load(key):
+    include = list(config[key + ":include"]) if key + ":include" in config else ["*"]
+    exclude = list(config[key + ":exclude"]) if key + ":exclude" in config else []
+
+    # print("{}: includes={}, excludes={}".format(key, len(includes), len(excludes)))
+
+    return [PathSpec.from_lines('gitwildmatch', include),
+            PathSpec.from_lines('gitwildmatch', exclude)]
+
+checks = {k: load(k) for k in keys}
+
+tree = load("tree")
+
+tree = set(tree[0].match_tree(basedir)) -\
+       set(tree[1].match_tree(basedir))
+
+try:
+    for f in sorted(tree):
+        print(f, end="")
+        for k, v in checks.items():
+            include = v[0].match_file(f) and not v[1].match_file(f)
+            i = "yes" if include else "no"
+            print("\t{}={}".format(k, i), end="")
+        print()
+except IOError as e:
+    # This will happen if stdout is closed. Not a problem we care about fixing.
+    pass