From c4db4fadba740feedcc6771b29d7a073d8b2255c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Mon, 14 Mar 2016 20:56:10 +0100 Subject: [PATCH 01/10] Implement suppresion of line numbering in continuation lines. --- bin/fypp | 238 +++++++++++++++++++++++++------------------- test/test_fypp.py | 244 ++++++++++++++++++++++++---------------------- 2 files changed, 264 insertions(+), 218 deletions(-) diff --git a/bin/fypp b/bin/fypp index e48b89f..c03e0ab 100755 --- a/bin/fypp +++ b/bin/fypp @@ -923,7 +923,7 @@ class Builder: '''Should be called to signalize a comment directive. The content of the comment is not needed by the builder, but it needs - the span of the comment to generate proper synclines if needed. + the span of the comment to generate proper line numbers if needed. Args: span (tuple of int): Start and end line of the directive. @@ -1007,18 +1007,22 @@ class Renderer: Args: evaluator (Evaluator, optional): Evaluator to use when rendering eval directives. If None (default), Evaluator() is used. - synclines (bool, optional): Whether synclines should be generated, + linenums (bool, optional): Whether linenums should be generated, defaults to False. + contlinenums (bool, optional): Whether linenums for continuation + should be generated, defaults to False. linefolder (callable): Callable to use when folding a line. ''' - def __init__(self, evaluator=None, synclines=False, linefolder=None): + def __init__(self, evaluator=None, linenums=False, contlinenums=False, + linefolder=None): self._evaluator = Evaluator() if evaluator is None else evaluator self._evaluator.updateenv(_DATE_=time.strftime('%Y-%m-%d'), _TIME_=time.strftime('%H:%M:%S')) self._curfile = None self._diversions = 0 - self._synclines = synclines + self._linenums = linenums + self._contlinenums = contlinenums if linefolder is None: self._linefolder = lambda line: [line] else: @@ -1130,15 +1134,15 @@ class Renderer: condition, exc) raise FyppError(msg, self._curfile, span) if cond: - if self._synclines and not self._diversions and multiline: - out.append(syncline(span[1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(span[1], self._curfile)) outcont, ievalcont, pevalcont = self._render(content) ieval += _shiftinds(ievalcont, len(out)) peval += pevalcont out += outcont break - if self._synclines and not self._diversions and multiline: - out.append(syncline(spans[-1][1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(spans[-1][1], self._curfile)) return out, ieval, peval @@ -1160,14 +1164,14 @@ class Renderer: else: loopscope = {varname: value for varname, value in zip(loopvars, var)} - if self._synclines and not self._diversions and multiline: - out.append(syncline(spans[0][1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(spans[0][1], self._curfile)) outcont, ievalcont, pevalcont = self._render(content, loopscope) ieval += _shiftinds(ievalcont, len(out)) peval += pevalcont out += outcont - if self._synclines and not self._diversions and multiline: - out.append(syncline(spans[1][1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(spans[1][1], self._curfile)) return out, ieval, peval @@ -1197,8 +1201,8 @@ class Renderer: msg = "exception occured when defining macro '{}'\n{}"\ .format(name, exc) raise FyppError(msg, self._curfile, spans[0]) - if self._synclines and not self._diversions: - result = syncline(spans[1][1], self._curfile) + if self._linenums and not self._diversions: + result = linenumdir(spans[1][1], self._curfile) return result @@ -1212,22 +1216,22 @@ class Renderer: .format(name, valstr, exc) raise FyppError(msg, self._curfile, span) multiline = (span[0] != span[1]) - if self._synclines and not self._diversions and multiline: - result = syncline(span[1], self._curfile) + if self._linenums and not self._diversions and multiline: + result = linenumdir(span[1], self._curfile) return result def _get_comment(self, span): - if self._synclines and not self._diversions: - return syncline(span[1], self._curfile) + if self._linenums and not self._diversions: + return linenumdir(span[1], self._curfile) else: return '' def _get_muted_content(self, spans, content): self._render(content) - if self._synclines and not self._diversions: - return syncline(spans[-1][1], self._curfile) + if self._linenums and not self._diversions: + return linenumdir(spans[-1][1], self._curfile) else: return '' @@ -1247,8 +1251,8 @@ class Renderer: result = '' self._curfile = fname self._evaluator.updateenv(_FILE_=fname) - if self._synclines and not self._diversions: - result = syncline(span[1], self._curfile) + if self._linenums and not self._diversions: + result = linenumdir(span[1], self._curfile) return result @@ -1258,85 +1262,109 @@ class Renderer: def _postprocess_eval_lines(self, output, eval_inds, eval_pos): - lastproc = -1 + ilastproc = -1 for ieval, ind in enumerate(eval_inds): span, fname = eval_pos[ieval] - if ind <= lastproc: + if ind <= ilastproc: continue - - # find last eol before expr. evaluation - iprev = ind - 1 - while iprev >= 0: - eolprev = output[iprev].rfind('\n') - if eolprev != -1: - break - iprev -= 1 - else: - iprev = 0 - eolprev = -1 - - # find first eol after expr. evaluation - inext = ind + 1 - while inext < len(output): - eolnext = output[inext].find('\n') - if eolnext != -1: - break - inext += 1 - else: - inext = len(output) - 1 - eolnext = len(output[-1]) - 1 - - # Create full line containing the evaluated expression - curline_parts = [] - if iprev != ind: - curline_parts = [output[iprev][eolprev + 1:]] - output[iprev] = output[iprev][:eolprev + 1] - curline_parts.extend(output[iprev + 1:ind]) - curline_parts.extend(output[ind]) - curline_parts.extend(output[ind + 1:inext]) - if inext != ind: - curline_parts.append(output[inext][:eolnext + 1]) - output[inext] = output[inext][eolnext + 1:] - curline = ''.join(curline_parts) + iprev, eolprev = self._find_last_eol(output, ind) + inext, eolnext = self._find_next_eol(output, ind) + curline = self._glue_line(output, ind, iprev, eolprev, inext, + eolnext) output[iprev + 1:inext] = [''] * (inext - iprev - 1) + output[ind] = self._postprocess_eval_line(curline, fname, span) + ilastproc = inext - # Split lines and add synclines if needed - lines = self._splitlines(curline) - # if curline ended with '\n', last element of lines is '' - nrlines = len(lines) if lines[-1] else len(lines) - 1 - if self._synclines and (nrlines > 1 or span[0] != span[1]): - oldlines = lines - syncl = syncline(span[0], fname) - lines = [oldlines[0] + '\n'] - for line in oldlines[1:-1]: - lines.append(syncl) - lines.append(line + '\n') - if oldlines[-1]: - # Line did not end on '\n' -> it was last line of input - # -> no syncline is needed. - lines.append(oldlines[-1]) - elif span[1] > span[0] + 1: - # Syncline needed, if expression evaluation goes - # over multiple lines. Otherwise the syncline inserted - # before the last line of the evaluation (pointing to - # the line of the directive) ensures correct line numbering - # for the following content. - lines.append(syncline(span[1], fname)) - output[ind] = ''.join(lines) - else: - output[ind] = '\n'.join(lines) - lastproc = inext + @staticmethod + def _find_last_eol(output, ind): + 'Find last newline before current position.' + iprev = ind - 1 + while iprev >= 0: + eolprev = output[iprev].rfind('\n') + if eolprev != -1: + break + iprev -= 1 + else: + iprev = 0 + eolprev = -1 + return iprev, eolprev - def _splitlines(self, txt): - lines = txt.split('\n') - result = [] - for line in lines: - if _COMMENTLINE_REGEXP.match(line) is None: - result += self._linefolder(line) - else: - result.append(line) - return result + + @staticmethod + def _find_next_eol(output, ind): + 'Find last newline before current position.' + # find first eol after expr. evaluation + inext = ind + 1 + while inext < len(output): + eolnext = output[inext].find('\n') + if eolnext != -1: + break + inext += 1 + else: + inext = len(output) - 1 + eolnext = len(output[-1]) - 1 + return inext, eolnext + + + @staticmethod + def _glue_line(output, ind, iprev, eolprev, inext, eolnext): + 'Create line from parts between specified boundaries.' + curline_parts = [] + if iprev != ind: + curline_parts = [output[iprev][eolprev + 1:]] + output[iprev] = output[iprev][:eolprev + 1] + curline_parts.extend(output[iprev + 1:ind]) + curline_parts.extend(output[ind]) + curline_parts.extend(output[ind + 1:inext]) + if inext != ind: + curline_parts.append(output[inext][:eolnext + 1]) + output[inext] = output[inext][eolnext + 1:] + return ''.join(curline_parts) + + + def _postprocess_eval_line(self, evalline, fname, span): + lines = evalline.split('\n') + # If line ended on '\n', last element is ''. We remove it and + # add the trailing newline later manually. + trailing_newline = (lines[-1] == '') + if trailing_newline: + del lines[-1] + lnum = linenumdir(span[0], fname) if self._linenums else '' + clnum = lnum if self._contlinenums else '' + linenumsep = '\n' + lnum + clinenumsep = '\n' + clnum + foldedlines = [self._foldline(line) for line in lines] + outlines = [clinenumsep.join(lines) for lines in foldedlines] + result = linenumsep.join(outlines) + # Add missing trailing newline + if trailing_newline: + trailing = '\n' + if self._linenums: + # Last line was folded, but no linenums were generated for + # the continuation lines -> current line position is not + # in sync with the one calculated from the last line number + unsync = ( + len(foldedlines) and len(foldedlines[-1]) > 1 + and not self._contlinenums) + # Eval directive in source consists of more than one line + multiline = span[1] - span[0] > 1 + if unsync or multiline: + # For inline eval directives span[0] == span[1] + # -> next line is span[0] + 1 and not span[1] as for + # line eval directives + nextline = max(span[1], span[0] + 1) + trailing += linenumdir(nextline, fname) + else: + trailing = '' + return result + trailing + + + def _foldline(self, line): + if _COMMENTLINE_REGEXP.match(line) is None: + return self._linefolder(line) + else: + return [line] class Evaluator: @@ -1669,8 +1697,8 @@ class Processor: return ''.join(output) -def syncline(linenr, fname): - '''Returns a syncline directive. +def linenumdir(linenr, fname): + '''Returns a line numbering directive. Args: linenr (int): Line nr (starting with 0). @@ -1855,8 +1883,11 @@ class Fypp: suffix = '' if fixed_format else '&' linefolder = LineFolder(linelength, indentation, folding, prefix, suffix) - renderer = Renderer(evaluator, synclines=self._args.synclines, - linefolder=linefolder) + linenums = self._args.line_numbering + contlinenums = (self._args.line_numbering_mode != 'nocontlines') + renderer = Renderer( + evaluator, linenums=linenums, contlinenums=contlinenums, + linefolder=linefolder) self._preprocessor = Processor(parser, builder, renderer) @@ -1901,9 +1932,16 @@ class Fypp: msg = 'add directory to the search paths for include files' parser.add_argument('-I', '--include', action='append', dest='includes', metavar='INCDIR', help=msg) - msg = 'include CPP style sync-lines in the output' - parser.add_argument('-s', '--synclines', action='store_true', + msg = 'put line numbering directives to the output' + parser.add_argument('-n', '--line-numbering', action='store_true', default=False, help=msg) + msg = 'line numbering mode, \'full\' (default): line numbering '\ + 'directives generated whenever source and output lines are out '\ + 'of sync, \'nocontlines\': line numbering directives omitted '\ + 'for continuation lines' + parser.add_argument('-N', '--line-numbering-mode', metavar='MODE', + choices=['full', 'nocontlines'], default='full', + help=msg) msg = 'maximal line length (default: 132), lines modified by the '\ 'preprocessor are folded if becoming longer' parser.add_argument('-l', '--line-length', type=int, default=132, diff --git a/test/test_fypp.py b/test/test_fypp.py index 0b53d26..7b8225f 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -3,14 +3,10 @@ import unittest import fypp -def _strsyncl(linenr, fname=None): +def _linenum(linenr, fname=None): if fname is None: fname = fypp.STRING - return fypp.syncline(linenr, fname) - -def _filesyncl(fname, linenr): - return fypp.syncline(linenr, fname) - + return fypp.linenumdir(linenr, fname) def _defvar(var, val): return '-D{}={}'.format(var, val) @@ -27,7 +23,10 @@ def _indentation(ind): def _folding(fold): return '-f{}'.format(fold) -_SYNCL_FLAG = '-s' +def _linenumbering(nummode): + return '-N{}'.format(nummode) + +_LINENUM_FLAG = '-n' _FIXED_FORMAT_FLAG = '--fixed-format' @@ -427,232 +426,241 @@ def _folding(fold): ), ] -SYNCLINE_TESTS = [ - # This test (but only this) must be changed, if syncline format changes. - ('explicit_str_syncline_test', [_SYNCL_FLAG], +LINENUM_TESTS = [ + # This test (but only this) must be changed, if linenum directive changes. + ('explicit_str_linenum_test', [_LINENUM_FLAG], '', '# 1 ""\n', ), # - ('trivial', [_SYNCL_FLAG], + ('trivial', [_LINENUM_FLAG], 'Test\n', - _strsyncl(0) + 'Test\n' + _linenum(0) + 'Test\n' ), # - ('if_true', [_SYNCL_FLAG], + ('if_true', [_LINENUM_FLAG], '#:if 1 < 2\nTrue\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True\n' + _strsyncl(3) + 'Done\n' + _linenum(0) + _linenum(1) + 'True\n' + _linenum(3) + 'Done\n' ), # - ('if_false', [_SYNCL_FLAG], + ('if_false', [_LINENUM_FLAG], '#:if 1 > 2\nTrue\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'Done\n' + _linenum(0) + _linenum(3) + 'Done\n' ), # - ('if_else_true', [_SYNCL_FLAG], + ('if_else_true', [_LINENUM_FLAG], '#:if 1 < 2\nTrue\n#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(1) + 'True\n' + _linenum(5) + 'Done\n' ), # - ('if_else_false', [_SYNCL_FLAG], + ('if_else_false', [_LINENUM_FLAG], '#:if 1 > 2\nTrue\n#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'False\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(3) + 'False\n' + _linenum(5) + 'Done\n' ), - ('if_elif_true1', [_SYNCL_FLAG], + ('if_elif_true1', [_LINENUM_FLAG], '#:if 1 == 1\nTrue1\n#:elif 1 == 2\nTrue2\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True1\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(1) + 'True1\n' + _linenum(5) + 'Done\n' ), # - ('if_elif_true2', [_SYNCL_FLAG], + ('if_elif_true2', [_LINENUM_FLAG], '#:if 2 == 1\nTrue1\n#:elif 2 == 2\nTrue2\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'True2\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(3) + 'True2\n' + _linenum(5) + 'Done\n' ), # - ('if_elif_false', [_SYNCL_FLAG], + ('if_elif_false', [_LINENUM_FLAG], '#:if 0 == 1\nTrue1\n#:elif 0 == 2\nTrue2\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(5) + 'Done\n' ), # - ('if_elif_else_true1', [_SYNCL_FLAG], + ('if_elif_else_true1', [_LINENUM_FLAG], '#:if 1 == 1\nTrue1\n#:elif 1 == 2\nTrue2\n' '#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True1\n' + _strsyncl(7) + 'Done\n' + _linenum(0) + _linenum(1) + 'True1\n' + _linenum(7) + 'Done\n' ), # - ('if_elif_else_true2', [_SYNCL_FLAG], + ('if_elif_else_true2', [_LINENUM_FLAG], '#:if 2 == 1\nTrue1\n#:elif 2 == 2\nTrue2\n' '#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'True2\n' + _strsyncl(7) + 'Done\n' + _linenum(0) + _linenum(3) + 'True2\n' + _linenum(7) + 'Done\n' ), # - ('if_elif_else_false', [_SYNCL_FLAG], + ('if_elif_else_false', [_LINENUM_FLAG], '#:if 0 == 1\nTrue1\n#:elif 0 == 2\nTrue2\n' '#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(5) + 'False\n' + _strsyncl(7) + 'Done\n' + _linenum(0) + _linenum(5) + 'False\n' + _linenum(7) + 'Done\n' ), # - ('inline_if_true', [_SYNCL_FLAG], + ('inline_if_true', [_LINENUM_FLAG], '#{if 1 < 2}#True#{endif}#Done\n', - _strsyncl(0) + 'TrueDone\n' + _linenum(0) + 'TrueDone\n' ), # - ('inline_if_false', [_SYNCL_FLAG], + ('inline_if_false', [_LINENUM_FLAG], '#{if 1 > 2}#True#{endif}#Done\n', - _strsyncl(0) + 'Done\n' + _linenum(0) + 'Done\n' ), # - ('inline_if_else_true', [_SYNCL_FLAG], + ('inline_if_else_true', [_LINENUM_FLAG], '#{if 1 < 2}#True#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'TrueDone\n' + _linenum(0) + 'TrueDone\n' ), # - ('inline_if_else_false', [_SYNCL_FLAG], + ('inline_if_else_false', [_LINENUM_FLAG], '#{if 1 > 2}#True#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'FalseDone\n' + _linenum(0) + 'FalseDone\n' ), - ('inline_if_elif_true1', [_SYNCL_FLAG], + ('inline_if_elif_true1', [_LINENUM_FLAG], '#{if 1 == 1}#True1#{elif 1 == 2}#True2#{endif}#Done\n', - _strsyncl(0) + 'True1Done\n' + _linenum(0) + 'True1Done\n' ), # - ('inline_if_elif_true2', [_SYNCL_FLAG], + ('inline_if_elif_true2', [_LINENUM_FLAG], '#{if 2 == 1}#True1#{elif 2 == 2}#True2#{endif}#Done\n', - _strsyncl(0) + 'True2Done\n' + _linenum(0) + 'True2Done\n' ), # - ('inline_if_elif_false', [_SYNCL_FLAG], + ('inline_if_elif_false', [_LINENUM_FLAG], '#{if 0 == 1}#True1#{elif 0 == 2}#True2#{endif}#Done\n', - _strsyncl(0) + 'Done\n' + _linenum(0) + 'Done\n' ), # - ('inline_if_elif_else_true1', [_SYNCL_FLAG], + ('inline_if_elif_else_true1', [_LINENUM_FLAG], '#{if 1 == 1}#True1#{elif 1 == 2}#True2#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'True1Done\n' + _linenum(0) + 'True1Done\n' ), # - ('inline_if_elif_else_true2', [_SYNCL_FLAG], + ('inline_if_elif_else_true2', [_LINENUM_FLAG], '#{if 2 == 1}#True1#{elif 2 == 2}#True2#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'True2Done\n' + _linenum(0) + 'True2Done\n' ), # - ('inline_if_elif_else_false', [_SYNCL_FLAG], + ('inline_if_elif_else_false', [_LINENUM_FLAG], '#{if 0 == 1}#True1#{elif 0 == 2}#True2#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'FalseDone\n' + _linenum(0) + 'FalseDone\n' ), # - ('linesub_oneline', [_SYNCL_FLAG], + ('linesub_oneline', [_LINENUM_FLAG], 'A\n$: 1 + 1\nB\n', - _strsyncl(0) + 'A\n2\nB\n' + _linenum(0) + 'A\n2\nB\n' ), # - ('linesub_contlines', [_SYNCL_FLAG, _defvar('TESTVAR', 1)], + ('linesub_contlines', [_LINENUM_FLAG, _defvar('TESTVAR', 1)], '$: TESTVAR & \n & + 1\nDone\n', - _strsyncl(0) + '2\n' + _strsyncl(2) + 'Done\n' + _linenum(0) + '2\n' + _linenum(2) + 'Done\n' ), # - ('linesub_contlines2', [_SYNCL_FLAG, _defvar('TESTVAR', 1)], + ('linesub_contlines2', [_LINENUM_FLAG, _defvar('TESTVAR', 1)], '$: TEST& \n &VAR & \n & + 1\nDone\n', - _strsyncl(0) + '2\n' + _strsyncl(3) + 'Done\n' + _linenum(0) + '2\n' + _linenum(3) + 'Done\n' ), # - ('exprsub_single_line', [_SYNCL_FLAG, _defvar('TESTVAR', 1)], + ('exprsub_single_line', [_LINENUM_FLAG, _defvar('TESTVAR', 1)], 'A${TESTVAR}$B${TESTVAR + 1}$C', - _strsyncl(0) + 'A1B2C' + _linenum(0) + 'A1B2C' ), # - ('exprsub_multi_line', [_SYNCL_FLAG], + ('exprsub_multi_line', [_LINENUM_FLAG], '${"line1\\nline2"}$\nDone\n', - _strsyncl(0) + 'line1\n' + _strsyncl(0) + 'line2\nDone\n' + _linenum(0) + 'line1\n' + _linenum(0) + 'line2\nDone\n' ), # - ('macrosubs', [_SYNCL_FLAG], + ('macrosubs', [_LINENUM_FLAG], '#:def macro(var)\nMACRO|${var}$|\n#:enddef\n${macro(1)}$', - _strsyncl(0) + _strsyncl(3) + 'MACRO|1|' + _linenum(0) + _linenum(3) + 'MACRO|1|' ), # - ('recursive_macrosubs', [_SYNCL_FLAG], + ('recursive_macrosubs', [_LINENUM_FLAG], '#:def macro(var)\nMACRO|${var}$|\n#:enddef\n${macro(macro(1))}$', - _strsyncl(0) + _strsyncl(3) + 'MACRO|MACRO|1||' + _linenum(0) + _linenum(3) + 'MACRO|MACRO|1||' ), # - ('macrosubs_multiline', [_SYNCL_FLAG], + ('macrosubs_multiline', [_LINENUM_FLAG], '#:def macro(c)\nMACRO1|${c}$|\nMACRO2|${c}$|\n#:enddef\n${macro(\'A\')}$' '\n', - _strsyncl(0) + _strsyncl(4) + 'MACRO1|A|\n' + _strsyncl(4) + 'MACRO2|A|\n' + _linenum(0) + _linenum(4) + 'MACRO1|A|\n' + _linenum(4) + 'MACRO2|A|\n' ), # - ('recursive_macrosubs_multiline', [_SYNCL_FLAG], + ('recursive_macrosubs_multiline', [_LINENUM_FLAG], '#:def f(c)\nLINE1|${c}$|\nLINE2|${c}$|\n#:enddef\n$: f(f("A"))\n', - (_strsyncl(0) + _strsyncl(4) + 'LINE1|LINE1|A|\n' + _strsyncl(4) - + 'LINE2|A||\n' + _strsyncl(4) + 'LINE2|LINE1|A|\n' + _strsyncl(4) + (_linenum(0) + _linenum(4) + 'LINE1|LINE1|A|\n' + _linenum(4) + + 'LINE2|A||\n' + _linenum(4) + 'LINE2|LINE1|A|\n' + _linenum(4) + 'LINE2|A||\n') ), # - ('multiline_macrocall', [_SYNCL_FLAG], + ('multiline_macrocall', [_LINENUM_FLAG], '#:def macro(c)\nMACRO|${c}$|\n#:enddef\n$: mac& \n &ro(\'A\')\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'MACRO|A|\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(3) + 'MACRO|A|\n' + _linenum(5) + 'Done\n' ), # - ('call_directive_2_args', [_SYNCL_FLAG], + ('call_directive_2_args', [_LINENUM_FLAG], '#:def mymacro(val1, val2)\n|${val1}$|${val2}$|\n#:enddef\n'\ '#:call mymacro\nL1\nL2\n#:nextarg\nL3\n#:endcall\n', - _strsyncl(0) + _strsyncl(3) + '|L1\n' + _strsyncl(3) + 'L2|L3|\n'\ - + _strsyncl(9), + _linenum(0) + _linenum(3) + '|L1\n' + _linenum(3) + 'L2|L3|\n'\ + + _linenum(9), ), # - ('for', [_SYNCL_FLAG], + ('for', [_LINENUM_FLAG], '#:for i in (1, 2)\n${i}$\n#:endfor\nDone\n', - (_strsyncl(0) + _strsyncl(1) + '1\n' + _strsyncl(1) + '2\n' - + _strsyncl(3) + 'Done\n') + (_linenum(0) + _linenum(1) + '1\n' + _linenum(1) + '2\n' + + _linenum(3) + 'Done\n') ), # - ('inline_for', [_SYNCL_FLAG], + ('inline_for', [_LINENUM_FLAG], '#{for i in (1, 2)}#${i}$#{endfor}#Done\n', - _strsyncl(0) + '12Done\n' + _linenum(0) + '12Done\n' ), # - ('setvar', [_SYNCL_FLAG], + ('setvar', [_LINENUM_FLAG], '#:setvar x 2\n$: x\n', - _strsyncl(0) + _strsyncl(1) + '2\n', + _linenum(0) + _linenum(1) + '2\n', ), # - ('inline_setvar', [_SYNCL_FLAG], + ('inline_setvar', [_LINENUM_FLAG], '#{setvar x 2}#${x}$Done\n', - _strsyncl(0) + '2Done\n', + _linenum(0) + '2Done\n', ), # - ('comment_single', [_SYNCL_FLAG], + ('comment_single', [_LINENUM_FLAG], ' #! Comment here\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'Done\n' + _linenum(0) + _linenum(1) + 'Done\n' ), # - ('comment_multiple', [_SYNCL_FLAG], + ('comment_multiple', [_LINENUM_FLAG], ' #! Comment1\n#! Comment2\nDone\n', - _strsyncl(0) + _strsyncl(2) + 'Done\n', - ), - # - ('mute', [_SYNCL_FLAG], - 'A\n#:mute\nB\n#:setvar VAR 2\n#:endmute\nVAR=${VAR}$\n', - _strsyncl(0) + 'A\n' + _strsyncl(5) + 'VAR=2\n' + _linenum(0) + _linenum(2) + 'Done\n', ), # - ('mute', [_SYNCL_FLAG], + ('mute', [_LINENUM_FLAG], 'A\n#:mute\nB\n#:setvar VAR 2\n#:endmute\nVAR=${VAR}$\n', - _strsyncl(0) + 'A\n' + _strsyncl(5) + 'VAR=2\n' + _linenum(0) + 'A\n' + _linenum(5) + 'VAR=2\n' ), # - ('direct_call', [_SYNCL_FLAG], + ('direct_call', [_LINENUM_FLAG], '#:def mymacro(val)\n|${val}$|\n#:enddef\n'\ '@:mymacro a < b\n', - _strsyncl(0) + _strsyncl(3) + '|a < b|\n', + _linenum(0) + _linenum(3) + '|a < b|\n', ), # - ('direct_call_contline', [_SYNCL_FLAG], + ('direct_call_contline', [_LINENUM_FLAG], '#:def mymacro(val)\n|${val}$|\n#:enddef\n'\ '@:mymacro a &\n &< b&\n &\nDone\n', - _strsyncl(0) + _strsyncl(3) + '|a < b|\n' + _strsyncl(6) + 'Done\n', + _linenum(0) + _linenum(3) + '|a < b|\n' + _linenum(6) + 'Done\n', + ), + # + ('smart_folding', + [_LINENUM_FLAG, _linelen(15), _indentation(4), _folding('smart')], + ' ${3}$456 89 123456 8\nDone\n', + _linenum(0) + ' 3456 89&\n' + _linenum(0) + ' & 123456&\n' \ + + _linenum(0) + ' & 8\n' + 'Done\n' + ), + # + ('smart_folding_nocontlines', + [_LINENUM_FLAG, _linenumbering('nocontlines'), _linelen(15), + _indentation(4), _folding('smart')], + ' ${3}$456 89 123456 8\nDone\n', + _linenum(0) + ' 3456 89&\n' + ' & 123456&\n' \ + + ' & 8\n' + _linenum(1) + 'Done\n' ), - ] INCLUDE_TESTS = [ @@ -677,28 +685,28 @@ def _folding(fold): 'FYPP2\n' ), # - ('search_include_syncl', [_SYNCL_FLAG, _incdir('include')], + ('search_include_linenum', [_LINENUM_FLAG, _incdir('include')], '#:include "fypp1.inc"\n$: incmacro(1)\n', - (_strsyncl(0) + _filesyncl('include/fypp1.inc', 0) - + 'INCL1\n' + _filesyncl('include/fypp1.inc', 4) - + 'INCL5\n' + _strsyncl(1) + 'INCMACRO(1)\n') + (_linenum(0) + _linenum(0, 'include/fypp1.inc') + + 'INCL1\n' + _linenum(4, 'include/fypp1.inc') + + 'INCL5\n' + _linenum(1) + 'INCMACRO(1)\n') ), # - ('nested_include_in_incpath_syncl', [_SYNCL_FLAG, _incdir('include')], + ('nested_include_in_incpath_linenum', [_LINENUM_FLAG, _incdir('include')], '#:include "subfolder/include_fypp1.inc"\n', - (_strsyncl(0) + _strsyncl(0, 'include/subfolder/include_fypp1.inc') - + _strsyncl(0, 'include/fypp1.inc') + 'INCL1\n' - + _strsyncl(4, 'include/fypp1.inc') + 'INCL5\n' - + _strsyncl(1, 'include/subfolder/include_fypp1.inc') - + _strsyncl(1)) + (_linenum(0) + _linenum(0, 'include/subfolder/include_fypp1.inc') + + _linenum(0, 'include/fypp1.inc') + 'INCL1\n' + + _linenum(4, 'include/fypp1.inc') + 'INCL5\n' + + _linenum(1, 'include/subfolder/include_fypp1.inc') + + _linenum(1)) ), # - ('nested_include_in_folder_of_incfile', [_SYNCL_FLAG, _incdir('include')], + ('nested_include_in_folder_of_incfile', [_LINENUM_FLAG, _incdir('include')], '#:include "subfolder/include_fypp2.inc"\n', - (_strsyncl(0) + _strsyncl(0, 'include/subfolder/include_fypp2.inc') - + _strsyncl(0, 'include/subfolder/fypp2.inc') + (_linenum(0) + _linenum(0, 'include/subfolder/include_fypp2.inc') + + _linenum(0, 'include/subfolder/fypp2.inc') + 'FYPP2\n' - + _strsyncl(1, 'include/subfolder/include_fypp2.inc') + _strsyncl(1)) + + _linenum(1, 'include/subfolder/include_fypp2.inc') + _linenum(1)) ), ] @@ -1009,8 +1017,8 @@ class SimpleTest(unittest.TestCase): pass -class SynclineTest(unittest.TestCase): - '''Container for tests with syncline output.''' +class LineNumberingTest(unittest.TestCase): + '''Container for tests with line numbering directives.''' pass @@ -1095,7 +1103,7 @@ def add_test_methods(tests, testcase, methodfactory): add_test_methods(SIMPLE_TESTS, SimpleTest, test_output_method) -add_test_methods(SYNCLINE_TESTS, SynclineTest, test_output_method) +add_test_methods(LINENUM_TESTS, LineNumberingTest, test_output_method) add_test_methods(INCLUDE_TESTS, IncludeTest, test_output_method) add_test_methods(EXCEPTION_TESTS, ExceptionTest, test_exception_method) From 44d73bf761ce4c0134cc7bf9bec3102b31caa44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Mon, 14 Mar 2016 21:25:43 +0100 Subject: [PATCH 02/10] Refactor code and options for folding and line numbering. --- bin/fypp | 56 ++++++++++++++++++++++++++++++----------------- test/test_fypp.py | 14 +++++++----- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/bin/fypp b/bin/fypp index c03e0ab..84f39f7 100755 --- a/bin/fypp +++ b/bin/fypp @@ -1707,8 +1707,7 @@ def linenumdir(linenr, fname): return '# {} "{}"\n'.format(linenr + 1, fname) - -class LineFolder: +class FortranLineFolder: '''Implements line folding with Fortran continuation lines. @@ -1717,7 +1716,6 @@ class LineFolder: indent (int, optional): Indentation for continuation lines (default: 4). method (str, optional): Folding method with following options: - * ``none``: no folding, * ``brute``: folding with maximal length of continuation lines, * ``simple``: indents with respect of indentation of first line, * ``smart``: like ``simple``, but tries to fold at whitespaces. @@ -1741,7 +1739,7 @@ class LineFolder: self._indent = indent self._prefix = ' ' * self._indent + prefix self._suffix = suffix - if method not in ['brute', 'smart', 'simple', 'none']: + if method not in ['brute', 'smart', 'simple']: raise FyppError('invalid folding type') if method == 'brute': self._inherit_indent = False @@ -1752,9 +1750,6 @@ class LineFolder: elif method == 'smart': self._inherit_indent = True self._fold_position_finder = self._get_smart_fold_pos - else: - # No folding occurs, line is returned unaltered - self._maxlen = -1 def __call__(self, line): @@ -1762,7 +1757,7 @@ class LineFolder: Can be directly called to return the list of folded lines:: - linefolder = LineFolder(maxlen=10) + linefolder = FortranLineFolder(maxlen=10) linefolder(' print *, "some Fortran line"') Args: @@ -1818,6 +1813,21 @@ class LineFolder: return end +class DummyLineFolder: + + '''Implements a dummy line folder returning the line unaltered.''' + + def __call__(self, line): + '''Returns the entire line without any folding. + + Returns: + list of str: Components of folded line. They should be + assembled via ``\\n.join()`` to obtain the string + representation. + ''' + return [line] + + def _shiftinds(inds, shift): return [ind + shift for ind in inds] @@ -1876,13 +1886,16 @@ class Fypp: builder = Builder() fixed_format = self._args.fixed_format - folding = 'brute' if fixed_format else self._args.folding_method - linelength = 72 if fixed_format else self._args.line_length - indentation = 5 if fixed_format else self._args.indentation - prefix = '&' - suffix = '' if fixed_format else '&' - linefolder = LineFolder(linelength, indentation, folding, prefix, - suffix) + if self._args.linefolding: + folding = 'brute' if fixed_format else self._args.folding_mode + linelength = 72 if fixed_format else self._args.line_length + indentation = 5 if fixed_format else self._args.indentation + prefix = '&' + suffix = '' if fixed_format else '&' + linefolder = FortranLineFolder(linelength, indentation, folding, + prefix, suffix) + else: + linefolder = DummyLineFolder() linenums = self._args.line_numbering contlinenums = (self._args.line_numbering_mode != 'nocontlines') renderer = Renderer( @@ -1946,12 +1959,15 @@ class Fypp: 'preprocessor are folded if becoming longer' parser.add_argument('-l', '--line-length', type=int, default=132, metavar='LEN', help=msg) - msg = 'folding method, \'smart\' (default): indentation context and '\ - 'whitespace aware, \'simple\': indentation context aware, '\ - '\'brute\' : mechnical folding, none\': no folding' - parser.add_argument('-f', '--folding-method', metavar='METHOD', - choices=['smart', 'simple', 'brute', 'none'], + msg = 'line folding mode, \'smart\' (default): indentation context '\ + 'and whitespace aware, \'simple\': indentation context aware, '\ + '\'brute\': mechnical folding' + parser.add_argument('-f', '--folding-mode', metavar='MODE', + choices=['smart', 'simple', 'brute'], default='smart', help=msg) + msg = 'suppress line folding' + parser.add_argument('-F', '--no-folding', action='store_false', + dest='linefolding', default=True, help=msg) msg = 'indentation to use for continuation lines (default 4)' parser.add_argument('--indentation', type=int, metavar='IND', default=4, help=msg) diff --git a/test/test_fypp.py b/test/test_fypp.py index 7b8225f..1c4403b 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -30,6 +30,8 @@ def _linenumbering(nummode): _FIXED_FORMAT_FLAG = '--fixed-format' +_NO_FOLDING_FLAG = '-F' + SIMPLE_TESTS = [ ('if_true', [_defvar('TESTVAR', 1)], @@ -321,7 +323,7 @@ def _linenumbering(nummode): # ('direct_call_2_args_escape', [], '#:def mymacro(val1, val2)\n|${val1}$|${val2}$|\n#:enddef\n'\ - '@:mymacro """L1""" @\@ L2 @@ L3\n', + '@:mymacro """L1""" @\\@ L2 @@ L3\n', '|"""L1""" @@ L2|L3|\n', ), # @@ -397,7 +399,7 @@ def _linenumbering(nummode): ' ! Should be not folded\nShould be&\n & folded\n' ), # - ('no_folding', [_linelen(15), _indentation(4), _folding('none')], + ('no_folding', [_linelen(15), _indentation(4), _NO_FOLDING_FLAG], ' ${3}$456 89 123456 8', ' 3456 89 123456 8', ), @@ -647,15 +649,15 @@ def _linenumbering(nummode): _linenum(0) + _linenum(3) + '|a < b|\n' + _linenum(6) + 'Done\n', ), # - ('smart_folding', + ('smart_folding', [_LINENUM_FLAG, _linelen(15), _indentation(4), _folding('smart')], ' ${3}$456 89 123456 8\nDone\n', _linenum(0) + ' 3456 89&\n' + _linenum(0) + ' & 123456&\n' \ + _linenum(0) + ' & 8\n' + 'Done\n' ), # - ('smart_folding_nocontlines', - [_LINENUM_FLAG, _linenumbering('nocontlines'), _linelen(15), + ('smart_folding_nocontlines', + [_LINENUM_FLAG, _linenumbering('nocontlines'), _linelen(15), _indentation(4), _folding('smart')], ' ${3}$456 89 123456 8\nDone\n', _linenum(0) + ' 3456 89&\n' + ' & 123456&\n' \ @@ -695,7 +697,7 @@ def _linenumbering(nummode): ('nested_include_in_incpath_linenum', [_LINENUM_FLAG, _incdir('include')], '#:include "subfolder/include_fypp1.inc"\n', (_linenum(0) + _linenum(0, 'include/subfolder/include_fypp1.inc') - + _linenum(0, 'include/fypp1.inc') + 'INCL1\n' + + _linenum(0, 'include/fypp1.inc') + 'INCL1\n' + _linenum(4, 'include/fypp1.inc') + 'INCL5\n' + _linenum(1, 'include/subfolder/include_fypp1.inc') + _linenum(1)) From 6a9b171d111659b84064fdc96d8f47545d5937f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Tue, 15 Mar 2016 21:09:19 +0100 Subject: [PATCH 03/10] Extend documentation and test suite. --- README.rst | 16 ++--- bin/fypp | 10 +-- docs/fypp.rst | 177 +++++++++++++++++++++++++++++----------------- test/test_fypp.py | 20 ++++++ 4 files changed, 147 insertions(+), 76 deletions(-) diff --git a/README.rst b/README.rst index 5f2a1f4..71518a9 100644 --- a/README.rst +++ b/README.rst @@ -2,12 +2,13 @@ Fypp — Python powered Fortran metaprogramming ============================================= -Fypp is a Python powered Fortran preprocessor. It extends Fortran with -condititional compiling and template metaprogramming capabilities. Instead of -introducing its own expression syntax, it uses Python expressions in its -preprocessor directives, offering the consistency and flexibility of Python when -formulating metaprogramming tasks. It puts strong emphasis on robustness and on -neat integration into Fortran developing toolchains. +Fypp is a Python powered preprocessor. It can be used for any programming +languages but its primary aim is to offer a Fortran preprocessor, which helps to +extend Fortran with condititional compiling and template metaprogramming +capabilities. Instead of introducing its own expression syntax, it uses Python +expressions in its preprocessor directives, offering the consistency and +versatility of Python when formulating metaprogramming tasks. It puts strong +emphasis on robustness and on neat integration into developing toolchains. The project is `hosted on bitbucket `_. @@ -57,7 +58,7 @@ Main features use serial #:endif -* Iterated output (e.g. for Fortran templates):: +* Iterated output (e.g. for generating Fortran templates):: interface myfunc #:for dtype in [ 'real', 'dreal', 'complex', 'dcomplex' ] @@ -165,7 +166,6 @@ environment variable, by just issuing :: fypp - Running ======= diff --git a/bin/fypp b/bin/fypp index 84f39f7..c31563a 100755 --- a/bin/fypp +++ b/bin/fypp @@ -48,14 +48,16 @@ using following classes: * `Processor`_: Connects `Parser`, `Builder`, `Renderer` and `Evaluator` with each other and returns for a given input the preprocessed output. -* `Fypp`_: Command line preprocessor. Drives `Processor` according to the - command line arguments. +* `Fypp`_: Command line preprocessor. Initializes and drives + `Processor` according to the command line arguments. If any expected error occurs during processing, `FyppError`_ is raised. Additional to the ones above, following class is used for fine tuning: -* `LineFolder`_: Folds overlong lines by using Fortran continuation lines. +* `FortranLineFolder`_: Folds overlong lines by using Fortran continuation + lines. + ''' from __future__ import print_function @@ -177,7 +179,7 @@ class Parser: Args: includedirs (list): List of directories, in which include files should - be searched for, when they are not found in the default location. + be searched for, when they are not found at the default location. ''' def __init__(self, includedirs=None): diff --git a/docs/fypp.rst b/docs/fypp.rst index cc067eb..d0df4e6 100644 --- a/docs/fypp.rst +++ b/docs/fypp.rst @@ -4,24 +4,25 @@ Introduction ************ -Fypp is a Python powered Fortran preprocessor. It extends Fortran with -condititional compiling and template metaprogramming capabilities. Instead of -introducing its own expression syntax, it uses Python expressions in its -preprocessor directives, offering the consistency and flexibility of Python when -formulating metaprogramming tasks. It puts strong emphasis on robustness and on -neat integration into Fortran developing toolchains. +Fypp is a Python powered preprocessor. It can be used for any programming +languages but its primary aim is to offer a Fortran preprocessor, which helps to +extend Fortran with condititional compiling and template metaprogramming +capabilities. Instead of introducing its own expression syntax, it uses Python +expressions in its preprocessor directives, offering the consistency and +versatility of Python when formulating metaprogramming tasks. It puts strong +emphasis on robustness and on neat integration into developing toolchains. Fypp was inspired by the `pyratemp `_ templating engine [#]_. Although it shares many concepts with pyratemp, it was written from -scratch focusing on the special needs when preprocessing Fortran source -files. Fypp natively supports the output of synchronization line directives, -which are used by many compilers to generate compiler messages with correct line -numbers. Unlike most cpp/fpp-like preprocessors, Fypp also supports iterations, -multiline macros, continuation lines in preprocessor directives and automatic -line folding. It generally tries to extend the modern Fortran language with some -useful features without tempting you to use it for tasks, which could/should be -done in Fortran itself. +scratch focusing on the special needs when preprocessing source code. Fypp +natively supports the output of line numbering directives, which are used by +many compilers to generate compiler messages with correct line numbers. Unlike +most cpp/fpp preprocessors or the coco preprocessor, Fypp also supports +iterations, multiline macros, continuation lines in preprocessor directives and +automatic line folding. It generally tries to extend the modern Fortran language +with some useful features without tempting you to use it for tasks which +could/should be done in Fortran itself. The project is `hosted on bitbucket `_ with documentation available on `readthedocs.org @@ -74,7 +75,7 @@ more detail in individual sections further down. use serial #:endif -* Iterated output (e.g. for Fortran templates):: +* Iterated output (e.g. for generating Fortran templates):: interface myfunc #:for dtype in [ 'real', 'dreal', 'complex', 'dcomplex' ] @@ -205,9 +206,9 @@ Running The Fypp command line tool reads a file, preprocesses it and writes it to another file, so you would typically invoke it like:: - fypp source.fypp source.f90 + fypp source.F90 source.f90 -which would process `source.fypp` and write the result to `source.f90`. If +which would process `source.F90` and write the result to `source.f90`. If input and output files are not specified, information is read from stdin and written to stdout. @@ -233,7 +234,7 @@ an inline form: * Line form, starting with ``#:`` (hashmark colon):: #:if 1 > 2 - Some fortran code + Some code #:endif * Inline form, enclosed between ``#{`` and ``}#``:: @@ -264,7 +265,7 @@ line. While both forms can be used at the same time, for a particular construct they must be consistent, e.g. a directive opened as line directive can not be closed with an inline directive and vica versa. -Whitespaces in preprocessor commands are ignored, if they appear after the +Whitespaces in preprocessor commands are ignored if they appear after the opening colon or curly brace or before the closing curly brace. So the following examples are pairwise equivalent:: @@ -295,7 +296,7 @@ choose any indentation strategy you like for the directives:: : end program test -Preprocessor directives can be nested arbitrarily:: +Preprocessor directives can be arbitrarily nested:: #:if DEBUG > 0 #:if DO_LOGGING @@ -324,8 +325,8 @@ be, therefore, syntactically and semantically correct Python expressions. Although, this may require some additional quotations as compared to other preprocessor languages :: - #:if defined('DEBUG') #! defined() takes a Python string as argument - #:for dtype in [ 'real(dp)', 'integer', 'logical' ] # dtype runs over strings + #:if defined('DEBUG') #! The Python function defined() expects a string argument + #:for dtype in [ 'real(dp)', 'integer', 'logical' ] #! dtype runs over strings it enables consistent expressions with (hopefully) least surprises (once you know, how to formulate the expression in Python, you exactly know, how to write @@ -339,10 +340,10 @@ command line options or via preprocessor directives) before. For example :: can only be evaluated, if the variable `DEBUG` had been already defined. -Python expressions are evaluted in an isolated Python environment. It contains a -restricted set of Python built-in functions and a few predefined variables and -functions (see below). There are no modules loaded by default, and for safety -reasons, no modules can be loaded once the preprocessing has started. +Python expressions are evaluted in an isolated Python environment, which +contains a restricted set of Python built-in functions and a few predefined +variables and functions (see below). There are no modules loaded by default, and +for safety reasons, no modules can be loaded once the preprocessing has started. Initializing the environment @@ -388,9 +389,9 @@ Predefined variables and functions The isolated Python environment for the expression evaluation contains following predefined read-only variables: -* ``_LINE_``: number of the line where the eval directive was found +* ``_LINE_``: number of current line -* ``_FILE_``: name of the file in which the eval directive was found :: +* ``_FILE_``: name of curernt file :: print *, "This is line nr. ${_LINE_}$ in file '${_FILE_}$'" @@ -403,19 +404,19 @@ predefined read-only variables: Additionally following predefined functions are provided: * ``defined(VARNAME)``: Returns ``True`` if a variable with a given name has - been already defined. The variable name must be provided as string :: + been already defined. The variable name must be provided as string:: #:if defined('WITH_MPI') * ``getvar(VARNAME, DEFAULTVALUE)``: Returns the value of a variable or a default value if the variable is not defined. The variable name must be - provided as string. :: + provided as string:: #:if getvar('DEBUG', 0) * ``setvar(VARNAME, VALUE)``: Sets a variable to given value. It is identical to the ``#:setvar`` control directive. The variable name must be provided as - string :: + string:: $:setvar('i', 12) print *, "VAR I: ${i}$" @@ -424,9 +425,9 @@ Additionally following predefined functions are provided: Eval directive ============== -A result of a Python expression can be inserted into the code by using the eval +A result of a Python expression can be inserted into the code by using eval directives ``$:`` (line form) or ``${`` and ``}$`` (inline form). The expression -is evaluated using Python's built-in function `eval()` . If it evaluates to +is evaluated using Python's built-in function `eval()`. If it evaluates to `None`, no output is produced. Otherwise the result is converted to a string and written to the output. The eval directive has both, a line and an inline variant:: @@ -465,7 +466,7 @@ The `setvar` directive can be also used in the inline form:: ============== Conditional output can be generated using the `if` directive. The condition must -be a Python expression which can be converted to a Boolean. If the condition +be a Python expression, which can be converted to a `bool`. If the condition evaluates to `True`, the enclosed code is written to the output, otherwise it is ignored. @@ -537,7 +538,7 @@ The `for` directive expects a loop variable and an iterable expression, separated by the ``in`` keyword. The code within the `for` directive is outputed for every iteration with the current value of the loop variable, which can be inserted using eval directives. If the iterable consists of iterables -(e.g. tuples), usual indexing can be used to access the components, or a +(e.g. tuples), usual indexing can be used to access their components, or a variable tuple to unpack them directly in the loop header:: #:setvar kinds_names [ ('sp', 'real'), ('dp', 'dreal') ] @@ -571,11 +572,11 @@ The `for` directive can be used also in its inline form:: =============== Parametrized macros can be defined with the `def` directive. This defines a -regular Python callable which returns the rendered content of the macro body +regular Python callable, which returns the rendered content of the macro body when called. The macro arguments are converted to local variables containing the actual arguments as values. The macro can be either called from within an -eval-directive or via the `call` control directive or its abreviated form -(direct call). +eval-directive, via the `call` control directive or its abreviated form, the +direct call. Given the macro definition :: @@ -605,14 +606,13 @@ would all yield :: error stop end if -if the variabl `DEBUG` had a value greater than zero or an empty string +if the variable `DEBUG` had a value greater than zero or an empty string otherwise. -When called from within an eval-directive, additional to the regular macro -arguments, arbitrary optional parameters can be passed. Those optional -parameters will be converted to local variables when the macro content is -rendered. For example given the defintion of the ``assertTrue()`` macro from -above, the call :: +When called from within an eval-directive, arbitrary optional parameters can be +passed additional to the regular macro arguments. The optional parameters are +converted to local variables when the macro content is rendered. For example +given the defintion of the ``assertTrue()`` macro from above, the call :: $:assertTrue('x > y', DEBUG=1) @@ -662,7 +662,7 @@ The `def` directive can also be used in its short form:: ================ When a Python callable (e.g. regular Python function, macro) is called with -string arguments only (e.g. Fortran code), it can be called using the `call` +string arguments only (e.g. source code), it can be called using the `call` directive to avoid extra quoting of the arguments:: #:def debug_code(code) @@ -699,9 +699,9 @@ each other:: print *, "No debugging" #:endcall -The lines in the body of the `call` directive can contain directives -themselves. However, any variables defined within the body of the `call` -directive will be local variables, existing only during the evaluation of that +The lines in the body of the `call` directive may contain directives +themselves. However, any variable defined within the body of the `call` +directive will be a local variable existing only during the evaluation of that branch of the directive (and not being available during the call itself). The `call` directive can also be used in its inline form. As this easily leads @@ -761,8 +761,8 @@ string to the callable. `include` directive =================== -The `include` directive enables you to collect your preprocessor macro and -variable definitions in a separate files and include them whenever needed. The +The `include` directive allows you to collect your preprocessor macros and +variable definitions in separate files and include them whenever needed. The include directive expects a quoted string with a file name:: #:include 'mydefs.fypp' @@ -779,7 +779,7 @@ The `include` directive does not have an inline form. Empty lines between Fypp definitions makes the code easier to read. However, being outside of Fypp-directives, those empty lines will be written unaltered to -the output file. This can be especially disturbing, if various macro definition +the output. This can be especially disturbing if various macro definition files are included, as the resulting output would eventually contian a lot of empty lines. With the `mute` directive, the output can be suspended. While everything is still processed as normal, no output is written for the code @@ -847,15 +847,13 @@ Line folding ============ The Fortran standard only allows source lines up to 132 characters. In order to -emit standard conforming code, Fypp folds all lines in the output, which it had -manipulated before (all lines containing eval directives). Lines, which were -just copied to the output, are left unaltered. The maximal line length can be +emit standard conforming code, Fypp folds all lines in the output which it had +manipulated before (all lines containing eval directives). Lines which were +just copied to the output are left unaltered. The maximal line length can be chosen by the ``-l`` command line option. The indentation of the continuation lines can be tuned with the ``--indentation`` option, and the folding strategy can be selected by the ``-f`` option with following possibilities: -* ``none``: Switches line folding off. - * ``brute``: Continuation lines are indented relative to the beginning of the line, and each line is folded at the maximal line position. @@ -867,13 +865,15 @@ can be selected by the ``-f`` option with following possibilities: becoming too short, it defaults to ``simple`` if no whitespace occurs in the last third of the line. +The ``-F`` option can be used to turn off line folding. + .. warning:: Fypp is not aware of the Fortran semantics represented by the lines it folds. Fypp applies the line folding rather mechanically (only considering the the position of the whitespace characters). Lines containing eval directives and -lines within macro definition should, therefore, not contain any Fortran style +lines within macro definitions should, therefore, not contain any Fortran style comments (started by ``!``) *within* the line, as folding within the comment would result in invalid Fortran code. For comments within such lines, Fypps comment directive (``#!``) can be used instead:: @@ -882,10 +882,10 @@ comment directive (``#!``) can be used instead:: print *, "DO NOT DO THIS!" ! Warning: Line may be folded within the comment print *, "This is OK." #! Preprocessor comment is safe as it will be stripped -If the comment starts at the beginning of the line (preceeded by optional -whitespace characters only), the folding is suppressed, though. This enables you -to define macros, which emit non-negligible comment lines (e.g. containing -source code documentation or OpenMP directives):: +For comments starting at the beginning of the line (preceeded by optional +whitespace characters only) the folding is suppressed, though. This enables you +to define macros with non-negligible comment lines (e.g. with source code +documentation or OpenMP directives):: #:def macro(DTYPE) !> This functions calculates something (version ${DTYPE}$) @@ -907,11 +907,60 @@ closing delimiter as well:: $\: 1 + 2 #\{if 1 > 2}\# + @\:myMacro arg1 Fypp will not recognize the escaped strings as directives, but will remove the backslash between the delimiter characters in the output. If you put more than one backslash between the delimiters, only one will be removed. + +Line numbering directives +========================= + +In order to support compilers in emitting messages with correct line numbers +with respect to the original source file, Fypp can put line number directives +(a.k.a. linemarkers) in its output. This can be enabled by using the command +line option ``-n``. Given a file ``test.F90`` with the content :: + + program test + #:if defined('MPI') + use mpi + #:else + use openmpi + #:endif + : + end program test + +the command :: + + fypp -n -DMPI test.F90 + +produces the output :: + + # 1 "test.F90" + program test + # 3 "test.F90" + use mpi + # 7 "test.F90" + : + end program test + +If during compilation of this output an error occured in the line ``use mpi`` +(e.g. the mpi module can not be found), the compiler would know that this line +corresponds to line number 3 in the original file ``test.F90`` and could emit an +according error message. + +The line numbering directives can be fine tuned with the ``-N`` option, which +accepts following mode arguments: + +* ``full`` (default): Line numbering directives are emitted whenever lines are + removed from the original source file or extra lines are added to it. + +* ``nocontlines``: Same as full, but line numbering directives are ommitted + before continuation lines. (Some compilers, like the NAG Fortran compiler, + have difficulties with line numbering directives before continuation lines). + + ***************** API documentation ***************** @@ -977,10 +1026,10 @@ FyppError :members: -LineFolder -========== +FortranLineFolder +================= -.. autoclass:: LineFolder +.. autoclass:: FortranLineFolder :members: :special-members: __call__ diff --git a/test/test_fypp.py b/test/test_fypp.py index 1c4403b..77ff69a 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -347,6 +347,21 @@ def _linenumbering(nummode): '2Done\n', ), # + ('setvar_function', [], + '$:setvar("x", 2)\n${x}$\nDone\n', + "\n2\nDone\n", + ), + # + ('getvar_existing_value', [_defvar('VAR', '\"VAL\"')], + '$:getvar("VAR", "DEFAULT")\n', + 'VAL\n', + ), + # + ('getvar_default_value', [], + '$:getvar("VAR", "DEFAULT")\n', + 'DEFAULT\n', + ), + # ('mute', [], 'A\n#:mute\nB\n#:setvar VAR 2\n#:endmute\nVAR=${VAR}$\n', 'A\nVAR=2\n' @@ -387,6 +402,11 @@ def _linenumbering(nummode): r'$\\{1 + 1}\$' ), # + ('escape_direct_call', [], + '@\\:assertTrue x > y\n', + '@:assertTrue x > y\n' + ), + # ('fold_lines', [_linelen(10), _indentation(2), _folding('simple')], 'This line is not folded\nThis line ${1 + 1}$ is folded\n', 'This line is not folded\nThis line&\n & 2 is &\n &folded\n' From ae2a242454e1f5ed916a5a4f874510616e2ea7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Wed, 16 Mar 2016 20:37:17 +0100 Subject: [PATCH 04/10] Make Fypp class independent of ArgumentParser. --- bin/fypp | 309 +++++++++++++++++++++++++++++----------------- docs/fypp.rst | 41 +++--- test/test_fypp.py | 9 +- 3 files changed, 231 insertions(+), 128 deletions(-) diff --git a/bin/fypp b/bin/fypp index c31563a..b7fe871 100755 --- a/bin/fypp +++ b/bin/fypp @@ -43,13 +43,18 @@ using following classes: * `Renderer`_: Renders a tree built by the builder. * `Evaluator`_: Evaluates Python expressions in a designated environment. It is - used by the Renderer when rendering eval directives. + used by `Renderer`_ when rendering eval directives. -* `Processor`_: Connects `Parser`, `Builder`, `Renderer` and `Evaluator` with - each other and returns for a given input the preprocessed output. +* `Processor`_: Connects `Parser`_, `Builder`_, `Renderer`_ and `Evaluator`_ + with each other and returns for a given input the preprocessed output. -* `Fypp`_: Command line preprocessor. Initializes and drives - `Processor` according to the command line arguments. +* `Fypp`_: The actual Fypp preprocessor. It initializes and drives + `Processor`_. + +* `FyppOptions`_: Contains customizable settings controling the behaviour of + `Fypp`_. Alternatively, the function `get_option_parser()`_ can be used to + obtain an argument parser, which can create settings based on command line + arguments. If any expected error occurs during processing, `FyppError`_ is raised. @@ -1838,90 +1843,118 @@ def _shiftinds(inds, shift): # Command line tool ################################################################################ -_FYPP_DESC = '''Preprocess Fortran source files with Fypp directives.''' +class Fypp: + '''Fypp preprocessor. -class Fypp: + You can invoke it like :: - '''Represents the Fypp command line tool. + tool = Fypp() + tool.process_file('file.in', 'file.out') - You can invoke it like:: + to initialize Fypp with default options, process `file.in` and write the + result to `file.out`. If the input should be read from a string, the + ``process_text()`` method can be used:: tool = Fypp() - tool.process_cmdline_files() + output = tool.process_text('#:if DEBUG > 0\\nprint *, "DEBUG"\\n#:endif\\n') + + If you want to fine tune Fypps behaviour, pass a customized `FyppOptions`_ + instance at initialization:: - to parse the command line arguments (in ``sys.argv``) and run Fypp using - the input/output files and options as specified there. + options = FyppOptions() + options.fixed_format = True + tool = Fypp(options) - The command line arguments can also be passed directly as a list:: + Alternatively, you can use the command line parser + ``argparse.ArgumentParser`` to set options for Fypp. The function + ``get_option_parser()`` returns you a default argument parser. You can then + use its ``parse_args()`` method to obtain settings by reading the command + line arguments:: - tool = Fypp(['-DDEBUG=0', 'input.F90', 'output.F90']) - tool.process_cmdline_files() + options = FyppOptions() + argparser = get_option_parser() + options = argparser.parse_args(namespace=options) + tool = fypp.Fypp(options) - If the input should be read from a string and the result of the - preprocessing should be also returned as a string (instead of - using input/output files), the ``process_text()`` method can be - used:: + The command line arguments can also be passed directly as a list when + calling ``parse_args()``:: - tool = Fypp(['-DDEBUG=0']) - result = tool.process_text('#:if DEBUG > 0\\nprint *, "DEBUG"\\n#:endif\\n') + options = FyppOptions() + args = ['-DDEBUG=0', 'input.F90', 'output.F90'] + argparser = get_option_parser() + options = argparser.parse_args(args=args, namespace=options) + tool = fypp.Fypp(options) Args: - cmdline_args (list of str, optional): Command line arguments. If None - (default), the arguments are read from sys.argv using Argparser. + options (object): Object containing the settings for Fypp. You typically + would pass a customized `FyppOptions`_ instance or a ``Namespace`` + object as returned by an argument parser. If not present, the + default settings in `FyppOptions`_ are used. ''' - def __init__(self, cmdline_args=None): - self._argparser = self._get_cmdline_parser() - self._args = self._argparser.parse_args(cmdline_args) + def __init__(self, options=None): + if options is None: + options = FyppOptions() inieval = Evaluator(restricted=False) - if self._args.modules: - self._import_modules(self._args.modules, inieval) - if self._args.inifiles: - self._exec_inifiles(self._args.inifiles, inieval) + if options.modules: + self._import_modules(options.modules, inieval) + if options.inifiles: + self._exec_inifiles(options.inifiles, inieval) evaluator = Evaluator(env=inieval.env, restricted=True) - if self._args.defines: - self._apply_definitions(self._args.defines, evaluator) - parser = Parser(self._args.includes) + if options.defines: + self._apply_definitions(options.defines, evaluator) + parser = Parser(options.includes) builder = Builder() - fixed_format = self._args.fixed_format - if self._args.linefolding: - folding = 'brute' if fixed_format else self._args.folding_mode - linelength = 72 if fixed_format else self._args.line_length - indentation = 5 if fixed_format else self._args.indentation + fixed_format = options.fixed_format + linefolding = not options.no_folding + if linefolding: + folding = 'brute' if fixed_format else options.folding_mode + linelength = 72 if fixed_format else options.line_length + indentation = 5 if fixed_format else options.indentation prefix = '&' suffix = '' if fixed_format else '&' linefolder = FortranLineFolder(linelength, indentation, folding, prefix, suffix) else: linefolder = DummyLineFolder() - linenums = self._args.line_numbering - contlinenums = (self._args.line_numbering_mode != 'nocontlines') + linenums = options.line_numbering + contlinenums = (options.line_numbering_mode != 'nocontlines') renderer = Renderer( evaluator, linenums=linenums, contlinenums=contlinenums, linefolder=linefolder) self._preprocessor = Processor(parser, builder, renderer) - def process_cmdline_files(self, env=None): - '''Processes the input file and write result to the output file which - were specified as the command line arguments. + def process_file(self, infile, outfile=None, env=None): + '''Processes input file and writes result to output file. Args: - env (dict): Additional definitions for the evaluator. + infile (str): Name of the file to read and process. If its value is + '-', input is read from stdin. + outfile (str, optional): Name of the file to write the result to. + If its value is '-', result is written to stdout. If not + present, result will be returned as string. + env (dict, optional): Additional definitions for the evaluator. + + Returns: + str: Result of processed input, if no outfile was specified. ''' - infile = STDIN if self._args.infile == '-' else self._args.infile + infile = STDIN if infile == '-' else infile output = self._preprocessor.process_file(infile, env) - if self._args.outfile == '-': - outfile = sys.stdout + if outfile is None: + return output else: - outfile = open(self._args.outfile, 'w') - outfile.write(output) - if outfile != sys.stdout: - outfile.close() + if outfile == '-': + outfile = sys.stdout + else: + outfile = open(outfile, 'w') + outfile.write(output) + if outfile != sys.stdout: + outfile.close() def process_text(self, txt, env=None): @@ -1929,6 +1962,7 @@ class Fypp: Args: txt (str): String to process. + env (dict, optional): Additional definitions for the evaluator. Returns: str: Processed content. @@ -1936,64 +1970,6 @@ class Fypp: return self._preprocessor.process_text(txt, env) - @staticmethod - def _get_cmdline_parser(): - parser = ArgumentParser(description=_FYPP_DESC) - msg = 'define variable, value is interpreted as ' \ - 'Python expression (e.g \'-DDEBUG=1\' sets DEBUG to the ' \ - 'integer 1) or set to None if ommitted' - parser.add_argument('-D', '--define', action='append', dest='defines', - metavar='VAR[=VALUE]', help=msg) - msg = 'add directory to the search paths for include files' - parser.add_argument('-I', '--include', action='append', dest='includes', - metavar='INCDIR', help=msg) - msg = 'put line numbering directives to the output' - parser.add_argument('-n', '--line-numbering', action='store_true', - default=False, help=msg) - msg = 'line numbering mode, \'full\' (default): line numbering '\ - 'directives generated whenever source and output lines are out '\ - 'of sync, \'nocontlines\': line numbering directives omitted '\ - 'for continuation lines' - parser.add_argument('-N', '--line-numbering-mode', metavar='MODE', - choices=['full', 'nocontlines'], default='full', - help=msg) - msg = 'maximal line length (default: 132), lines modified by the '\ - 'preprocessor are folded if becoming longer' - parser.add_argument('-l', '--line-length', type=int, default=132, - metavar='LEN', help=msg) - msg = 'line folding mode, \'smart\' (default): indentation context '\ - 'and whitespace aware, \'simple\': indentation context aware, '\ - '\'brute\': mechnical folding' - parser.add_argument('-f', '--folding-mode', metavar='MODE', - choices=['smart', 'simple', 'brute'], - default='smart', help=msg) - msg = 'suppress line folding' - parser.add_argument('-F', '--no-folding', action='store_false', - dest='linefolding', default=True, help=msg) - msg = 'indentation to use for continuation lines (default 4)' - parser.add_argument('--indentation', type=int, metavar='IND', - default=4, help=msg) - msg = 'import python module before starting the processing' - parser.add_argument('-m', '--module', action='append', dest='modules', - metavar='MOD', help=msg) - msg = 'execute python initialization script before starting processing' - parser.add_argument('-i', '--ini-file', action='append', - dest='inifiles', metavar='INI', help=msg) - msg = 'produce fixed format output (any settings for options '\ - '--line-length, --folding-method and --indentation are ignored)' - parser.add_argument('--fixed-format', action='store_true', - default=False, help=msg) - versionstr = '%(prog)s ' + VERSION - parser.add_argument('-v', '--version', action='version', - version=versionstr) - msg = "input file to be processed (default: '-', stdin)" - parser.add_argument('infile', nargs='?', default='-', help=msg) - msg = "output file where processed content will be written (default: " \ - "'-', stdout)" - parser.add_argument('outfile', nargs='?', default='-', help=msg) - return parser - - @staticmethod def _apply_definitions(defines, evaluator): for define in defines: @@ -2041,11 +2017,122 @@ class Fypp: raise FyppError(msg) +class FyppOptions: + + '''Container for Fypp options with default values. + + Attributes: + defines (list of str): List of variable definitions in the form of + 'VARNAME=VALUE'. Default: [] + includes (list of str): List of paths to search when looking for include + files. Default: [] + line_numbering (bool): Whether line numbering directives should appear + in the output. Default: False + line_numbering_mode (str): Line numbering mode 'full' or 'nocontlines'. + Default: 'full'. + line_length (int): Length of output lines. Default: 132. + folding_mode (str): Folding mode 'smart', 'simple' or 'brute'. Default: + 'smart'. + no_folding (bool): Whether folding should be suppresed. Default: False. + indentation (int): Indentation in continuation lines. Default: 4. + modules (list of str): Modules to import at initialization. Default: []. + inifiles (list of str): Python files to execute at initialization. + Default: [] + fixed_format (bool): Whether input file is in fixed format. + Default: False. + ''' + + def __init__(self): + self.defines = [] + self.includes = [] + self.line_numbering = False + self.line_numbering_mode = 'full' + self.line_length = 132 + self.folding_mode = 'smart' + self.no_folding = False + self.indentation = 4 + self.modules = [] + self.inifiles = [] + self.fixed_format = False + + +def get_option_parser(): + '''Returns an option parser for the Fypp command line tool. + + Returns: + ArgumentParser: Parser which can create a namespace object with + Fypp settings based on command line arguments. + ''' + fypp_name = 'fypp' + fypp_desc = 'Preprocess source files with Fypp directives.' + parser = ArgumentParser(prog=fypp_name, description=fypp_desc) + msg = 'define variable, value is interpreted as ' \ + 'Python expression (e.g \'-DDEBUG=1\' sets DEBUG to the ' \ + 'integer 1) or set to None if ommitted' + parser.add_argument('-D', '--define', action='append', dest='defines', + metavar='VAR[=VALUE]', help=msg) + msg = 'add directory to the search paths for include files' + parser.add_argument('-I', '--include', action='append', dest='includes', + metavar='INCDIR', help=msg) + msg = 'put line numbering directives to the output' + parser.add_argument('-n', '--line-numbering', action='store_true', + default=False, help=msg) + msg = 'line numbering mode, \'full\' (default): line numbering '\ + 'directives generated whenever source and output lines are out '\ + 'of sync, \'nocontlines\': line numbering directives omitted '\ + 'for continuation lines' + parser.add_argument('-N', '--line-numbering-mode', metavar='MODE', + choices=['full', 'nocontlines'], default='full', + help=msg) + msg = 'maximal line length (default: 132), lines modified by the '\ + 'preprocessor are folded if becoming longer' + parser.add_argument('-l', '--line-length', type=int, default=132, + metavar='LEN', help=msg) + msg = 'line folding mode, \'smart\' (default): indentation context '\ + 'and whitespace aware, \'simple\': indentation context aware, '\ + '\'brute\': mechnical folding' + parser.add_argument('-f', '--folding-mode', metavar='MODE', + choices=['smart', 'simple', 'brute'], + default='smart', help=msg) + msg = 'suppress line folding' + parser.add_argument('-F', '--no-folding', action='store_true', + dest='no_folding', default=False, help=msg) + msg = 'indentation to use for continuation lines (default 4)' + parser.add_argument('--indentation', type=int, metavar='IND', + default=4, help=msg) + msg = 'import python module before starting the processing' + parser.add_argument('-m', '--module', action='append', dest='modules', + metavar='MOD', help=msg) + msg = 'execute python initialization script before starting processing' + parser.add_argument('-i', '--ini-file', action='append', + dest='inifiles', metavar='INI', help=msg) + msg = 'produce fixed format output (any settings for options '\ + '--line-length, --folding-method and --indentation are ignored)' + parser.add_argument('--fixed-format', action='store_true', + default=False, help=msg) + versionstr = '%(prog)s ' + VERSION + parser.add_argument('-v', '--version', action='version', + version=versionstr) + return parser + + +def _add_io_arguments(parser): + msg = "input file to be processed (default: '-', stdin)" + parser.add_argument('infile', nargs='?', default='-', help=msg) + msg = "output file where processed content will be written (default: " \ + "'-', stdout)" + parser.add_argument('outfile', nargs='?', default='-', help=msg) + + def run_fypp(): '''Run the Fypp command line tool.''' + options = FyppOptions() + argparser = get_option_parser() + _add_io_arguments(argparser) + args = argparser.parse_args(namespace=options) try: - tool = Fypp() - tool.process_cmdline_files() + tool = Fypp(args) + tool.process_file(args.infile, args.outfile) except FyppError as exc: sys.stderr.write(str(exc)) sys.stderr.write('\n') diff --git a/docs/fypp.rst b/docs/fypp.rst index d0df4e6..b4e9fa7 100644 --- a/docs/fypp.rst +++ b/docs/fypp.rst @@ -977,6 +977,33 @@ fypp module .. automodule:: fypp +Fypp +==== + +.. autoclass:: Fypp + :members: + + +FyppOptions +=========== + +.. autoclass:: FyppOptions + :members: + + +get_option_parser() +=================== + +.. autofunction:: get_option_parser() + + +FyppError +========= + +.. autoclass:: FyppError + :members: + + Parser ====== @@ -1012,20 +1039,6 @@ Processor :members: -Fypp -==== - -.. autoclass:: Fypp - :members: - - -FyppError -========= - -.. autoclass:: FyppError - :members: - - FortranLineFolder ================= diff --git a/test/test_fypp.py b/test/test_fypp.py index 77ff69a..de8750f 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -1068,8 +1068,9 @@ def test_output_method(args, inp, out): def test_output(self): '''Tests whether Fypp result matches expected output.''' - - tool = fypp.Fypp(args) + options = fypp.FyppOptions() + argparser = fypp.get_option_parser() + tool = fypp.Fypp(argparser.parse_args(args, namespace=options)) result = tool.process_text(inp) self.assertEqual(out, result) return test_output @@ -1092,8 +1093,10 @@ def test_exception_method(args, inp, exc, fname, span): def test_exception(self): '''Tests whether Fypp throws the correct exception.''' + options = fypp.FyppOptions() + argparser = fypp.get_option_parser() with self.assertRaises(exc) as ctx: - tool = fypp.Fypp(args) + tool = fypp.Fypp(argparser.parse_args(args, namespace=options)) _ = tool.process_text(inp) raised = ctx.exception if fname is None: From 3094ef5b4fbfbfef739159c984fc7d723a035abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Wed, 16 Mar 2016 20:48:48 +0100 Subject: [PATCH 05/10] Bump version number to 1.0. --- bin/fypp | 2 +- docs/conf.py | 4 ++-- docs/fypp.rst | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/fypp b/bin/fypp index b7fe871..efb3367 100755 --- a/bin/fypp +++ b/bin/fypp @@ -78,7 +78,7 @@ import time from argparse import ArgumentParser -VERSION = '0.12' +VERSION = '1.0' STDIN = '' diff --git a/docs/conf.py b/docs/conf.py index 6e66bb2..c7ae245 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,10 +53,10 @@ # built documents. # # The short X.Y version. -version = '0.12' +version = '1.0' # The full version, including alpha/beta/rc tags. -release = '0.12' +release = '1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/fypp.rst b/docs/fypp.rst index b4e9fa7..c75d514 100644 --- a/docs/fypp.rst +++ b/docs/fypp.rst @@ -29,7 +29,7 @@ documentation available on `readthedocs.org `_. Fypp is released under the *BSD 2-clause license*. -This document describes Fypp Version 0.12. +This document describes Fypp Version 1.0. Features diff --git a/setup.py b/setup.py index df1f33c..af11e2e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='fypp', - version='0.12', + version='1.0', description='Python powered Fortran preprocessor', long_description=long_description, From 3c59bb31c0872800ca6af61a0db4dd31930e9197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Wed, 16 Mar 2016 21:02:44 +0100 Subject: [PATCH 06/10] Add explicit links to PyPI in the documentation. --- README.rst | 3 ++- docs/fypp.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 71518a9..7c405d6 100644 --- a/README.rst +++ b/README.rst @@ -122,7 +122,8 @@ above. Automatic install ----------------- -You can use Pythons installer `pip` to install the last stable release of Fypp +You can use Pythons command line installer ``pip`` to download the stable +release of Fypp from `PyPI `_ and install it on your system:: pip install fypp diff --git a/docs/fypp.rst b/docs/fypp.rst index c75d514..ec39af7 100644 --- a/docs/fypp.rst +++ b/docs/fypp.rst @@ -142,7 +142,8 @@ above. Automatic install ----------------- -You can use Pythons installer `pip` to install the last stable release of Fypp +You can use Pythons command line installer ``pip`` to download the stable +release of Fypp from `PyPI `_ and install it on your system:: pip install fypp From c23a4484f41dff95c4f31cbf9e60ce7fda41fb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Fri, 18 Mar 2016 12:46:14 +0100 Subject: [PATCH 07/10] Fix false error when include occured within a directive. --- bin/fypp | 177 ++++++++++++++++++++++++++++++---------------- test/test_fypp.py | 10 +++ 2 files changed, 128 insertions(+), 59 deletions(-) diff --git a/bin/fypp b/bin/fypp index efb3367..07c2d1c 100755 --- a/bin/fypp +++ b/bin/fypp @@ -146,7 +146,6 @@ class FyppError(Exception): span (tuple of int): Beginning and end line of the region where error occured or None if not available. - Attributes: msg (str): Error message. fname (str or None): File name or None if not available. @@ -188,11 +187,17 @@ class Parser: ''' def __init__(self, includedirs=None): + + # Directories to search for include files if includedirs is None: self._includedirs = [] else: self._includedirs = includedirs + + # Name of current file self._curfile = None + + # Directory of current file self._curdir = None @@ -202,26 +207,26 @@ class Parser: Args: fobj (str or file): Name of a file or a file like object. ''' - closefile = False if isinstance(fobj, str): if fobj == STDIN: - inpfile = sys.stdin - self._curfile = fobj - self._curdir = os.getcwd() + self._includefile(None, sys.stdin, STDIN, os.getcwd()) else: - inpfile = open(fobj, 'r') - closefile = True - self._curfile = fobj - self._curdir = os.path.dirname(fobj) + with open(fobj, 'r') as inp: + self._includefile(None, inp, fobj, os.path.dirname(fobj)) else: - inpfile = fobj - self._curfile = FILEOBJ - self._curdir = os.getcwd() - self.handle_open_file((0, 0), self._curfile) - self._parse(inpfile.read()) - self.handle_close_file(self._curfile) - if closefile: - inpfile.close() + self._includefile(None, fobj, FILEOBJ, os.getcwd()) + + + def _includefile(self, span, fobj, fname, curdir): + oldfile = self._curfile + olddir = self._curdir + self._curfile = fname + self._curdir = curdir + self.handle_include(span, fname) + self._parse(fobj.read()) + self.handle_endinclude(span, fname) + self._curfile = oldfile + self._curdir = olddir def parse(self, txt): @@ -232,32 +237,35 @@ class Parser: ''' self._curfile = STRING self._curdir = os.getcwd() - self.handle_open_file((0, 0), self._curfile) + self.handle_include(None, self._curfile) self._parse(txt) - self.handle_close_file(self._curfile) + self.handle_endinclude(None, self._curfile) - def handle_open_file(self, span, fname): - '''Called when parser starts to parse a new file. + def handle_include(self, span, fname): + '''Called when parser starts to process a new file. It is a dummy methond and should be overriden for actual use. Args: - span (tuple of int): Start and end line of the directive. + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' - self._log_event('open file', span, filename=fname) + self._log_event('include', span, filename=fname) - def handle_close_file(self, fname): + def handle_endinclude(self, span, fname): '''Called when parser finished processing a file. It is a dummy method and should be overriden for actual use. Args: + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' - self._log_event('close file', filename=fname) + self._log_event('endinclude', span, filename=fname) def handle_setvar(self, span, name, expr): @@ -634,10 +642,8 @@ class Parser: else: msg = "include file '{}' not found".format(fname) raise FyppError(msg, self._curfile, span) - oldfile = self._curfile - self.parsefile(fpath) - self._curfile = oldfile - self.handle_open_file(span, self._curfile) + with open(fpath, 'r') as fobj: + self._includefile(span, fobj, fpath, os.path.dirname(fpath)) def _process_mute(self, span): @@ -678,57 +684,83 @@ class Builder: ''' def __init__(self): + # The tree, which should be built. self._tree = [] - self._path = [] - self._open_files = [] + + # List of all open constructs self._open_blocks = [] - # Nr. of open blocks before current file was opened + + # Nodes to which the open blocks have to be appended when closed + self._path = [] + + # Nr. of open blocks when file was opened. Used for checking whether all + # blocks have been closed, when file processing finishes. self._nr_prev_blocks = [] + + # Current node, to which content should be added self._curnode = self._tree + + # Current file self._curfile = None def reset(self): '''Resets the builder so that it starts to build a new tree.''' self._tree = [] - self._path = [] - self._open_files = [] self._open_blocks = [] + self._path = [] self._nr_prev_blocks = [] self._curnode = self._tree self._curfile = None - def handle_open_file(self, span, fname): + def handle_include(self, span, fname): '''Should be called to signalize change to new file. Args: - span (tuple of int): Start and end line of the directive. + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' self._curfile = fname - self._curnode.append(('include', span, fname)) - self._open_files.append(fname) + self._path.append(self._curnode) + self._curnode = [] + self._open_blocks.append(('include', [span], fname, None)) self._nr_prev_blocks.append(len(self._open_blocks)) - def handle_close_file(self, fname): + def handle_endinclude(self, span, fname): '''Should be called when processing of a file finished. Args: + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' - oldfname = self._open_files.pop(-1) - if fname != oldfname: - msg = 'internal error: mismatching file name in close_file event'\ - " (expected: '{}', got '{}')".format(oldfname, fname) - raise FyppError(msg, fname) - if len(self._open_blocks) > self._nr_prev_blocks[-1]: + nprev_blocks = self._nr_prev_blocks.pop(-1) + if len(self._open_blocks) > nprev_blocks: directive, spans = self._open_blocks[-1][0:2] msg = '{} directive in line {} still unclosed when reaching end '\ 'of file'.format(directive, spans[0][0] + 1) raise FyppError(msg, self._curfile) - del self._nr_prev_blocks[-1] + block = self._open_blocks.pop(-1) + directive, spans = block[0:2] + if directive != 'include': + msg = 'internal error: last open block is not \'include\' when '\ + 'closing file \'{}\''.format(fname) + raise FyppError(msg) + if span != spans[0]: + msg = 'internal error: span for include and endinclude differ ('\ + '{} vs {}'.format(span, spans[0]) + raise FyppError(msg) + oldfname, _ = block[2:4] + if fname != oldfname: + msg = 'internal error: mismatching file name in close_file event'\ + " (expected: '{}', got '{}')".format(oldfname, fname) + raise FyppError(msg, fname) + block = directive, spans, fname, self._curnode + self._curnode = self._path.pop(-1) + self._curnode.append(block) def handle_if(self, span, cond): @@ -1023,13 +1055,24 @@ class Renderer: def __init__(self, evaluator=None, linenums=False, contlinenums=False, linefolder=None): + # Evaluator to use for Python expressions self._evaluator = Evaluator() if evaluator is None else evaluator self._evaluator.updateenv(_DATE_=time.strftime('%Y-%m-%d'), _TIME_=time.strftime('%H:%M:%S')) + + # Name of current file being processed self._curfile = None + + # Number of diversions, when > 0 we are within a macro call self._diversions = 0 + + # Whether line numbering directives should be emitted self._linenums = linenums + + # Whether line numbering directives in continuation lines are needed. self._contlinenums = contlinenums + + # Callable to be used for folding lines if linefolder is None: self._linefolder = lambda line: [line] else: @@ -1093,8 +1136,10 @@ class Renderer: eval_pos += peval output += out elif cmd == 'include': - result = self._register_file(*node[1:3]) - output.append(result) + out, ieval, peval = self._get_included_content(*node[1:4]) + eval_inds += _shiftinds(ieval, len(output)) + eval_pos += peval + output += out elif cmd == 'comment': output.append(self._get_comment(*node[1:2])) elif cmd == 'mute': @@ -1198,6 +1243,23 @@ class Renderer: return out, ieval, peval + def _get_included_content(self, spans, fname, content): + out = [] + oldfile = self._curfile + self._curfile = fname + self._evaluator.updateenv(_FILE_=self._curfile) + if self._linenums and not self._diversions: + out += linenumdir(0, self._curfile) + outcont, ieval, peval = self._render(content) + ieval = _shiftinds(ieval, len(out)) + out += outcont + self._curfile = oldfile + self._evaluator.updateenv(_FILE_=self._curfile) + if self._linenums and not self._diversions and spans[0] is not None: + out += linenumdir(spans[0][1], self._curfile) + return out, ieval, peval + + def _define_macro(self, spans, name, args, content): result = '' try: @@ -1254,15 +1316,6 @@ class Renderer: raise FyppError(msg, self._curfile) - def _register_file(self, span, fname): - result = '' - self._curfile = fname - self._evaluator.updateenv(_FILE_=fname) - if self._linenums and not self._diversions: - result = linenumdir(span[1], self._curfile) - return result - - def _update_linenr(self, linenr): if not self._diversions: self._evaluator.updateenv(_LINE_=linenr + 1) @@ -1461,8 +1514,12 @@ class Evaluator: } def __init__(self, env=None, restricted=True): + # Definitions (environment) to use when evaluating expressions self._env = env.copy() if env is not None else {} + + # Stack for environments to implement nested scopes self._envstack = [] + if restricted: builtindict = {} builtindict.update(self.RESTRICTED_BUILTINS) @@ -1472,6 +1529,8 @@ class Evaluator: builtindict['defined'] = self._func_defined builtindict['setvar'] = self._func_setvar builtindict['getvar'] = self._func_getvar + + # Permitted builtins when evaluating expressions self._builtins = {'__builtins__': builtindict} @@ -1648,8 +1707,8 @@ class Processor: else: self._renderer = renderer - self._parser.handle_open_file = self._builder.handle_open_file - self._parser.handle_close_file = self._builder.handle_close_file + self._parser.handle_include = self._builder.handle_include + self._parser.handle_endinclude = self._builder.handle_endinclude self._parser.handle_if = self._builder.handle_if self._parser.handle_else = self._builder.handle_else self._parser.handle_elif = self._builder.handle_elif diff --git a/test/test_fypp.py b/test/test_fypp.py index de8750f..299f8a8 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -730,6 +730,16 @@ def _linenumbering(nummode): + 'FYPP2\n' + _linenum(1, 'include/subfolder/include_fypp2.inc') + _linenum(1)) ), + # + ('muted_include', [_incdir('include')], + 'START\n#:mute\n#:include \'fypp1.inc\'\n#:endmute\nDONE\n', + 'START\nDONE\n' + ), + # + ('muted_include_linenum', [_LINENUM_FLAG, _incdir('include')], + 'START\n#:mute\n#:include \'fypp1.inc\'\n#:endmute\nDONE\n', + _linenum(0) + 'START\n' + _linenum(4) + 'DONE\n' + ), ] EXCEPTION_TESTS = [ From 0ff6b4bdf9364292aeb8b86c3c0a7636a6fe6ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Sun, 20 Mar 2016 12:51:36 +0100 Subject: [PATCH 08/10] Minor changes in documentation. --- README.rst | 28 +++++++------ docs/fypp.rst | 109 +++++++++++++++++++++++++++----------------------- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/README.rst b/README.rst index 7c405d6..95adf32 100644 --- a/README.rst +++ b/README.rst @@ -34,10 +34,12 @@ Main features Fortran standard):: #:def assertTrue(cond) + #:if DEBUG > 0 if (.not. ${cond}$) then print *, "Assert failed in file ${_FILE_}$, line ${_LINE_}$" error stop end if + #:endif #:enddef ! Invoked via direct call (needs no quotation) @@ -110,34 +112,32 @@ Main features #:mute #:include "macrodefs.fypp" #:endmute - Installing ========== -Fypp needs a working Python interpreter, either version 2.7 or version 3.2 or -above. +Fypp needs a Python interpreter of version 2.7, 3.2 or above. Automatic install ----------------- -You can use Pythons command line installer ``pip`` to download the stable -release of Fypp from `PyPI `_ and install it -on your system:: +Use Pythons command line installer ``pip`` in order to download the stable +release from the `Fypp page on PyPI `_ and +install it on your system:: pip install fypp -This installs the command line tool ``fypp`` as well as the Python module -``fypp``. Latter you can import if you want to access the functionality of Fypp -directly from within your Python scripts. +This installs both, the command line tool ``fypp`` and the Python module +``fypp.py``. Latter you can import if you want to access the functionality of +Fypp directly from within your Python scripts. Manual install -------------- -Alternatively, you can download the source code from the `Fypp project website -`_ :: +For a manual install, you can download the source code from the `Fypp project +website `_ :: git clone https://aradi@bitbucket.org/aradi/fypp.git @@ -166,6 +166,8 @@ environment variable, by just issuing :: fypp +The python module ``fypp.py`` can be found in ``FYP_SOURCE_FOLDER/src``. + Running ======= @@ -173,9 +175,9 @@ Running The Fypp command line tool reads a file, preprocesses it and writes it to another file, so you would typically invoke it like:: - fypp source.fypp source.f90 + fypp source.F90 source.f90 -which would process `source.fypp` and write the result to `source.f90`. If +which would process `source.F90` and write the result to `source.f90`. If input and output files are not specified, information is read from stdin and written to stdout. diff --git a/docs/fypp.rst b/docs/fypp.rst index ec39af7..4edc2ab 100644 --- a/docs/fypp.rst +++ b/docs/fypp.rst @@ -35,8 +35,8 @@ This document describes Fypp Version 1.0. Features ======== -Below you find a summary over Fypps main features. Each of them is described in -more detail in individual sections further down. +Below you find a summary over Fypps main features. Each of them is described +more in detail in the individual sections further down. * Definition and evaluation of preprocessor variables:: @@ -51,10 +51,12 @@ more detail in individual sections further down. Fortran standard):: #:def assertTrue(cond) + #:if DEBUG > 0 if (.not. ${cond}$) then print *, "Assert failed in file ${_FILE_}$, line ${_LINE_}$" error stop end if + #:endif #:enddef ! Invoked via direct call (needs no quotation) @@ -136,28 +138,27 @@ Getting started Installing ========== -Fypp needs a working Python interpreter, either version 2.7 or version 3.2 or -above. +Fypp needs a Python interpreter of version 2.7, 3.2 or above. Automatic install ----------------- -You can use Pythons command line installer ``pip`` to download the stable -release of Fypp from `PyPI `_ and install it -on your system:: +Use Pythons command line installer ``pip`` in order to download the stable +release from the `Fypp page on PyPI `_ and +install it on your system:: pip install fypp -This installs the command line tool ``fypp`` as well as the Python module -``fypp``. Latter you can import if you want to access the functionality of Fypp -directly from within your Python scripts. +This installs both, the command line tool ``fypp`` and the Python module +``fypp.py``. Latter you can import if you want to access the functionality of +Fypp directly from within your Python scripts. Manual install -------------- -Alternatively, you can download the source code from the `Fypp project website -`_ :: +For a manual install, you can download the source code from the `Fypp project +website `_ :: git clone https://aradi@bitbucket.org/aradi/fypp.git @@ -186,6 +187,8 @@ environment variable, by just issuing :: fypp +The python module ``fypp.py`` can be found in ``FYP_SOURCE_FOLDER/src``. + Testing ======= @@ -194,7 +197,7 @@ You can test Fypp on your system by running :: ./test/runtests.sh -in its source tree. This will execute various unit tests to check whether Fypp +in the source tree. This will execute various unit tests to check whether Fypp works as expected. If you want to run the tests with a specific Python interpreter, you can specify it as argument to the script:: @@ -259,12 +262,12 @@ an inline form: The line form must always start at the beginning of a line (preceded by optional -whitespace characters only) and it goes until the end of the line. The inline -form can appear anywhere, but if the construct consists of several directives +whitespace characters only) and it ends at the end of the line. The inline form +can appear anywhere, but if the construct consists of several directives (e.g. ``#{if ...}#`` and ``#{endif}#``), all of them must appear on the same -line. While both forms can be used at the same time, for a particular construct -they must be consistent, e.g. a directive opened as line directive can not be -closed with an inline directive and vica versa. +line. While both forms can be used at the same time, they must be consistent for +a particular construct, e.g. a directive opened as line directive can not be +closed by an inline directive and vica versa. Whitespaces in preprocessor commands are ignored if they appear after the opening colon or curly brace or before the closing curly brace. So the following @@ -282,8 +285,8 @@ examples are pairwise equivalent:: ${time.strftime('%Y-%m-%d')}$ ${ time.strftime('%Y-%m-%d') }$ -Starting whitespaces before line directives are also ignored, enabling you to -choose any indentation strategy you like for the directives:: +Starting whitespaces before line directives are ignored, enabling you to choose +any indentation strategy you like for the directives:: program test : @@ -332,14 +335,15 @@ to other preprocessor languages :: it enables consistent expressions with (hopefully) least surprises (once you know, how to formulate the expression in Python, you exactly know, how to write it for Fypp). Also, note, that variable names, macros etc. are in Python (and -therefore als for Fypp) case sensitive. +therefore also for Fypp) case sensitive. -If you access a variable in an expression, it must have been defined (either via -command line options or via preprocessor directives) before. For example :: +When you access a variable in an expression, it must have been already defined +before, either via command line options or via preprocessor directives. For +example the directive :: #:if DEBUG > 0 -can only be evaluated, if the variable `DEBUG` had been already defined. +can only be evaluated, if the variable `DEBUG` had been already defined before. Python expressions are evaluted in an isolated Python environment, which contains a restricted set of Python built-in functions and a few predefined @@ -350,8 +354,8 @@ for safety reasons, no modules can be loaded once the preprocessing has started. Initializing the environment ---------------------------- -If a Python module is needed during the preprocessing, it can be imported via -the command line option (``-m``) before the preprocessing starts:: +If a Python module is required for the preprocessing, it can be imported before +the preprocessing starts via the command line option (``-m``):: fypp -m time @@ -372,23 +376,23 @@ Initial values for preprocessor variables can be set via the command line option The assigned value for a given variable is evaluated in Python. If no value is provided, `None` is assigned. -When complex initialization is needed (e.g. user defined Python functions are -needed during preprocessing), initialization scripts can be specified via the -command line option ``-i``:: +When complex initialization is needed (e.g. user defined Python functions should +be defined), initialization scripts can be specified via the command line option +``-i``:: fypp -i ini1.py -i ini2.py The preprocessor executes the content of each initialization script in the isolated environment via Pythons `exec()` command before processing any input. If modules had been also specified via the ``-m`` option, they are -imported before the initialization scripts are executed. +imported before the execution of the initialization scripts. Predefined variables and functions ---------------------------------- The isolated Python environment for the expression evaluation contains following -predefined read-only variables: +predefined (read-only) variables: * ``_LINE_``: number of current line @@ -535,30 +539,33 @@ single and double precision reals:: end function sin2_${rkind}$ #:endfor -The `for` directive expects a loop variable and an iterable expression, +The `for` directive expects a Python loop variable expression and an iterable separated by the ``in`` keyword. The code within the `for` directive is outputed for every iteration with the current value of the loop variable, which can be inserted using eval directives. If the iterable consists of iterables (e.g. tuples), usual indexing can be used to access their components, or a variable tuple to unpack them directly in the loop header:: - #:setvar kinds_names [ ('sp', 'real'), ('dp', 'dreal') ] - + #:setvar kinds ['sp', 'dp'] + #:setvar names ['real', 'dreal'] + #! create kinds_names as [('sp', 'real'), ('dp', 'dreal')] + #:setvar kinds_names list(zip(kinds, names)) + #! Acces by indexing interface sin2 #:for kind_name in kinds_names module procedure sin2_${kind_name[1]}$ #:endfor end interface sin2 - + #! Unpacking in the loop header #:for kind, name in kinds_names function sin2_${name}$(xx) result(res) real(${kind}$), intent(in) :: xx real(${kind}$) :: res - + res = sin(xx) * sin(xx) - + end function sin2_${name}$ #:endfor @@ -714,6 +721,8 @@ to code being hard to read, it should be usually avoided:: ! This form is more readable print *, ${choose_code('a(:)', 'size(a)')}$ +If the arguments need no further processing, the direct call directive can be +also used as an alternative to the line form (see next section). Direct call directive @@ -812,7 +821,7 @@ Comment directive Comment lines can be added by using the ``#!`` preprocessor directive. The comment line (including the newlines at their end) will be ignored by the -prepropessor and not appear in the ouput:: +prepropessor and will not appear in the ouput:: #! This will not show up in the output @@ -869,15 +878,14 @@ can be selected by the ``-f`` option with following possibilities: The ``-F`` option can be used to turn off line folding. -.. warning:: Fypp is not aware of the Fortran semantics represented by the lines - it folds. +.. warning:: Fypp is not aware of the Fortran semantics of the lines it folds. -Fypp applies the line folding rather mechanically (only considering the the -position of the whitespace characters). Lines containing eval directives and -lines within macro definitions should, therefore, not contain any Fortran style -comments (started by ``!``) *within* the line, as folding within the comment -would result in invalid Fortran code. For comments within such lines, Fypps -comment directive (``#!``) can be used instead:: +Fypp applies the line folding mechanically (only considering the position of the +whitespace characters). Lines containing eval directives and lines within macro +definitions should, therefore, not contain any Fortran style comments (started +by ``!``) *within* the line, as folding within the comment would result in +invalid Fortran code. For comments within such lines, Fypps comment directive +(``#!``) can be used instead:: #:def macro() print *, "DO NOT DO THIS!" ! Warning: Line may be folded within the comment @@ -901,10 +909,9 @@ documentation or OpenMP directives):: Escaping ======== -If you want to prevent Fypp to interprete something as control or eval -directive, put a backslash (``\``) between the first and second delimiter -character. In case of inline directives, do it for the opening and the -closing delimiter as well:: +If you want to prevent Fypp to interprete something as a directive, put a +backslash (``\``) between the first and second delimiter character. In case of +inline directives, do it for both, the opening and the closing delimiter:: $\: 1 + 2 #\{if 1 > 2}\# @@ -954,7 +961,7 @@ according error message. The line numbering directives can be fine tuned with the ``-N`` option, which accepts following mode arguments: -* ``full`` (default): Line numbering directives are emitted whenever lines are +* ``full``: Line numbering directives are emitted whenever lines are removed from the original source file or extra lines are added to it. * ``nocontlines``: Same as full, but line numbering directives are ommitted From 4da652cc70d402259fcc9796431e562143ef1ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Sun, 20 Mar 2016 21:00:51 +0100 Subject: [PATCH 09/10] Implement optional creation of parent folders for output file. --- bin/fypp | 302 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 168 insertions(+), 134 deletions(-) diff --git a/bin/fypp b/bin/fypp index 07c2d1c..88ed81b 100755 --- a/bin/fypp +++ b/bin/fypp @@ -74,6 +74,7 @@ else: import types import re import os +import errno import time from argparse import ArgumentParser @@ -211,8 +212,9 @@ class Parser: if fobj == STDIN: self._includefile(None, sys.stdin, STDIN, os.getcwd()) else: - with open(fobj, 'r') as inp: - self._includefile(None, inp, fobj, os.path.dirname(fobj)) + inpfp = _open_input_file(fobj) + self._includefile(None, inpfp, fobj, os.path.dirname(fobj)) + inpfp.close() else: self._includefile(None, fobj, FILEOBJ, os.getcwd()) @@ -642,8 +644,9 @@ class Parser: else: msg = "include file '{}' not found".format(fname) raise FyppError(msg, self._curfile, span) - with open(fpath, 'r') as fobj: - self._includefile(span, fobj, fpath, os.path.dirname(fpath)) + inpfp = _open_input_file(fpath) + self._includefile(span, inpfp, fpath, os.path.dirname(fpath)) + inpfp.close() def _process_mute(self, span): @@ -1773,135 +1776,6 @@ def linenumdir(linenr, fname): return '# {} "{}"\n'.format(linenr + 1, fname) -class FortranLineFolder: - - '''Implements line folding with Fortran continuation lines. - - Args: - maxlen (int, optional): Maximal line length (default: 132). - indent (int, optional): Indentation for continuation lines (default: 4). - method (str, optional): Folding method with following options: - - * ``brute``: folding with maximal length of continuation lines, - * ``simple``: indents with respect of indentation of first line, - * ``smart``: like ``simple``, but tries to fold at whitespaces. - - prefix (str, optional): String to use at the beginning of a continuation - line (default: '&'). - suffix (str, optional): String to use at the end of the line preceeding - a continuation line (default: '&') - ''' - - def __init__(self, maxlen=132, indent=4, method='smart', prefix='&', - suffix='&'): - # Line length should be long enough that contintuation lines can host at - # east one character apart of indentation and two continuation signs - minmaxlen = indent + len(prefix) + len(suffix) + 1 - if maxlen < minmaxlen: - msg = 'Maximal line length less than {} when using an indentation' \ - 'of {}'.format(minmaxlen, indent) - raise FyppError(msg) - self._maxlen = maxlen - self._indent = indent - self._prefix = ' ' * self._indent + prefix - self._suffix = suffix - if method not in ['brute', 'smart', 'simple']: - raise FyppError('invalid folding type') - if method == 'brute': - self._inherit_indent = False - self._fold_position_finder = self._get_maximal_fold_pos - elif method == 'simple': - self._inherit_indent = True - self._fold_position_finder = self._get_maximal_fold_pos - elif method == 'smart': - self._inherit_indent = True - self._fold_position_finder = self._get_smart_fold_pos - - - def __call__(self, line): - '''Folds a line. - - Can be directly called to return the list of folded lines:: - - linefolder = FortranLineFolder(maxlen=10) - linefolder(' print *, "some Fortran line"') - - Args: - line (str): Line to fold. - - Returns: - list of str: Components of folded line. They should be - assembled via ``\\n.join()`` to obtain the string - representation. - ''' - if self._maxlen < 0 or len(line) <= self._maxlen: - return [line] - if self._inherit_indent: - indent = len(line) - len(line.lstrip()) - prefix = ' ' * indent + self._prefix - else: - indent = 0 - prefix = self._prefix - suffix = self._suffix - return self._split_line(line, self._maxlen, prefix, suffix, - self._fold_position_finder) - - - @staticmethod - def _split_line(line, maxlen, prefix, suffix, fold_position_finder): - # length of continuation lines with 1 or two continuation chars. - maxlen1 = maxlen - len(prefix) - maxlen2 = maxlen1 - len(suffix) - start = 0 - end = fold_position_finder(line, start, maxlen - len(suffix)) - result = [line[start:end] + suffix] - while end < len(line) - maxlen1: - start = end - end = fold_position_finder(line, start, start + maxlen2) - result.append(prefix + line[start:end] + suffix) - result.append(prefix + line[end:]) - return result - - - @staticmethod - def _get_maximal_fold_pos(_, __, end): - return end - - - @staticmethod - def _get_smart_fold_pos(line, start, end): - linelen = end - start - ispace = line.rfind(' ', start, end) - # The space we waste for smart folding should be max. 1/3rd of the line - if ispace != -1 and ispace >= start + (2 * linelen) // 3: - return ispace - else: - return end - - -class DummyLineFolder: - - '''Implements a dummy line folder returning the line unaltered.''' - - def __call__(self, line): - '''Returns the entire line without any folding. - - Returns: - list of str: Components of folded line. They should be - assembled via ``\\n.join()`` to obtain the string - representation. - ''' - return [line] - - -def _shiftinds(inds, shift): - return [ind + shift for ind in inds] - - -################################################################################ -# Command line tool -################################################################################ - class Fypp: '''Fypp preprocessor. @@ -1982,6 +1856,7 @@ class Fypp: linefolder = DummyLineFolder() linenums = options.line_numbering contlinenums = (options.line_numbering_mode != 'nocontlines') + self._create_parent_folder = options.create_parent_folder renderer = Renderer( evaluator, linenums=linenums, contlinenums=contlinenums, linefolder=linefolder) @@ -2010,7 +1885,7 @@ class Fypp: if outfile == '-': outfile = sys.stdout else: - outfile = open(outfile, 'w') + outfile = _open_output_file(outfile, self._create_parent_folder) outfile.write(output) if outfile != sys.stdout: outfile.close() @@ -2099,6 +1974,8 @@ class FyppOptions: Default: [] fixed_format (bool): Whether input file is in fixed format. Default: False. + create_parent_folder (bool): Whether the parent folder for the output + file should be created if it does not exist. Default: False. ''' def __init__(self): @@ -2113,6 +1990,128 @@ class FyppOptions: self.modules = [] self.inifiles = [] self.fixed_format = False + self.create_parent_folder = False + + +class FortranLineFolder: + + '''Implements line folding with Fortran continuation lines. + + Args: + maxlen (int, optional): Maximal line length (default: 132). + indent (int, optional): Indentation for continuation lines (default: 4). + method (str, optional): Folding method with following options: + + * ``brute``: folding with maximal length of continuation lines, + * ``simple``: indents with respect of indentation of first line, + * ``smart``: like ``simple``, but tries to fold at whitespaces. + + prefix (str, optional): String to use at the beginning of a continuation + line (default: '&'). + suffix (str, optional): String to use at the end of the line preceeding + a continuation line (default: '&') + ''' + + def __init__(self, maxlen=132, indent=4, method='smart', prefix='&', + suffix='&'): + # Line length should be long enough that contintuation lines can host at + # east one character apart of indentation and two continuation signs + minmaxlen = indent + len(prefix) + len(suffix) + 1 + if maxlen < minmaxlen: + msg = 'Maximal line length less than {} when using an indentation' \ + 'of {}'.format(minmaxlen, indent) + raise FyppError(msg) + self._maxlen = maxlen + self._indent = indent + self._prefix = ' ' * self._indent + prefix + self._suffix = suffix + if method not in ['brute', 'smart', 'simple']: + raise FyppError('invalid folding type') + if method == 'brute': + self._inherit_indent = False + self._fold_position_finder = self._get_maximal_fold_pos + elif method == 'simple': + self._inherit_indent = True + self._fold_position_finder = self._get_maximal_fold_pos + elif method == 'smart': + self._inherit_indent = True + self._fold_position_finder = self._get_smart_fold_pos + + + def __call__(self, line): + '''Folds a line. + + Can be directly called to return the list of folded lines:: + + linefolder = FortranLineFolder(maxlen=10) + linefolder(' print *, "some Fortran line"') + + Args: + line (str): Line to fold. + + Returns: + list of str: Components of folded line. They should be + assembled via ``\\n.join()`` to obtain the string + representation. + ''' + if self._maxlen < 0 or len(line) <= self._maxlen: + return [line] + if self._inherit_indent: + indent = len(line) - len(line.lstrip()) + prefix = ' ' * indent + self._prefix + else: + indent = 0 + prefix = self._prefix + suffix = self._suffix + return self._split_line(line, self._maxlen, prefix, suffix, + self._fold_position_finder) + + + @staticmethod + def _split_line(line, maxlen, prefix, suffix, fold_position_finder): + # length of continuation lines with 1 or two continuation chars. + maxlen1 = maxlen - len(prefix) + maxlen2 = maxlen1 - len(suffix) + start = 0 + end = fold_position_finder(line, start, maxlen - len(suffix)) + result = [line[start:end] + suffix] + while end < len(line) - maxlen1: + start = end + end = fold_position_finder(line, start, start + maxlen2) + result.append(prefix + line[start:end] + suffix) + result.append(prefix + line[end:]) + return result + + + @staticmethod + def _get_maximal_fold_pos(_, __, end): + return end + + + @staticmethod + def _get_smart_fold_pos(line, start, end): + linelen = end - start + ispace = line.rfind(' ', start, end) + # The space we waste for smart folding should be max. 1/3rd of the line + if ispace != -1 and ispace >= start + (2 * linelen) // 3: + return ispace + else: + return end + + +class DummyLineFolder: + + '''Implements a dummy line folder returning the line unaltered.''' + + def __call__(self, line): + '''Returns the entire line without any folding. + + Returns: + list of str: Components of folded line. They should be + assembled via ``\\n.join()`` to obtain the string + representation. + ''' + return [line] def get_option_parser(): @@ -2169,6 +2168,9 @@ def get_option_parser(): '--line-length, --folding-method and --indentation are ignored)' parser.add_argument('--fixed-format', action='store_true', default=False, help=msg) + msg = 'create parent folders of the output file if they do not exist' + parser.add_argument('-p', '--create-parents', action='store_true', + default=False, dest='create_parent_folder', help=msg) versionstr = '%(prog)s ' + VERSION parser.add_argument('-v', '--version', action='version', version=versionstr) @@ -2198,5 +2200,37 @@ def run_fypp(): sys.exit(1) +def _shiftinds(inds, shift): + return [ind + shift for ind in inds] + + +def _open_input_file(inpfile): + try: + inpfp = open(inpfile, 'r') + except IOError as ex: + msg = "Failed to open file '{}' for read\n{}".format(inpfile, ex) + raise FyppError(msg) + return inpfp + + +def _open_output_file(outfile, create_parents=False): + if create_parents: + parentdir = os.path.abspath(os.path.dirname(outfile)) + if not os.path.exists(parentdir): + try: + os.makedirs(parentdir) + except OSError as ex: + if ex.errno != errno.EEXIST: + msg = "Folder '{}' can not be created\n{}"\ + .format(parentdir, ex) + raise FyppError(msg) + try: + outfp = open(outfile, 'w') + except IOError as ex: + msg = "Failed to open file '{}' for write\n{}".format(outfile, ex) + raise FyppError(msg) + return outfp + + if __name__ == '__main__': run_fypp() From bcbb699063f87e1aba70a7c85892fdbe0d8182ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Thu, 24 Mar 2016 19:19:55 +0100 Subject: [PATCH 10/10] Fix wrong line number offset in line eval directives. --- bin/fypp | 2 +- test/test_fypp.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/fypp b/bin/fypp index 88ed81b..df252bb 100755 --- a/bin/fypp +++ b/bin/fypp @@ -1156,7 +1156,7 @@ class Renderer: def _get_eval(self, span, expr): - self._update_linenr(span[1]) + self._update_linenr(span[0]) try: result = self._evaluator.evaluate(expr) except Exception as exc: diff --git a/test/test_fypp.py b/test/test_fypp.py index 299f8a8..1983075 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -377,6 +377,11 @@ def _linenumbering(nummode): fypp.STRING ), # + ('builtin_var_line_in_lineeval', [], + '$:_LINE_\n', + '1\n' + ), + # ('escaped_control_inline', [], r'A#\{if False}\#B#\{endif}\#', 'A#{if False}#B#{endif}#'