diff --git a/README.rst b/README.rst index 5f2a1f4..95adf32 100644 --- a/README.rst +++ b/README.rst @@ -2,12 +2,13 @@ Fypp — Python powered Fortran metaprogramming ============================================= -Fypp is a Python powered Fortran preprocessor. It extends Fortran with -condititional compiling and template metaprogramming capabilities. Instead of -introducing its own expression syntax, it uses Python expressions in its -preprocessor directives, offering the consistency and flexibility of Python when -formulating metaprogramming tasks. It puts strong emphasis on robustness and on -neat integration into Fortran developing toolchains. +Fypp is a Python powered preprocessor. It can be used for any programming +languages but its primary aim is to offer a Fortran preprocessor, which helps to +extend Fortran with condititional compiling and template metaprogramming +capabilities. Instead of introducing its own expression syntax, it uses Python +expressions in its preprocessor directives, offering the consistency and +versatility of Python when formulating metaprogramming tasks. It puts strong +emphasis on robustness and on neat integration into developing toolchains. The project is `hosted on bitbucket `_. @@ -33,10 +34,12 @@ Main features Fortran standard):: #:def assertTrue(cond) + #:if DEBUG > 0 if (.not. ${cond}$) then print *, "Assert failed in file ${_FILE_}$, line ${_LINE_}$" error stop end if + #:endif #:enddef ! Invoked via direct call (needs no quotation) @@ -57,7 +60,7 @@ Main features use serial #:endif -* Iterated output (e.g. for Fortran templates):: +* Iterated output (e.g. for generating Fortran templates):: interface myfunc #:for dtype in [ 'real', 'dreal', 'complex', 'dcomplex' ] @@ -109,33 +112,32 @@ Main features #:mute #:include "macrodefs.fypp" #:endmute - Installing ========== -Fypp needs a working Python interpreter, either version 2.7 or version 3.2 or -above. +Fypp needs a Python interpreter of version 2.7, 3.2 or above. Automatic install ----------------- -You can use Pythons installer `pip` to install the last stable release of Fypp -on your system:: +Use Pythons command line installer ``pip`` in order to download the stable +release from the `Fypp page on PyPI `_ and +install it on your system:: pip install fypp -This installs the command line tool ``fypp`` as well as the Python module -``fypp``. Latter you can import if you want to access the functionality of Fypp -directly from within your Python scripts. +This installs both, the command line tool ``fypp`` and the Python module +``fypp.py``. Latter you can import if you want to access the functionality of +Fypp directly from within your Python scripts. Manual install -------------- -Alternatively, you can download the source code from the `Fypp project website -`_ :: +For a manual install, you can download the source code from the `Fypp project +website `_ :: git clone https://aradi@bitbucket.org/aradi/fypp.git @@ -164,6 +166,7 @@ environment variable, by just issuing :: fypp +The python module ``fypp.py`` can be found in ``FYP_SOURCE_FOLDER/src``. Running @@ -172,9 +175,9 @@ Running The Fypp command line tool reads a file, preprocesses it and writes it to another file, so you would typically invoke it like:: - fypp source.fypp source.f90 + fypp source.F90 source.f90 -which would process `source.fypp` and write the result to `source.f90`. If +which would process `source.F90` and write the result to `source.f90`. If input and output files are not specified, information is read from stdin and written to stdout. diff --git a/bin/fypp b/bin/fypp index e48b89f..df252bb 100755 --- a/bin/fypp +++ b/bin/fypp @@ -43,19 +43,26 @@ using following classes: * `Renderer`_: Renders a tree built by the builder. * `Evaluator`_: Evaluates Python expressions in a designated environment. It is - used by the Renderer when rendering eval directives. + used by `Renderer`_ when rendering eval directives. -* `Processor`_: Connects `Parser`, `Builder`, `Renderer` and `Evaluator` with - each other and returns for a given input the preprocessed output. +* `Processor`_: Connects `Parser`_, `Builder`_, `Renderer`_ and `Evaluator`_ + with each other and returns for a given input the preprocessed output. -* `Fypp`_: Command line preprocessor. Drives `Processor` according to the - command line arguments. +* `Fypp`_: The actual Fypp preprocessor. It initializes and drives + `Processor`_. + +* `FyppOptions`_: Contains customizable settings controling the behaviour of + `Fypp`_. Alternatively, the function `get_option_parser()`_ can be used to + obtain an argument parser, which can create settings based on command line + arguments. If any expected error occurs during processing, `FyppError`_ is raised. Additional to the ones above, following class is used for fine tuning: -* `LineFolder`_: Folds overlong lines by using Fortran continuation lines. +* `FortranLineFolder`_: Folds overlong lines by using Fortran continuation + lines. + ''' from __future__ import print_function @@ -67,11 +74,12 @@ else: import types import re import os +import errno import time from argparse import ArgumentParser -VERSION = '0.12' +VERSION = '1.0' STDIN = '' @@ -139,7 +147,6 @@ class FyppError(Exception): span (tuple of int): Beginning and end line of the region where error occured or None if not available. - Attributes: msg (str): Error message. fname (str or None): File name or None if not available. @@ -177,15 +184,21 @@ class Parser: Args: includedirs (list): List of directories, in which include files should - be searched for, when they are not found in the default location. + be searched for, when they are not found at the default location. ''' def __init__(self, includedirs=None): + + # Directories to search for include files if includedirs is None: self._includedirs = [] else: self._includedirs = includedirs + + # Name of current file self._curfile = None + + # Directory of current file self._curdir = None @@ -195,26 +208,27 @@ class Parser: Args: fobj (str or file): Name of a file or a file like object. ''' - closefile = False if isinstance(fobj, str): if fobj == STDIN: - inpfile = sys.stdin - self._curfile = fobj - self._curdir = os.getcwd() + self._includefile(None, sys.stdin, STDIN, os.getcwd()) else: - inpfile = open(fobj, 'r') - closefile = True - self._curfile = fobj - self._curdir = os.path.dirname(fobj) + inpfp = _open_input_file(fobj) + self._includefile(None, inpfp, fobj, os.path.dirname(fobj)) + inpfp.close() else: - inpfile = fobj - self._curfile = FILEOBJ - self._curdir = os.getcwd() - self.handle_open_file((0, 0), self._curfile) - self._parse(inpfile.read()) - self.handle_close_file(self._curfile) - if closefile: - inpfile.close() + self._includefile(None, fobj, FILEOBJ, os.getcwd()) + + + def _includefile(self, span, fobj, fname, curdir): + oldfile = self._curfile + olddir = self._curdir + self._curfile = fname + self._curdir = curdir + self.handle_include(span, fname) + self._parse(fobj.read()) + self.handle_endinclude(span, fname) + self._curfile = oldfile + self._curdir = olddir def parse(self, txt): @@ -225,32 +239,35 @@ class Parser: ''' self._curfile = STRING self._curdir = os.getcwd() - self.handle_open_file((0, 0), self._curfile) + self.handle_include(None, self._curfile) self._parse(txt) - self.handle_close_file(self._curfile) + self.handle_endinclude(None, self._curfile) - def handle_open_file(self, span, fname): - '''Called when parser starts to parse a new file. + def handle_include(self, span, fname): + '''Called when parser starts to process a new file. It is a dummy methond and should be overriden for actual use. Args: - span (tuple of int): Start and end line of the directive. + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' - self._log_event('open file', span, filename=fname) + self._log_event('include', span, filename=fname) - def handle_close_file(self, fname): + def handle_endinclude(self, span, fname): '''Called when parser finished processing a file. It is a dummy method and should be overriden for actual use. Args: + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' - self._log_event('close file', filename=fname) + self._log_event('endinclude', span, filename=fname) def handle_setvar(self, span, name, expr): @@ -627,10 +644,9 @@ class Parser: else: msg = "include file '{}' not found".format(fname) raise FyppError(msg, self._curfile, span) - oldfile = self._curfile - self.parsefile(fpath) - self._curfile = oldfile - self.handle_open_file(span, self._curfile) + inpfp = _open_input_file(fpath) + self._includefile(span, inpfp, fpath, os.path.dirname(fpath)) + inpfp.close() def _process_mute(self, span): @@ -671,57 +687,83 @@ class Builder: ''' def __init__(self): + # The tree, which should be built. self._tree = [] - self._path = [] - self._open_files = [] + + # List of all open constructs self._open_blocks = [] - # Nr. of open blocks before current file was opened + + # Nodes to which the open blocks have to be appended when closed + self._path = [] + + # Nr. of open blocks when file was opened. Used for checking whether all + # blocks have been closed, when file processing finishes. self._nr_prev_blocks = [] + + # Current node, to which content should be added self._curnode = self._tree + + # Current file self._curfile = None def reset(self): '''Resets the builder so that it starts to build a new tree.''' self._tree = [] - self._path = [] - self._open_files = [] self._open_blocks = [] + self._path = [] self._nr_prev_blocks = [] self._curnode = self._tree self._curfile = None - def handle_open_file(self, span, fname): + def handle_include(self, span, fname): '''Should be called to signalize change to new file. Args: - span (tuple of int): Start and end line of the directive. + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' self._curfile = fname - self._curnode.append(('include', span, fname)) - self._open_files.append(fname) + self._path.append(self._curnode) + self._curnode = [] + self._open_blocks.append(('include', [span], fname, None)) self._nr_prev_blocks.append(len(self._open_blocks)) - def handle_close_file(self, fname): + def handle_endinclude(self, span, fname): '''Should be called when processing of a file finished. Args: + span (tuple of int): Start and end line of the include directive + or None if called the first time for the main input. fname (str): Name of the file. ''' - oldfname = self._open_files.pop(-1) - if fname != oldfname: - msg = 'internal error: mismatching file name in close_file event'\ - " (expected: '{}', got '{}')".format(oldfname, fname) - raise FyppError(msg, fname) - if len(self._open_blocks) > self._nr_prev_blocks[-1]: + nprev_blocks = self._nr_prev_blocks.pop(-1) + if len(self._open_blocks) > nprev_blocks: directive, spans = self._open_blocks[-1][0:2] msg = '{} directive in line {} still unclosed when reaching end '\ 'of file'.format(directive, spans[0][0] + 1) raise FyppError(msg, self._curfile) - del self._nr_prev_blocks[-1] + block = self._open_blocks.pop(-1) + directive, spans = block[0:2] + if directive != 'include': + msg = 'internal error: last open block is not \'include\' when '\ + 'closing file \'{}\''.format(fname) + raise FyppError(msg) + if span != spans[0]: + msg = 'internal error: span for include and endinclude differ ('\ + '{} vs {}'.format(span, spans[0]) + raise FyppError(msg) + oldfname, _ = block[2:4] + if fname != oldfname: + msg = 'internal error: mismatching file name in close_file event'\ + " (expected: '{}', got '{}')".format(oldfname, fname) + raise FyppError(msg, fname) + block = directive, spans, fname, self._curnode + self._curnode = self._path.pop(-1) + self._curnode.append(block) def handle_if(self, span, cond): @@ -923,7 +965,7 @@ class Builder: '''Should be called to signalize a comment directive. The content of the comment is not needed by the builder, but it needs - the span of the comment to generate proper synclines if needed. + the span of the comment to generate proper line numbers if needed. Args: span (tuple of int): Start and end line of the directive. @@ -1007,18 +1049,33 @@ class Renderer: Args: evaluator (Evaluator, optional): Evaluator to use when rendering eval directives. If None (default), Evaluator() is used. - synclines (bool, optional): Whether synclines should be generated, + linenums (bool, optional): Whether linenums should be generated, defaults to False. + contlinenums (bool, optional): Whether linenums for continuation + should be generated, defaults to False. linefolder (callable): Callable to use when folding a line. ''' - def __init__(self, evaluator=None, synclines=False, linefolder=None): + def __init__(self, evaluator=None, linenums=False, contlinenums=False, + linefolder=None): + # Evaluator to use for Python expressions self._evaluator = Evaluator() if evaluator is None else evaluator self._evaluator.updateenv(_DATE_=time.strftime('%Y-%m-%d'), _TIME_=time.strftime('%H:%M:%S')) + + # Name of current file being processed self._curfile = None + + # Number of diversions, when > 0 we are within a macro call self._diversions = 0 - self._synclines = synclines + + # Whether line numbering directives should be emitted + self._linenums = linenums + + # Whether line numbering directives in continuation lines are needed. + self._contlinenums = contlinenums + + # Callable to be used for folding lines if linefolder is None: self._linefolder = lambda line: [line] else: @@ -1082,8 +1139,10 @@ class Renderer: eval_pos += peval output += out elif cmd == 'include': - result = self._register_file(*node[1:3]) - output.append(result) + out, ieval, peval = self._get_included_content(*node[1:4]) + eval_inds += _shiftinds(ieval, len(output)) + eval_pos += peval + output += out elif cmd == 'comment': output.append(self._get_comment(*node[1:2])) elif cmd == 'mute': @@ -1097,7 +1156,7 @@ class Renderer: def _get_eval(self, span, expr): - self._update_linenr(span[1]) + self._update_linenr(span[0]) try: result = self._evaluator.evaluate(expr) except Exception as exc: @@ -1130,15 +1189,15 @@ class Renderer: condition, exc) raise FyppError(msg, self._curfile, span) if cond: - if self._synclines and not self._diversions and multiline: - out.append(syncline(span[1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(span[1], self._curfile)) outcont, ievalcont, pevalcont = self._render(content) ieval += _shiftinds(ievalcont, len(out)) peval += pevalcont out += outcont break - if self._synclines and not self._diversions and multiline: - out.append(syncline(spans[-1][1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(spans[-1][1], self._curfile)) return out, ieval, peval @@ -1160,14 +1219,14 @@ class Renderer: else: loopscope = {varname: value for varname, value in zip(loopvars, var)} - if self._synclines and not self._diversions and multiline: - out.append(syncline(spans[0][1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(spans[0][1], self._curfile)) outcont, ievalcont, pevalcont = self._render(content, loopscope) ieval += _shiftinds(ievalcont, len(out)) peval += pevalcont out += outcont - if self._synclines and not self._diversions and multiline: - out.append(syncline(spans[1][1], self._curfile)) + if self._linenums and not self._diversions and multiline: + out.append(linenumdir(spans[1][1], self._curfile)) return out, ieval, peval @@ -1187,6 +1246,23 @@ class Renderer: return out, ieval, peval + def _get_included_content(self, spans, fname, content): + out = [] + oldfile = self._curfile + self._curfile = fname + self._evaluator.updateenv(_FILE_=self._curfile) + if self._linenums and not self._diversions: + out += linenumdir(0, self._curfile) + outcont, ieval, peval = self._render(content) + ieval = _shiftinds(ieval, len(out)) + out += outcont + self._curfile = oldfile + self._evaluator.updateenv(_FILE_=self._curfile) + if self._linenums and not self._diversions and spans[0] is not None: + out += linenumdir(spans[0][1], self._curfile) + return out, ieval, peval + + def _define_macro(self, spans, name, args, content): result = '' try: @@ -1197,8 +1273,8 @@ class Renderer: msg = "exception occured when defining macro '{}'\n{}"\ .format(name, exc) raise FyppError(msg, self._curfile, spans[0]) - if self._synclines and not self._diversions: - result = syncline(spans[1][1], self._curfile) + if self._linenums and not self._diversions: + result = linenumdir(spans[1][1], self._curfile) return result @@ -1212,22 +1288,22 @@ class Renderer: .format(name, valstr, exc) raise FyppError(msg, self._curfile, span) multiline = (span[0] != span[1]) - if self._synclines and not self._diversions and multiline: - result = syncline(span[1], self._curfile) + if self._linenums and not self._diversions and multiline: + result = linenumdir(span[1], self._curfile) return result def _get_comment(self, span): - if self._synclines and not self._diversions: - return syncline(span[1], self._curfile) + if self._linenums and not self._diversions: + return linenumdir(span[1], self._curfile) else: return '' def _get_muted_content(self, spans, content): self._render(content) - if self._synclines and not self._diversions: - return syncline(spans[-1][1], self._curfile) + if self._linenums and not self._diversions: + return linenumdir(spans[-1][1], self._curfile) else: return '' @@ -1243,100 +1319,115 @@ class Renderer: raise FyppError(msg, self._curfile) - def _register_file(self, span, fname): - result = '' - self._curfile = fname - self._evaluator.updateenv(_FILE_=fname) - if self._synclines and not self._diversions: - result = syncline(span[1], self._curfile) - return result - - def _update_linenr(self, linenr): if not self._diversions: self._evaluator.updateenv(_LINE_=linenr + 1) def _postprocess_eval_lines(self, output, eval_inds, eval_pos): - lastproc = -1 + ilastproc = -1 for ieval, ind in enumerate(eval_inds): span, fname = eval_pos[ieval] - if ind <= lastproc: + if ind <= ilastproc: continue - - # find last eol before expr. evaluation - iprev = ind - 1 - while iprev >= 0: - eolprev = output[iprev].rfind('\n') - if eolprev != -1: - break - iprev -= 1 - else: - iprev = 0 - eolprev = -1 - - # find first eol after expr. evaluation - inext = ind + 1 - while inext < len(output): - eolnext = output[inext].find('\n') - if eolnext != -1: - break - inext += 1 - else: - inext = len(output) - 1 - eolnext = len(output[-1]) - 1 - - # Create full line containing the evaluated expression - curline_parts = [] - if iprev != ind: - curline_parts = [output[iprev][eolprev + 1:]] - output[iprev] = output[iprev][:eolprev + 1] - curline_parts.extend(output[iprev + 1:ind]) - curline_parts.extend(output[ind]) - curline_parts.extend(output[ind + 1:inext]) - if inext != ind: - curline_parts.append(output[inext][:eolnext + 1]) - output[inext] = output[inext][eolnext + 1:] - curline = ''.join(curline_parts) + iprev, eolprev = self._find_last_eol(output, ind) + inext, eolnext = self._find_next_eol(output, ind) + curline = self._glue_line(output, ind, iprev, eolprev, inext, + eolnext) output[iprev + 1:inext] = [''] * (inext - iprev - 1) + output[ind] = self._postprocess_eval_line(curline, fname, span) + ilastproc = inext - # Split lines and add synclines if needed - lines = self._splitlines(curline) - # if curline ended with '\n', last element of lines is '' - nrlines = len(lines) if lines[-1] else len(lines) - 1 - if self._synclines and (nrlines > 1 or span[0] != span[1]): - oldlines = lines - syncl = syncline(span[0], fname) - lines = [oldlines[0] + '\n'] - for line in oldlines[1:-1]: - lines.append(syncl) - lines.append(line + '\n') - if oldlines[-1]: - # Line did not end on '\n' -> it was last line of input - # -> no syncline is needed. - lines.append(oldlines[-1]) - elif span[1] > span[0] + 1: - # Syncline needed, if expression evaluation goes - # over multiple lines. Otherwise the syncline inserted - # before the last line of the evaluation (pointing to - # the line of the directive) ensures correct line numbering - # for the following content. - lines.append(syncline(span[1], fname)) - output[ind] = ''.join(lines) - else: - output[ind] = '\n'.join(lines) - lastproc = inext + @staticmethod + def _find_last_eol(output, ind): + 'Find last newline before current position.' + iprev = ind - 1 + while iprev >= 0: + eolprev = output[iprev].rfind('\n') + if eolprev != -1: + break + iprev -= 1 + else: + iprev = 0 + eolprev = -1 + return iprev, eolprev - def _splitlines(self, txt): - lines = txt.split('\n') - result = [] - for line in lines: - if _COMMENTLINE_REGEXP.match(line) is None: - result += self._linefolder(line) - else: - result.append(line) - return result + + @staticmethod + def _find_next_eol(output, ind): + 'Find last newline before current position.' + # find first eol after expr. evaluation + inext = ind + 1 + while inext < len(output): + eolnext = output[inext].find('\n') + if eolnext != -1: + break + inext += 1 + else: + inext = len(output) - 1 + eolnext = len(output[-1]) - 1 + return inext, eolnext + + + @staticmethod + def _glue_line(output, ind, iprev, eolprev, inext, eolnext): + 'Create line from parts between specified boundaries.' + curline_parts = [] + if iprev != ind: + curline_parts = [output[iprev][eolprev + 1:]] + output[iprev] = output[iprev][:eolprev + 1] + curline_parts.extend(output[iprev + 1:ind]) + curline_parts.extend(output[ind]) + curline_parts.extend(output[ind + 1:inext]) + if inext != ind: + curline_parts.append(output[inext][:eolnext + 1]) + output[inext] = output[inext][eolnext + 1:] + return ''.join(curline_parts) + + + def _postprocess_eval_line(self, evalline, fname, span): + lines = evalline.split('\n') + # If line ended on '\n', last element is ''. We remove it and + # add the trailing newline later manually. + trailing_newline = (lines[-1] == '') + if trailing_newline: + del lines[-1] + lnum = linenumdir(span[0], fname) if self._linenums else '' + clnum = lnum if self._contlinenums else '' + linenumsep = '\n' + lnum + clinenumsep = '\n' + clnum + foldedlines = [self._foldline(line) for line in lines] + outlines = [clinenumsep.join(lines) for lines in foldedlines] + result = linenumsep.join(outlines) + # Add missing trailing newline + if trailing_newline: + trailing = '\n' + if self._linenums: + # Last line was folded, but no linenums were generated for + # the continuation lines -> current line position is not + # in sync with the one calculated from the last line number + unsync = ( + len(foldedlines) and len(foldedlines[-1]) > 1 + and not self._contlinenums) + # Eval directive in source consists of more than one line + multiline = span[1] - span[0] > 1 + if unsync or multiline: + # For inline eval directives span[0] == span[1] + # -> next line is span[0] + 1 and not span[1] as for + # line eval directives + nextline = max(span[1], span[0] + 1) + trailing += linenumdir(nextline, fname) + else: + trailing = '' + return result + trailing + + + def _foldline(self, line): + if _COMMENTLINE_REGEXP.match(line) is None: + return self._linefolder(line) + else: + return [line] class Evaluator: @@ -1426,8 +1517,12 @@ class Evaluator: } def __init__(self, env=None, restricted=True): + # Definitions (environment) to use when evaluating expressions self._env = env.copy() if env is not None else {} + + # Stack for environments to implement nested scopes self._envstack = [] + if restricted: builtindict = {} builtindict.update(self.RESTRICTED_BUILTINS) @@ -1437,6 +1532,8 @@ class Evaluator: builtindict['defined'] = self._func_defined builtindict['setvar'] = self._func_setvar builtindict['getvar'] = self._func_getvar + + # Permitted builtins when evaluating expressions self._builtins = {'__builtins__': builtindict} @@ -1613,8 +1710,8 @@ class Processor: else: self._renderer = renderer - self._parser.handle_open_file = self._builder.handle_open_file - self._parser.handle_close_file = self._builder.handle_close_file + self._parser.handle_include = self._builder.handle_include + self._parser.handle_endinclude = self._builder.handle_endinclude self._parser.handle_if = self._builder.handle_if self._parser.handle_else = self._builder.handle_else self._parser.handle_elif = self._builder.handle_elif @@ -1669,8 +1766,8 @@ class Processor: return ''.join(output) -def syncline(linenr, fname): - '''Returns a syncline directive. +def linenumdir(linenr, fname): + '''Returns a line numbering directive. Args: linenr (int): Line nr (starting with 0). @@ -1679,8 +1776,224 @@ def syncline(linenr, fname): return '# {} "{}"\n'.format(linenr + 1, fname) +class Fypp: + + '''Fypp preprocessor. + + You can invoke it like :: + + tool = Fypp() + tool.process_file('file.in', 'file.out') + + to initialize Fypp with default options, process `file.in` and write the + result to `file.out`. If the input should be read from a string, the + ``process_text()`` method can be used:: + + tool = Fypp() + output = tool.process_text('#:if DEBUG > 0\\nprint *, "DEBUG"\\n#:endif\\n') + + If you want to fine tune Fypps behaviour, pass a customized `FyppOptions`_ + instance at initialization:: + + options = FyppOptions() + options.fixed_format = True + tool = Fypp(options) + + Alternatively, you can use the command line parser + ``argparse.ArgumentParser`` to set options for Fypp. The function + ``get_option_parser()`` returns you a default argument parser. You can then + use its ``parse_args()`` method to obtain settings by reading the command + line arguments:: + + options = FyppOptions() + argparser = get_option_parser() + options = argparser.parse_args(namespace=options) + tool = fypp.Fypp(options) + + The command line arguments can also be passed directly as a list when + calling ``parse_args()``:: + + options = FyppOptions() + args = ['-DDEBUG=0', 'input.F90', 'output.F90'] + argparser = get_option_parser() + options = argparser.parse_args(args=args, namespace=options) + tool = fypp.Fypp(options) + + + Args: + options (object): Object containing the settings for Fypp. You typically + would pass a customized `FyppOptions`_ instance or a ``Namespace`` + object as returned by an argument parser. If not present, the + default settings in `FyppOptions`_ are used. + + ''' + + def __init__(self, options=None): + if options is None: + options = FyppOptions() + inieval = Evaluator(restricted=False) + if options.modules: + self._import_modules(options.modules, inieval) + if options.inifiles: + self._exec_inifiles(options.inifiles, inieval) + evaluator = Evaluator(env=inieval.env, restricted=True) + if options.defines: + self._apply_definitions(options.defines, evaluator) + parser = Parser(options.includes) + builder = Builder() + + fixed_format = options.fixed_format + linefolding = not options.no_folding + if linefolding: + folding = 'brute' if fixed_format else options.folding_mode + linelength = 72 if fixed_format else options.line_length + indentation = 5 if fixed_format else options.indentation + prefix = '&' + suffix = '' if fixed_format else '&' + linefolder = FortranLineFolder(linelength, indentation, folding, + prefix, suffix) + else: + linefolder = DummyLineFolder() + linenums = options.line_numbering + contlinenums = (options.line_numbering_mode != 'nocontlines') + self._create_parent_folder = options.create_parent_folder + renderer = Renderer( + evaluator, linenums=linenums, contlinenums=contlinenums, + linefolder=linefolder) + self._preprocessor = Processor(parser, builder, renderer) + -class LineFolder: + def process_file(self, infile, outfile=None, env=None): + '''Processes input file and writes result to output file. + + Args: + infile (str): Name of the file to read and process. If its value is + '-', input is read from stdin. + outfile (str, optional): Name of the file to write the result to. + If its value is '-', result is written to stdout. If not + present, result will be returned as string. + env (dict, optional): Additional definitions for the evaluator. + + Returns: + str: Result of processed input, if no outfile was specified. + ''' + infile = STDIN if infile == '-' else infile + output = self._preprocessor.process_file(infile, env) + if outfile is None: + return output + else: + if outfile == '-': + outfile = sys.stdout + else: + outfile = _open_output_file(outfile, self._create_parent_folder) + outfile.write(output) + if outfile != sys.stdout: + outfile.close() + + + def process_text(self, txt, env=None): + '''Processes a string. + + Args: + txt (str): String to process. + env (dict, optional): Additional definitions for the evaluator. + + Returns: + str: Processed content. + ''' + return self._preprocessor.process_text(txt, env) + + + @staticmethod + def _apply_definitions(defines, evaluator): + for define in defines: + words = define.split('=', 2) + name = words[0] + value = None + if len(words) > 1: + try: + value = evaluator.evaluate(words[1]) + except Exception as exc: + msg = "exception at evaluating '{}' in definition for " \ + "'{}'\n{}".format(words[1], name, exc) + raise FyppError(msg) + evaluator.define(name, value) + + + @staticmethod + def _import_modules(modules, evaluator): + for module in modules: + try: + evaluator.execute('import ' + module) + except Exception as ex: + msg = "exception occured during import of module '{}'\n{}"\ + .format(module, ex) + raise FyppError(msg) + + + @staticmethod + def _exec_inifiles(inifiles, evaluator): + for inifile in inifiles: + try: + inifp = open(inifile, 'r') + source = inifp.read() + inifp.close() + except IOError as ex: + msg = "IO error occured at reading file '{}'\n{}"\ + .format(inifile, ex) + raise FyppError(msg) + try: + code = compile(source, inifile, 'exec', dont_inherit=-1) + evaluator.execute(code) + except Exception as ex: + msg = "exception occured when executing ini-file '{}'\n{}"\ + .format(inifile, ex) + raise FyppError(msg) + + +class FyppOptions: + + '''Container for Fypp options with default values. + + Attributes: + defines (list of str): List of variable definitions in the form of + 'VARNAME=VALUE'. Default: [] + includes (list of str): List of paths to search when looking for include + files. Default: [] + line_numbering (bool): Whether line numbering directives should appear + in the output. Default: False + line_numbering_mode (str): Line numbering mode 'full' or 'nocontlines'. + Default: 'full'. + line_length (int): Length of output lines. Default: 132. + folding_mode (str): Folding mode 'smart', 'simple' or 'brute'. Default: + 'smart'. + no_folding (bool): Whether folding should be suppresed. Default: False. + indentation (int): Indentation in continuation lines. Default: 4. + modules (list of str): Modules to import at initialization. Default: []. + inifiles (list of str): Python files to execute at initialization. + Default: [] + fixed_format (bool): Whether input file is in fixed format. + Default: False. + create_parent_folder (bool): Whether the parent folder for the output + file should be created if it does not exist. Default: False. + ''' + + def __init__(self): + self.defines = [] + self.includes = [] + self.line_numbering = False + self.line_numbering_mode = 'full' + self.line_length = 132 + self.folding_mode = 'smart' + self.no_folding = False + self.indentation = 4 + self.modules = [] + self.inifiles = [] + self.fixed_format = False + self.create_parent_folder = False + + +class FortranLineFolder: '''Implements line folding with Fortran continuation lines. @@ -1689,7 +2002,6 @@ class LineFolder: indent (int, optional): Indentation for continuation lines (default: 4). method (str, optional): Folding method with following options: - * ``none``: no folding, * ``brute``: folding with maximal length of continuation lines, * ``simple``: indents with respect of indentation of first line, * ``smart``: like ``simple``, but tries to fold at whitespaces. @@ -1713,7 +2025,7 @@ class LineFolder: self._indent = indent self._prefix = ' ' * self._indent + prefix self._suffix = suffix - if method not in ['brute', 'smart', 'simple', 'none']: + if method not in ['brute', 'smart', 'simple']: raise FyppError('invalid folding type') if method == 'brute': self._inherit_indent = False @@ -1724,9 +2036,6 @@ class LineFolder: elif method == 'smart': self._inherit_indent = True self._fold_position_finder = self._get_smart_fold_pos - else: - # No folding occurs, line is returned unaltered - self._maxlen = -1 def __call__(self, line): @@ -1734,7 +2043,7 @@ class LineFolder: Can be directly called to return the list of folded lines:: - linefolder = LineFolder(maxlen=10) + linefolder = FortranLineFolder(maxlen=10) linefolder(' print *, "some Fortran line"') Args: @@ -1790,211 +2099,138 @@ class LineFolder: return end -def _shiftinds(inds, shift): - return [ind + shift for ind in inds] - - -################################################################################ -# Command line tool -################################################################################ - -_FYPP_DESC = '''Preprocess Fortran source files with Fypp directives.''' - - -class Fypp: - - '''Represents the Fypp command line tool. - - You can invoke it like:: - - tool = Fypp() - tool.process_cmdline_files() - - to parse the command line arguments (in ``sys.argv``) and run Fypp using - the input/output files and options as specified there. - - The command line arguments can also be passed directly as a list:: - - tool = Fypp(['-DDEBUG=0', 'input.F90', 'output.F90']) - tool.process_cmdline_files() +class DummyLineFolder: - If the input should be read from a string and the result of the - preprocessing should be also returned as a string (instead of - using input/output files), the ``process_text()`` method can be - used:: + '''Implements a dummy line folder returning the line unaltered.''' - tool = Fypp(['-DDEBUG=0']) - result = tool.process_text('#:if DEBUG > 0\\nprint *, "DEBUG"\\n#:endif\\n') - - - Args: - cmdline_args (list of str, optional): Command line arguments. If None - (default), the arguments are read from sys.argv using Argparser. - - ''' - - def __init__(self, cmdline_args=None): - self._argparser = self._get_cmdline_parser() - self._args = self._argparser.parse_args(cmdline_args) - inieval = Evaluator(restricted=False) - if self._args.modules: - self._import_modules(self._args.modules, inieval) - if self._args.inifiles: - self._exec_inifiles(self._args.inifiles, inieval) - evaluator = Evaluator(env=inieval.env, restricted=True) - if self._args.defines: - self._apply_definitions(self._args.defines, evaluator) - parser = Parser(self._args.includes) - builder = Builder() - - fixed_format = self._args.fixed_format - folding = 'brute' if fixed_format else self._args.folding_method - linelength = 72 if fixed_format else self._args.line_length - indentation = 5 if fixed_format else self._args.indentation - prefix = '&' - suffix = '' if fixed_format else '&' - linefolder = LineFolder(linelength, indentation, folding, prefix, - suffix) - renderer = Renderer(evaluator, synclines=self._args.synclines, - linefolder=linefolder) - self._preprocessor = Processor(parser, builder, renderer) - - - def process_cmdline_files(self, env=None): - '''Processes the input file and write result to the output file which - were specified as the command line arguments. - - Args: - env (dict): Additional definitions for the evaluator. - ''' - infile = STDIN if self._args.infile == '-' else self._args.infile - output = self._preprocessor.process_file(infile, env) - if self._args.outfile == '-': - outfile = sys.stdout - else: - outfile = open(self._args.outfile, 'w') - outfile.write(output) - if outfile != sys.stdout: - outfile.close() - - - def process_text(self, txt, env=None): - '''Processes a string. - - Args: - txt (str): String to process. + def __call__(self, line): + '''Returns the entire line without any folding. Returns: - str: Processed content. + list of str: Components of folded line. They should be + assembled via ``\\n.join()`` to obtain the string + representation. ''' - return self._preprocessor.process_text(txt, env) + return [line] - @staticmethod - def _get_cmdline_parser(): - parser = ArgumentParser(description=_FYPP_DESC) - msg = 'define variable, value is interpreted as ' \ - 'Python expression (e.g \'-DDEBUG=1\' sets DEBUG to the ' \ - 'integer 1) or set to None if ommitted' - parser.add_argument('-D', '--define', action='append', dest='defines', - metavar='VAR[=VALUE]', help=msg) - msg = 'add directory to the search paths for include files' - parser.add_argument('-I', '--include', action='append', dest='includes', - metavar='INCDIR', help=msg) - msg = 'include CPP style sync-lines in the output' - parser.add_argument('-s', '--synclines', action='store_true', - default=False, help=msg) - msg = 'maximal line length (default: 132), lines modified by the '\ - 'preprocessor are folded if becoming longer' - parser.add_argument('-l', '--line-length', type=int, default=132, - metavar='LEN', help=msg) - msg = 'folding method, \'smart\' (default): indentation context and '\ - 'whitespace aware, \'simple\': indentation context aware, '\ - '\'brute\' : mechnical folding, none\': no folding' - parser.add_argument('-f', '--folding-method', metavar='METHOD', - choices=['smart', 'simple', 'brute', 'none'], - default='smart', help=msg) - msg = 'indentation to use for continuation lines (default 4)' - parser.add_argument('--indentation', type=int, metavar='IND', - default=4, help=msg) - msg = 'import python module before starting the processing' - parser.add_argument('-m', '--module', action='append', dest='modules', - metavar='MOD', help=msg) - msg = 'execute python initialization script before starting processing' - parser.add_argument('-i', '--ini-file', action='append', - dest='inifiles', metavar='INI', help=msg) - msg = 'produce fixed format output (any settings for options '\ - '--line-length, --folding-method and --indentation are ignored)' - parser.add_argument('--fixed-format', action='store_true', - default=False, help=msg) - versionstr = '%(prog)s ' + VERSION - parser.add_argument('-v', '--version', action='version', - version=versionstr) - msg = "input file to be processed (default: '-', stdin)" - parser.add_argument('infile', nargs='?', default='-', help=msg) - msg = "output file where processed content will be written (default: " \ - "'-', stdout)" - parser.add_argument('outfile', nargs='?', default='-', help=msg) - return parser - - - @staticmethod - def _apply_definitions(defines, evaluator): - for define in defines: - words = define.split('=', 2) - name = words[0] - value = None - if len(words) > 1: - try: - value = evaluator.evaluate(words[1]) - except Exception as exc: - msg = "exception at evaluating '{}' in definition for " \ - "'{}'\n{}".format(words[1], name, exc) - raise FyppError(msg) - evaluator.define(name, value) - - - @staticmethod - def _import_modules(modules, evaluator): - for module in modules: - try: - evaluator.execute('import ' + module) - except Exception as ex: - msg = "exception occured during import of module '{}'\n{}"\ - .format(module, ex) - raise FyppError(msg) - +def get_option_parser(): + '''Returns an option parser for the Fypp command line tool. - @staticmethod - def _exec_inifiles(inifiles, evaluator): - for inifile in inifiles: - try: - inifp = open(inifile, 'r') - source = inifp.read() - inifp.close() - except IOError as ex: - msg = "IO error occured at reading file '{}'\n{}"\ - .format(inifile, ex) - raise FyppError(msg) - try: - code = compile(source, inifile, 'exec', dont_inherit=-1) - evaluator.execute(code) - except Exception as ex: - msg = "exception occured when executing ini-file '{}'\n{}"\ - .format(inifile, ex) - raise FyppError(msg) + Returns: + ArgumentParser: Parser which can create a namespace object with + Fypp settings based on command line arguments. + ''' + fypp_name = 'fypp' + fypp_desc = 'Preprocess source files with Fypp directives.' + parser = ArgumentParser(prog=fypp_name, description=fypp_desc) + msg = 'define variable, value is interpreted as ' \ + 'Python expression (e.g \'-DDEBUG=1\' sets DEBUG to the ' \ + 'integer 1) or set to None if ommitted' + parser.add_argument('-D', '--define', action='append', dest='defines', + metavar='VAR[=VALUE]', help=msg) + msg = 'add directory to the search paths for include files' + parser.add_argument('-I', '--include', action='append', dest='includes', + metavar='INCDIR', help=msg) + msg = 'put line numbering directives to the output' + parser.add_argument('-n', '--line-numbering', action='store_true', + default=False, help=msg) + msg = 'line numbering mode, \'full\' (default): line numbering '\ + 'directives generated whenever source and output lines are out '\ + 'of sync, \'nocontlines\': line numbering directives omitted '\ + 'for continuation lines' + parser.add_argument('-N', '--line-numbering-mode', metavar='MODE', + choices=['full', 'nocontlines'], default='full', + help=msg) + msg = 'maximal line length (default: 132), lines modified by the '\ + 'preprocessor are folded if becoming longer' + parser.add_argument('-l', '--line-length', type=int, default=132, + metavar='LEN', help=msg) + msg = 'line folding mode, \'smart\' (default): indentation context '\ + 'and whitespace aware, \'simple\': indentation context aware, '\ + '\'brute\': mechnical folding' + parser.add_argument('-f', '--folding-mode', metavar='MODE', + choices=['smart', 'simple', 'brute'], + default='smart', help=msg) + msg = 'suppress line folding' + parser.add_argument('-F', '--no-folding', action='store_true', + dest='no_folding', default=False, help=msg) + msg = 'indentation to use for continuation lines (default 4)' + parser.add_argument('--indentation', type=int, metavar='IND', + default=4, help=msg) + msg = 'import python module before starting the processing' + parser.add_argument('-m', '--module', action='append', dest='modules', + metavar='MOD', help=msg) + msg = 'execute python initialization script before starting processing' + parser.add_argument('-i', '--ini-file', action='append', + dest='inifiles', metavar='INI', help=msg) + msg = 'produce fixed format output (any settings for options '\ + '--line-length, --folding-method and --indentation are ignored)' + parser.add_argument('--fixed-format', action='store_true', + default=False, help=msg) + msg = 'create parent folders of the output file if they do not exist' + parser.add_argument('-p', '--create-parents', action='store_true', + default=False, dest='create_parent_folder', help=msg) + versionstr = '%(prog)s ' + VERSION + parser.add_argument('-v', '--version', action='version', + version=versionstr) + return parser + + +def _add_io_arguments(parser): + msg = "input file to be processed (default: '-', stdin)" + parser.add_argument('infile', nargs='?', default='-', help=msg) + msg = "output file where processed content will be written (default: " \ + "'-', stdout)" + parser.add_argument('outfile', nargs='?', default='-', help=msg) def run_fypp(): '''Run the Fypp command line tool.''' + options = FyppOptions() + argparser = get_option_parser() + _add_io_arguments(argparser) + args = argparser.parse_args(namespace=options) try: - tool = Fypp() - tool.process_cmdline_files() + tool = Fypp(args) + tool.process_file(args.infile, args.outfile) except FyppError as exc: sys.stderr.write(str(exc)) sys.stderr.write('\n') sys.exit(1) +def _shiftinds(inds, shift): + return [ind + shift for ind in inds] + + +def _open_input_file(inpfile): + try: + inpfp = open(inpfile, 'r') + except IOError as ex: + msg = "Failed to open file '{}' for read\n{}".format(inpfile, ex) + raise FyppError(msg) + return inpfp + + +def _open_output_file(outfile, create_parents=False): + if create_parents: + parentdir = os.path.abspath(os.path.dirname(outfile)) + if not os.path.exists(parentdir): + try: + os.makedirs(parentdir) + except OSError as ex: + if ex.errno != errno.EEXIST: + msg = "Folder '{}' can not be created\n{}"\ + .format(parentdir, ex) + raise FyppError(msg) + try: + outfp = open(outfile, 'w') + except IOError as ex: + msg = "Failed to open file '{}' for write\n{}".format(outfile, ex) + raise FyppError(msg) + return outfp + + if __name__ == '__main__': run_fypp() diff --git a/docs/conf.py b/docs/conf.py index 6e66bb2..c7ae245 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,10 +53,10 @@ # built documents. # # The short X.Y version. -version = '0.12' +version = '1.0' # The full version, including alpha/beta/rc tags. -release = '0.12' +release = '1.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/fypp.rst b/docs/fypp.rst index cc067eb..4edc2ab 100644 --- a/docs/fypp.rst +++ b/docs/fypp.rst @@ -4,38 +4,39 @@ Introduction ************ -Fypp is a Python powered Fortran preprocessor. It extends Fortran with -condititional compiling and template metaprogramming capabilities. Instead of -introducing its own expression syntax, it uses Python expressions in its -preprocessor directives, offering the consistency and flexibility of Python when -formulating metaprogramming tasks. It puts strong emphasis on robustness and on -neat integration into Fortran developing toolchains. +Fypp is a Python powered preprocessor. It can be used for any programming +languages but its primary aim is to offer a Fortran preprocessor, which helps to +extend Fortran with condititional compiling and template metaprogramming +capabilities. Instead of introducing its own expression syntax, it uses Python +expressions in its preprocessor directives, offering the consistency and +versatility of Python when formulating metaprogramming tasks. It puts strong +emphasis on robustness and on neat integration into developing toolchains. Fypp was inspired by the `pyratemp `_ templating engine [#]_. Although it shares many concepts with pyratemp, it was written from -scratch focusing on the special needs when preprocessing Fortran source -files. Fypp natively supports the output of synchronization line directives, -which are used by many compilers to generate compiler messages with correct line -numbers. Unlike most cpp/fpp-like preprocessors, Fypp also supports iterations, -multiline macros, continuation lines in preprocessor directives and automatic -line folding. It generally tries to extend the modern Fortran language with some -useful features without tempting you to use it for tasks, which could/should be -done in Fortran itself. +scratch focusing on the special needs when preprocessing source code. Fypp +natively supports the output of line numbering directives, which are used by +many compilers to generate compiler messages with correct line numbers. Unlike +most cpp/fpp preprocessors or the coco preprocessor, Fypp also supports +iterations, multiline macros, continuation lines in preprocessor directives and +automatic line folding. It generally tries to extend the modern Fortran language +with some useful features without tempting you to use it for tasks which +could/should be done in Fortran itself. The project is `hosted on bitbucket `_ with documentation available on `readthedocs.org `_. Fypp is released under the *BSD 2-clause license*. -This document describes Fypp Version 0.12. +This document describes Fypp Version 1.0. Features ======== -Below you find a summary over Fypps main features. Each of them is described in -more detail in individual sections further down. +Below you find a summary over Fypps main features. Each of them is described +more in detail in the individual sections further down. * Definition and evaluation of preprocessor variables:: @@ -50,10 +51,12 @@ more detail in individual sections further down. Fortran standard):: #:def assertTrue(cond) + #:if DEBUG > 0 if (.not. ${cond}$) then print *, "Assert failed in file ${_FILE_}$, line ${_LINE_}$" error stop end if + #:endif #:enddef ! Invoked via direct call (needs no quotation) @@ -74,7 +77,7 @@ more detail in individual sections further down. use serial #:endif -* Iterated output (e.g. for Fortran templates):: +* Iterated output (e.g. for generating Fortran templates):: interface myfunc #:for dtype in [ 'real', 'dreal', 'complex', 'dcomplex' ] @@ -135,27 +138,27 @@ Getting started Installing ========== -Fypp needs a working Python interpreter, either version 2.7 or version 3.2 or -above. +Fypp needs a Python interpreter of version 2.7, 3.2 or above. Automatic install ----------------- -You can use Pythons installer `pip` to install the last stable release of Fypp -on your system:: +Use Pythons command line installer ``pip`` in order to download the stable +release from the `Fypp page on PyPI `_ and +install it on your system:: pip install fypp -This installs the command line tool ``fypp`` as well as the Python module -``fypp``. Latter you can import if you want to access the functionality of Fypp -directly from within your Python scripts. +This installs both, the command line tool ``fypp`` and the Python module +``fypp.py``. Latter you can import if you want to access the functionality of +Fypp directly from within your Python scripts. Manual install -------------- -Alternatively, you can download the source code from the `Fypp project website -`_ :: +For a manual install, you can download the source code from the `Fypp project +website `_ :: git clone https://aradi@bitbucket.org/aradi/fypp.git @@ -184,6 +187,8 @@ environment variable, by just issuing :: fypp +The python module ``fypp.py`` can be found in ``FYP_SOURCE_FOLDER/src``. + Testing ======= @@ -192,7 +197,7 @@ You can test Fypp on your system by running :: ./test/runtests.sh -in its source tree. This will execute various unit tests to check whether Fypp +in the source tree. This will execute various unit tests to check whether Fypp works as expected. If you want to run the tests with a specific Python interpreter, you can specify it as argument to the script:: @@ -205,9 +210,9 @@ Running The Fypp command line tool reads a file, preprocesses it and writes it to another file, so you would typically invoke it like:: - fypp source.fypp source.f90 + fypp source.F90 source.f90 -which would process `source.fypp` and write the result to `source.f90`. If +which would process `source.F90` and write the result to `source.f90`. If input and output files are not specified, information is read from stdin and written to stdout. @@ -233,7 +238,7 @@ an inline form: * Line form, starting with ``#:`` (hashmark colon):: #:if 1 > 2 - Some fortran code + Some code #:endif * Inline form, enclosed between ``#{`` and ``}#``:: @@ -257,14 +262,14 @@ an inline form: The line form must always start at the beginning of a line (preceded by optional -whitespace characters only) and it goes until the end of the line. The inline -form can appear anywhere, but if the construct consists of several directives +whitespace characters only) and it ends at the end of the line. The inline form +can appear anywhere, but if the construct consists of several directives (e.g. ``#{if ...}#`` and ``#{endif}#``), all of them must appear on the same -line. While both forms can be used at the same time, for a particular construct -they must be consistent, e.g. a directive opened as line directive can not be -closed with an inline directive and vica versa. +line. While both forms can be used at the same time, they must be consistent for +a particular construct, e.g. a directive opened as line directive can not be +closed by an inline directive and vica versa. -Whitespaces in preprocessor commands are ignored, if they appear after the +Whitespaces in preprocessor commands are ignored if they appear after the opening colon or curly brace or before the closing curly brace. So the following examples are pairwise equivalent:: @@ -280,8 +285,8 @@ examples are pairwise equivalent:: ${time.strftime('%Y-%m-%d')}$ ${ time.strftime('%Y-%m-%d') }$ -Starting whitespaces before line directives are also ignored, enabling you to -choose any indentation strategy you like for the directives:: +Starting whitespaces before line directives are ignored, enabling you to choose +any indentation strategy you like for the directives:: program test : @@ -295,7 +300,7 @@ choose any indentation strategy you like for the directives:: : end program test -Preprocessor directives can be nested arbitrarily:: +Preprocessor directives can be arbitrarily nested:: #:if DEBUG > 0 #:if DO_LOGGING @@ -324,32 +329,33 @@ be, therefore, syntactically and semantically correct Python expressions. Although, this may require some additional quotations as compared to other preprocessor languages :: - #:if defined('DEBUG') #! defined() takes a Python string as argument - #:for dtype in [ 'real(dp)', 'integer', 'logical' ] # dtype runs over strings + #:if defined('DEBUG') #! The Python function defined() expects a string argument + #:for dtype in [ 'real(dp)', 'integer', 'logical' ] #! dtype runs over strings it enables consistent expressions with (hopefully) least surprises (once you know, how to formulate the expression in Python, you exactly know, how to write it for Fypp). Also, note, that variable names, macros etc. are in Python (and -therefore als for Fypp) case sensitive. +therefore also for Fypp) case sensitive. -If you access a variable in an expression, it must have been defined (either via -command line options or via preprocessor directives) before. For example :: +When you access a variable in an expression, it must have been already defined +before, either via command line options or via preprocessor directives. For +example the directive :: #:if DEBUG > 0 -can only be evaluated, if the variable `DEBUG` had been already defined. +can only be evaluated, if the variable `DEBUG` had been already defined before. -Python expressions are evaluted in an isolated Python environment. It contains a -restricted set of Python built-in functions and a few predefined variables and -functions (see below). There are no modules loaded by default, and for safety -reasons, no modules can be loaded once the preprocessing has started. +Python expressions are evaluted in an isolated Python environment, which +contains a restricted set of Python built-in functions and a few predefined +variables and functions (see below). There are no modules loaded by default, and +for safety reasons, no modules can be loaded once the preprocessing has started. Initializing the environment ---------------------------- -If a Python module is needed during the preprocessing, it can be imported via -the command line option (``-m``) before the preprocessing starts:: +If a Python module is required for the preprocessing, it can be imported before +the preprocessing starts via the command line option (``-m``):: fypp -m time @@ -370,27 +376,27 @@ Initial values for preprocessor variables can be set via the command line option The assigned value for a given variable is evaluated in Python. If no value is provided, `None` is assigned. -When complex initialization is needed (e.g. user defined Python functions are -needed during preprocessing), initialization scripts can be specified via the -command line option ``-i``:: +When complex initialization is needed (e.g. user defined Python functions should +be defined), initialization scripts can be specified via the command line option +``-i``:: fypp -i ini1.py -i ini2.py The preprocessor executes the content of each initialization script in the isolated environment via Pythons `exec()` command before processing any input. If modules had been also specified via the ``-m`` option, they are -imported before the initialization scripts are executed. +imported before the execution of the initialization scripts. Predefined variables and functions ---------------------------------- The isolated Python environment for the expression evaluation contains following -predefined read-only variables: +predefined (read-only) variables: -* ``_LINE_``: number of the line where the eval directive was found +* ``_LINE_``: number of current line -* ``_FILE_``: name of the file in which the eval directive was found :: +* ``_FILE_``: name of curernt file :: print *, "This is line nr. ${_LINE_}$ in file '${_FILE_}$'" @@ -403,19 +409,19 @@ predefined read-only variables: Additionally following predefined functions are provided: * ``defined(VARNAME)``: Returns ``True`` if a variable with a given name has - been already defined. The variable name must be provided as string :: + been already defined. The variable name must be provided as string:: #:if defined('WITH_MPI') * ``getvar(VARNAME, DEFAULTVALUE)``: Returns the value of a variable or a default value if the variable is not defined. The variable name must be - provided as string. :: + provided as string:: #:if getvar('DEBUG', 0) * ``setvar(VARNAME, VALUE)``: Sets a variable to given value. It is identical to the ``#:setvar`` control directive. The variable name must be provided as - string :: + string:: $:setvar('i', 12) print *, "VAR I: ${i}$" @@ -424,9 +430,9 @@ Additionally following predefined functions are provided: Eval directive ============== -A result of a Python expression can be inserted into the code by using the eval +A result of a Python expression can be inserted into the code by using eval directives ``$:`` (line form) or ``${`` and ``}$`` (inline form). The expression -is evaluated using Python's built-in function `eval()` . If it evaluates to +is evaluated using Python's built-in function `eval()`. If it evaluates to `None`, no output is produced. Otherwise the result is converted to a string and written to the output. The eval directive has both, a line and an inline variant:: @@ -465,7 +471,7 @@ The `setvar` directive can be also used in the inline form:: ============== Conditional output can be generated using the `if` directive. The condition must -be a Python expression which can be converted to a Boolean. If the condition +be a Python expression, which can be converted to a `bool`. If the condition evaluates to `True`, the enclosed code is written to the output, otherwise it is ignored. @@ -533,30 +539,33 @@ single and double precision reals:: end function sin2_${rkind}$ #:endfor -The `for` directive expects a loop variable and an iterable expression, +The `for` directive expects a Python loop variable expression and an iterable separated by the ``in`` keyword. The code within the `for` directive is outputed for every iteration with the current value of the loop variable, which can be inserted using eval directives. If the iterable consists of iterables -(e.g. tuples), usual indexing can be used to access the components, or a +(e.g. tuples), usual indexing can be used to access their components, or a variable tuple to unpack them directly in the loop header:: - #:setvar kinds_names [ ('sp', 'real'), ('dp', 'dreal') ] - + #:setvar kinds ['sp', 'dp'] + #:setvar names ['real', 'dreal'] + #! create kinds_names as [('sp', 'real'), ('dp', 'dreal')] + #:setvar kinds_names list(zip(kinds, names)) + #! Acces by indexing interface sin2 #:for kind_name in kinds_names module procedure sin2_${kind_name[1]}$ #:endfor end interface sin2 - + #! Unpacking in the loop header #:for kind, name in kinds_names function sin2_${name}$(xx) result(res) real(${kind}$), intent(in) :: xx real(${kind}$) :: res - + res = sin(xx) * sin(xx) - + end function sin2_${name}$ #:endfor @@ -571,11 +580,11 @@ The `for` directive can be used also in its inline form:: =============== Parametrized macros can be defined with the `def` directive. This defines a -regular Python callable which returns the rendered content of the macro body +regular Python callable, which returns the rendered content of the macro body when called. The macro arguments are converted to local variables containing the actual arguments as values. The macro can be either called from within an -eval-directive or via the `call` control directive or its abreviated form -(direct call). +eval-directive, via the `call` control directive or its abreviated form, the +direct call. Given the macro definition :: @@ -605,14 +614,13 @@ would all yield :: error stop end if -if the variabl `DEBUG` had a value greater than zero or an empty string +if the variable `DEBUG` had a value greater than zero or an empty string otherwise. -When called from within an eval-directive, additional to the regular macro -arguments, arbitrary optional parameters can be passed. Those optional -parameters will be converted to local variables when the macro content is -rendered. For example given the defintion of the ``assertTrue()`` macro from -above, the call :: +When called from within an eval-directive, arbitrary optional parameters can be +passed additional to the regular macro arguments. The optional parameters are +converted to local variables when the macro content is rendered. For example +given the defintion of the ``assertTrue()`` macro from above, the call :: $:assertTrue('x > y', DEBUG=1) @@ -662,7 +670,7 @@ The `def` directive can also be used in its short form:: ================ When a Python callable (e.g. regular Python function, macro) is called with -string arguments only (e.g. Fortran code), it can be called using the `call` +string arguments only (e.g. source code), it can be called using the `call` directive to avoid extra quoting of the arguments:: #:def debug_code(code) @@ -699,9 +707,9 @@ each other:: print *, "No debugging" #:endcall -The lines in the body of the `call` directive can contain directives -themselves. However, any variables defined within the body of the `call` -directive will be local variables, existing only during the evaluation of that +The lines in the body of the `call` directive may contain directives +themselves. However, any variable defined within the body of the `call` +directive will be a local variable existing only during the evaluation of that branch of the directive (and not being available during the call itself). The `call` directive can also be used in its inline form. As this easily leads @@ -713,6 +721,8 @@ to code being hard to read, it should be usually avoided:: ! This form is more readable print *, ${choose_code('a(:)', 'size(a)')}$ +If the arguments need no further processing, the direct call directive can be +also used as an alternative to the line form (see next section). Direct call directive @@ -761,8 +771,8 @@ string to the callable. `include` directive =================== -The `include` directive enables you to collect your preprocessor macro and -variable definitions in a separate files and include them whenever needed. The +The `include` directive allows you to collect your preprocessor macros and +variable definitions in separate files and include them whenever needed. The include directive expects a quoted string with a file name:: #:include 'mydefs.fypp' @@ -779,7 +789,7 @@ The `include` directive does not have an inline form. Empty lines between Fypp definitions makes the code easier to read. However, being outside of Fypp-directives, those empty lines will be written unaltered to -the output file. This can be especially disturbing, if various macro definition +the output. This can be especially disturbing if various macro definition files are included, as the resulting output would eventually contian a lot of empty lines. With the `mute` directive, the output can be suspended. While everything is still processed as normal, no output is written for the code @@ -811,7 +821,7 @@ Comment directive Comment lines can be added by using the ``#!`` preprocessor directive. The comment line (including the newlines at their end) will be ignored by the -prepropessor and not appear in the ouput:: +prepropessor and will not appear in the ouput:: #! This will not show up in the output @@ -847,15 +857,13 @@ Line folding ============ The Fortran standard only allows source lines up to 132 characters. In order to -emit standard conforming code, Fypp folds all lines in the output, which it had -manipulated before (all lines containing eval directives). Lines, which were -just copied to the output, are left unaltered. The maximal line length can be +emit standard conforming code, Fypp folds all lines in the output which it had +manipulated before (all lines containing eval directives). Lines which were +just copied to the output are left unaltered. The maximal line length can be chosen by the ``-l`` command line option. The indentation of the continuation lines can be tuned with the ``--indentation`` option, and the folding strategy can be selected by the ``-f`` option with following possibilities: -* ``none``: Switches line folding off. - * ``brute``: Continuation lines are indented relative to the beginning of the line, and each line is folded at the maximal line position. @@ -867,25 +875,26 @@ can be selected by the ``-f`` option with following possibilities: becoming too short, it defaults to ``simple`` if no whitespace occurs in the last third of the line. +The ``-F`` option can be used to turn off line folding. -.. warning:: Fypp is not aware of the Fortran semantics represented by the lines - it folds. -Fypp applies the line folding rather mechanically (only considering the the -position of the whitespace characters). Lines containing eval directives and -lines within macro definition should, therefore, not contain any Fortran style -comments (started by ``!``) *within* the line, as folding within the comment -would result in invalid Fortran code. For comments within such lines, Fypps -comment directive (``#!``) can be used instead:: +.. warning:: Fypp is not aware of the Fortran semantics of the lines it folds. + +Fypp applies the line folding mechanically (only considering the position of the +whitespace characters). Lines containing eval directives and lines within macro +definitions should, therefore, not contain any Fortran style comments (started +by ``!``) *within* the line, as folding within the comment would result in +invalid Fortran code. For comments within such lines, Fypps comment directive +(``#!``) can be used instead:: #:def macro() print *, "DO NOT DO THIS!" ! Warning: Line may be folded within the comment print *, "This is OK." #! Preprocessor comment is safe as it will be stripped -If the comment starts at the beginning of the line (preceeded by optional -whitespace characters only), the folding is suppressed, though. This enables you -to define macros, which emit non-negligible comment lines (e.g. containing -source code documentation or OpenMP directives):: +For comments starting at the beginning of the line (preceeded by optional +whitespace characters only) the folding is suppressed, though. This enables you +to define macros with non-negligible comment lines (e.g. with source code +documentation or OpenMP directives):: #:def macro(DTYPE) !> This functions calculates something (version ${DTYPE}$) @@ -900,18 +909,66 @@ source code documentation or OpenMP directives):: Escaping ======== -If you want to prevent Fypp to interprete something as control or eval -directive, put a backslash (``\``) between the first and second delimiter -character. In case of inline directives, do it for the opening and the -closing delimiter as well:: +If you want to prevent Fypp to interprete something as a directive, put a +backslash (``\``) between the first and second delimiter character. In case of +inline directives, do it for both, the opening and the closing delimiter:: $\: 1 + 2 #\{if 1 > 2}\# + @\:myMacro arg1 Fypp will not recognize the escaped strings as directives, but will remove the backslash between the delimiter characters in the output. If you put more than one backslash between the delimiters, only one will be removed. + +Line numbering directives +========================= + +In order to support compilers in emitting messages with correct line numbers +with respect to the original source file, Fypp can put line number directives +(a.k.a. linemarkers) in its output. This can be enabled by using the command +line option ``-n``. Given a file ``test.F90`` with the content :: + + program test + #:if defined('MPI') + use mpi + #:else + use openmpi + #:endif + : + end program test + +the command :: + + fypp -n -DMPI test.F90 + +produces the output :: + + # 1 "test.F90" + program test + # 3 "test.F90" + use mpi + # 7 "test.F90" + : + end program test + +If during compilation of this output an error occured in the line ``use mpi`` +(e.g. the mpi module can not be found), the compiler would know that this line +corresponds to line number 3 in the original file ``test.F90`` and could emit an +according error message. + +The line numbering directives can be fine tuned with the ``-N`` option, which +accepts following mode arguments: + +* ``full``: Line numbering directives are emitted whenever lines are + removed from the original source file or extra lines are added to it. + +* ``nocontlines``: Same as full, but line numbering directives are ommitted + before continuation lines. (Some compilers, like the NAG Fortran compiler, + have difficulties with line numbering directives before continuation lines). + + ***************** API documentation ***************** @@ -928,6 +985,33 @@ fypp module .. automodule:: fypp +Fypp +==== + +.. autoclass:: Fypp + :members: + + +FyppOptions +=========== + +.. autoclass:: FyppOptions + :members: + + +get_option_parser() +=================== + +.. autofunction:: get_option_parser() + + +FyppError +========= + +.. autoclass:: FyppError + :members: + + Parser ====== @@ -963,24 +1047,10 @@ Processor :members: -Fypp -==== - -.. autoclass:: Fypp - :members: - - -FyppError -========= - -.. autoclass:: FyppError - :members: - - -LineFolder -========== +FortranLineFolder +================= -.. autoclass:: LineFolder +.. autoclass:: FortranLineFolder :members: :special-members: __call__ diff --git a/setup.py b/setup.py index df1f33c..af11e2e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='fypp', - version='0.12', + version='1.0', description='Python powered Fortran preprocessor', long_description=long_description, diff --git a/test/test_fypp.py b/test/test_fypp.py index 0b53d26..1983075 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -3,14 +3,10 @@ import unittest import fypp -def _strsyncl(linenr, fname=None): +def _linenum(linenr, fname=None): if fname is None: fname = fypp.STRING - return fypp.syncline(linenr, fname) - -def _filesyncl(fname, linenr): - return fypp.syncline(linenr, fname) - + return fypp.linenumdir(linenr, fname) def _defvar(var, val): return '-D{}={}'.format(var, val) @@ -27,10 +23,15 @@ def _indentation(ind): def _folding(fold): return '-f{}'.format(fold) -_SYNCL_FLAG = '-s' +def _linenumbering(nummode): + return '-N{}'.format(nummode) + +_LINENUM_FLAG = '-n' _FIXED_FORMAT_FLAG = '--fixed-format' +_NO_FOLDING_FLAG = '-F' + SIMPLE_TESTS = [ ('if_true', [_defvar('TESTVAR', 1)], @@ -322,7 +323,7 @@ def _folding(fold): # ('direct_call_2_args_escape', [], '#:def mymacro(val1, val2)\n|${val1}$|${val2}$|\n#:enddef\n'\ - '@:mymacro """L1""" @\@ L2 @@ L3\n', + '@:mymacro """L1""" @\\@ L2 @@ L3\n', '|"""L1""" @@ L2|L3|\n', ), # @@ -346,6 +347,21 @@ def _folding(fold): '2Done\n', ), # + ('setvar_function', [], + '$:setvar("x", 2)\n${x}$\nDone\n', + "\n2\nDone\n", + ), + # + ('getvar_existing_value', [_defvar('VAR', '\"VAL\"')], + '$:getvar("VAR", "DEFAULT")\n', + 'VAL\n', + ), + # + ('getvar_default_value', [], + '$:getvar("VAR", "DEFAULT")\n', + 'DEFAULT\n', + ), + # ('mute', [], 'A\n#:mute\nB\n#:setvar VAR 2\n#:endmute\nVAR=${VAR}$\n', 'A\nVAR=2\n' @@ -361,6 +377,11 @@ def _folding(fold): fypp.STRING ), # + ('builtin_var_line_in_lineeval', [], + '$:_LINE_\n', + '1\n' + ), + # ('escaped_control_inline', [], r'A#\{if False}\#B#\{endif}\#', 'A#{if False}#B#{endif}#' @@ -386,6 +407,11 @@ def _folding(fold): r'$\\{1 + 1}\$' ), # + ('escape_direct_call', [], + '@\\:assertTrue x > y\n', + '@:assertTrue x > y\n' + ), + # ('fold_lines', [_linelen(10), _indentation(2), _folding('simple')], 'This line is not folded\nThis line ${1 + 1}$ is folded\n', 'This line is not folded\nThis line&\n & 2 is &\n &folded\n' @@ -398,7 +424,7 @@ def _folding(fold): ' ! Should be not folded\nShould be&\n & folded\n' ), # - ('no_folding', [_linelen(15), _indentation(4), _folding('none')], + ('no_folding', [_linelen(15), _indentation(4), _NO_FOLDING_FLAG], ' ${3}$456 89 123456 8', ' 3456 89 123456 8', ), @@ -427,232 +453,241 @@ def _folding(fold): ), ] -SYNCLINE_TESTS = [ - # This test (but only this) must be changed, if syncline format changes. - ('explicit_str_syncline_test', [_SYNCL_FLAG], +LINENUM_TESTS = [ + # This test (but only this) must be changed, if linenum directive changes. + ('explicit_str_linenum_test', [_LINENUM_FLAG], '', '# 1 ""\n', ), # - ('trivial', [_SYNCL_FLAG], + ('trivial', [_LINENUM_FLAG], 'Test\n', - _strsyncl(0) + 'Test\n' + _linenum(0) + 'Test\n' ), # - ('if_true', [_SYNCL_FLAG], + ('if_true', [_LINENUM_FLAG], '#:if 1 < 2\nTrue\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True\n' + _strsyncl(3) + 'Done\n' + _linenum(0) + _linenum(1) + 'True\n' + _linenum(3) + 'Done\n' ), # - ('if_false', [_SYNCL_FLAG], + ('if_false', [_LINENUM_FLAG], '#:if 1 > 2\nTrue\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'Done\n' + _linenum(0) + _linenum(3) + 'Done\n' ), # - ('if_else_true', [_SYNCL_FLAG], + ('if_else_true', [_LINENUM_FLAG], '#:if 1 < 2\nTrue\n#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(1) + 'True\n' + _linenum(5) + 'Done\n' ), # - ('if_else_false', [_SYNCL_FLAG], + ('if_else_false', [_LINENUM_FLAG], '#:if 1 > 2\nTrue\n#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'False\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(3) + 'False\n' + _linenum(5) + 'Done\n' ), - ('if_elif_true1', [_SYNCL_FLAG], + ('if_elif_true1', [_LINENUM_FLAG], '#:if 1 == 1\nTrue1\n#:elif 1 == 2\nTrue2\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True1\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(1) + 'True1\n' + _linenum(5) + 'Done\n' ), # - ('if_elif_true2', [_SYNCL_FLAG], + ('if_elif_true2', [_LINENUM_FLAG], '#:if 2 == 1\nTrue1\n#:elif 2 == 2\nTrue2\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'True2\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(3) + 'True2\n' + _linenum(5) + 'Done\n' ), # - ('if_elif_false', [_SYNCL_FLAG], + ('if_elif_false', [_LINENUM_FLAG], '#:if 0 == 1\nTrue1\n#:elif 0 == 2\nTrue2\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(5) + 'Done\n' ), # - ('if_elif_else_true1', [_SYNCL_FLAG], + ('if_elif_else_true1', [_LINENUM_FLAG], '#:if 1 == 1\nTrue1\n#:elif 1 == 2\nTrue2\n' '#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'True1\n' + _strsyncl(7) + 'Done\n' + _linenum(0) + _linenum(1) + 'True1\n' + _linenum(7) + 'Done\n' ), # - ('if_elif_else_true2', [_SYNCL_FLAG], + ('if_elif_else_true2', [_LINENUM_FLAG], '#:if 2 == 1\nTrue1\n#:elif 2 == 2\nTrue2\n' '#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'True2\n' + _strsyncl(7) + 'Done\n' + _linenum(0) + _linenum(3) + 'True2\n' + _linenum(7) + 'Done\n' ), # - ('if_elif_else_false', [_SYNCL_FLAG], + ('if_elif_else_false', [_LINENUM_FLAG], '#:if 0 == 1\nTrue1\n#:elif 0 == 2\nTrue2\n' '#:else\nFalse\n#:endif\nDone\n', - _strsyncl(0) + _strsyncl(5) + 'False\n' + _strsyncl(7) + 'Done\n' + _linenum(0) + _linenum(5) + 'False\n' + _linenum(7) + 'Done\n' ), # - ('inline_if_true', [_SYNCL_FLAG], + ('inline_if_true', [_LINENUM_FLAG], '#{if 1 < 2}#True#{endif}#Done\n', - _strsyncl(0) + 'TrueDone\n' + _linenum(0) + 'TrueDone\n' ), # - ('inline_if_false', [_SYNCL_FLAG], + ('inline_if_false', [_LINENUM_FLAG], '#{if 1 > 2}#True#{endif}#Done\n', - _strsyncl(0) + 'Done\n' + _linenum(0) + 'Done\n' ), # - ('inline_if_else_true', [_SYNCL_FLAG], + ('inline_if_else_true', [_LINENUM_FLAG], '#{if 1 < 2}#True#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'TrueDone\n' + _linenum(0) + 'TrueDone\n' ), # - ('inline_if_else_false', [_SYNCL_FLAG], + ('inline_if_else_false', [_LINENUM_FLAG], '#{if 1 > 2}#True#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'FalseDone\n' + _linenum(0) + 'FalseDone\n' ), - ('inline_if_elif_true1', [_SYNCL_FLAG], + ('inline_if_elif_true1', [_LINENUM_FLAG], '#{if 1 == 1}#True1#{elif 1 == 2}#True2#{endif}#Done\n', - _strsyncl(0) + 'True1Done\n' + _linenum(0) + 'True1Done\n' ), # - ('inline_if_elif_true2', [_SYNCL_FLAG], + ('inline_if_elif_true2', [_LINENUM_FLAG], '#{if 2 == 1}#True1#{elif 2 == 2}#True2#{endif}#Done\n', - _strsyncl(0) + 'True2Done\n' + _linenum(0) + 'True2Done\n' ), # - ('inline_if_elif_false', [_SYNCL_FLAG], + ('inline_if_elif_false', [_LINENUM_FLAG], '#{if 0 == 1}#True1#{elif 0 == 2}#True2#{endif}#Done\n', - _strsyncl(0) + 'Done\n' + _linenum(0) + 'Done\n' ), # - ('inline_if_elif_else_true1', [_SYNCL_FLAG], + ('inline_if_elif_else_true1', [_LINENUM_FLAG], '#{if 1 == 1}#True1#{elif 1 == 2}#True2#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'True1Done\n' + _linenum(0) + 'True1Done\n' ), # - ('inline_if_elif_else_true2', [_SYNCL_FLAG], + ('inline_if_elif_else_true2', [_LINENUM_FLAG], '#{if 2 == 1}#True1#{elif 2 == 2}#True2#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'True2Done\n' + _linenum(0) + 'True2Done\n' ), # - ('inline_if_elif_else_false', [_SYNCL_FLAG], + ('inline_if_elif_else_false', [_LINENUM_FLAG], '#{if 0 == 1}#True1#{elif 0 == 2}#True2#{else}#False#{endif}#Done\n', - _strsyncl(0) + 'FalseDone\n' + _linenum(0) + 'FalseDone\n' ), # - ('linesub_oneline', [_SYNCL_FLAG], + ('linesub_oneline', [_LINENUM_FLAG], 'A\n$: 1 + 1\nB\n', - _strsyncl(0) + 'A\n2\nB\n' + _linenum(0) + 'A\n2\nB\n' ), # - ('linesub_contlines', [_SYNCL_FLAG, _defvar('TESTVAR', 1)], + ('linesub_contlines', [_LINENUM_FLAG, _defvar('TESTVAR', 1)], '$: TESTVAR & \n & + 1\nDone\n', - _strsyncl(0) + '2\n' + _strsyncl(2) + 'Done\n' + _linenum(0) + '2\n' + _linenum(2) + 'Done\n' ), # - ('linesub_contlines2', [_SYNCL_FLAG, _defvar('TESTVAR', 1)], + ('linesub_contlines2', [_LINENUM_FLAG, _defvar('TESTVAR', 1)], '$: TEST& \n &VAR & \n & + 1\nDone\n', - _strsyncl(0) + '2\n' + _strsyncl(3) + 'Done\n' + _linenum(0) + '2\n' + _linenum(3) + 'Done\n' ), # - ('exprsub_single_line', [_SYNCL_FLAG, _defvar('TESTVAR', 1)], + ('exprsub_single_line', [_LINENUM_FLAG, _defvar('TESTVAR', 1)], 'A${TESTVAR}$B${TESTVAR + 1}$C', - _strsyncl(0) + 'A1B2C' + _linenum(0) + 'A1B2C' ), # - ('exprsub_multi_line', [_SYNCL_FLAG], + ('exprsub_multi_line', [_LINENUM_FLAG], '${"line1\\nline2"}$\nDone\n', - _strsyncl(0) + 'line1\n' + _strsyncl(0) + 'line2\nDone\n' + _linenum(0) + 'line1\n' + _linenum(0) + 'line2\nDone\n' ), # - ('macrosubs', [_SYNCL_FLAG], + ('macrosubs', [_LINENUM_FLAG], '#:def macro(var)\nMACRO|${var}$|\n#:enddef\n${macro(1)}$', - _strsyncl(0) + _strsyncl(3) + 'MACRO|1|' + _linenum(0) + _linenum(3) + 'MACRO|1|' ), # - ('recursive_macrosubs', [_SYNCL_FLAG], + ('recursive_macrosubs', [_LINENUM_FLAG], '#:def macro(var)\nMACRO|${var}$|\n#:enddef\n${macro(macro(1))}$', - _strsyncl(0) + _strsyncl(3) + 'MACRO|MACRO|1||' + _linenum(0) + _linenum(3) + 'MACRO|MACRO|1||' ), # - ('macrosubs_multiline', [_SYNCL_FLAG], + ('macrosubs_multiline', [_LINENUM_FLAG], '#:def macro(c)\nMACRO1|${c}$|\nMACRO2|${c}$|\n#:enddef\n${macro(\'A\')}$' '\n', - _strsyncl(0) + _strsyncl(4) + 'MACRO1|A|\n' + _strsyncl(4) + 'MACRO2|A|\n' + _linenum(0) + _linenum(4) + 'MACRO1|A|\n' + _linenum(4) + 'MACRO2|A|\n' ), # - ('recursive_macrosubs_multiline', [_SYNCL_FLAG], + ('recursive_macrosubs_multiline', [_LINENUM_FLAG], '#:def f(c)\nLINE1|${c}$|\nLINE2|${c}$|\n#:enddef\n$: f(f("A"))\n', - (_strsyncl(0) + _strsyncl(4) + 'LINE1|LINE1|A|\n' + _strsyncl(4) - + 'LINE2|A||\n' + _strsyncl(4) + 'LINE2|LINE1|A|\n' + _strsyncl(4) + (_linenum(0) + _linenum(4) + 'LINE1|LINE1|A|\n' + _linenum(4) + + 'LINE2|A||\n' + _linenum(4) + 'LINE2|LINE1|A|\n' + _linenum(4) + 'LINE2|A||\n') ), # - ('multiline_macrocall', [_SYNCL_FLAG], + ('multiline_macrocall', [_LINENUM_FLAG], '#:def macro(c)\nMACRO|${c}$|\n#:enddef\n$: mac& \n &ro(\'A\')\nDone\n', - _strsyncl(0) + _strsyncl(3) + 'MACRO|A|\n' + _strsyncl(5) + 'Done\n' + _linenum(0) + _linenum(3) + 'MACRO|A|\n' + _linenum(5) + 'Done\n' ), # - ('call_directive_2_args', [_SYNCL_FLAG], + ('call_directive_2_args', [_LINENUM_FLAG], '#:def mymacro(val1, val2)\n|${val1}$|${val2}$|\n#:enddef\n'\ '#:call mymacro\nL1\nL2\n#:nextarg\nL3\n#:endcall\n', - _strsyncl(0) + _strsyncl(3) + '|L1\n' + _strsyncl(3) + 'L2|L3|\n'\ - + _strsyncl(9), + _linenum(0) + _linenum(3) + '|L1\n' + _linenum(3) + 'L2|L3|\n'\ + + _linenum(9), ), # - ('for', [_SYNCL_FLAG], + ('for', [_LINENUM_FLAG], '#:for i in (1, 2)\n${i}$\n#:endfor\nDone\n', - (_strsyncl(0) + _strsyncl(1) + '1\n' + _strsyncl(1) + '2\n' - + _strsyncl(3) + 'Done\n') + (_linenum(0) + _linenum(1) + '1\n' + _linenum(1) + '2\n' + + _linenum(3) + 'Done\n') ), # - ('inline_for', [_SYNCL_FLAG], + ('inline_for', [_LINENUM_FLAG], '#{for i in (1, 2)}#${i}$#{endfor}#Done\n', - _strsyncl(0) + '12Done\n' + _linenum(0) + '12Done\n' ), # - ('setvar', [_SYNCL_FLAG], + ('setvar', [_LINENUM_FLAG], '#:setvar x 2\n$: x\n', - _strsyncl(0) + _strsyncl(1) + '2\n', + _linenum(0) + _linenum(1) + '2\n', ), # - ('inline_setvar', [_SYNCL_FLAG], + ('inline_setvar', [_LINENUM_FLAG], '#{setvar x 2}#${x}$Done\n', - _strsyncl(0) + '2Done\n', + _linenum(0) + '2Done\n', ), # - ('comment_single', [_SYNCL_FLAG], + ('comment_single', [_LINENUM_FLAG], ' #! Comment here\nDone\n', - _strsyncl(0) + _strsyncl(1) + 'Done\n' + _linenum(0) + _linenum(1) + 'Done\n' ), # - ('comment_multiple', [_SYNCL_FLAG], + ('comment_multiple', [_LINENUM_FLAG], ' #! Comment1\n#! Comment2\nDone\n', - _strsyncl(0) + _strsyncl(2) + 'Done\n', + _linenum(0) + _linenum(2) + 'Done\n', ), # - ('mute', [_SYNCL_FLAG], + ('mute', [_LINENUM_FLAG], 'A\n#:mute\nB\n#:setvar VAR 2\n#:endmute\nVAR=${VAR}$\n', - _strsyncl(0) + 'A\n' + _strsyncl(5) + 'VAR=2\n' + _linenum(0) + 'A\n' + _linenum(5) + 'VAR=2\n' ), # - ('mute', [_SYNCL_FLAG], - 'A\n#:mute\nB\n#:setvar VAR 2\n#:endmute\nVAR=${VAR}$\n', - _strsyncl(0) + 'A\n' + _strsyncl(5) + 'VAR=2\n' - ), - # - ('direct_call', [_SYNCL_FLAG], + ('direct_call', [_LINENUM_FLAG], '#:def mymacro(val)\n|${val}$|\n#:enddef\n'\ '@:mymacro a < b\n', - _strsyncl(0) + _strsyncl(3) + '|a < b|\n', + _linenum(0) + _linenum(3) + '|a < b|\n', ), # - ('direct_call_contline', [_SYNCL_FLAG], + ('direct_call_contline', [_LINENUM_FLAG], '#:def mymacro(val)\n|${val}$|\n#:enddef\n'\ '@:mymacro a &\n &< b&\n &\nDone\n', - _strsyncl(0) + _strsyncl(3) + '|a < b|\n' + _strsyncl(6) + 'Done\n', + _linenum(0) + _linenum(3) + '|a < b|\n' + _linenum(6) + 'Done\n', + ), + # + ('smart_folding', + [_LINENUM_FLAG, _linelen(15), _indentation(4), _folding('smart')], + ' ${3}$456 89 123456 8\nDone\n', + _linenum(0) + ' 3456 89&\n' + _linenum(0) + ' & 123456&\n' \ + + _linenum(0) + ' & 8\n' + 'Done\n' + ), + # + ('smart_folding_nocontlines', + [_LINENUM_FLAG, _linenumbering('nocontlines'), _linelen(15), + _indentation(4), _folding('smart')], + ' ${3}$456 89 123456 8\nDone\n', + _linenum(0) + ' 3456 89&\n' + ' & 123456&\n' \ + + ' & 8\n' + _linenum(1) + 'Done\n' ), - ] INCLUDE_TESTS = [ @@ -677,28 +712,38 @@ def _folding(fold): 'FYPP2\n' ), # - ('search_include_syncl', [_SYNCL_FLAG, _incdir('include')], + ('search_include_linenum', [_LINENUM_FLAG, _incdir('include')], '#:include "fypp1.inc"\n$: incmacro(1)\n', - (_strsyncl(0) + _filesyncl('include/fypp1.inc', 0) - + 'INCL1\n' + _filesyncl('include/fypp1.inc', 4) - + 'INCL5\n' + _strsyncl(1) + 'INCMACRO(1)\n') + (_linenum(0) + _linenum(0, 'include/fypp1.inc') + + 'INCL1\n' + _linenum(4, 'include/fypp1.inc') + + 'INCL5\n' + _linenum(1) + 'INCMACRO(1)\n') ), # - ('nested_include_in_incpath_syncl', [_SYNCL_FLAG, _incdir('include')], + ('nested_include_in_incpath_linenum', [_LINENUM_FLAG, _incdir('include')], '#:include "subfolder/include_fypp1.inc"\n', - (_strsyncl(0) + _strsyncl(0, 'include/subfolder/include_fypp1.inc') - + _strsyncl(0, 'include/fypp1.inc') + 'INCL1\n' - + _strsyncl(4, 'include/fypp1.inc') + 'INCL5\n' - + _strsyncl(1, 'include/subfolder/include_fypp1.inc') - + _strsyncl(1)) + (_linenum(0) + _linenum(0, 'include/subfolder/include_fypp1.inc') + + _linenum(0, 'include/fypp1.inc') + 'INCL1\n' + + _linenum(4, 'include/fypp1.inc') + 'INCL5\n' + + _linenum(1, 'include/subfolder/include_fypp1.inc') + + _linenum(1)) ), # - ('nested_include_in_folder_of_incfile', [_SYNCL_FLAG, _incdir('include')], + ('nested_include_in_folder_of_incfile', [_LINENUM_FLAG, _incdir('include')], '#:include "subfolder/include_fypp2.inc"\n', - (_strsyncl(0) + _strsyncl(0, 'include/subfolder/include_fypp2.inc') - + _strsyncl(0, 'include/subfolder/fypp2.inc') + (_linenum(0) + _linenum(0, 'include/subfolder/include_fypp2.inc') + + _linenum(0, 'include/subfolder/fypp2.inc') + 'FYPP2\n' - + _strsyncl(1, 'include/subfolder/include_fypp2.inc') + _strsyncl(1)) + + _linenum(1, 'include/subfolder/include_fypp2.inc') + _linenum(1)) + ), + # + ('muted_include', [_incdir('include')], + 'START\n#:mute\n#:include \'fypp1.inc\'\n#:endmute\nDONE\n', + 'START\nDONE\n' + ), + # + ('muted_include_linenum', [_LINENUM_FLAG, _incdir('include')], + 'START\n#:mute\n#:include \'fypp1.inc\'\n#:endmute\nDONE\n', + _linenum(0) + 'START\n' + _linenum(4) + 'DONE\n' ), ] @@ -1009,8 +1054,8 @@ class SimpleTest(unittest.TestCase): pass -class SynclineTest(unittest.TestCase): - '''Container for tests with syncline output.''' +class LineNumberingTest(unittest.TestCase): + '''Container for tests with line numbering directives.''' pass @@ -1038,8 +1083,9 @@ def test_output_method(args, inp, out): def test_output(self): '''Tests whether Fypp result matches expected output.''' - - tool = fypp.Fypp(args) + options = fypp.FyppOptions() + argparser = fypp.get_option_parser() + tool = fypp.Fypp(argparser.parse_args(args, namespace=options)) result = tool.process_text(inp) self.assertEqual(out, result) return test_output @@ -1062,8 +1108,10 @@ def test_exception_method(args, inp, exc, fname, span): def test_exception(self): '''Tests whether Fypp throws the correct exception.''' + options = fypp.FyppOptions() + argparser = fypp.get_option_parser() with self.assertRaises(exc) as ctx: - tool = fypp.Fypp(args) + tool = fypp.Fypp(argparser.parse_args(args, namespace=options)) _ = tool.process_text(inp) raised = ctx.exception if fname is None: @@ -1095,7 +1143,7 @@ def add_test_methods(tests, testcase, methodfactory): add_test_methods(SIMPLE_TESTS, SimpleTest, test_output_method) -add_test_methods(SYNCLINE_TESTS, SynclineTest, test_output_method) +add_test_methods(LINENUM_TESTS, LineNumberingTest, test_output_method) add_test_methods(INCLUDE_TESTS, IncludeTest, test_output_method) add_test_methods(EXCEPTION_TESTS, ExceptionTest, test_exception_method)