Refactor scanner in Py2.7-compatible form (temporarily) and enable full testing

.github/workflows/testing.yaml

@@ -1,22 +1,52 @@
 name: f90nml
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  build:
+  legacy:
+    strategy:
+      matrix:
+        python: ["2.7", "3.5", "3.6", "3.7"]
     runs-on: ubuntu-latest
+    container:
+      image: python:${{ matrix.python }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          python --version
+          pip --version
+          pip install --upgrade pip
+          pip install -r tests/requirements_test.txt
+          pip install pytest pytest-cov
+
+      - name: Run tests
+        run: pytest --cov --cov-branch --cov-report=xml
+
+      - name: Upload coverage to Codecov (bash uploader fallback)
+        if: matrix.python == '2.7' || matrix.python == '3.5'
+        run: |
+          curl -s https://codecov.io/bash | bash -s -- \
+            -t ${{ secrets.CODECOV_TOKEN }} \
+            -f coverage.xml \
+            -F python${{ matrix.python }} \
+            -Z
+
+      - name: Upload coverage to Codecov
+        if: matrix.python != '2.7' && matrix.python != '3.5'
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: marshallward/f90nml
+
+  modern:
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os:
-          - macos-latest
-          - windows-latest
-          - ubuntu-18.04
-          - ubuntu-20.04
-        python:
-          - "2.7"
-          - "3.5"
-          - "3.6"
-          - "3.7"
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version:
           - "3.8"
           - "3.9"
           - "3.10"
@@ -26,26 +56,24 @@ jobs:
           - "pypy-3.9"
           - "pypy-3.10"
     steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r tests/requirements_test.txt
-        pip install pytest pytest-cov
-
-    - name: Main test
-      shell: bash
-      run: |
-        pytest --cov --cov-branch --cov-report=xml
-
-    - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v5
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        slug: marshallward/f90nml
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/requirements_test.txt
+          pip install pytest pytest-cov
+
+      - name: Run tests
+        run: pytest --cov --cov-branch --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: marshallward/f90nml

f90nml/scanner.py

@@ -36,38 +36,46 @@ def notchar(chars, ref=charset):
 # DFA scanner
 
 M = {}
-M['start'] = (
-    {c: 'blank' for c in blank}
-    | {c: 'cmt' for c in comment_tokens}
-    | {c: 'name' for c in alpha + '_'}
-    | {c: 'num' for c in digit}
-    | {"'": 'str_a'}
-    | {'"': 'str_q'}
-    | {'.': 'dec'}
-    | {'+': 'op_plus'}
-    | {'-': 'op_minus'}
-    | {c: 'op' for c in notchar('+-."\'' + comment_tokens, special)}
-)
+
+M['start'] = {}
+for d in (
+    {c: 'blank' for c in blank},
+    {c: 'cmt' for c in comment_tokens},
+    {c: 'name' for c in alpha + '_'},
+    {c: 'num' for c in digit},
+    {"'": 'str_a'},
+    {'"': 'str_q'},
+    {'.': 'dec'},
+    {'+': 'op_plus'},
+    {'-': 'op_minus'},
+    {c: 'op' for c in notchar('+-."\'' + comment_tokens, special)},
+):
+    M['start'].update(d)
 
 # Blank (whitespace) tokens
 
-M['blank'] = (
-    {c: 'blank' for c in blank}
-    | {c: 'cmt' for c in comment_tokens}
-    | {c: 'end' for c in notchar(blank + comment_tokens)}
-)
+M['blank'] = {}
+for d in (
+    {c: 'blank' for c in blank},
+    {c: 'cmt' for c in comment_tokens},
+    {c: 'end' for c in notchar(blank + comment_tokens)},
+):
+    M['blank'].update(d)
 
 # This doesn't actually get used more than once, but it is correct.
-M['cmt'] = (
-    {c: 'cmt' for c in notchar('\n')}
-    | {'\n': 'end'}
-)
+M['cmt'] = {c: 'cmt' for c in notchar('\n')}
+M['cmt']['\n'] = 'end'
 
 
 # Identifiers (keywords, functions, variables, ...)
 # NOTE: We permit identifiers to start with _ for preprocessor support
-M['name'] = {c: 'name' for c in alnum}
-M['name'] |= {c: 'end' for c in notchar(alnum)}
+M['name'] = {}
+for d in (
+    {c: 'name' for c in alnum},
+    {c: 'end' for c in notchar(alnum)},
+):
+    M['name'].update(d)
+
 if non_delimited_strings:
     M['name']["'"] = 'name'
     M['name']['"'] = 'name'
@@ -91,116 +99,135 @@ def notchar(chars, ref=charset):
 
 # Literal numeric
 # NOTE: Decimals must be separate due to logicals (.true./.false.)
-M['num'] = (
-    {c: 'num' for c in digit}
-    | {c: 'num_float_e' for c in 'eEdD'}
-    | {'.': 'num_frac'}
-    | {c: 'num_float_sign' for c in '+-'}
-    | {'_': 'num_kind'}
-    | {c: 'end' for c in notchar(digit + '+-._eEdD')}
-)
-
-M['num_frac'] = (
-    {c: 'num_frac' for c in digit}
-    | {c: 'num_float_e' for c in 'eEdD'}
-    | {c: 'num_float_sign' for c in '+-'}
-    | {'_': 'num_kind'}
-    | {c: 'end' for c in notchar(digit + '+-_eEdD')}
-)
+M['num'] = {}
+for d in (
+    {c: 'num' for c in digit},
+    {c: 'num_float_e' for c in 'eEdD'},
+    {'.': 'num_frac'},
+    {c: 'num_float_sign' for c in '+-'},
+    {'_': 'num_kind'},
+    {c: 'end' for c in notchar(digit + '+-._eEdD')},
+):
+    M['num'].update(d)
+
+M['num_frac'] = {}
+for d in (
+    {c: 'num_frac' for c in digit},
+    {c: 'num_float_e' for c in 'eEdD'},
+    {c: 'num_float_sign' for c in '+-'},
+    {'_': 'num_kind'},
+    {c: 'end' for c in notchar(digit + '+-_eEdD')},
+):
+    M['num_frac'].update(d)
 
 # Numeric E notation token
-M['num_float_e'] = (
-    {c: 'num_float_sign' for c in '+-'}
-    | {c: 'num_float_exp' for c in digit}
+M['num_float_e'] = {}
+for d in (
+    {c: 'num_float_sign' for c in '+-'},
+    {c: 'num_float_exp' for c in digit},
     # Error: ^[0-9+-]
-)
+):
+    M['num_float_e'].update(d)
 
 # Numeric E notation exponent sign
-M['num_float_sign'] = (
-    {c: 'num_float_exp' for c in digit}
+M['num_float_sign'] = {c: 'num_float_exp' for c in digit}
     # Error: ^[0-9]
-)
 
 # Numeric E notation exponent
-M['num_float_exp'] = (
-    {c: 'num_float_exp' for c in digit}
-    | {'_': 'num_kind'}
-    | {c: 'end' for c in notchar(digit + '_')}
-)
+M['num_float_exp'] = {}
+for d in (
+    {c: 'num_float_exp' for c in digit},
+    {'_': 'num_kind'},
+    {c: 'end' for c in notchar(digit + '_')},
+):
+    M['num_float_exp'].update(d)
 
 # Numeric kind token (_)
-M['num_kind'] = (
-    {c: 'num_kind_name' for c in alpha}
-    | {c: 'num_kind_int' for c in digit}
-)
+M['num_kind'] = {}
+for d in (
+    {c: 'num_kind_name' for c in alpha},
+    {c: 'num_kind_int' for c in digit},
+):
+    M['num_kind'].update(d)
 
 # Numeric kind as a variable name
 # NOTE: This is identical to name, but might be useful for tokenization
-M['num_kind_name'] = (
-    {c: 'num_kind_name' for c in alnum}
-    | {c: 'end' for c in notchar(alnum)}
-)
+M['num_kind_name'] = {}
+for d in (
+    {c: 'num_kind_name' for c in alnum},
+    {c: 'end' for c in notchar(alnum)},
+):
+    M['num_kind_name'].update(d)
 
 # Numeric kind as coded integer
 # XXX: Why is this alnum?  Shouldn't it be digit?
-M['num_kind_int'] = (
-    {c: 'num_kind_int' for c in alnum}
-    | {c: 'end' for c in notchar(alnum)}
-)
-
+M['num_kind_int'] = {}
+for d in (
+    {c: 'num_kind_int' for c in alnum},
+    {c: 'end' for c in notchar(alnum)},
+):
+    M['num_kind_int'].update(d)
 
 # ----
 # Old numeric stuff.. not sure how it holds up
 
 # Decimal mark
 # TODO: Fix me!  This only represents the leading decimal mark.
-M['dec'] = (
-    {c: 'num' for c in digit}
-    | {'_': 'num_kind'}
-    | {c: 'op_keyword' for c in notchar('eEdD', ref=alpha)}
-    | {c: 'op_kw_test' for c in 'eEdD'}
-    | {c: 'end' for c in notchar(digit + alpha + '_')}
-)
+M['dec'] = {}
+for d in (
+    {c: 'num' for c in digit},
+    {'_': 'num_kind'},
+    {c: 'op_keyword' for c in notchar('eEdD', ref=alpha)},
+    {c: 'op_kw_test' for c in 'eEdD'},
+    {c: 'end' for c in notchar(digit + alpha + '_')},
+):
+    M['dec'].update(d)
 
 # TODO: These permit "+." and "-." which are not valid!
 # TODO: "name" is only handled for +-inf and +-nan.  It could be tightened but
 #   the DFA will be unpretty.
 
-M['op_plus'] = (
-    {c: 'num' for c in digit}
-    | {'.': 'num_frac'}
-    | {c: 'name' for c in alpha}
-)
-
-M['op_minus'] = (
-    {c: 'num' for c in digit}
-    | {'.': 'num_frac'}
-    | {c: 'name' for c in alpha}
-)
-
-M['op_kw_test'] = (
-    {c: 'op_keyword' for c in alpha}
-    | {c: 'num_float_sign' for c in '+-'}
-    | {c: 'num_float_exp' for c in digit}
+M['op_plus'] = {}
+for d in (
+    {c: 'num' for c in digit},
+    {'.': 'num_frac'},
+    {c: 'name' for c in alpha},
+):
+    M['op_plus'].update(d)
+
+M['op_minus'] = {}
+for d in (
+    {c: 'num' for c in digit},
+    {'.': 'num_frac'},
+    {c: 'name' for c in alpha},
+):
+    M['op_minus'].update(d)
+
+M['op_kw_test'] = {}
+for d in (
+    {c: 'op_keyword' for c in alpha},
+    {c: 'num_float_sign' for c in '+-'},
+    {c: 'num_float_exp' for c in digit},
     # Error: ^[a-z0-9+-]
-)
+):
+    M['op_kw_test'].update(d)
 
 # End decimal
 #---
 
 
 # Single-character tokens (operators, declaration, etc)
-M['op'] = (
-    {c: 'end' for c in charset}
-)
+M['op'] = {c: 'end' for c in charset}
 
 
 # TODO: We don't have keyword operators, just .true. and .false. values.
-M['op_keyword'] = (
-    {c: 'op_keyword' for c in alpha}
-    | {'.': 'op'}
-    | {c: 'end' for c in notchar(alpha + '.')}
-)
+M['op_keyword'] = {}
+for d in (
+    {c: 'op_keyword' for c in alpha},
+    {'.': 'op'},
+    {c: 'end' for c in notchar(alpha + '.')},
+):
+    M['op_keyword'].update(d)
 
 
 def scan(file):