From dc9a4fb9176fc37368470f1bf44f0ba83079d137 Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Sun, 22 Oct 2023 11:43:12 -0600 Subject: [PATCH 1/9] fix: Add link to XLSForm design doc --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 347c025bf..216c4df77 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -89,6 +89,7 @@ nav: - External Data: about/externaldata.md - Data conflation: about/conflation.md - The Config File: about/configuring.md + - XLSForm Design: about/xlsforms.md # - odk2geojson: api/odk2geojson.md # - odk2csv: api/odk2csv.md # - odk2osm: api/odk2osm.md From 25b5a8ebec4d22052ad3213f13ecef4fcc54e819 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:38:06 +0100 Subject: [PATCH 2/9] [pre-commit.ci] pre-commit autoupdate (#208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/commitizen-tools/commitizen: 3.10.1 → 3.12.0](https://github.com/commitizen-tools/commitizen/compare/3.10.1...3.12.0) - [github.com/psf/black: 23.9.1 → 23.10.0](https://github.com/psf/black/compare/23.9.1...23.10.0) - [github.com/astral-sh/ruff-pre-commit: v0.0.292 → v0.1.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.292...v0.1.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4a57e9db..22f557201 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,21 @@ repos: # Versioning: Commit messages & changelog - repo: https://github.com/commitizen-tools/commitizen - rev: 3.10.1 + rev: 3.12.0 hooks: - id: commitizen stages: [commit-msg] # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black # Lint / autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.0.292" + rev: "v0.1.1" hooks: - id: ruff args: [--exit-non-zero-on-fix] From 58096f8eff23751eb417e66eed845db7ffd66f1e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Oct 2023 04:21:40 +0800 Subject: [PATCH 3/9] [pre-commit.ci] pre-commit autoupdate (#209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.10.0 → 23.10.1](https://github.com/psf/black/compare/23.10.0...23.10.1) - [github.com/astral-sh/ruff-pre-commit: v0.1.1 → v0.1.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.1...v0.1.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 22f557201..4fef43405 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,14 +8,14 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 23.10.0 + rev: 23.10.1 hooks: - id: black # Lint / autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.1.1" + rev: "v0.1.3" hooks: - id: ruff args: [--exit-non-zero-on-fix] From ec352b3965530c8289c1a319840e6cab54cca8e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:41:01 +0700 Subject: [PATCH 4/9] [pre-commit.ci] pre-commit autoupdate (#210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.3 → v0.1.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.3...v0.1.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4fef43405..a671a72e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: # Lint / autoformat: Python code - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: "v0.1.3" + rev: "v0.1.4" hooks: - id: ruff args: [--exit-non-zero-on-fix] From b6884081183bd80307c027187c96009fca33cb71 Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Wed, 8 Nov 2023 10:41:30 -0700 Subject: [PATCH 5/9] fix: Don't call OsmFile.footer, it's now handled by a destructor --- osm_fieldwork/json2osm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm_fieldwork/json2osm.py b/osm_fieldwork/json2osm.py index bf645c70b..1742bce6c 100755 --- a/osm_fieldwork/json2osm.py +++ b/osm_fieldwork/json2osm.py @@ -420,7 +420,7 @@ def json2osm(input_file, yaml_file=None): log.debug("Writing final OSM XML file...") jsonin.writeOSM(feature) - jsonin.finishOSM() + # jsonin.finishOSM() log.info(f"Wrote OSM XML file: {osmoutfile}") return osmoutfile From 7de0f6abf543e5e3bb226bf3e36826aa48c076ce Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Wed, 8 Nov 2023 11:08:08 -0700 Subject: [PATCH 6/9] fix: Use geojson instead of json to read in the boundary file --- osm_fieldwork/basemapper.py | 6 +++--- tests/test_basemap.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/osm_fieldwork/basemapper.py b/osm_fieldwork/basemapper.py index 79063a40c..3cf2c3841 100755 --- a/osm_fieldwork/basemapper.py +++ b/osm_fieldwork/basemapper.py @@ -20,7 +20,7 @@ import argparse import concurrent.futures -import json +import geojson import logging import queue import re @@ -292,10 +292,10 @@ def makeBbox( log.debug(f"Reading geojson file: {boundary}") with open(boundary, "r") as f: - poly = json.load(f) + poly = geojson.load(f) if "features" in poly: geometry = shape(poly["features"][0]["geometry"]) - if "geometry" in poly: + elif "geometry" in poly: geometry = shape(poly["geometry"]) else: geometry = shape(poly) diff --git a/tests/test_basemap.py b/tests/test_basemap.py index 1e291f8da..55dc8cd3c 100755 --- a/tests/test_basemap.py +++ b/tests/test_basemap.py @@ -29,7 +29,7 @@ log = logging.getLogger(__name__) rootdir = os.path.dirname(os.path.abspath(__file__)) -infile = f"{rootdir}/testdata/Rollinsville.geojson" +boundary = f"{rootdir}/testdata/Rollinsville.geojson" outfile = f"{rootdir}/testdata/rollinsville.mbtiles" base = "./tiles" # boundary = open(infile, "r") @@ -45,7 +45,7 @@ def test_create(): """See if the file got loaded.""" hits = 0 - basemap = BaseMapper(infile, base, "topo", False) + basemap = BaseMapper(boundary, base, "topo", False) tiles = list() for level in [8, 9, 10, 11, 12]: basemap.getTiles(level) @@ -65,6 +65,5 @@ def test_create(): assert hits == 2 - if __name__ == "__main__": test_create() From 40bd40cacd13ff2158b4bb6925cd65e64dce837a Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Wed, 8 Nov 2023 11:17:28 -0700 Subject: [PATCH 7/9] fix: Fix bug with referencing an out of scope variable --- osm_fieldwork/basemapper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/osm_fieldwork/basemapper.py b/osm_fieldwork/basemapper.py index 3cf2c3841..5e642582e 100755 --- a/osm_fieldwork/basemapper.py +++ b/osm_fieldwork/basemapper.py @@ -101,6 +101,7 @@ def dlthread( # Create the subdirectories as pySmartDL doesn't do it for us Path(dest).mkdir(parents=True, exist_ok=True) + dl = None try: if site["source"] == "topo": filespec += "." + site["suffix"] @@ -112,7 +113,10 @@ def dlthread( log.debug("%s exists!" % (outfile)) except Exception as e: log.error(e) - log.error("Couldn't download from %r: %s" % (filespec, dl.get_errors())) + if dl: + log.error(f"Couldn't download {filespec}: {dl.get_errors()}") + else: + log.error(f"Couldn't download {filespec}") class BaseMapper(object): From 6a1dee2dd26a0bc05594017995c2d0038f60b6f3 Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Thu, 9 Nov 2023 14:36:39 -0700 Subject: [PATCH 8/9] fix: Always escape the value for embedded quotes --- osm_fieldwork/osmfile.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/osm_fieldwork/osmfile.py b/osm_fieldwork/osmfile.py index 3490a493e..b18356bab 100755 --- a/osm_fieldwork/osmfile.py +++ b/osm_fieldwork/osmfile.py @@ -200,6 +200,7 @@ def createWay( for ref in way["refs"]: osm += '\n ' % ref + import epdb; epdb.st() if "tags" in way: for key, value in way["tags"].items(): if value is None: @@ -208,8 +209,9 @@ def createWay( continue if key not in attrs: newkey = escape(key) - osm += "\n " % (newkey, str(value)) - if modified: + newval = escape(str(value)) + osm += f"\n " + if modified and key != 'note': osm += '\n ' osm += "\n" @@ -296,8 +298,10 @@ def createNode( if not value: continue if key not in attrs: - osm += "\n " % (key, str(value)) - if modified: + newkey = escape(key) + newval = escape(str(value)) + osm += f"\n " + if modified and key != 'note': osm += '\n ' osm += "\n \n" else: From de309edfcfa3f220187db7eaf47ad3abc1640dbe Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Thu, 9 Nov 2023 14:40:15 -0700 Subject: [PATCH 9/9] fix: Make less verbose --- osm_fieldwork/json2osm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osm_fieldwork/json2osm.py b/osm_fieldwork/json2osm.py index 1742bce6c..02183794a 100755 --- a/osm_fieldwork/json2osm.py +++ b/osm_fieldwork/json2osm.py @@ -267,7 +267,7 @@ def parse( tags[key] = v total.append(tags) - log.debug(f"Finsished parsing JSON file {filespec}") + # log.debug(f"Finished parsing JSON file {filespec}") return total def createEntry( @@ -417,8 +417,8 @@ def json2osm(input_file, yaml_file=None): log.warning(f"Bad record! {feature}") continue # Skip bad records - log.debug("Writing final OSM XML file...") jsonin.writeOSM(feature) + # log.debug("Writing final OSM XML file...") # jsonin.finishOSM() log.info(f"Wrote OSM XML file: {osmoutfile}")