diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f4c265..32f2682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project attempts to match the major and minor versions of [stactools](https://github.com/stac-utils/stactools) and increments the patch number as needed. +## [Unreleased] + +### Added + +- Support for NAIP 2022 + +### Changed + +- Updated repository structure to match [stactools](https://github.com/stac-utils/stactools) + ## [v0.4.0] - 2023-05-18 ### Added diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev index 1b4acdb..5ee0a80 100644 --- a/docker/Dockerfile-dev +++ b/docker/Dockerfile-dev @@ -3,5 +3,6 @@ FROM stactools-naip:latest RUN conda install -c conda-forge pandoc COPY . /src/stactools-naip -RUN pip install -r /src/stactools-naip/requirements-dev.txt WORKDIR /src/stactools-naip +RUN pip install .[dev] + diff --git a/pyproject.toml b/pyproject.toml index c5d1fe6..638f295 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,42 @@ +[project] +name = "stactools-naip" +dynamic = ["version"] +description = "STAC item creation tools for NAIP dataset." +readme = "README.md" +authors = [{name = "stac-utils", email = "stac@radiant.earth"}] +keywords = ["stactools", "pystac", "catalog", "STAC"] +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" +] +requires-python = ">=3.8" +dependencies = [ + "stactools>=0.4.3", + "pystac>=1.5" +] + +[tool.setuptools.dynamic] +version = { attr = "stactools.naip.__version__" } + +[project.urls] +homepage = "https://github.com/stactools-naips/stactools-naip" +documentation = "https://stactools-naip.readthedocs.io/en/latest/" + +[project.optional-dependencies] +dev = [ + "black~=23.3", + "codespell~=2.2", + "coverage~=7.4.0", + "pre-commit~=3.3", + "flake8~=7.0.0", + "pytest-cov~=3.0", + "pytest~=7.3", +] + [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 61e0943..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -black -codespell -coverage -flake8 -pre-commit -pytest -pytest-cov \ No newline at end of file diff --git a/scripts/update b/scripts/update index 74b4e6b..405509b 100755 --- a/scripts/update +++ b/scripts/update @@ -18,7 +18,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then usage else python -m pip install --upgrade pip - pip install . --no-binary rasterio - pip install -r requirements-dev.txt + pip install -e '.[dev]' fi fi diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index b89e2f6..0000000 --- a/setup.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[metadata] -name = stactools-naip -version = attr: stactools.naip.__version__ -description = PROVIDE DESCRIPTION HERE -long_description = file: README.md -long_description_content_type = text/markdown -author = stac-utils -author_email = stac@radiant.earth -url = https://github.com/stactools-naips/stactools-naip -project_urls = - Documentation = https://stactools-naip.readthedocs.io/en/latest/ - Issues = https://github.com/stactools-naips/stactools-naip/issues -keywords = - stactools - pystac - catalog - STAC -classifiers = - Development Status :: 4 - Beta - License :: OSI Approved :: Apache Software License - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - -[options] -python_requires = >=3.8 -package_dir = - =src -packages = find_namespace: -install_requires = - stactools >= 0.4.3 - pystac >= 1.5 # for GridExtension class -[options.packages.find] -where = src diff --git a/src/stactools/naip/stac.py b/src/stactools/naip/stac.py index b46858b..4a2d43c 100644 --- a/src/stactools/naip/stac.py +++ b/src/stactools/naip/stac.py @@ -146,7 +146,25 @@ def create_item( ) if fgdc_metadata_href: - if year in ["2020", "2021"]: + if year < "2020": + fgdc_metadata_text = read_text(fgdc_metadata_href) + fgdc = parse_fgdc_metadata(fgdc_metadata_text) + try: + resource_desc = fgdc["Distribution_Information"]["Resource_Description"] + dt = str_to_datetime( + fgdc["Identification_Information"]["Time_Period_of_Content"][ + "Time_Period_Information" + ]["Single_Date/Time"]["Calendar_Date"] + ) + except KeyError: + res = maybe_extract_id_and_date(cog_href) + if res is not None: + resource_desc, dt = res + else: + raise Exception( + f"Failed to extract item resource_desc and dt: {cog_href}" + ) + else: first_xpath = "gmd:fileIdentifier/gco:CharacterString" second_xpath = "idinfo/citation/citeinfo/title" @@ -171,28 +189,6 @@ def create_item( raise Exception( f"Failed to extract item resource_desc and dt: {cog_href}" ) - - elif year < "2020": - fgdc_metadata_text = read_text(fgdc_metadata_href) - fgdc = parse_fgdc_metadata(fgdc_metadata_text) - try: - resource_desc = fgdc["Distribution_Information"]["Resource_Description"] - dt = str_to_datetime( - fgdc["Identification_Information"]["Time_Period_of_Content"][ - "Time_Period_Information" - ]["Single_Date/Time"]["Calendar_Date"] - ) - except KeyError: - res = maybe_extract_id_and_date(cog_href) - if res is not None: - resource_desc, dt = res - else: - raise Exception( - f"Failed to extract item resource_desc and dt: {cog_href}" - ) - else: - raise Exception(f"Metadata for year {year} is not supported.") - else: res = maybe_extract_id_and_date(cog_href) if res is not None: diff --git a/tests/data-files/m_4112001_sw_10_060_20220716.tif b/tests/data-files/m_4112001_sw_10_060_20220716.tif new file mode 100644 index 0000000..1b92658 Binary files /dev/null and b/tests/data-files/m_4112001_sw_10_060_20220716.tif differ diff --git a/tests/data-files/m_4112001_sw_10_060_20220716.xml b/tests/data-files/m_4112001_sw_10_060_20220716.xml new file mode 100755 index 0000000..9a11bfb --- /dev/null +++ b/tests/data-files/m_4112001_sw_10_060_20220716.xml @@ -0,0 +1,515 @@ + + + + m_4112001_sw_10_060_20220716 + + + eng; USA + + + utf8 + + + dataset + + + + + United States Department of Agriculture, Farm Production and Conservation Business Center, Geospatial Enterprise Operations (FPAC-BC-GEO) + + + + + + + 801-844-2922 + + + Nothing + + + + + + + 125 S. State Street Suite 6416 + + + Salt Lake City + + + Utah + + + 84138 + + + USA + + + GEO.Sales@usda.gov + + + + + + + pointOfContact + + + + + 2022-09-07 + + + + + 2 + + + + + column + + + 9590 + + + 0.6 + + + + + + + row + + + 12450 + + + 0.6 + + + + + point + + + 0 + + + + + + + + + + + D_North_American_1983/UTM zone 10 + + + + + 2008-11-12 + + + publication + + + + + + + European Petroleum Survey Group + + + + + + + http://www.epsg-registry.org/export.htm?gml=urn:ogc:def:crs:EPSG::26910 + + + + + + + + + + + + urn:ogc:def:crs:EPSG:26910 + + + + + + + + + + + + + m_4112001_sw_10_060_20220716.tif + + + + + 2022-09-07 + + + publication + + + + + + + Hexagon - Leica Geosystems + + + originator + + + + + + + United States Department of Agriculture, Farm Production and Conservation Business Center, Geospatial Enterprise Operations (FPAC-BC-GEO) + + + owner + + + + + + + This four band orthorectified geotiff was flown for the National Agriculture Imagery Program (NAIP). The imagery has a minimum 60cm GSD, and is 8-bits per band, with tonal ranges per pixel between 0 - 255. + + + The National Agriculture Imagery Program (NAIP) is a multi-agency funded, state-based collection program that acquires high resolution, orthorectified imagery during peak agricultural growing seasons. The primary product is the digital orthophoto quarter quadrangle (DOQQ) file. + + + + + United States Department of Agriculture, Farm Production and Conservation Business Center, Geospatial Enterprise Operations (FPAC-BC-GEO) + + + + + + + 801-844-2922 + + + + + + + 125 S. State Street Suite 6416 + + + Salt Lake City + + + Utah + + + 84138 + + + USA + + + GEO.Sales@usda.gov + + + + + + + pointOfContact + + + + + + + National Agriculture Imagery Program (NAIP) + + + Farming + + + Digital Ortho Rectified Image + + + Ortho Rectification + + + theme + + + + + + + CA + + + Modoc + + + Modoc CO CA FSA + + + 06049 + + + place + + + + + Geographic Names Information System + + + + + + + + eng + + + Leica HxMap running on Microsoft Windows 10 + + + + + + + -121.004450 + + + -120.932985 + + + 41.872018 + + + 41.940483 + + + + + + + + + + + + + + + United States Department of Agriculture, Farm Production and Conservation Business Center, Geospatial Enterprise Operations (FPAC-BC-GEO) + + + + + + + 801-844-2922 + + + Nothing + + + + + + + 125 S. State Street Suite 6416 + + + Salt Lake City + + + Utah + + + 84138 + + + USA + + + GEO.Sales@usda.gov + + + + + + + distributor + + + + + + + + + + + + + coverage + + + + + + + Imagery was inspected by United States Department of Agriculture, Farm Production and Conservation Business Center, Geospatial Enterprise Operations (FPAC-BC-GEO) , visually and via custom computer scripts, against contract requirements and specifications. Final inspection reports may be obtained from GEO's QA department. + + + + + + + + + + + + Digital imagery was collected at a nominal GSD of 15cm using 6 Cessna 441's, 2 Reims-Cessna F406, 4 Piper PA31's, one Piper PAY2 and one Rockwell AC69 aircraft flying at an average flight height of 4470m AGL. Aircraft flew with Leica Geosystem's ContentMaper digital sensors with firmware 6.01. Each sensor collected 4 image bands Red, Green, Blue and Near-infrared in an integrated frame image with an across-track field of view 67.1 degrees. The ContentMapper spectral ranges are; Red 580-660nm, Green 480-590nm, Blue 420-510nm and Near-infrared at 720-850nm. The BSI CMOS arrays have a pixel size of 3.76 microns and deliver an integrated frame image with 40000x8200 format. The CMOS arrays have a dynamic range of 83db and the A/D converters have a resolution of 14bits. The ContentMapper is a frame-based sensor the ground footprint of the imagery is approximately 6km wide x 1.23 km high at a nominal 15cm GSD. The maximum flightline length is limited to approximately 130km. The factory calibrations and IMU alignments for each sensor (Serial Numbers: Hx5001, Hx5002, Hx5003, Hx5004, Hx5005, Hx5006, Hx5007, Hx5008, Hx5009, Hx5010) were tested and verified by in-situ test flights before the start of the project. The Leica MissionPro Flight Planning Software is used to develop the flight acquisition plans. Flight acquisition sub blocks are designed first to define the GNSS base station logistics, and to break the project up into manageable acquisition units. The flight acquisition sub blocks are designed based on the specified acquisition season, native UTM zone of the DOQQs, flight line length limitations (to ensure sufficient performance of the IMU solution) as well as air traffic restrictions in the area. Once the sub blocks have been delineated they are brought into MissionPro for flight line design. The design parameters used in MissionPro will be 20% lateral overlap and 15cm resolution. The flight lines have been designed with a north/south orientation or east/west where required for efficiency. The design takes into account the latitude of the state, which affects line spacing due to convergence as well as the terrain. SRTM elevation data is used in the MissionPro design to ensure the 50cm GSD is achieved over all types of terrain. The raw data was downloaded from the sensors after each flight using Leica HxMap software. The imagery was then georeferenced using the 500Hz GPS/INS data creating an exterior orientation for each frame image (x/y/z/o/p/k). Leica HxMap APM software was used to automatically generate tiepoint measurements between the frame images for each line and to tie all flight lines together. The resulting point data and exterior orientation data were used to perform a full bundle adjustment using HxMap Triangulation software. Blunders were removed, and additional tie points measured in weak areas to ensure a robust solution. Once the point data was clean and point coverage was acceptable, photo-identifiable GPS-surveyed ground control points were introduced into the block adjustment. The bundle adjustment process produces revised exterior orientation data for the sensor with GPS/INS, datum, and sensor calibration errors modeled and removed. Using the revised exterior orientation from the bundle adjustment, orthorectified image frames were created with HxMap software and the 2018 or newer HxIP DEM. The HxMap orthorectification software applies an atmospheric-BRDF radiometric correction to the imagery. This correction compensates for atmospheric absorption, solar illumination angle and bi-directional reflectance. The orthorectified frames were then overlaid with each other and the ground control to check accuracy. Once the accuracy of the orthorectified image strips were validated the strips were then imported into the HxMap Mosaic software which was used for the final radiometric balance, mosaic, and DOQQ sheet creation. The final DOQQ sheets, with a 300m buffer and a ground pixel resolution of 60cm were then combined and compressed to create the county wide CCMs. + + + + + United States Department of Agriculture, Farm Production and Conservation Business Center, Geospatial Enterprise Operations (FPAC-BC-GEO) + + + + + + + 801-844-2922 + + + Nothing + + + + + + + 125 S. State Street Suite 6416 + + + Salt Lake City + + + Utah + + + 84138 + + + USA + + + GEO.Sales@usda.gov + + + + + + + distributor + + + + + + + + + + + + + irregular + + + This metadata was automatically generated from the FGDC Content Standards for Digital Geospatial Metadata standard version FGDC-STD-001-1998 and FGDC-STD-012-2002 using the January 2013 version of the FGDC RSE to ISO 19115-2 transform. + + + + + + + + + The general scope of this task order is to procure vertical, high resolution, direct digital, 4-band aerial imagery in several formats for the inventory of natural resources for the National Agriculture Imagery Program (NAIP). The United States Department of Agriculture (USDA) shall use the imagery acquired under this task order to collect and measure natural resource data by means of photo interpretation and use of Geographic Information Systems (GIS) technologies. + + + + + General scope + + + + + completed + + + + + + + diff --git a/tests/test_stac.py b/tests/test_stac.py index 6c55884..2aadec5 100644 --- a/tests/test_stac.py +++ b/tests/test_stac.py @@ -144,3 +144,14 @@ def test_create_item_xml_ext(self): item.datetime, datetime.strptime("2020-04-15T16:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), ) + + def test_create_item_2022(self): + item = create_item( + "ca", + "2022", + test_data.get_path("data-files/m_4112001_sw_10_060_20220716.tif"), + test_data.get_path("data-files/m_4112001_sw_10_060_20220716.xml"), + ) + self.assertEqual(item.id, "ca_m_4112001_sw_10_060_20220716") + schemas = item.validate() + self.assertTrue(schemas)