diff --git a/nansat/__init__.py b/nansat/__init__.py index f04dd608..b99e72d4 100644 --- a/nansat/__init__.py +++ b/nansat/__init__.py @@ -15,10 +15,14 @@ # but WITHOUT ANY WARRANTY without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. from __future__ import absolute_import +import logging.config import os import sys +import os.path import warnings import importlib +import yaml + pixfun_module_name = 'nansat._pixfun_py{0}'.format(sys.version_info[0]) # check if pixel functions were compiled using setup_tools diff --git a/nansat/mappers/mapper_aapp_l1b.py b/nansat/mappers/mapper_aapp_l1b.py index 08d1f63c..34ee642f 100644 --- a/nansat/mappers/mapper_aapp_l1b.py +++ b/nansat/mappers/mapper_aapp_l1b.py @@ -10,11 +10,15 @@ import sys import struct import datetime +import logging from nansat.exceptions import WrongMapperError from nansat.geolocation import Geolocation from nansat.vrt import VRT +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) + satIDs = {4: 'NOAA-15', 2: 'NOAA-16', 6: 'NOAA-17', 7: 'NOAA-18', 8: 'NOAA-19', 11: 'Metop-B (Metop-1)', 12: 'Metop-A (Metop-2)', 13: 'Metop-C (Metop-3)'} @@ -66,8 +70,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): numCalibratedScanLines = int(struct.unpack(' 3B)') - print('###########################################') + LOGGER.warning('############################################') + LOGGER.warning('WARNING: channel 3 switches ') + LOGGER.warning('between daytime and nighttime (3A <-> 3B)') + LOGGER.warning('###########################################') ########################### # Make Geolocation Arrays diff --git a/nansat/mappers/mapper_geostationary.py b/nansat/mappers/mapper_geostationary.py index 6d8549a5..4ee2745f 100644 --- a/nansat/mappers/mapper_geostationary.py +++ b/nansat/mappers/mapper_geostationary.py @@ -9,11 +9,14 @@ from datetime import datetime from numpy import array, arange import warnings +import logging from nansat.exceptions import WrongMapperError from nansat.vrt import VRT from nansat.node import Node +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) def arrays2LUTString(a, b): LUTString = '' @@ -176,24 +179,24 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): for sat in satDict: if sat['name'] == satellite: - print('This is ' + satellite) + LOGGER.info(f'This is {satellite}') wavelengths = sat['wavelengths'] try: scale = sat['scale'] offset = sat['offset'] except: - print("No scale and offset found") + LOGGER.info("No scale and offset found") scale = None offset = None try: LUT = sat['LUT'] except: - print("No LUT found") + LOGGER.info("No LUT found") LUT = [""]*len(wavelengths) try: NODATA = sat['NODATA'] except: - print("No NODATA values found") + LOGGER.info("No NODATA values found") NODATA = [""]*len(wavelengths) if wavelengths is None: @@ -220,8 +223,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): try: gdal.Open(bandSource) except: - print('Warning: band missing for wavelength ' + - str(wavelength) + 'nm') + LOGGER.warning(f"band missing for wavelength {wavelength} nm") continue src = {'SourceFilename': bandSource, 'SourceBand': 1, 'LUT': LUT[i], 'NODATA': NODATA[i]} diff --git a/nansat/mappers/mapper_globcolour_l3b.py b/nansat/mappers/mapper_globcolour_l3b.py index 0897228c..4b244f56 100644 --- a/nansat/mappers/mapper_globcolour_l3b.py +++ b/nansat/mappers/mapper_globcolour_l3b.py @@ -11,6 +11,7 @@ import json import tempfile import shutil +import logging import numpy as np @@ -22,6 +23,8 @@ from nansat.vrt import VRT from nansat.mappers.globcolour import Globcolour +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) class Mapper(VRT, Globcolour): ''' Create VRT with mapping of WKV for MERIS Level 2 (FR or RR)''' @@ -73,7 +76,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, latlonGrid=None, self.band_vrts = {'mask': [], 'lonlat': []} mask = None for simFile in simFiles: - print('sim: ', simFile) + LOGGER.info(f'sim: {simFile}') # copy simFile to a temporary file tmpf = tempfile.mkstemp() shutil.copyfile(simFile, tmpf[1]) diff --git a/nansat/mappers/mapper_globcolour_l3m.py b/nansat/mappers/mapper_globcolour_l3m.py index 20756579..ecc9f0f6 100644 --- a/nansat/mappers/mapper_globcolour_l3m.py +++ b/nansat/mappers/mapper_globcolour_l3m.py @@ -9,6 +9,7 @@ import datetime import os.path import glob +import logging import numpy as np @@ -17,6 +18,8 @@ from nansat.utils import gdal, ogr from nansat.exceptions import WrongMapperError +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) class Mapper(VRT, Globcolour): """Mapper for GLOBCOLOR L3M products""" @@ -25,7 +28,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): ''' GLOBCOLOR L3M VRT ''' try: - print("=>%s<=" % gdalMetadata['NC_GLOBAL#title']) + LOGGER.info(f"=>{gdalMetadata['NC_GLOBAL#title']}<=") except (TypeError, KeyError): raise WrongMapperError @@ -35,15 +38,15 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): # get list of similar (same date) files in the directory iDir, iFile = os.path.split(filename) iFileName, iFileExt = os.path.splitext(iFile) - print('idir:', iDir, iFile, iFileName[0:30], iFileExt[0:8]) + LOGGER.info(f'idir: {iDir} {iFile} {iFileName[0:30]} {iFileExt[0:8]}') simFilesMask = os.path.join(iDir, iFileName[0:30] + '*.nc') simFiles = glob.glob(simFilesMask) - print('simFilesMask, simFiles', simFilesMask, simFiles) + LOGGER.info(f'simFilesMask:{simFilesMask}, simFiles:{simFiles}') metaDict = [] for simFile in simFiles: - print('simFile', simFile) + LOGGER.info(f'simFile:{simFile}') # open file, get metadata and get parameter name simSupDataset = gdal.Open(simFile) simSubDatasets = simSupDataset.GetSubDatasets() @@ -56,7 +59,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): simBandMetadata = simBand.GetMetadata() simVarname = simBandMetadata['NETCDF_VARNAME'] # get WKV - print(' simVarname', simVarname) + LOGGER.info(f' simVarname:{simVarname}') if simVarname in self.varname2wkv: simWKV = self.varname2wkv[simVarname] break @@ -93,17 +96,17 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): (simWavelength, solarIrradiance) } - print(' metaEntry', metaEntry) + LOGGER.info(f' metaEntry:{metaEntry}') metaDict.append(metaEntry) if metaEntry2 is not None: - print(' metaEntry2', metaEntry2) + LOGGER.info(f' metaEntry2:{metaEntry2}') metaDict.append(metaEntry2) - print('simSubDatasets', simValidSupDataset.GetSubDatasets()) + LOGGER.info(f'simSubDatasets:{simValidSupDataset.GetSubDatasets()}') for simSubDataset in simValidSupDataset.GetSubDatasets(): - print('simSubDataset', simSubDataset) + LOGGER.info(f'simSubDataset:{simSubDataset}') if '_flags ' in simSubDataset[1]: - print(' mask simSubDataset', simSubDataset[1]) + LOGGER.info(f' mask simSubDataset:{simSubDataset[1]}') flags = gdal.Open(simSubDataset[0]).ReadAsArray() mask = np.ones(flags.shape) * 64 mask[np.bitwise_and(flags, np.power(2, 0)) > 0] = 1 diff --git a/nansat/mappers/mapper_metno_local_hires_seaice.py b/nansat/mappers/mapper_metno_local_hires_seaice.py index 8eb7bb9c..a06ce6ea 100644 --- a/nansat/mappers/mapper_metno_local_hires_seaice.py +++ b/nansat/mappers/mapper_metno_local_hires_seaice.py @@ -18,12 +18,15 @@ import sys import os from datetime import datetime, timedelta +import logging from nansat.utils import gdal, ogr from nansat.exceptions import WrongMapperError from nansat.vrt import VRT import nansat.mappers.mapper_generic as mg +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) class Mapper(mg.Mapper): """Create VRT with mapping of WKV for Met.no seaice""" @@ -52,8 +55,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): filename = (ice_folder_name + 'ice_conc_svalbard_' + valid_time.strftime('%Y%m%d1500.nc')) if os.path.exists(filename): - print('Found file:') - print(filename) + LOGGER.info(f'Found file:{filename}') gdal_dataset = gdal.Open(filename) gdal_metadata = gdalDataset.GetMetadata() mg.Mapper.__init__(self, filename, gdal_dataset, gdal_metadata) diff --git a/nansat/mappers/mapper_ncep_wind_online.py b/nansat/mappers/mapper_ncep_wind_online.py index 860046c0..d630b537 100644 --- a/nansat/mappers/mapper_ncep_wind_online.py +++ b/nansat/mappers/mapper_ncep_wind_online.py @@ -20,11 +20,15 @@ import os import sys from datetime import datetime, timedelta +import logging from nansat.vrt import VRT from nansat.exceptions import WrongMapperError from nansat.nansat import Nansat +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) + # Place to store downloads - this can be changed via the "outFolder" argument # to Mapper.__init__ downloads = os.path.join(os.path.expanduser('~'), 'ncep_gfs_downloads') @@ -49,7 +53,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, time_str = filename[len(keyword_base)+1::] time = datetime.strptime(time_str, '%Y%m%d%H%M') - print(time) + LOGGER.info(f'{time}') ######################################## # Find and download online grib file @@ -86,13 +90,13 @@ def __init__(self, filename, gdalDataset, gdalMetadata, '%.2d' % forecast_hour + '.10m.uv.grib2')) if os.path.exists(out_filename): - print('NCEP wind is already downloaded: ' + out_filename) + LOGGER.info(f'NCEP wind is already downloaded: {out_filename}') else: os.system('curl -so ' + out_filename + ' ' + url) if os.path.exists(out_filename): - print('Downloaded ' + out_filename) + LOGGER.info(f'Downloaded {out_filename}') else: - print('NRT GRIB file not available: ' + url) + LOGGER.info(f'NRT GRIB file not available: {url}') ######################################################### # If NRT file not available, search in long term archive ######################################################### @@ -103,7 +107,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, '%.3d' % forecast_hour) filename = basename + '.grb2' out_filename = os.path.join(outFolder, filename) - print('Downloading ' + url + filename) + LOGGER.info(f'Downloading {url}{filename}') # Download subset of grib file mapper_dir = os.path.dirname(os.path.abspath(__file__)) @@ -121,9 +125,9 @@ def __init__(self, filename, gdalDataset, gdalMetadata, get_grib + ' ' + url + filename + ' ' + out_filename) os.system(command) if os.path.isfile(out_filename): - print('Downloaded ' + filename + ' to ' + outFolder) + LOGGER.info(f'Downloaded {filename} to {outFolder}') else: - print('Already downloaded %s' % out_filename) + LOGGER.info(f'Already downloaded {out_filename}') if not os.path.isfile(out_filename): sys.exit('No NCEP wind files found for requested time') diff --git a/nansat/mappers/mapper_obpg_l3.py b/nansat/mappers/mapper_obpg_l3.py index 85ffcb8f..014d9485 100644 --- a/nansat/mappers/mapper_obpg_l3.py +++ b/nansat/mappers/mapper_obpg_l3.py @@ -9,6 +9,7 @@ import datetime import os.path import glob +import logging import numpy as np @@ -18,6 +19,9 @@ from nansat.exceptions import WrongMapperError +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) + class Mapper(VRT): ''' Mapper for Level-3 Standard Mapped Image from http://oceancolor.gsfc.nasa.gov''' @@ -85,7 +89,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): tmpGdalMetadata = tmpGdalDataset.GetMetadata() simParameter = tmpGdalMetadata['Parameter'] except: - print('No parameter: %s not a supported SMI file') + LOGGER.info(f"No parameter: {tmpGdalMetadata['Parameter']} not a supported SMI file") continue else: # set params of the similar file diff --git a/nansat/mappers/mapper_ocean_productivity.py b/nansat/mappers/mapper_ocean_productivity.py index bbb0f82c..1a902c81 100644 --- a/nansat/mappers/mapper_ocean_productivity.py +++ b/nansat/mappers/mapper_ocean_productivity.py @@ -9,6 +9,7 @@ import datetime import os.path import glob +import logging import numpy as np @@ -17,6 +18,9 @@ from nansat.exceptions import WrongMapperError +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) + class Mapper(VRT): ''' Mapper for Ocean Productivity website http://www.science.oregonstate.edu/ocean.productivity/''' @@ -43,7 +47,7 @@ def __init__(self, filename, gdalDataset, gdalMetadata, **kwargs): except: raise WrongMapperError - print('Ocean Productivity website data') + LOGGER.info(f"Ocean Productivity website data") # get list of similar (same date) files in the directory iDir, iFile = os.path.split(filename) iFileName, iFileExt = os.path.splitext(iFile) diff --git a/nansat/mappers/mapper_radarsat2.py b/nansat/mappers/mapper_radarsat2.py index d5cd9022..80402c41 100644 --- a/nansat/mappers/mapper_radarsat2.py +++ b/nansat/mappers/mapper_radarsat2.py @@ -9,6 +9,7 @@ import zipfile import json from dateutil.parser import parse +import logging import numpy as np try: @@ -27,6 +28,9 @@ from nansat.utils import initial_bearing, gdal from nansat.exceptions import WrongMapperError, NansatReadError +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) + class Mapper(VRT): ''' Create VRT with mapping of WKV for Radarsat2 ''' @@ -169,7 +173,7 @@ def __init__(self, inputFileName, gdalDataset, gdalMetadata, sat_heading = initial_bearing(lon[:, 1:], lat[:, 1:], lon[:, :-1], lat[:, :-1]) + 90 else: - print('Can not decode pass direction: ' + str(passDirection)) + LOGGER.warning(f'Can not decode pass direction: {passDirection}') # Calculate SAR look direction look_direction = sat_heading + antennaPointing diff --git a/nansat/mappers/mapper_viirs_l1.py b/nansat/mappers/mapper_viirs_l1.py index c8d7f676..4875311e 100644 --- a/nansat/mappers/mapper_viirs_l1.py +++ b/nansat/mappers/mapper_viirs_l1.py @@ -10,6 +10,8 @@ import glob from datetime import datetime, timedelta from math import ceil +import logging + try: from scipy.ndimage.filters import gaussian_filter except: @@ -22,6 +24,8 @@ from nansat.utils import gdal, ogr from nansat.exceptions import WrongMapperError, NansatReadError +LOGGER = logging.getLogger("nansat.mappers."+__name__) +LOGGER.addHandler(logging.NullHandler()) class Mapper(VRT): ''' VRT with mapping of WKV for VIIRS Level 1B ''' @@ -53,14 +57,14 @@ def __init__(self, filename, gdalDataset, gdalMetadata, metaDict = [] for ifile in ifiles: ifilename = os.path.split(ifile)[1] - print(ifilename) + LOGGER.info(f"{ifilename}") bNumber = int(ifilename[3:5]) - print(bNumber) + LOGGER.info(f"{bNumber}") bWavelength = viirsWavelengths[bNumber] - print(bWavelength) + LOGGER.info(f"{bWavelength}") SourceFilename = ('HDF5:"%s"://All_Data/VIIRS-M%d-SDR_All/Radiance' % (ifile, bNumber)) - print(SourceFilename) + LOGGER.info(f"{SourceFilename}") metaEntry = {'src': {'SourceFilename': SourceFilename, 'SourceBand': 1}, 'dst': {'wkv': 'toa_outgoing_spectral_radiance', diff --git a/nansat/nansat.py b/nansat/nansat.py index 5042c72f..30f19cfd 100644 --- a/nansat/nansat.py +++ b/nansat/nansat.py @@ -156,7 +156,7 @@ def __init__(self, filename='', mapper='', log_level=30, **kwargs): self._init_empty(filename, log_level) # Create VRT object with mapping of variables - self.vrt = self._get_mapper(mapper, **kwargs) + self.vrt = self._get_mapper(mapper, log_level, **kwargs) def __getitem__(self, band_id): """Returns the band as a NumPy array, by overloading [] @@ -523,7 +523,7 @@ def list_bands(self, do_print=True): outString += ' %s: %s\n' % (i, bands[b][i]) if do_print: # print to screeen - print(outString) + self.logger.info(outString) else: return outString @@ -970,8 +970,8 @@ def write_geotiffimage(self, filename, band_id=1): outDataset.GetRasterBand(1).SetColorTable(colorTable) except: # Happens after reprojection, a possible bug? - print('Could not set color table') - print(colorTable) + self.logger.warning('Could not set color table %s' % colorTable) + outDataset = None self.vrt.copyproj(filename) @@ -1076,7 +1076,7 @@ def _get_dataset_metadata(self): return gdal_dataset, metadata - def _get_mapper(self, mappername, **kwargs): + def _get_mapper(self, mappername, log_level, **kwargs): """Create VRT file in memory (VSI-file) with variable mapping If mappername is given only this mapper will be used, @@ -1115,7 +1115,7 @@ def _get_mapper(self, mappername, **kwargs): # if nansat mappers were not imported yet global nansatMappers if nansatMappers is None: - nansatMappers = _import_mappers() + nansatMappers = _import_mappers(log_level) # open GDAL dataset. It will be parsed to all mappers for testing gdal_dataset, metadata = self._get_dataset_metadata() @@ -1585,7 +1585,7 @@ def _import_mappers(log_level=None): value: class Mapper(VRT) from the mappers module """ - logger = add_logger('import_mappers', logLevel=log_level) + logger = add_logger("nansat.import_mappers", logLevel=log_level) # import built-in mappers import nansat.mappers mapper_packages = [nansat.mappers] diff --git a/nansat/utils.py b/nansat/utils.py index b1cf3cb9..42e4f7d8 100644 --- a/nansat/utils.py +++ b/nansat/utils.py @@ -16,7 +16,7 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. from __future__ import absolute_import -import os +import os, sys import warnings import logging from dateutil.parser import parse @@ -178,7 +178,7 @@ def haversine(lon1, lat1, lon2, lat2): def add_logger(logName='', logLevel=None): - """ Creates and returns logger with default formatting for Nansat + """ Creates and returns logger with proper log levels Parameters ----------- @@ -205,11 +205,10 @@ def add_logger(logName='', logLevel=None): # otherwise create and add a new handler if len(logger.handlers) == 0: # create console handler and set level to debug - ch = logging.StreamHandler() + ch = logging.StreamHandler(sys.stdout) # create formatter - formatter = logging.Formatter('%(asctime)s|%(levelno)s|%(module)s|' - '%(funcName)s|%(message)s', - datefmt='%I:%M:%S') + formatter = logging.Formatter('%(asctime)s - %(name)s - %(threadName)s - ' + '%(levelname)s - %(message)s') # add formatter to ch ch.setFormatter(formatter) # add ch to logger