Proto api

.devcontainer/Dockerfile

@@ -0,0 +1,17 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.134.1/containers/python-3/.devcontainer/base.Dockerfile
+ARG VARIANT="3"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update \
+#     && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends \
+#         pandoc \
+#         texlive-xetex \
+#         texlive-fonts-recommended \
+#         texlive-plain-generic

.devcontainer/devcontainer.json

@@ -0,0 +1,49 @@
+// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.134.1/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		// Update 'VARIANT' to pick a Python version: 3, 3.6, 3.7, 3.8 
+		"args": {
+			"VARIANT": "3"
+		}
+	},
+	// Set *default* container specific settings.json values on container create.
+	"settings": {
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"python.pythonPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+		"python.formatting.provider": "black",
+		"editor.formatOnSave": true,
+		"python.dataScience.alwaysTrustNotebooks": true,
+		"jupyter.alwaysTrustNotebooks": true,
+		"jupyter.textOutputLimit": 0
+	},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-azuretools.vscode-docker",
+		"github.copilot",
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [
+		8888,
+		8050
+	],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "pip3 install --user -r requirements.txt",
+	// Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root.
+	// "remoteUser": "vscode"
+}

.gitattributes

@@ -0,0 +1,2 @@
+tmee/data_out/data_transformed/TMEE_ETL_out.csv filter=lfs diff=lfs merge=lfs -text
+TMEE_ETL_out.csv filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +1,5 @@
 
+
 # Created by https://www.toptal.com/developers/gitignore/api/windows,jupyternotebooks,microsoftoffice
 # Edit at https://www.toptal.com/developers/gitignore?templates=windows,jupyternotebooks,microsoftoffice
 
@@ -11,6 +12,41 @@
 
 # python cache (created by imports executed in notebooks)
 __pycache__
+*/__pycache__/*
+*.pyc
+
+# Debug file used for temporary debugging
+debug_file.py
+india_covid_dash.py
+india_covid_dash_.py
+EDA_covid19_trends.py
+EDA_covid19_v0.py
+ETL_steps*.py
+ETL_steps.txt
+EDA_ICO.py
+main_extract_transform_*.py
+sitan_extract.py
+query_keys_SDMX.py
+test_imports.py
+test_nb.py
+test_nb2.py
+
+# Gecko driver log file
+*.log
+
+# exclude all archive folders
+z_archive
+data/*/z_archive
+
+# Exclude all data output by the ETL script (commented by Alberto below)
+# data_out/
+# (Alberto: treated for tmee/data_out end of file)
+
+# this is legacy, because the data folder has been renamed to data_out
+data/*
+
+# exclude folder created by visual studio 2019
+#.vs/
 
 # IPython
 profile_default/
@@ -67,3 +103,142 @@ $RECYCLE.BIN/
 *.lnk
 
 # End of https://www.toptal.com/developers/gitignore/api/windows,jupyternotebooks,microsoftoffice
+data/data_normalized/S_101_normalized.xlsx
+data/data_normalized/S_101_normalized.xlsx
+data/data_raw/S-155, S-156 nrgi_data.csv
+data/data_raw/S-166 Registering Property.xlsx
+data/data_raw/S-167 Pct_IP_CommunityLands.zip
+data/data_raw/S-189 idmc_displacement_all_dataset.xlsx
+data/data_raw/S-3-Ratifications_20200828165211.xls
+data/data_raw/S-37, S-38, S-39.xlsx
+data/data_raw/S-49 WORLD_Dataset_Poverty_3.9.15_0/WORLD Poverty Public Use Data Dictionary.pdf
+data/data_raw/S-49 WORLD_Dataset_Poverty_3.9.15_0/WORLD_Dataset_Poverty_3.9.15.dta
+data/data_raw/S-49 WORLD_Dataset_Poverty_3.9.15_0/WORLD_Dataset_Poverty_3.9.15.xls
+data/data_raw/S-59-Ratifications_20200828165032.xls
+data/data_raw/S-59-Ratifications_20200828165128.xls
+data/data_raw/S-59-Ratifications_20200828165147.xls
+data/data_raw/S_10, S_13, S_36, S_45/Childhood Public Use Data Dictionary_4.16.15.pdf
+data/data_raw/S_10, S_13, S_36, S_45/WORLD_Dataset_Childhood_4.16.15.dta
+data/data_raw/S_10, S_13, S_36, S_45/WORLD_Dataset_Childhood_4.16.15.xls
+data/data_raw/S_10, S_13, S_36, S_45/WORLD_Dataset_Childhood_4.16.15.zip
+data/data_raw/S_40, S_41, S_63, S_64, S_65, S_66/WORLD Adult Labor Public Use Data Dictionary_9.17.2018.pdf
+data/data_raw/S_40, S_41, S_63, S_64, S_65, S_66/WORLD_Dataset_Adult_Labor_9.17.2018.dta
+data/data_raw/S_40, S_41, S_63, S_64, S_65, S_66/WORLD_Dataset_Adult_Labor_9.17.2018.xls
+data/data_raw/S_40, S_41, S_63, S_64, S_65, S_66/WORLD_Dataset_Adult_Labor_9.17.2018_0.zip
+data/data_raw/S_42, S_43, S_44/Workplace Discrimination Public Use Data Dictionary_13July20.pdf
+data/data_raw/S_42, S_43, S_44/WORLD_Dataset_Discrimination_at_Work_14July20_2.zip
+data/data_raw/S_42, S_43, S_44/WORLD_discrimination_at_work.dta
+data/data_raw/S_42, S_43, S_44/WORLD_discrimination_at_work.xls
+data/data_raw/S_8, S_9/All WOrld Policy Analysis Centre Data Catalogues.pdf
+data/data_raw/S_8, S_9/Child Labor Public Use Data Dictionary 10.28.19.pdf
+data/data_raw/S_8, S_9/WORLD_child_labor.dta
+data/data_raw/S_8, S_9/WORLD_child_labor.xls
+data/data_raw/S_8, S_9/WORLD_Dataset_Child_Labor_10.28.19.zip
+
+# ignore local VS code settings (this should be applied from dev container)
+settings.json
+
+# ignore folder tests
+tests
+
+# ignore from validation
+/tmee/validation/*.csv
+/tmee/validation/*.py
+/tmee/validation/ipynb_2_py
+
+# ignore python files with mapping dictionaries (written by ETL)
+# these dictionaries have been copied into codes_2_map.py
+seasons_map.py
+country_map.py
+country_names_map.py
+
+
+# Ignore noteboooks prototypes
+API_calls.ipynb
+column_mapper.ipynb
+proto_sqlite.ipynb
+prototype_API_Transform.ipynb
+wrap_API_calls.ipynb
+wrap_API_calls2.ipynb
+proto_extract_transform.ipynb
+excel_tmee.ipynb
+sdg_16_2_1.ipynb
+sitan_extract.ipynb
+sitan_invoice.ipynb
+EDA_covid19_trends.ipynb
+EDA_covid19_v0.ipynb
+ETL_steps*.ipynb
+main_legacy.ipynb
+main_legacy_proto.ipynb
+prototype_legacy.ipynb
+prototype_API.ipynb
+prototype_API_v2.ipynb
+prototype_transform.ipynb
+prototype_validation.ipynb
+main_extract_transform - Copy.ipynb
+test_imports.ipynb
+sharepoint_auth.ipynb
+test_pandera.ipynb
+try_sql_excel.ipynb
+validation_analytics.ipynb
+test_nb.ipynb
+main_validation - Copy.ipynb
+proto_who_imm_scrape.ipynb
+bdss_uis.ipynb
+bdss_uis_sdg_ger.ipynb
+bdss_uis_unicef.ipynb
+
+
+# ignore folder below (all excel files)
+populate_data_dictionary
+
+# ignore some versioning excel files
+indicator_dictionary_TM_v01.xlsx
+sitan_report*.xlsx
+cross_cut*.xlsx
+report_*.xlsx
+discarded_report.xlsx
+Full_TMEE_Database.xlsx
+styled.xlsx
+content_legacy.csv
+
+# ignore csv files from particular folders
+/tmee/data_out/data_raw/*.csv
+/tmee/data_out/data_transformed/*.csv
+# track from data_out: TMEE_ETL_out --> this allows to know how the DB is updated along history
+# commented: lack of git LFS space
+# !TMEE_ETL_out.csv
+# ignore xml files from particular folders
+/tmee/data_out/data_raw/*.xml
+# ignore json files from particular folders
+/tmee/data_out/data_raw/*.json
+# ignore excel files from codelist
+/tmee/data_out/codelists/*.xlsx
+# ignore csv files from codelist
+/tmee/data_out/codelists/*.csv
+# track from codelists: indicators, age and units
+!CL_TMEE_INDICATORS.csv
+!CL_TMEE_UNITS.csv
+!CL_TMEE_AGE.csv
+
+# ignore test folder population (india ICO project)
+/tmee/population/
+# ignore folder ba_sdmx (flavio project bosnia sdmx)
+/tmee/ba_sdmx/
+# ignore folder ebse_dev (ebse project kpis)
+/tmee/ebse/
+# ignore folder dash_pages (parse TM dashboard pages)
+/tmee/tm_dash/
+
+# Ignore history download folder
+noYreq
+extractTest
+loopTest
+older
+
+# Ignore data_raw, transformed and data_2_load folders in tmee folder
+/tmee/data_raw/
+/tmee/data_transformed/
+/tmee/data_2_load/
+
+# End of https://www.toptal.com/developers/gitignore/api/windows,jupyternotebooks,microsoftoffice

.vs/VSWorkspaceState.json

@@ -0,0 +1,12 @@
+{
+  "ExpandedNodes": [
+    "",
+    "\\tmee"
+  ],
+  "SelectedNode": [
+    "\\tmee\\prototype_API_Transform.ipynb",
+    "\\tmee\\main_extract_transform.ipynb",
+    "\\tmee\\validation\\test_pandera.ipynb"
+  ],
+  "PreviewInSolutionExplorer": false
+}

.vscode/.ropeproject/config.py

@@ -0,0 +1,114 @@
+# The default ``config.py``
+# flake8: noqa
+
+
+def set_prefs(prefs):
+    """This function is called before opening the project"""
+
+    # Specify which files and folders to ignore in the project.
+    # Changes to ignored resources are not added to the history and
+    # VCSs.  Also they are not returned in `Project.get_files()`.
+    # Note that ``?`` and ``*`` match all characters but slashes.
+    # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc'
+    # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc'
+    # '.svn': matches 'pkg/.svn' and all of its children
+    # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o'
+    # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o'
+    prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject',
+                                  '.hg', '.svn', '_svn', '.git', '.tox']
+
+    # Specifies which files should be considered python files.  It is
+    # useful when you have scripts inside your project.  Only files
+    # ending with ``.py`` are considered to be python files by
+    # default.
+    # prefs['python_files'] = ['*.py']
+
+    # Custom source folders:  By default rope searches the project
+    # for finding source folders (folders that should be searched
+    # for finding modules).  You can add paths to that list.  Note
+    # that rope guesses project source folders correctly most of the
+    # time; use this if you have any problems.
+    # The folders should be relative to project root and use '/' for
+    # separating folders regardless of the platform rope is running on.
+    # 'src/my_source_folder' for instance.
+    # prefs.add('source_folders', 'src')
+
+    # You can extend python path for looking up modules
+    # prefs.add('python_path', '~/python/')
+
+    # Should rope save object information or not.
+    prefs['save_objectdb'] = True
+    prefs['compress_objectdb'] = False
+
+    # If `True`, rope analyzes each module when it is being saved.
+    prefs['automatic_soa'] = True
+    # The depth of calls to follow in static object analysis
+    prefs['soa_followed_calls'] = 0
+
+    # If `False` when running modules or unit tests "dynamic object
+    # analysis" is turned off.  This makes them much faster.
+    prefs['perform_doa'] = True
+
+    # Rope can check the validity of its object DB when running.
+    prefs['validate_objectdb'] = True
+
+    # How many undos to hold?
+    prefs['max_history_items'] = 32
+
+    # Shows whether to save history across sessions.
+    prefs['save_history'] = True
+    prefs['compress_history'] = False
+
+    # Set the number spaces used for indenting.  According to
+    # :PEP:`8`, it is best to use 4 spaces.  Since most of rope's
+    # unit-tests use 4 spaces it is more reliable, too.
+    prefs['indent_size'] = 4
+
+    # Builtin and c-extension modules that are allowed to be imported
+    # and inspected by rope.
+    prefs['extension_modules'] = []
+
+    # Add all standard c-extensions to extension_modules list.
+    prefs['import_dynload_stdmods'] = True
+
+    # If `True` modules with syntax errors are considered to be empty.
+    # The default value is `False`; When `False` syntax errors raise
+    # `rope.base.exceptions.ModuleSyntaxError` exception.
+    prefs['ignore_syntax_errors'] = False
+
+    # If `True`, rope ignores unresolvable imports.  Otherwise, they
+    # appear in the importing namespace.
+    prefs['ignore_bad_imports'] = False
+
+    # If `True`, rope will insert new module imports as
+    # `from <package> import <module>` by default.
+    prefs['prefer_module_from_imports'] = False
+
+    # If `True`, rope will transform a comma list of imports into
+    # multiple separate import statements when organizing
+    # imports.
+    prefs['split_imports'] = False
+
+    # If `True`, rope will remove all top-level import statements and
+    # reinsert them at the top of the module when making changes.
+    prefs['pull_imports_to_top'] = True
+
+    # If `True`, rope will sort imports alphabetically by module name instead
+    # of alphabetically by import statement, with from imports after normal
+    # imports.
+    prefs['sort_imports_alphabetically'] = False
+
+    # Location of implementation of
+    # rope.base.oi.type_hinting.interfaces.ITypeHintingFactory In general
+    # case, you don't have to change this value, unless you're an rope expert.
+    # Change this value to inject you own implementations of interfaces
+    # listed in module rope.base.oi.type_hinting.providers.interfaces
+    # For example, you can add you own providers for Django Models, or disable
+    # the search type-hinting in a class hierarchy, etc.
+    prefs['type_hinting_factory'] = (
+        'rope.base.oi.type_hinting.factory.default_type_hinting_factory')
+
+
+def project_opened(project):
+    """This function is called after opening the project"""
+    # Do whatever you like here!