de-duplication

.gitignore

@@ -0,0 +1,155 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+.idea

AOI_based_ipf_submitter.py

@@ -121,6 +121,11 @@ def submit_ipf_scraper(acq, tag, endpoint):
             "from": "value",
             "value": endpoint
         },
+        {
+            "name": "force",
+            "from": "value",
+            "value": False
+        },
         {
             "name": "ds_cfg",
             "from": "value",
@@ -164,7 +169,7 @@ def submit_ipf_scraper(acq, tag, endpoint):
         print("Date:" + acq.get("metadata").get("sensingStart"))
         acq_date = acq.get("metadata").get("sensingStart")
         start_time = dateutil.parser.parse(acq_date)
-        if start_time.replace(tzinfo=None) < datetime.now() - timedelta(days=1):
+        if start_time.replace(tzinfo=None) < datetime.now() - timedelta(days=2):
             endpoint = "asf"
         else:
             endpoint = "scihub"

deprecate_acquisition.py

@@ -0,0 +1,47 @@
+import elasticsearch
+from hysds.celery import app
+
+es_url = app.conf["GRQ_ES_URL"]
+# _index = None
+# _type = None
+
+ES = elasticsearch.Elasticsearch(es_url)
+
+
+def deprecate_document(index, _id):
+    """
+    Update the ES document with new information
+    :param index: name of elasticsearch index
+    :param _id: id of product delivered to ASF
+    :param delivery_time: delivery time to ASF to stamp to delivered product (can we delete this note)
+    :param ingest_time: ingestion time to ASF to stamp to delivered product (can we delete this note)
+    :param delivery_status: status of delivery to stamp to delivered product (can we delete this note)
+    :param product_tagging:
+    :return:
+    """
+    '''
+
+    Note: borrowed from user_tags
+    @param product_id - 
+    @param delivery_time - 
+    '''
+
+    new_doc = dict()
+    doc = dict()
+    metadata = dict()
+    metadata["tags"] = "deprecated"
+    doc["metadata"] = metadata
+    new_doc["doc"] = doc
+
+    ES.update(index=index, doc_type="acquisition-S1-IW_SLC", id=_id, body=new_doc)
+    return True
+
+
+if __name__ == "__main__":
+    '''
+    Main program that find IPF version for acquisition
+    '''
+    # txt = open("deprecate_acq.txt", "r")
+    # for acq in txt:
+    #     acq_id = acq.strip()
+    #     update_document(_id=acq_id)

docker/hysds-io.json.ipf-scraper-asf

@@ -23,6 +23,11 @@
         "from": "value",
         "value": "asf"
     },
+    {
+        "name": "force",
+        "from": "submitter",
+        "type": "boolean"
+    },
     {
         "name": "ds_cfg",
         "from": "value",

docker/hysds-io.json.ipf-scraper-scihub

@@ -23,6 +23,11 @@
         "from": "value",
         "value": "scihub"
     },
+    {
+        "name": "force",
+        "from": "submitter",
+        "type": "boolean"
+    },
     {
         "name": "ds_cfg",
         "from": "value",

docker/job-spec.json.ipf-scraper-asf

@@ -31,6 +31,10 @@
         "destination": "context"
 
     },
+    {
+        "name": "force",
+        "destination": "context"
+    },
     {
         "name": "ds_cfg",
         "destination": "positional"

docker/job-spec.json.ipf-scraper-scihub

@@ -31,6 +31,10 @@
         "destination": "context"
 
     },
+    {
+        "name": "force",
+        "destination": "context"
+    },
     {
         "name": "ds_cfg",
         "destination": "positional"