Add code generator ABC and post operation

BenGalewsky · BenGalewsky · commit 7a9a38ced6c5 · 2022-06-15T15:48:21.000-05:00
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,39 @@
+name: CI/CD
+
+on:
+  push:
+    branches:
+      - "*"
+    tags:
+      - "*"
+
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        pip install --no-cache-dir ".[test]"
+        pip list
+    - name: Lint with Flake8
+      run: |
+        flake8
+    - name: Test with pytest
+      run: |
+        python -m coverage run -m pytest -r sx
+    - name: Report coverage with Codecov
+      if: github.event_name == 'push'
+      run: |
+        codecov --token=${{ secrets.CODECOV_TOKEN }}
diff --git a/servicex_codegen/__init__.py b/servicex_codegen/__init__.py
@@ -31,6 +31,44 @@
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
-#
-#
-#
+import os
+
+from flask import Flask
+from flask_restful import Api
+
+from servicex_codegen.post_operation import GeneratedCode
+
+
+def handle_invalid_usage(error: BaseException):
+    from flask import jsonify
+    response = jsonify({"message": str(error)})
+    response.status_code = 400
+    return response
+
+
+def create_app(test_config=None, provided_translator=None):
+    """Create and configure an instance of the Flask application."""
+    app = Flask(__name__, instance_relative_config=True)
+
+    # ensure the instance folder exists
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    if not test_config:
+        app.config.from_envvar('APP_CONFIG_FILE')
+    else:
+        app.config.from_mapping(test_config)
+
+    with app.app_context():
+        translator = provided_translator
+
+        api = Api(app)
+        GeneratedCode.make_api(translator)
+
+        api.add_resource(GeneratedCode, '/servicex/generated-code')
+
+    app.errorhandler(Exception)(handle_invalid_usage)
+
+    return app
diff --git a/servicex_codegen/code_generator.py b/servicex_codegen/code_generator.py
@@ -0,0 +1,88 @@
+#  Copyright (c) 2022 , IRIS-HEP
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+#   * Neither the name of the copyright holder nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+from abc import ABC, abstractmethod
+import os
+import zipfile
+from collections import namedtuple
+from tempfile import TemporaryDirectory
+
+GeneratedFileResult = namedtuple('GeneratedFileResult', 'hash output_dir')
+
+
+class GenerateCodeException(BaseException):
+    """Custom exception for top level code generation exceptions"""
+
+    def __init__(self, message: str):
+        BaseException.__init__(self, message)
+
+
+class CodeGenerator(ABC):
+    def zipdir(self, path: str, zip_handle: zipfile.ZipFile) -> None:
+        """Given a `path` to a directory, zip up its contents into a zip file.
+
+        Arguments:
+            path        Path to a local directory. The contents will be put into the zip file
+            zip_handle  The zip file handle to write into.
+        """
+        for root, _, files in os.walk(path):
+            for file in files:
+                zip_handle.write(os.path.join(root, file), file)
+
+    @abstractmethod
+    def generate_code(self,  query, cache_path: str):
+        pass
+
+    def translate_query_to_zip(self, query: str) -> bytes:
+        """Translate a text ast into a zip file as a memory stream
+
+        Arguments:
+            code            Text `qastle` version of the input ast generated by func_adl
+
+        Returns
+            bytes       Data that if written as a binary output would be a zip file.
+        """
+
+        # Generate the python code
+        with TemporaryDirectory() as tempdir:
+            r = self.generate_code(query, tempdir)
+
+            # Zip up everything in the directory - we are going to ship it as back as part
+            # of the message.
+            z_filename = os.path.join(str(tempdir), 'joined.zip')
+            zip_h = zipfile.ZipFile(z_filename, 'w', zipfile.ZIP_DEFLATED)
+            self.zipdir(r.output_dir, zip_h)
+            zip_h.close()
+
+            with open(z_filename, 'rb') as b_in:
+                return b_in.read()
diff --git a/servicex_codegen/post_operation.py b/servicex_codegen/post_operation.py
@@ -0,0 +1,60 @@
+#  Copyright (c) 2022 , IRIS-HEP
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+#   * Neither the name of the copyright holder nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+from flask import request, Response
+from flask_restful import Resource
+from servicex_codegen.code_generator import CodeGenerator
+
+
+class GeneratedCode(Resource):
+    @classmethod
+    def make_api(cls, code_generator: CodeGenerator):
+        cls.code_generator = code_generator
+        return cls
+
+    def post(self):
+        try:
+            code = request.data.decode('utf8')
+            zip_data = self.code_generator.generate_code(code)
+
+            # Send the response back to you-know-what.
+            response = Response(
+                response=zip_data,
+                status=200, mimetype='application/octet-stream')
+            return response
+        except BaseException as e:
+            print(str(e))
+            import traceback
+            import sys
+            traceback.print_exc(file=sys.stdout)
+            return {'Message': str(e)}, 500
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 99
diff --git a/setup.py b/setup.py
@@ -67,7 +67,13 @@
     install_requires=[
         "Flask==1.1.2",
         "Flask-RESTful==0.3.8",
-        "Flask-WTF==0.14.3"
+        "Flask-WTF==0.14.3",
+        # Incompatibility between flask and the latest itsdangerous
+        "itsdangerous==2.0.1",
+        # Avoid import error in Flask
+        "werkzeug==2.0.3",
+        "jinja2==3.0.3"
+
     ],
     extras_require={
         "test": [
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1,36 @@
+#  Copyright (c) 2022 , IRIS-HEP
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+#   * Neither the name of the copyright holder nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#
+#
+#
diff --git a/tests/test_code_generator.py b/tests/test_code_generator.py
@@ -0,0 +1,70 @@
+#  Copyright (c) 2022 , IRIS-HEP
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions are met:
+#
+#   * Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#
+#   * Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#
+#   * Neither the name of the copyright holder nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+import io
+import os
+import zipfile
+from tempfile import TemporaryDirectory
+
+from servicex_codegen.code_generator import CodeGenerator, GeneratedFileResult
+
+
+def get_zipfile_data(zip_data: bytes):
+    with zipfile.ZipFile(io.BytesIO(zip_data)) as thezip:
+        for zipinfo in thezip.infolist():
+            with thezip.open(zipinfo) as thefile:
+                yield zipinfo.filename, thefile
+
+
+def check_zip_file(zip_data: bytes, expected_file_count):
+    names = []
+    for name, data in get_zipfile_data(zip_data):
+        names.append(name)
+        print(name)
+    assert len(names) == expected_file_count
+
+
+class TestCodeGenerator:
+    def test_translate_query_to_zip(self, mocker):
+        mocker.patch.object(CodeGenerator, "__abstractmethods__", new_callable=set)
+        code_gen = CodeGenerator()
+
+        with TemporaryDirectory() as tempdir, \
+                open(os.path.join(tempdir, "baz.txt"), 'w'),\
+                open(os.path.join(tempdir, "foo.txt"), 'w'):
+            code_gen.generate_code = mocker.Mock(
+                return_value=GeneratedFileResult(hash="31415", output_dir=tempdir)
+            )
+
+            zip_bytes = code_gen.translate_query_to_zip("select * from foo")
+            check_zip_file(zip_bytes, expected_file_count=2)
diff --git a/tests/test_post_operation.py b/tests/test_post_operation.py
