diff --git a/README.md b/README.md index a974491e..9f5f0696 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,6 @@ See [CONTRIBUTING.md](./docs/CONTRIBUTING.md) for information related to develop cellPACK uses AWS and Firebase Firestore as remote databases to store packing results and recipes. Follow [ setup instructions](./docs/REMOTE_DATABASES.md) for access. ## Docker -cellPACK can be run in Docker containers for both AWS ECS and AWS Batch. Follow the [instructions](./docs/DOCKER.md) to set up the Docker environment. +cellPACK can be run in Docker containers for running the server on AWS ECS. Follow the [instructions](./docs/DOCKER.md) to set up the Docker environment. **MIT license** \ No newline at end of file diff --git a/docker/Dockerfile.batch b/docker/Dockerfile.batch deleted file mode 100644 index 3fbec014..00000000 --- a/docker/Dockerfile.batch +++ /dev/null @@ -1,15 +0,0 @@ -### Build image ### -FROM python:3.9 - -WORKDIR /cellpack -COPY . /cellpack - -RUN python -m pip install --upgrade pip --root-user-action=ignore -RUN python -m pip install . --root-user-action=ignore - -# Install AWS CLI -RUN apt-get update && apt-get install -y awscli - -COPY docker/entrypoint-batch.sh /usr/local/bin/ -RUN ["chmod", "+x", "/usr/local/bin/entrypoint-batch.sh"] -ENTRYPOINT ["/usr/local/bin/entrypoint-batch.sh"] \ No newline at end of file diff --git a/docker/entrypoint-batch.sh b/docker/entrypoint-batch.sh deleted file mode 100644 index a66d45a3..00000000 --- a/docker/entrypoint-batch.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -if [ -z "$recipe" ]; then - echo "Required recipe parameter is missing, please include recipe in Docker run script, ie: -e recipe=path/to/recipe" - exit; -else - echo "recipe passed in: '$recipe'" -fi - -cd /cellpack - -if [ -z "$config" ]; then - echo "Config parameter not included, using default value" - pack -r $recipe -d - exit; -else - echo "config passed in: '$config'" -fi - -pack -r $recipe -c $config -d diff --git a/docker/server.py b/docker/server.py index 00523eb5..9bcac051 100644 --- a/docker/server.py +++ b/docker/server.py @@ -1,5 +1,4 @@ import asyncio -import uuid from aiohttp import web from cellpack.autopack.DBRecipeHandler import DataDoc, DBUploader from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS @@ -26,19 +25,18 @@ def job_exists(self, dedup_hash): job_status, _ = db.get_doc_by_id("job_status", dedup_hash) return job_status is not None - async def run_packing(self, job_id, recipe=None, config=None, body=None): - self.update_job_status(job_id, "RUNNING") + async def run_packing(self, dedup_hash, recipe, config=None): + self.update_job_status(dedup_hash, "RUNNING") try: - # Pack JSON recipe in body if provided, otherwise use recipe path - pack(recipe=(body if body else recipe), config_path=config, docker=True, hash=job_id) + pack(recipe=recipe, config_path=config, docker=True, hash=dedup_hash) except Exception as e: - self.update_job_status(job_id, "FAILED", error_message=str(e)) + self.update_job_status(dedup_hash, "FAILED", error_message=str(e)) - def update_job_status(self, job_id, status, result_path=None, error_message=None): + def update_job_status(self, dedup_hash, status, result_path=None, error_message=None): db = self._get_firebase_handler() if db: db_uploader = DBUploader(db) - db_uploader.upload_job_status(job_id, status, result_path, error_message) + db_uploader.upload_job_status(dedup_hash, status, result_path, error_message) async def hello_world(self, request: web.Request) -> web.Response: return web.Response(text="Hello from the cellPACK server") @@ -48,35 +46,29 @@ async def health_check(self, request: web.Request) -> web.Response: return web.Response() async def pack_handler(self, request: web.Request) -> web.Response: - recipe = request.rel_url.query.get("recipe") or "" - body = None - - # If request has a body, attempt to parse it as JSON and use as recipe - # otherwise rely on recipe query param - if (request.can_read_body and request.content_length - and request.content_length > 0): - try: - body = await request.json() - except Exception: - body = None - - if not recipe and not body: + if not (request.can_read_body and request.content_length and request.content_length > 0): raise web.HTTPBadRequest( - "Pack requests must include a recipe, either as a query param or in the request body" + "Pack requests must include a recipe in the request body" ) - config = request.rel_url.query.get("config") - if body: - dedup_hash = DataDoc.generate_hash(body) - if self.job_exists(dedup_hash): - return web.json_response({"jobId": dedup_hash}) - job_id = dedup_hash - else: - job_id = str(uuid.uuid4()) + try: + recipe = await request.json() + except Exception: + raise web.HTTPBadRequest( + "Pack requests must include a valid JSON recipe in the request body" + ) + + dedup_hash = DataDoc.generate_hash(recipe) + if self.job_exists(dedup_hash): + # We've already packed this recipe, return job id immediately + # to avoid redundant packing + return web.json_response({"jobId": dedup_hash}) + + config = request.rel_url.query.get("config") # Initiate packing task to run in background packing_task = asyncio.create_task( - self.run_packing(job_id, recipe, config, body) + self.run_packing(dedup_hash, recipe, config) ) # Keep track of task references to prevent them from being garbage @@ -86,7 +78,7 @@ async def pack_handler(self, request: web.Request) -> web.Response: # return job id immediately, rather than wait for task to complete, # to avoid timeout issues with API gateway - return web.json_response({"jobId": job_id}) + return web.json_response({"jobId": dedup_hash}) async def init_app() -> web.Application: diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 48e4b6dd..a590792d 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -1,18 +1,13 @@ # Docker +The CellPACK server, which is used for running packings for [CellPACK Studio](https://github.com/AllenCell/cellpack-client), is run in a Docker container using a Docker image built in this repo. +## Prerequisites * Install [docker](https://docs.docker.com/v17.09/engine/installation/) * Clone the repository locally, if you haven't already: `git clone https://github.com/mesoscope/cellpack.git` * Ensure that you have valid AWS access key and secret to access the `cellpack-results` S3 bucket, usually stored in a `~/.aws/credentials` file. If you have multiple accounts in your credentials files, ensure that the desured account is the `default` option. -* We have two Dockerfiles in /docker, one that builds the Docker image to be run in AWS ECS, and one to be run in AWS Batch. To build one of the images, run: `docker build -f [DOCKERFILE-NAME] -t [CONTAINER-NAME] .` Rebuild the container if new files are added or changes are made to the codebase. -## AWS Batch Docker Image -1. Build image, running `docker build -f docker/Dockerfile.batch -t [CONTAINER-NAME] .` -2. Run packings in the container, running: `docker run -v ~/.aws:/root/.aws -e recipe=examples/recipes/v2/one_sphere.json -e config=examples/packing-configs/run.json [CONTAINER-NAME]` -3. Verify that the packing results are saved in the `cellpack-results` S3 bucket. You should see a botocore logging message indicating that the credentials were successfully loaded. - -## AWS ECS Docker Image +## Building and Running the Docker Container 1. Build image, running `docker build -f docker/Dockerfile.ecs -t [CONTAINER-NAME] .` 2. Run packings in the container, running: `docker run -v ~/.aws:/root/.aws -p 80:80 [CONTAINER-NAME]` 3. Try hitting the test endpoint on the server, by navigating to `http://0.0.0.0:80/hello` in your browser. -4. Try running a packing on the server, by hitting the `http://0.0.0.0:80/start-packing?recipe=firebase:recipes/one_sphere_v_1.0.0` in your browser. -5. Verify that the packing result path was uploaded to the firebase results table, with the job id specified in the response from the request in step 4.The result simularium file can be found at the s3 path specified there. \ No newline at end of file +4. Try running a packing on the server, install and run [CellPACK Studio](https://github.com/AllenCell/cellpack-client) locally, with the [`SUBMIT_PACKING_ECS` constant](https://github.com/AllenCell/cellpack-client/blob/main/src/constants/aws.ts) pointing to your local Docker instance