adding HPC file transfer capability

Author: KastanDay

codeflare_helpers/data_movement_select_destination.md

@@ -0,0 +1,36 @@
+## Select the destination location
+
+=== "Destination: Clowder Dataset"
+    Select from available datasets in Clowder, or create a new dataset to be the destination.
+
+    ## Select the destination Clowder dataset.
+    === "expand(cat .v2_available_clowder_datasets.txt | tr -d '[]')"
+        ```shell
+        echo "👉 Selected destination dataset: ${choice}"
+        ```
+
+=== "Destination: HPC [default: none]"
+    SSH to your HPC resource and copy data from any path there to any destination. You can supply a list of files, or a path to a directory. 
+    ```shell
+    python ./CodeFlare-Extractors/codeflare_helpers/select_path_on_hpc.py
+    echo "👉 Starting copy from Clowder 🐱 to HPC 🏢."
+    ```
+
+    # Enter the destination path on HPC. The dataset will be copied to a folder inside this path.
+    === "Enter the destination path on HPC (e.g. name@hpc_cluster:/data_path/) [default: ~/]"
+        details here.
+        ```shell
+        echo "👉 Selected destination path: ${choice}"
+        # export HPC_DESTINATION_PATH=${choice}
+        ```
+
+=== "Destination: AWS S3"
+    Select from available buckets in AWS S3. Transfer the entire bucket, or a list of files. Ensure '$ aws s3 ls' works on your machine.
+
+    # Enter the destination S3 bucket name and sub-folder. The dataset will be copied to a folder inside this path.
+    === "Enter the destination path on S3 (e.g. s3://my-bucket/data/destination/) [default: s3://my-bucket]"
+        details here.
+        ```shell
+        echo "👉 Selected destination path: ${choice}"
+        # export HPC_DESTINATION_PATH=${choice}
+        ```

codeflare_helpers/launch_clowder.sh

@@ -1,23 +1,23 @@
 #!/bin/bash
 
 # no sudo for mac Docker, yes sudo for linux.
-echo "Starting Clowder with extractors"
+echo "  👉 Starting Clowder with extractors"
 if [ $(uname) = "Darwin" ]; then
-    echo "Platform is MacOS"
+    # echo "Platform is MacOS"
     docker-compose -f docker-compose.yml -f docker-compose.extractors.yml up -d
     echo "🌐 Starting Clowder (http://localhost:8000) in your default browser"
-    open http://localhost:8000
+    open --url http://localhost:8000 --background --hide
     echo "❓ It may take up to a minute for the containers to start up, please try refreshing the page a few times."
 
 elif [ $(uname -r | grep -c "microsoft") -eq 1 ]; then
-    echo "Platform is WSL2"
+    # echo "Platform is WSL2"
     sudo docker-compose -f docker-compose.yml -f docker-compose.extractors.yml up -d
     echo "🌐 Starting Clowder (http://localhost:8000) in your default browser"
     wslview http://localhost:8000
     echo "❓ It may take up to a minute for the containers to start up, please try refreshing the page a few times."
 
 elif [ $(uname) = "Linux" ]; then
-    echo "Platform is Linux"
+    # echo "Platform is Linux"
     sudo docker-compose -f docker-compose.yml -f docker-compose.extractors.yml up -d
     echo "🌐 Starting Clowder (http://localhost:8000) in your default browser"
     if ! which xdg-open > /dev/null; then

codeflare_helpers/select_path_on_hpc.py

@@ -0,0 +1,43 @@
+import os
+from typing import List
+
+from pyclowder.client import ClowderClient
+from termcolor import colored
+
+
+def main():
+    # Initialize the client
+    client = ClowderClient(host="http://localhost:8000", key="bbe2dce9-c3eb-4188-8a70-1d6e676beeb1")
+
+    # client.list_datasets()
+    # datasets = client.get('/datasets/')
+    # ds_list = [d['name'] for d in datasets]
+    # print(ds_list)
+    # save_to_file(ds_list, 'clowder_db')
+    # for i, dataset in enumerate(datasets):
+    #     print('\t', f'{i}.', dataset['name'])
+
+    # or enter a new dataset name, and create a new dataset
+
+    # works, but should use CodeFlare instead.
+    # n = input(
+    #     colored(
+    #         f"👉 Enter the destination filepath on the HPC resource. Your Clowder dataset will be transferred here, and arrive as a folder.",
+    #         "yellow",
+    #         attrs=["bold"]))
+    # print("You entered", n)
+    # os.environ['HPC_DESTINATION_PATH'] = n
+    return
+
+
+def save_to_file(sentences: List, filename: str):
+    # Open the file in write mode
+    with open(filename, "w") as file:
+        for sentence in sentences:
+            file.write(sentence)
+            # ASCII
+            #.encode(encoding='ascii', errors='ignore').decode()
+
+
+if __name__ == "__main__":
+    main()