Merge pull request #41 from mlibrary/move-reports-to-mayhem

changed dropbox to smb for digifeeds reports

Author: niquerio

.config/rclone/.keep

@@ -17,8 +17,9 @@ requirements.txt
 
 docs/_build
 bin/digifeeds/*.config
-.config/rclone/rclone.conf
-.config/rclone/*.json
+
+.config/rclone/*
+!.config/rclone/.keep
 
 tmp/*
 !tmp/.keep

.config/rclone/rclone.conf.example

@@ -19,7 +19,6 @@ AIM's python code repository
 
     This will:
     * set up the initial environment variables file
-    * set up the rclone config with the example file
     * build the docker image
     * install the python dependencies
     * Set up the database for digifeeds
@@ -28,11 +27,9 @@ AIM's python code repository
 
 3. Edit `.env` with actual environment variables
 
-4. Edit `.config/rclone/rclone.conf` with your actual values
+4. If using VSCode for editing, the repository is set up for use with dev containers. You will have to rebuild the container in there.
 
-5. If using VSCode for editing, the repository is set up for use with dev containers. You will have to rebuild the container in there.
-
-6. In the app container, use `poetry shell` to enable the virtual environment. Otherwise use:
+5. In the app container, use `poetry shell` to enable the virtual environment. Otherwise use:
 
 ```bash
  docker compose run --rm app poetry run YOUR_COMMAND
@@ -120,6 +117,14 @@ docker compose run --rm app poetry run aim digifeeds --help
 
 This will show the commands available for the digifeeds cli applciation.
 
+### Rclone
+
+This applicaiton uses [Rclone](https://rclone.org/) to work with remote storage. It uses the [rclone-python](https://pypi.org/project/rclone-python/) for working with rclone in python.  
+
+To configure new remotes for use with the application use environment variables. The names of the environment variables are on the documentation page for a given remote. We use environment variables because it so all of the configuration happens in `.env` instead of in multiple places.
+
+If you want to quickly add a remote to try something out, you can use `rclone configure` in the container terminal. The config file is ignored.
+
 ## Tests
 
 To run tests:

.gitignore

@@ -171,6 +171,6 @@ def generate_barcodes_in_s3_report():
     """
     Generates a report of barcodes that have been moved to the google pickup
     location in the last two weeks. It is based on the files in the processed
-    location in the s3 bucket. This report is sent to a dropbox folder.
+    location in the s3 bucket. This report is sent to a folder on Mayhem.
     """
     functions.generate_barcodes_added_in_last_two_weeks_report()

README.md

@@ -61,8 +61,8 @@ def generate_barcodes_added_in_last_two_weeks_report():
         write_barcodes_added_in_last_two_weeks_report(rf)
 
     today = datetime.today().strftime("%Y-%m-%d")
-    S.logger.info("writing report to dropbox")
+    S.logger.info("writing delivery report")
     rclone.copyto(
         in_path=report_file.name,
-        out_path=f"{S.digifeeds_reports_rclone_remote}:{today}_barcodes_in_s3_processed.tsv",
+        out_path=f"{S.digifeeds_delivery_reports_rclone_remote}:{today}_barcodes_in_s3_processed.tsv",
     )

aim/cli/digifeeds.py

@@ -89,8 +89,8 @@ class Services:
     #: The name of the rclone remote for the place where google pickups up the digifeeds files
     digifeeds_pickup_rclone_remote: str
 
-    #: The name of the rclone remote where reports from digifeeds are sent
-    digifeeds_reports_rclone_remote: str
+    #: The name of the rclone remote where reports about what has been sent to google are sent
+    digifeeds_delivery_reports_rclone_remote: str
 
     #: file path to store of the hathi_file_list update items
     hathifiles_store_path: str
@@ -129,8 +129,10 @@ class Services:
     or "digifeeds_bucket",
     digifeeds_pickup_rclone_remote=os.getenv("DIGIFEEDS_PICKUP_RCLONE_REMOTE")
     or "digifeeds_pickup",
-    digifeeds_reports_rclone_remote=os.getenv("DIGIFEEDS_REPORTS_RCLONE_REMOTE")
-    or "digifeeds_reports",
+    digifeeds_delivery_reports_rclone_remote=os.getenv(
+        "DIGIFEEDS_DELIVERY_REPORTS_RCLONE_REMOTE"
+    )
+    or "digifeeds_delivery_reports",
     hathifiles_store_path=os.getenv("HATHIFILES_STORE_PATH")
     or "tmp/hathi_file_list_store.json",
     hathifiles_webhook_url=os.getenv("HATHIFILES_WEBHOOK_URL")

aim/digifeeds/functions.py

@@ -2,3 +2,29 @@ UID=YOUR_UID
 GID=YOUR_GID
 POETRY_VERSION=1.5.1
 DEV=true
+
+RCLONE_CONFIG_CHAOS_TYPE=smb
+RCLONE_CONFIG_CHAOS_HOST=your_smb_host
+RCLONE_CONFIG_CHAOS_USER=your_smb_user
+RCLONE_CONFIG_CHAOS_PASS=your_smb_password
+RCLONE_CONFIG_CHAOS_DOMAIN=your_smb_domain
+
+RCLONE_CONFIG_DIGIFEEDS_DELIVERY_REPORTS_TYPE=alias
+RCLONE_CONFIG_DIGIFEEDS_DELIVERY_REPORTS_REMOTE=chaos:your_reports_path
+
+RCLONE_CONFIG_DIGIFEEDS_S3_TYPE=s3
+RCLONE_CONFIG_DIGIFEEDS_S3_PROVIDER=AWS
+RCLONE_CONFIG_DIGIFEEDS_S3_ACCESS_KEY_ID=your_s3_access_key_id
+RCLONE_CONFIG_DIGIFEEDS_S3_SECRET_ACCESS_KEY=your_s3_secret_access_key
+
+RCLONE_CONFIG_DIGIFEEDS_BUCKET_TYPE=alias
+RCLONE_CONFIG_DIGIFEEDS_BUCKET_REMOTE=digifeeds_s3:your_bucket_name
+
+RCLONE_CONFIG_DIGIFEEDS_PICKUP_TYPE=sftp
+RCLONE_CONFIG_DIGIFEEDS_PICKUP_HOST=your_sftp_host
+RCLONE_CONFIG_DIGIFEEDS_PICKUP_USER=your_sftp_user
+RCLONE_CONFIG_DIGIFEEDS_PICKUP_PORT=your_sftp_port
+RCLONE_CONFIG_DIGIFEEDS_PICKUP_PASS=your_sftp_password_hash
+RCLONE_CONFIG_DIGIFEEDS_PICKUP_SHELL_TYPE=cmd
+RCLONE_CONFIG_DIGIFEEDS_MD5SUM_COMMAND=none
+RCLONE_CONFIG_DIGIFEEDS_SHA1SUM_COMMAND=none

aim/services.py

@@ -1,22 +1,15 @@
 #!/usr/bin/env bash
 
 if [ -f ".env" ]; then
-  echo "🌎 .env exists. Leaving alone"
+    echo "🌎 .env exists. Leaving alone"
 else
-  echo "🌎 .env does not exist. Copying .env-example to .env"
-  cp env.example .env
-  YOUR_UID=$(id -u)
-  YOUR_GID=$(id -g)
-  echo "🙂 Setting your UID (${YOUR_UID}) and GID (${YOUR_UID}) in .env"
-  docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_UID/${YOUR_UID}/ .env)" >.env
-  docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_GID/${YOUR_GID}/ .env)" >.env
-fi
-
-if [ -f ".config/rclone/rclone.conf" ]; then
-  echo "📋 .config/rclone/rclone.conf exists. Leaving alone"
-else
-  echo "📋 .config/rclone/rclone.conf does not exist. Copying .config/rclone/rclone.conf.example to rclone_config"
-  cp .config/rclone/rclone.conf.example .config/rclone/rclone.conf
+    echo "🌎 .env does not exist. Copying .env-example to .env"
+    cp env.example .env
+    YOUR_UID=$(id -u)
+    YOUR_GID=$(id -g)
+    echo "🙂 Setting your UID (${YOUR_UID}) and GID (${YOUR_UID}) in .env"
+    docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_UID/${YOUR_UID}/ .env)" >.env
+    docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_GID/${YOUR_GID}/ .env)" >.env
 fi
 
 echo "🚢 Build docker images"