From b58266275745065b311e8f1e524881ae2962c6f6 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Thu, 5 Dec 2024 16:53:48 -0500 Subject: [PATCH 1/8] WIP: stub out new playbook for nostr_sync --- inventories/nostr_sync/inventory.yml | 6 ++++++ playbooks/nostr_sync_service.yml | 6 ++++++ roles/nostr_sync/README.md | 1 + roles/nostr_sync/meta/main.yml | 0 roles/nostr_sync/tasks/main.yml | 0 5 files changed, 13 insertions(+) create mode 100644 inventories/nostr_sync/inventory.yml create mode 100644 playbooks/nostr_sync_service.yml create mode 100644 roles/nostr_sync/README.md create mode 100644 roles/nostr_sync/meta/main.yml create mode 100644 roles/nostr_sync/tasks/main.yml diff --git a/inventories/nostr_sync/inventory.yml b/inventories/nostr_sync/inventory.yml new file mode 100644 index 0000000..af7590d --- /dev/null +++ b/inventories/nostr_sync/inventory.yml @@ -0,0 +1,6 @@ +--- +nostr_sync: + vars: + ansible_user: admin + hosts: + sync.nos.social: \ No newline at end of file diff --git a/playbooks/nostr_sync_service.yml b/playbooks/nostr_sync_service.yml new file mode 100644 index 0000000..57073e9 --- /dev/null +++ b/playbooks/nostr_sync_service.yml @@ -0,0 +1,6 @@ +- name: Configure the nostr-sync server + hosts: nostr-sync + vars: + ansible_user: admin + roles: + - nostr_sync diff --git a/roles/nostr_sync/README.md b/roles/nostr_sync/README.md new file mode 100644 index 0000000..2d80475 --- /dev/null +++ b/roles/nostr_sync/README.md @@ -0,0 +1 @@ +# Nostr Sync \ No newline at end of file diff --git a/roles/nostr_sync/meta/main.yml b/roles/nostr_sync/meta/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/roles/nostr_sync/tasks/main.yml b/roles/nostr_sync/tasks/main.yml new file mode 100644 index 0000000..e69de29 From df3c54c22b5c2073ace4151e5d111012938218f0 Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Thu, 5 Dec 2024 17:21:31 -0500 Subject: [PATCH 2/8] update hosts now that server is out there --- playbooks/{nostr_sync_service.yml => nostr_sync.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename playbooks/{nostr_sync_service.yml => nostr_sync.yml} (100%) diff --git a/playbooks/nostr_sync_service.yml b/playbooks/nostr_sync.yml similarity index 100% rename from playbooks/nostr_sync_service.yml rename to playbooks/nostr_sync.yml From 9aa3ec53bf1e64abb9cd5c75f5dd951019ed5c6f Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sun, 8 Dec 2024 18:47:33 -0500 Subject: [PATCH 3/8] Finalize playbook for sync-server --- inventories/nostr_sync/inventory.yml | 3 ++ playbooks/nostr_sync.yml | 4 +-- roles/nostr_sync/meta/main.yml | 0 roles/nostr_sync/tasks/main.yml | 51 ++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) delete mode 100644 roles/nostr_sync/meta/main.yml diff --git a/inventories/nostr_sync/inventory.yml b/inventories/nostr_sync/inventory.yml index af7590d..87c05e8 100644 --- a/inventories/nostr_sync/inventory.yml +++ b/inventories/nostr_sync/inventory.yml @@ -2,5 +2,8 @@ nostr_sync: vars: ansible_user: admin + ansible_become: true + admin_username: admin + venv_path: "/home/{{ admin_username }}/sync_venv" hosts: sync.nos.social: \ No newline at end of file diff --git a/playbooks/nostr_sync.yml b/playbooks/nostr_sync.yml index 57073e9..54e2621 100644 --- a/playbooks/nostr_sync.yml +++ b/playbooks/nostr_sync.yml @@ -1,6 +1,4 @@ - name: Configure the nostr-sync server - hosts: nostr-sync - vars: - ansible_user: admin + hosts: nostr_sync roles: - nostr_sync diff --git a/roles/nostr_sync/meta/main.yml b/roles/nostr_sync/meta/main.yml deleted file mode 100644 index e69de29..0000000 diff --git a/roles/nostr_sync/tasks/main.yml b/roles/nostr_sync/tasks/main.yml index e69de29..cd709a6 100644 --- a/roles/nostr_sync/tasks/main.yml +++ b/roles/nostr_sync/tasks/main.yml @@ -0,0 +1,51 @@ +--- +- name: Update cache + ansible.builtin.apt: + update_cache: true + +- name: Ensure Python and pip are installed + ansible.builtin.package: + name: + - python3 + - python3-pip + - python3-venv + state: present + +- name: Create venv + ansible.builtin.shell: + cmd: "python3 -m venv {{ venv_path }}" + +- name: Upgrade pip + ansible.builtin.command: + cmd: "{{ venv_path }}/bin/python -m pip install --upgrade pip" + creates: "{{ venv_path }}/bin/pip" + +- name: Clone our Repo + ansible.builtin.git: + repo: https://github.com/verse-pbc/nostr-sync-scripts + dest: /home/{{ admin_username }}/nostr-sync-scripts + version: 'main' + +- name: Install requirements in the virtual environment + ansible.builtin.command: + cmd: "{{ venv_path }}/bin/pip install -r /home{{ admin_username }}/nostr-sync-scripts/requirements.txt" + creates: "{{ venv_path }}/bin/activate" + +- name: Add grow_fedi_nhex Sync to Crontab + ansible.builtin.cron: + name: "grow_fedi_nhex" + hour: 1 + minute: 45 + job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sysnc-scripts/grow_fedi_nhex.py" + +- name: Add fedi_sync to Crontab + ansible.builtin.cron: + name: "fedi_sync" + hour: 2 + job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sysnc-scripts/fedi_sync.py" + +- name: Add news_sync to Crontab + ansible.builtin.cron: + name: "news_sync" + hour: 3 + job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sysnc-scripts/news_sync.py" From c61d244c560d88dad194e769fce59366fef0a8da Mon Sep 17 00:00:00 2001 From: Ben Moody Date: Sun, 8 Dec 2024 19:02:39 -0500 Subject: [PATCH 4/8] Add a PR template --- .github/PULL_REQUEST_TEMPLATE.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..3347a36 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +**Description/Goals** +_Why did you make this PR?_ + +**Summary of Changes** +_Example_ +* I updated the metrics inventory to target the correct host. + +**Tasks Before Merging** +_Do you need to remember to do anything BEFORE merging?_ \ No newline at end of file From f75b6c7131e7db7ea1b99d175902f3a67a8c787f Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 6 Jan 2025 15:20:48 -0500 Subject: [PATCH 5/8] Fixed nostr_sync setup --- roles/nostr_sync/tasks/main.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/roles/nostr_sync/tasks/main.yml b/roles/nostr_sync/tasks/main.yml index cd709a6..73c7509 100644 --- a/roles/nostr_sync/tasks/main.yml +++ b/roles/nostr_sync/tasks/main.yml @@ -9,12 +9,18 @@ - python3 - python3-pip - python3-venv + - pkg-config state: present - name: Create venv ansible.builtin.shell: cmd: "python3 -m venv {{ venv_path }}" +- name: Make venv activate script executable + ansible.builtin.file: + path: "{{ venv_path }}/bin/activate" + mode: '0755' + - name: Upgrade pip ansible.builtin.command: cmd: "{{ venv_path }}/bin/python -m pip install --upgrade pip" @@ -28,24 +34,23 @@ - name: Install requirements in the virtual environment ansible.builtin.command: - cmd: "{{ venv_path }}/bin/pip install -r /home{{ admin_username }}/nostr-sync-scripts/requirements.txt" - creates: "{{ venv_path }}/bin/activate" + cmd: "{{ venv_path }}/bin/pip install -r /home/{{ admin_username }}/nostr-sync-scripts/requirements.txt" - name: Add grow_fedi_nhex Sync to Crontab ansible.builtin.cron: name: "grow_fedi_nhex" hour: 1 minute: 45 - job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sysnc-scripts/grow_fedi_nhex.py" + job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/grow_fedi_nhex.py 2>&1 | logger -t grow_fedi_nhex" - name: Add fedi_sync to Crontab ansible.builtin.cron: name: "fedi_sync" hour: 2 - job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sysnc-scripts/fedi_sync.py" + job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/fedi_sync.py 2>&1 | logger -t fedi_sync" - name: Add news_sync to Crontab ansible.builtin.cron: name: "news_sync" hour: 3 - job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sysnc-scripts/news_sync.py" + job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/news_sync.py 2>&1 | logger -t news_sync" From 18ead1c8887a556b6781b9cd4e24412135998997 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Wed, 8 Jan 2025 16:55:53 -0500 Subject: [PATCH 6/8] Force git to pull the latest script from the repo --- roles/nostr_sync/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/nostr_sync/tasks/main.yml b/roles/nostr_sync/tasks/main.yml index 73c7509..c35aca8 100644 --- a/roles/nostr_sync/tasks/main.yml +++ b/roles/nostr_sync/tasks/main.yml @@ -31,6 +31,7 @@ repo: https://github.com/verse-pbc/nostr-sync-scripts dest: /home/{{ admin_username }}/nostr-sync-scripts version: 'main' + force: yes - name: Install requirements in the virtual environment ansible.builtin.command: From 6d073112c2b4ffee3600225f18ba8c7a418f7744 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Thu, 9 Jan 2025 16:45:12 -0500 Subject: [PATCH 7/8] Adjust syncs to happen every hour --- roles/nostr_sync/tasks/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/nostr_sync/tasks/main.yml b/roles/nostr_sync/tasks/main.yml index c35aca8..65202d0 100644 --- a/roles/nostr_sync/tasks/main.yml +++ b/roles/nostr_sync/tasks/main.yml @@ -40,18 +40,17 @@ - name: Add grow_fedi_nhex Sync to Crontab ansible.builtin.cron: name: "grow_fedi_nhex" - hour: 1 - minute: 45 + minute: 0 job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/grow_fedi_nhex.py 2>&1 | logger -t grow_fedi_nhex" - name: Add fedi_sync to Crontab ansible.builtin.cron: name: "fedi_sync" - hour: 2 + minute: 30 job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/fedi_sync.py 2>&1 | logger -t fedi_sync" - name: Add news_sync to Crontab ansible.builtin.cron: name: "news_sync" - hour: 3 + minute: 45 job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/news_sync.py 2>&1 | logger -t news_sync" From 965678d97f2767ccc0e53dc85f0d6e18e6083659 Mon Sep 17 00:00:00 2001 From: Daniel Cadenas Date: Wed, 22 Jan 2025 12:15:30 -0300 Subject: [PATCH 8/8] Use flock, clean pyc on playbook run --- roles/nostr_sync/tasks/main.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/roles/nostr_sync/tasks/main.yml b/roles/nostr_sync/tasks/main.yml index 65202d0..48af83e 100644 --- a/roles/nostr_sync/tasks/main.yml +++ b/roles/nostr_sync/tasks/main.yml @@ -3,13 +3,14 @@ ansible.builtin.apt: update_cache: true -- name: Ensure Python and pip are installed +- name: Ensure required packages are installed ansible.builtin.package: name: - python3 - python3-pip - python3-venv - pkg-config + - util-linux # for flock state: present - name: Create venv @@ -33,6 +34,22 @@ version: 'main' force: yes +- name: Clean up old .pyc files and __pycache__ directories + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_fileglob: + - "/home/{{ admin_username }}/nostr-sync-scripts/**/*.pyc" + - "/home/{{ admin_username }}/nostr-sync-scripts/**/__pycache__" + +- name: Create lock directory + ansible.builtin.file: + path: /var/lock/nostr-sync + state: directory + mode: '0755' + owner: "{{ admin_username }}" + group: "{{ admin_username }}" + - name: Install requirements in the virtual environment ansible.builtin.command: cmd: "{{ venv_path }}/bin/pip install -r /home/{{ admin_username }}/nostr-sync-scripts/requirements.txt" @@ -41,16 +58,16 @@ ansible.builtin.cron: name: "grow_fedi_nhex" minute: 0 - job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/grow_fedi_nhex.py 2>&1 | logger -t grow_fedi_nhex" + job: "cd /home/{{ admin_username }}/nostr-sync-scripts && flock -n /var/lock/nostr-sync/grow_fedi_nhex.lock {{ venv_path }}/bin/python grow_fedi_nhex.py 2>&1 | logger -t grow_fedi_nhex" - name: Add fedi_sync to Crontab ansible.builtin.cron: name: "fedi_sync" minute: 30 - job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/fedi_sync.py 2>&1 | logger -t fedi_sync" + job: "cd /home/{{ admin_username }}/nostr-sync-scripts && flock -n /var/lock/nostr-sync/fedi_sync.lock {{ venv_path }}/bin/python fedi_sync.py 2>&1 | logger -t fedi_sync" - name: Add news_sync to Crontab ansible.builtin.cron: name: "news_sync" minute: 45 - job: "{{ venv_path }}/bin/activate && {{ venv_path }}/bin/python /home/{{ admin_username }}/nostr-sync-scripts/news_sync.py 2>&1 | logger -t news_sync" + job: "cd /home/{{ admin_username }}/nostr-sync-scripts && flock -n /var/lock/nostr-sync/news_sync.lock {{ venv_path }}/bin/python news_sync.py 2>&1 | logger -t news_sync" \ No newline at end of file