diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..785e8d0 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + - package-ecosystem: gomod + directory: /cloudflare-ip-logger + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: docker + directory: /cloudflare-ip-logger + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..001f0ef --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,140 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + shellcheck: + name: ShellCheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run ShellCheck + uses: ludeeus/action-shellcheck@2.0.0 + env: + SHELLCHECK_OPTS: -e SC1091 -x + with: + severity: error + scandir: '.' + ignore_paths: >- + .git + .github + + python: + name: Python (compile + ruff) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Discover python files + id: pyfiles + run: | + mapfile -t files < <(git ls-files '*.py') + printf '%s\n' "${files[@]}" + { + echo 'files<> "$GITHUB_OUTPUT" + - name: Byte-compile + if: steps.pyfiles.outputs.files != '' + run: | + while IFS= read -r f; do + [ -z "$f" ] && continue + python -m py_compile "$f" + done <<< "${{ steps.pyfiles.outputs.files }}" + - name: Install ruff + run: pip install ruff==0.6.9 + - name: Lint with ruff + run: ruff check --output-format=github . + + go: + name: Go (cloudflare-ip-logger) + runs-on: ubuntu-latest + defaults: + run: + working-directory: cloudflare-ip-logger + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: cloudflare-ip-logger/go.mod + cache-dependency-path: cloudflare-ip-logger/go.sum + - name: Install SQLite headers + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + - name: go mod download + run: go mod download + - name: go vet + run: go vet ./... + - name: Check formatting + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "::error::Files not gofmt-formatted:" + echo "$unformatted" + exit 1 + fi + - name: go build + env: + CGO_ENABLED: '1' + run: | + go build -ldflags="-s -w" -o cf-ip-logger . + go build -o cf-log-parser ./cmd/logparser + - name: go test + run: go test ./... + + hadolint: + name: Hadolint (Dockerfile) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: hadolint/hadolint-action@v3.1.0 + with: + dockerfile: cloudflare-ip-logger/Dockerfile + config: cloudflare-ip-logger/.hadolint.yaml + failure-threshold: warning + + docker-build: + name: Docker build (cf-ip-logger) + runs-on: ubuntu-latest + needs: [go, hadolint] + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Build image (no push) + uses: docker/build-push-action@v6 + with: + context: cloudflare-ip-logger + push: false + load: true + tags: cf-ip-logger:ci + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Smoke-test image + run: | + docker run --rm --entrypoint /app/cf-ip-logger cf-ip-logger:ci -h 2>&1 || true + docker run --rm --entrypoint sh cf-ip-logger:ci -c 'test -x /app/cf-ip-logger && echo OK' + + yaml: + name: YAML lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install yamllint==1.35.1 + - run: yamllint -c .yamllint.yml . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5a7263a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,63 @@ +name: Release cf-ip-logger image + +on: + push: + tags: + - 'cf-ip-logger-v*' + workflow_dispatch: + inputs: + tag: + description: 'Version tag to publish (e.g. v0.1.0)' + required: true + +permissions: + contents: read + packages: write + +jobs: + publish: + name: Build & push multi-arch image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Resolve version + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + version="${{ inputs.tag }}" + else + version="${GITHUB_REF_NAME#cf-ip-logger-}" + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "Resolved version: ${version}" + + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/cf-ip-logger + tags: | + type=raw,value=${{ steps.version.outputs.version }} + type=raw,value=latest + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: cloudflare-ip-logger + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 1cd1224..460389e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,12 @@ Thumbs.db # Temp files *.tmp *.bak + +# Python +__pycache__/ +*.pyc +*.pyo + +# Go build artifacts (cloudflare-ip-logger) +cloudflare-ip-logger/cf-ip-logger +cloudflare-ip-logger/cf-log-parser diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000..1327a19 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,15 @@ +extends: relaxed + +ignore: | + .git/ + cloudflare-ip-logger/docker-compose.cloudflared.yml + +rules: + line-length: disable + trailing-spaces: disable + comments: disable + document-start: disable + empty-lines: + max: 2 + truthy: + check-keys: false diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5ab111d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,33 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Repository shape + +This repo is a *collection of independent utility scripts*, not a single application. Each top-level directory is a self-contained tool with its own README and (usually) a single entry-point script. There is no shared library, no top-level build, no test suite, and no package manifest at the root. Treat each subdirectory as its own micro-project. + +The only compiled component is `cloudflare-ip-logger/` (Go + SQLite, shipped as a Docker image). Everything else is bash or Python. + +## Conventions for adding/modifying scripts + +- New scripts start by copying `_template/` and renaming it. The template's `README.md` defines the documentation shape every tool follows (Overview / Prerequisites / Installation / Usage / Options table / Configuration table / How It Works / Troubleshooting / Changelog). +- After adding a new script directory, update the script table in the top-level `README.md` — the table is the canonical index and is expected to stay in sync with the directory listing. +- Bash scripts use `#!/usr/bin/env bash` (preferred) or `#!/bin/bash`. Several of the larger scripts (`HealthCheck`, `LinuxTroubleshooting`, `chown_throttled`, `File_retention`) carry a banner header comment with name + version — match that style when editing them. +- Python scripts target Python 3 and are written to be stdlib-only (no `requirements.txt` anywhere). Don't introduce third-party deps without flagging it; the "no external dependencies" property is called out explicitly in several READMEs (`dirsync`, `LinuxTroubleshooting`). +- Some directories use kebab-case (`cluster-ssh-key-setup`, `git-update`), others CamelCase (`HealthCheck`, `HungConnections`, `LinuxTroubleshooting`, `File_retention`). Don't rename existing ones — the README table links to them by name. + +## cloudflare-ip-logger (the one non-script component) + +A Go reverse proxy that sits behind `cloudflared` to capture real visitor IPs from the `CF-Connecting-IP` header before forwarding to backend services. Runs as a Docker container. + +- `main.go` — the proxy + REST API + dashboard server. Routes by `Host` header against `proxy-config.json`, persists to SQLite, also tails to a plain-text log. +- `cmd/logparser/main.go` — separate binary that parses `cloudflared`'s JSON logs and ingests them into the same SQLite DB. Used by `run-with-logging.sh` and the `cf-log-parser.service` systemd unit when running cloudflared alongside the logger. +- Build/run: `docker compose up -d --build` from the directory. The Dockerfile uses CGO (required by `mattn/go-sqlite3`), so cross-compilation needs the C toolchain. +- Data layout under `/data` (volume-mounted): `connections.db`, `connections.log`, `proxy-config.json`. The example config lives at `proxy-config.json.example` in the repo and is copied into the data dir on first run. +- Routing depends on cloudflared preserving the original `Host` header via `originRequest.httpHostHeader` — this is the most common misconfiguration, see the README's "Cloudflared Configuration" section. + +## Things that look like patterns but aren't + +- There is no shared bash helper library. Logging/colors/error-handling are reimplemented per-script. Don't try to refactor across scripts unless the user asks. +- `pwr-temp-monitor/` ships several `*_metrics.sh` scripts (pi/jetson/x86) that look similar but target different hardware — they're meant to be deployed selectively by `setup.sh`, not unified. +- `HungConnections/` intentionally has both `.sh` and `.py` implementations of the same tool. Keep them feature-equivalent if you change one. diff --git a/File_retention/file_retention.sh b/File_retention/file_retention.sh old mode 100644 new mode 100755 diff --git a/HealthCheck/system_health_check.sh b/HealthCheck/system_health_check.sh old mode 100644 new mode 100755 diff --git a/HungConnections/hung_connection_killer.py b/HungConnections/hung_connection_killer.py old mode 100644 new mode 100755 index 82f1e21..2baec56 --- a/HungConnections/hung_connection_killer.py +++ b/HungConnections/hung_connection_killer.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """ Hung Connection Killer ====================== @@ -374,7 +375,7 @@ def is_hung_connection(self, conn: Connection) -> tuple[bool, str]: # Without timer info, flag states that are typically problematic if conn.state == 'CLOSE_WAIT': # CLOSE_WAIT with no timer is suspicious - app should have closed - return True, f"CLOSE_WAIT without active timer (likely app not closing socket)" + return True, "CLOSE_WAIT without active timer (likely app not closing socket)" elif conn.state in ('FIN_WAIT1', 'FIN_WAIT2', 'CLOSING', 'LAST_ACK'): # These should have timers; if not, they may be stuck return True, f"{conn.state} state detected (typically indicates hung connection)" @@ -397,7 +398,7 @@ def should_skip_connection(self, conn: Connection) -> tuple[bool, str]: # Check port inclusions if self.include_only_ports: if conn.local_port not in self.include_only_ports and conn.remote_port not in self.include_only_ports: - return True, f"Port not in include list" + return True, "Port not in include list" # Check process exclusions if conn.process_name: @@ -450,9 +451,9 @@ def _kill_socket_ss(self, conn: Connection) -> bool: cmd = [ 'ss', '-K', 'dst', f'{conn.remote_addr}', - 'dport', f'eq', str(conn.remote_port), + 'dport', 'eq', str(conn.remote_port), 'src', f'{conn.local_addr}', - 'sport', f'eq', str(conn.local_port), + 'sport', 'eq', str(conn.local_port), ] returncode, stdout, stderr = self._run_command(cmd) diff --git a/HungConnections/hung_connection_killer.sh b/HungConnections/hung_connection_killer.sh old mode 100644 new mode 100755 diff --git a/LinuxTroubleshooting/linux_troubleshoot.sh b/LinuxTroubleshooting/linux_troubleshoot.sh old mode 100644 new mode 100755 diff --git a/README.md b/README.md index 1da83e3..7a61d96 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Misc Scripts +[![CI](https://github.com/bfritzinger/Misc_Scripts/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/bfritzinger/Misc_Scripts/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) + A collection of utility scripts for various system administration and automation tasks. ## Scripts @@ -18,7 +21,7 @@ A collection of utility scripts for various system administration and automation | Dot Files | Personal configuration files for setting up a new Linux system.| [README](./dotfiles/README.md) | | Hung Connections | A utility for detecting and terminating hung network connections on Unix-based systems. Available in both Python and Bash | [README](./HungConnections/README.md) | | Health Check | A single-file bash script that performs a comprehensive system health check and simultaneously exports every measured value to a **CSV** (for trend analysis) and a **JSON snapshot** (for tooling integration). Run it on a schedule and pipe the CSV into pandas, Grafana, Excel, or gnuplot to watch metrics evolve over time | [README](./HealthCheck/README.md) | -| Chowned Thottled | A performance-conscious bash script for recursively changing file ownership across multiple directories on high-throughput systems. Designed to run safely alongside active workloads by controlling CPU and I/O priority, batching filesystem operations to avoid argument list limits, and skipping files that are already correctly owned | [README](./chown_throttled/README.md) | +| Chown Throttled | A performance-conscious bash script for recursively changing file ownership across multiple directories on high-throughput systems. Designed to run safely alongside active workloads by controlling CPU and I/O priority, batching filesystem operations to avoid argument list limits, and skipping files that are already correctly owned | [README](./chown_throttled/README.md) | | Linux Troubleshooting | A comprehensive, interactive shell script for diagnosing and troubleshooting x86_64 Linux servers. Covers 15 diagnostic modules ranging from hardware inventory and network connection analysis to Kubernetes cluster health and security auditing — all from a single script with no external dependencies beyond standard Linux tooling | [README](./LinuxTroubleshooting/README.md) | | File Retention (Age Off) | A config-driven bash script for age-based file cleanup across multiple directories. Each directory can have its own retention policy, glob pattern, and recursion setting — all managed from a single config file without touching the script itself | [README](./File_retention/README.md) | | Dir Sync | A lightweight Python script that compares two directories and copies only the changed or new files (deltas) from source to destination. No external dependencies — stdlib only | [README](./dirsync/README.md) | @@ -47,37 +50,76 @@ Misc_Scripts/ ├── LICENSE ├── _template/ │ └── README.md +├── File_retention/ +│ ├── README.md +│ ├── file_retention.conf +│ └── file_retention.sh +├── HealthCheck/ +│ ├── README.md +│ ├── metrics-dashboard.html +│ └── system_health_check.sh +├── HungConnections/ +│ ├── README.md +│ ├── hung_conn_dashboard.html +│ ├── hung_connection_killer.py +│ └── hung_connection_killer.sh +├── LinuxTroubleshooting/ +│ ├── README.md +│ ├── linux_troubleshoot.sh +│ └── linux_troubleshoot_dashboard.html +├── alias-dist/ +│ ├── README.md +│ └── alias-dist.sh +├── chown_throttled/ +│ ├── README.md +│ └── chown_throttled.sh +├── cloudflare-ip-logger/ +│ ├── README.md +│ ├── Dockerfile +│ ├── cf-log-parser.service +│ ├── cmd/ +│ │ └── logparser/ +│ │ └── main.go +│ ├── docker-compose.cloudflared.yml +│ ├── docker-compose.yml +│ ├── go.mod +│ ├── main.go +│ ├── proxy-config.json.example +│ └── run-with-logging.sh ├── cluster-ssh-key-setup/ │ ├── README.md │ └── cluster-sshKey-setup.sh -└── cluster-system-update/ - ├── README.md - └── update-sys.sh -└── docker-container-update/ - ├── README.md - └── docker-container-update.sh -└── cloudflare-ip-logger/ - ├── cmd - ├── logparser - ├── main.go - ├── README.md - └── cd-log-parser.service - └── docker-compose.cloudflared.yml - └── docker-compose.yml - └── Dokerfile - └── go.mod - └── main.go - └── proxy-config.json.example - └── run-with-logging.sh -└── ollama-updater/ - ├── README.md - └── ollama-updater.py -└── git-update/ - ├── README.md - └── git-update.sh -└── pwr-tmp-monitor/ - ├── README.md - └── setup.sh +├── cluster-system-update/ +│ ├── README.md +│ └── update-sys.sh +├── dirsync/ +│ ├── README.md +│ └── dirsync.py +├── docker-container-update/ +│ ├── README.md +│ └── docker-container-update.sh +├── dotfiles/ +│ ├── README.md +│ └── bootstrap.sh +├── git-update/ +│ ├── README.md +│ └── git-update.sh +├── github-star-repos/ +│ ├── README.md +│ └── github-stars.py +├── ollama-updater/ +│ ├── README.md +│ └── ollama-updater.py +└── pwr-temp-monitor/ + ├── README.md + ├── NODE_EXPORTER_SETUP.md + ├── alertmanager.yml + ├── deploy_to_nodes.sh + ├── grafana-dashboard.json + ├── jetson_metrics.sh + ├── pi_alerts.yml + ├── pi_metrics.sh + ├── setup.sh └── x86_metrics.sh └── jetson_metrics.sh └── pi_metrics.sh @@ -151,6 +193,23 @@ Misc_Scripts/ 3. Update this main README to include your new script in the table above +## Continuous Integration + +Every push and pull request runs through a GitHub Actions pipeline that exercises the whole repo: + +| Job | Tool | What it covers | +|-----|------|----------------| +| `shellcheck` | [ShellCheck](https://www.shellcheck.net) | All `*.sh` scripts (errors only — warnings tracked separately) | +| `python` | `py_compile` + [ruff](https://docs.astral.sh/ruff/) | Byte-compiles every `.py` file and lints with the rules in `ruff.toml` | +| `go` | `go vet` / `gofmt` / `go build` / `go test` | The `cloudflare-ip-logger` Go module | +| `hadolint` | [Hadolint](https://github.com/hadolint/hadolint) | The `cloudflare-ip-logger` Dockerfile | +| `docker-build` | `docker buildx` | Builds the cf-ip-logger image as a smoke test | +| `yaml` | [yamllint](https://yamllint.readthedocs.io) | All YAML files (config in `.yamllint.yml`) | + +The release pipeline (`.github/workflows/release.yml`) fires on `cf-ip-logger-vX.Y.Z` tags and publishes a multi-arch (`linux/amd64`, `linux/arm64`) image to `ghcr.io//cf-ip-logger`. + +Dependabot watches Go modules, the Dockerfile base image, and the GitHub Actions versions weekly (`.github/dependabot.yml`). + ## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/alias-dist/alias-dist.sh b/alias-dist/alias-dist.sh old mode 100644 new mode 100755 index f828642..3114639 --- a/alias-dist/alias-dist.sh +++ b/alias-dist/alias-dist.sh @@ -1,462 +1,462 @@ -#!/bin/bash -# -# distribute-aliases.sh -# Distributes bash aliases to all nodes listed in /etc/hosts -# Supports parallel deployment for faster execution -# - -# ============================================================ -# CONFIGURATION - Add your aliases here -# ============================================================ -ALIASES=( - # ------------------------------ - # Navigation & Directory - # ------------------------------ - "alias ll='ls -lAhF --color=auto'" - "alias la='ls -A'" - "alias l='ls -CF'" - "alias ls='ls --color=auto'" - "alias ..='cd ..'" - "alias ...='cd ../..'" - "alias ....='cd ../../..'" - "alias ~='cd ~'" - "alias -- -='cd -'" - "alias mkdir='mkdir -pv'" - "alias tree='tree -C'" - - # ------------------------------ - # File Operations - # ------------------------------ - "alias cp='cp -iv'" - "alias mv='mv -iv'" - "alias rm='rm -Iv'" - "alias ln='ln -iv'" - "alias chmod='chmod -v'" - "alias chown='chown -v'" - "alias df='df -hT'" - "alias du='du -h'" - "alias dus='du -sh * | sort -h'" - "alias free='free -h'" - - # ------------------------------ - # Search & Find - # ------------------------------ - "alias grep='grep --color=auto'" - "alias egrep='egrep --color=auto'" - "alias fgrep='fgrep --color=auto'" - "alias ff='find . -type f -name'" - "alias fd='find . -type d -name'" - "alias h='history'" - "alias hg='history | grep'" - - # ------------------------------ - # System & Process - # ------------------------------ - "alias s='sudo'" - "alias ps='ps auxf'" - "alias psg='ps aux | grep -v grep | grep -i'" - "alias top='htop 2>/dev/null || top'" - "alias kill9='kill -9'" - "alias meminfo='free -h -l -t'" - "alias cpuinfo='lscpu'" - "alias temp='vcgencmd measure_temp 2>/dev/null || sensors 2>/dev/null || cat /sys/class/thermal/thermal_zone*/temp 2>/dev/null'" - - # ------------------------------ - # Networking - # ------------------------------ - "alias ports='ss -tulpn'" - "alias listening='ss -tulpn | grep LISTEN'" - "alias myip='curl -s ifconfig.me'" - "alias localip=\"ip -4 addr show | grep -oP '(?<=inet\\s)\\d+(\\.\\d+){3}' | grep -v 127.0.0.1\"" - "alias ping='ping -c 5'" - "alias pingg='ping google.com'" - "alias wget='wget -c'" - "alias speedtest='curl -s https://raw.githubusercontent.com/sivel/speedtest-cli/master/speedtest.py | python3 -'" - "alias flushdns='sudo systemd-resolve --flush-caches'" - - # ------------------------------ - # Package Management (Debian/Ubuntu) - # ------------------------------ - "alias update='sudo apt update && sudo apt upgrade -y'" - "alias install='sudo apt install'" - "alias remove='sudo apt remove'" - "alias autoremove='sudo apt autoremove -y'" - "alias search='apt search'" - "alias cleanup='sudo apt autoremove -y && sudo apt autoclean'" - - # ------------------------------ - # Systemd & Services - # ------------------------------ - "alias sc='sudo systemctl'" - "alias scstart='sudo systemctl start'" - "alias scstop='sudo systemctl stop'" - "alias screstart='sudo systemctl restart'" - "alias scstatus='sudo systemctl status'" - "alias scenable='sudo systemctl enable'" - "alias scdisable='sudo systemctl disable'" - "alias sclog='journalctl -xeu'" - "alias jlog='journalctl -f'" - "alias failed='systemctl --failed'" - - # ------------------------------ - # Docker - # ------------------------------ - "alias d='docker'" - "alias dc='docker compose'" - "alias dps='docker ps --format \"table {{.Names}}\\t{{.Status}}\\t{{.Ports}}\"'" - "alias dpsa='docker ps -a --format \"table {{.Names}}\\t{{.Status}}\\t{{.Ports}}\"'" - "alias dimg='docker images'" - "alias dlog='docker logs -f'" - "alias dexec='docker exec -it'" - "alias dstop='docker stop \$(docker ps -q)'" - "alias dprune='docker system prune -af'" - "alias dvprune='docker volume prune -f'" - "alias dstats='docker stats --no-stream'" - "alias dtop='docker stats --format \"table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\"'" - "alias dclean='docker system prune -af && docker volume prune -f'" - "alias dnet='docker network ls'" - "alias dinspect='docker inspect'" - "alias dip='docker inspect -f \"{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\"'" - - # ------------------------------ - # Kubernetes / k3s - # ------------------------------ - "alias k='kubectl'" - "alias kgp='kubectl get pods'" - "alias kgpa='kubectl get pods -A'" - "alias kgn='kubectl get nodes'" - "alias kgs='kubectl get svc'" - "alias kgd='kubectl get deployments'" - "alias kgi='kubectl get ingress'" - "alias kgpv='kubectl get pv'" - "alias kgpvc='kubectl get pvc'" - "alias kga='kubectl get all'" - "alias kgaa='kubectl get all -A'" - "alias kdesc='kubectl describe'" - "alias klog='kubectl logs -f'" - "alias kexec='kubectl exec -it'" - "alias kaf='kubectl apply -f'" - "alias kdf='kubectl delete -f'" - "alias kctx='kubectl config get-contexts'" - "alias kns='kubectl config set-context --current --namespace'" - "alias ktop='kubectl top'" - "alias kwatch='watch -n1 kubectl get pods'" - "alias krollout='kubectl rollout restart deployment'" - - # ------------------------------ - # Git - # ------------------------------ - "alias g='git'" - "alias gs='git status'" - "alias ga='git add'" - "alias gaa='git add -A'" - "alias gc='git commit -m'" - "alias gp='git push'" - "alias gpl='git pull'" - "alias gf='git fetch'" - "alias gb='git branch'" - "alias gco='git checkout'" - "alias gd='git diff'" - "alias glog='git log --oneline --graph --decorate -10'" - "alias gclone='git clone'" - - # ------------------------------ - # Editors & Config - # ------------------------------ - "alias nano='nano -l'" - "alias bashrc='nano ~/.bashrc && source ~/.bashrc'" - "alias src='source ~/.bashrc'" - "alias path='echo \$PATH | tr \":\" \"\\n\"'" - "alias hosts='sudo nano /etc/hosts'" - - # ------------------------------ - # Quick Shortcuts - # ------------------------------ - "alias c='clear'" - "alias q='exit'" - "alias now='date +\"%Y-%m-%d %H:%M:%S\"'" - "alias week='date +%V'" - "alias timestamp='date +%s'" - "alias weather='curl wttr.in/?0'" - "alias moon='curl wttr.in/Moon'" - "alias sha='shasum -a 256'" - "alias genpass='openssl rand -base64 20'" - "alias busy=\"cat /dev/urandom | hexdump -C | grep 'ca fe'\"" - - # ------------------------------ - # Safety Nets - # ------------------------------ - "alias reboot='sudo /sbin/reboot'" - "alias poweroff='sudo /sbin/poweroff'" - "alias shutdown='sudo /sbin/shutdown'" - - # ------------------------------ - # Tail & Watch - # ------------------------------ - "alias tf='tail -f'" - "alias t100='tail -100'" - "alias watch='watch -n 2'" - - # ------------------------------ - # Add your custom aliases below - # ------------------------------ - # "alias myalias='my command'" -) - -# ============================================================ -# SETTINGS -# ============================================================ - -# SSH user (change if different per host) -SSH_USER="${SSH_USER:-$(whoami)}" - -# Max parallel jobs (adjust based on your network) -MAX_PARALLEL="${MAX_PARALLEL:-10}" - -# SSH timeout in seconds -SSH_TIMEOUT=10 - -# Marker to identify our managed alias block -MARKER_START="# >>> DISTRIBUTED ALIASES START <<<" -MARKER_END="# >>> DISTRIBUTED ALIASES END <<<" - -# Temp directory for parallel results -TMPDIR=$(mktemp -d) -trap "rm -rf $TMPDIR" EXIT - -# ============================================================ -# FUNCTIONS -# ============================================================ - -get_hosts() { - grep -v '^#' /etc/hosts | \ - grep -v '^\s*$' | \ - grep -v -E 'localhost|broadcasthost' | \ - grep -v '127.0.0.1' | \ - grep -v '127.0.1.1' | \ - grep -v '::1' | \ - grep -v ':' | \ - awk '{print $1}' | \ - sort -u -} - -generate_alias_block() { - echo "$MARKER_START" - echo "# Distributed on: $(date)" - echo "# From: $(hostname)" - for alias_line in "${ALIASES[@]}"; do - # Skip comment-only lines for the actual file - if [[ ! "$alias_line" =~ ^[[:space:]]*# ]]; then - echo "$alias_line" - fi - done - echo "$MARKER_END" -} - -deploy_to_host() { - local host=$1 - local result_file="$TMPDIR/result_${host//\./_}" - local alias_block - alias_block=$(generate_alias_block) - - # Test SSH connectivity first - if ! ssh -o ConnectTimeout=$SSH_TIMEOUT -o BatchMode=yes -o StrictHostKeyChecking=no \ - "${SSH_USER}@${host}" "echo 'OK'" &>/dev/null; then - echo "SKIP:$host:Cannot connect" > "$result_file" - return 1 - fi - - # Remove existing alias block and add new one - ssh -o ConnectTimeout=$SSH_TIMEOUT -o BatchMode=yes "${SSH_USER}@${host}" bash -s </dev/null - - # Remove existing block if present - if grep -q "\$MARKER_START" "\$BASHRC" 2>/dev/null; then - sed -i "/\$MARKER_START/,/\$MARKER_END/d" "\$BASHRC" - fi - - # Append new alias block - cat >> "\$BASHRC" <<'ALIASBLOCK' -$alias_block -ALIASBLOCK - - echo "Deployed successfully" -EOF - - if [[ $? -eq 0 ]]; then - echo "OK:$host:Deployed ${#ALIASES[@]} aliases" > "$result_file" - return 0 - else - echo "FAIL:$host:SSH command failed" > "$result_file" - return 1 - fi -} - -run_parallel() { - local hosts=("$@") - local running=0 - local pids=() - - for host in "${hosts[@]}"; do - # Wait if we've hit max parallel jobs - while [[ $running -ge $MAX_PARALLEL ]]; do - for i in "${!pids[@]}"; do - if ! kill -0 "${pids[$i]}" 2>/dev/null; then - unset 'pids[i]' - ((running--)) - fi - done - pids=("${pids[@]}") # Re-index array - sleep 0.1 - done - - # Launch deployment in background - deploy_to_host "$host" & - pids+=($!) - ((running++)) - echo -ne "\rDeploying... [$running active jobs] " - done - - # Wait for all remaining jobs - echo -ne "\rWaiting for remaining jobs to complete... " - wait - echo -e "\rAll deployments complete. " -} - -print_results() { - local success=0 - local failed=0 - local skipped=0 - - echo "" - echo "========================================" - echo " Results" - echo "========================================" - - for result_file in "$TMPDIR"/result_*; do - [[ -f "$result_file" ]] || continue - local line=$(cat "$result_file") - local status=$(echo "$line" | cut -d: -f1) - local host=$(echo "$line" | cut -d: -f2) - local msg=$(echo "$line" | cut -d: -f3-) - - case "$status" in - OK) - echo -e " [\e[32m✓\e[0m] $host - $msg" - ((success++)) - ;; - SKIP) - echo -e " [\e[33m-\e[0m] $host - $msg" - ((skipped++)) - ;; - FAIL) - echo -e " [\e[31m✗\e[0m] $host - $msg" - ((failed++)) - ;; - esac - done - - echo "" - echo "========================================" - echo " Summary" - echo "========================================" - echo -e " Successful: \e[32m$success\e[0m" - echo -e " Skipped: \e[33m$skipped\e[0m" - echo -e " Failed: \e[31m$failed\e[0m" - echo "" - echo " Run 'source ~/.bashrc' or 'src' (after first deploy)" - echo " on each node to activate aliases." -} - -show_aliases() { - echo "" - echo "Aliases to deploy (${#ALIASES[@]} total):" - echo "----------------------------------------" - for alias_line in "${ALIASES[@]}"; do - echo " $alias_line" - done - echo "----------------------------------------" -} - -# ============================================================ -# MAIN -# ============================================================ - -echo "========================================" -echo " Alias Distribution Script" -echo "========================================" -echo " SSH User: $SSH_USER" -echo " Max Parallel: $MAX_PARALLEL" -echo " Aliases: ${#ALIASES[@]}" -echo "" - -# Parse arguments -DRY_RUN=false -SHOW_ALIASES=false -while [[ $# -gt 0 ]]; do - case $1 in - -n|--dry-run) DRY_RUN=true; shift ;; - -l|--list) SHOW_ALIASES=true; shift ;; - -p|--parallel) MAX_PARALLEL="$2"; shift 2 ;; - -u|--user) SSH_USER="$2"; shift 2 ;; - -h|--help) - echo "Usage: $0 [options]" - echo " -n, --dry-run Show what would be done without deploying" - echo " -l, --list List all aliases" - echo " -p, --parallel N Max parallel jobs (default: 10)" - echo " -u, --user USER SSH user (default: current user)" - echo " -h, --help Show this help" - exit 0 - ;; - *) echo "Unknown option: $1"; exit 1 ;; - esac -done - -if $SHOW_ALIASES; then - show_aliases - exit 0 -fi - -# Get list of hosts -HOSTS=($(get_hosts)) - -if [[ ${#HOSTS[@]} -eq 0 ]]; then - echo "No hosts found in /etc/hosts" - exit 1 -fi - -echo "Found ${#HOSTS[@]} hosts in /etc/hosts:" -printf ' - %s\n' "${HOSTS[@]}" -echo "" - -if $DRY_RUN; then - echo "[DRY RUN] Would deploy to the above hosts." - show_aliases - exit 0 -fi - -read -p "Deploy ${#ALIASES[@]} aliases to ${#HOSTS[@]} hosts? (y/N): " confirm -if [[ ! "$confirm" =~ ^[Yy]$ ]]; then - echo "Aborted." - exit 0 -fi - -echo "" -echo "Starting parallel deployment..." -START_TIME=$(date +%s) - -run_parallel "${HOSTS[@]}" - -END_TIME=$(date +%s) -ELAPSED=$((END_TIME - START_TIME)) - -print_results +#!/usr/bin/env bash +# +# distribute-aliases.sh +# Distributes bash aliases to all nodes listed in /etc/hosts +# Supports parallel deployment for faster execution +# + +# ============================================================ +# CONFIGURATION - Add your aliases here +# ============================================================ +ALIASES=( + # ------------------------------ + # Navigation & Directory + # ------------------------------ + "alias ll='ls -lAhF --color=auto'" + "alias la='ls -A'" + "alias l='ls -CF'" + "alias ls='ls --color=auto'" + "alias ..='cd ..'" + "alias ...='cd ../..'" + "alias ....='cd ../../..'" + "alias ~='cd ~'" + "alias -- -='cd -'" + "alias mkdir='mkdir -pv'" + "alias tree='tree -C'" + + # ------------------------------ + # File Operations + # ------------------------------ + "alias cp='cp -iv'" + "alias mv='mv -iv'" + "alias rm='rm -Iv'" + "alias ln='ln -iv'" + "alias chmod='chmod -v'" + "alias chown='chown -v'" + "alias df='df -hT'" + "alias du='du -h'" + "alias dus='du -sh * | sort -h'" + "alias free='free -h'" + + # ------------------------------ + # Search & Find + # ------------------------------ + "alias grep='grep --color=auto'" + "alias egrep='egrep --color=auto'" + "alias fgrep='fgrep --color=auto'" + "alias ff='find . -type f -name'" + "alias fd='find . -type d -name'" + "alias h='history'" + "alias hg='history | grep'" + + # ------------------------------ + # System & Process + # ------------------------------ + "alias s='sudo'" + "alias ps='ps auxf'" + "alias psg='ps aux | grep -v grep | grep -i'" + "alias top='htop 2>/dev/null || top'" + "alias kill9='kill -9'" + "alias meminfo='free -h -l -t'" + "alias cpuinfo='lscpu'" + "alias temp='vcgencmd measure_temp 2>/dev/null || sensors 2>/dev/null || cat /sys/class/thermal/thermal_zone*/temp 2>/dev/null'" + + # ------------------------------ + # Networking + # ------------------------------ + "alias ports='ss -tulpn'" + "alias listening='ss -tulpn | grep LISTEN'" + "alias myip='curl -s ifconfig.me'" + "alias localip=\"ip -4 addr show | grep -oP '(?<=inet\\s)\\d+(\\.\\d+){3}' | grep -v 127.0.0.1\"" + "alias ping='ping -c 5'" + "alias pingg='ping google.com'" + "alias wget='wget -c'" + "alias speedtest='curl -s https://raw.githubusercontent.com/sivel/speedtest-cli/master/speedtest.py | python3 -'" + "alias flushdns='sudo systemd-resolve --flush-caches'" + + # ------------------------------ + # Package Management (Debian/Ubuntu) + # ------------------------------ + "alias update='sudo apt update && sudo apt upgrade -y'" + "alias install='sudo apt install'" + "alias remove='sudo apt remove'" + "alias autoremove='sudo apt autoremove -y'" + "alias search='apt search'" + "alias cleanup='sudo apt autoremove -y && sudo apt autoclean'" + + # ------------------------------ + # Systemd & Services + # ------------------------------ + "alias sc='sudo systemctl'" + "alias scstart='sudo systemctl start'" + "alias scstop='sudo systemctl stop'" + "alias screstart='sudo systemctl restart'" + "alias scstatus='sudo systemctl status'" + "alias scenable='sudo systemctl enable'" + "alias scdisable='sudo systemctl disable'" + "alias sclog='journalctl -xeu'" + "alias jlog='journalctl -f'" + "alias failed='systemctl --failed'" + + # ------------------------------ + # Docker + # ------------------------------ + "alias d='docker'" + "alias dc='docker compose'" + "alias dps='docker ps --format \"table {{.Names}}\\t{{.Status}}\\t{{.Ports}}\"'" + "alias dpsa='docker ps -a --format \"table {{.Names}}\\t{{.Status}}\\t{{.Ports}}\"'" + "alias dimg='docker images'" + "alias dlog='docker logs -f'" + "alias dexec='docker exec -it'" + "alias dstop='docker stop \$(docker ps -q)'" + "alias dprune='docker system prune -af'" + "alias dvprune='docker volume prune -f'" + "alias dstats='docker stats --no-stream'" + "alias dtop='docker stats --format \"table {{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}\"'" + "alias dclean='docker system prune -af && docker volume prune -f'" + "alias dnet='docker network ls'" + "alias dinspect='docker inspect'" + "alias dip='docker inspect -f \"{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\"'" + + # ------------------------------ + # Kubernetes / k3s + # ------------------------------ + "alias k='kubectl'" + "alias kgp='kubectl get pods'" + "alias kgpa='kubectl get pods -A'" + "alias kgn='kubectl get nodes'" + "alias kgs='kubectl get svc'" + "alias kgd='kubectl get deployments'" + "alias kgi='kubectl get ingress'" + "alias kgpv='kubectl get pv'" + "alias kgpvc='kubectl get pvc'" + "alias kga='kubectl get all'" + "alias kgaa='kubectl get all -A'" + "alias kdesc='kubectl describe'" + "alias klog='kubectl logs -f'" + "alias kexec='kubectl exec -it'" + "alias kaf='kubectl apply -f'" + "alias kdf='kubectl delete -f'" + "alias kctx='kubectl config get-contexts'" + "alias kns='kubectl config set-context --current --namespace'" + "alias ktop='kubectl top'" + "alias kwatch='watch -n1 kubectl get pods'" + "alias krollout='kubectl rollout restart deployment'" + + # ------------------------------ + # Git + # ------------------------------ + "alias g='git'" + "alias gs='git status'" + "alias ga='git add'" + "alias gaa='git add -A'" + "alias gc='git commit -m'" + "alias gp='git push'" + "alias gpl='git pull'" + "alias gf='git fetch'" + "alias gb='git branch'" + "alias gco='git checkout'" + "alias gd='git diff'" + "alias glog='git log --oneline --graph --decorate -10'" + "alias gclone='git clone'" + + # ------------------------------ + # Editors & Config + # ------------------------------ + "alias nano='nano -l'" + "alias bashrc='nano ~/.bashrc && source ~/.bashrc'" + "alias src='source ~/.bashrc'" + "alias path='echo \$PATH | tr \":\" \"\\n\"'" + "alias hosts='sudo nano /etc/hosts'" + + # ------------------------------ + # Quick Shortcuts + # ------------------------------ + "alias c='clear'" + "alias q='exit'" + "alias now='date +\"%Y-%m-%d %H:%M:%S\"'" + "alias week='date +%V'" + "alias timestamp='date +%s'" + "alias weather='curl wttr.in/?0'" + "alias moon='curl wttr.in/Moon'" + "alias sha='shasum -a 256'" + "alias genpass='openssl rand -base64 20'" + "alias busy=\"cat /dev/urandom | hexdump -C | grep 'ca fe'\"" + + # ------------------------------ + # Safety Nets + # ------------------------------ + "alias reboot='sudo /sbin/reboot'" + "alias poweroff='sudo /sbin/poweroff'" + "alias shutdown='sudo /sbin/shutdown'" + + # ------------------------------ + # Tail & Watch + # ------------------------------ + "alias tf='tail -f'" + "alias t100='tail -100'" + "alias watch='watch -n 2'" + + # ------------------------------ + # Add your custom aliases below + # ------------------------------ + # "alias myalias='my command'" +) + +# ============================================================ +# SETTINGS +# ============================================================ + +# SSH user (change if different per host) +SSH_USER="${SSH_USER:-$(whoami)}" + +# Max parallel jobs (adjust based on your network) +MAX_PARALLEL="${MAX_PARALLEL:-10}" + +# SSH timeout in seconds +SSH_TIMEOUT=10 + +# Marker to identify our managed alias block +MARKER_START="# >>> DISTRIBUTED ALIASES START <<<" +MARKER_END="# >>> DISTRIBUTED ALIASES END <<<" + +# Temp directory for parallel results +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT + +# ============================================================ +# FUNCTIONS +# ============================================================ + +get_hosts() { + grep -v '^#' /etc/hosts | \ + grep -v '^\s*$' | \ + grep -v -E 'localhost|broadcasthost' | \ + grep -v '127.0.0.1' | \ + grep -v '127.0.1.1' | \ + grep -v '::1' | \ + grep -v ':' | \ + awk '{print $1}' | \ + sort -u +} + +generate_alias_block() { + echo "$MARKER_START" + echo "# Distributed on: $(date)" + echo "# From: $(hostname)" + for alias_line in "${ALIASES[@]}"; do + # Skip comment-only lines for the actual file + if [[ ! "$alias_line" =~ ^[[:space:]]*# ]]; then + echo "$alias_line" + fi + done + echo "$MARKER_END" +} + +deploy_to_host() { + local host=$1 + local result_file="$TMPDIR/result_${host//\./_}" + local alias_block + alias_block=$(generate_alias_block) + + # Test SSH connectivity first + if ! ssh -o ConnectTimeout=$SSH_TIMEOUT -o BatchMode=yes -o StrictHostKeyChecking=no \ + "${SSH_USER}@${host}" "echo 'OK'" &>/dev/null; then + echo "SKIP:$host:Cannot connect" > "$result_file" + return 1 + fi + + # Remove existing alias block and add new one + ssh -o ConnectTimeout=$SSH_TIMEOUT -o BatchMode=yes "${SSH_USER}@${host}" bash -s </dev/null + + # Remove existing block if present + if grep -q "\$MARKER_START" "\$BASHRC" 2>/dev/null; then + sed -i "/\$MARKER_START/,/\$MARKER_END/d" "\$BASHRC" + fi + + # Append new alias block + cat >> "\$BASHRC" <<'ALIASBLOCK' +$alias_block +ALIASBLOCK + + echo "Deployed successfully" +EOF + + if [[ $? -eq 0 ]]; then + echo "OK:$host:Deployed ${#ALIASES[@]} aliases" > "$result_file" + return 0 + else + echo "FAIL:$host:SSH command failed" > "$result_file" + return 1 + fi +} + +run_parallel() { + local hosts=("$@") + local running=0 + local pids=() + + for host in "${hosts[@]}"; do + # Wait if we've hit max parallel jobs + while [[ $running -ge $MAX_PARALLEL ]]; do + for i in "${!pids[@]}"; do + if ! kill -0 "${pids[$i]}" 2>/dev/null; then + unset 'pids[i]' + ((running--)) + fi + done + pids=("${pids[@]}") # Re-index array + sleep 0.1 + done + + # Launch deployment in background + deploy_to_host "$host" & + pids+=($!) + ((running++)) + echo -ne "\rDeploying... [$running active jobs] " + done + + # Wait for all remaining jobs + echo -ne "\rWaiting for remaining jobs to complete... " + wait + echo -e "\rAll deployments complete. " +} + +print_results() { + local success=0 + local failed=0 + local skipped=0 + + echo "" + echo "========================================" + echo " Results" + echo "========================================" + + for result_file in "$TMPDIR"/result_*; do + [[ -f "$result_file" ]] || continue + local line=$(cat "$result_file") + local status=$(echo "$line" | cut -d: -f1) + local host=$(echo "$line" | cut -d: -f2) + local msg=$(echo "$line" | cut -d: -f3-) + + case "$status" in + OK) + echo -e " [\e[32m✓\e[0m] $host - $msg" + ((success++)) + ;; + SKIP) + echo -e " [\e[33m-\e[0m] $host - $msg" + ((skipped++)) + ;; + FAIL) + echo -e " [\e[31m✗\e[0m] $host - $msg" + ((failed++)) + ;; + esac + done + + echo "" + echo "========================================" + echo " Summary" + echo "========================================" + echo -e " Successful: \e[32m$success\e[0m" + echo -e " Skipped: \e[33m$skipped\e[0m" + echo -e " Failed: \e[31m$failed\e[0m" + echo "" + echo " Run 'source ~/.bashrc' or 'src' (after first deploy)" + echo " on each node to activate aliases." +} + +show_aliases() { + echo "" + echo "Aliases to deploy (${#ALIASES[@]} total):" + echo "----------------------------------------" + for alias_line in "${ALIASES[@]}"; do + echo " $alias_line" + done + echo "----------------------------------------" +} + +# ============================================================ +# MAIN +# ============================================================ + +echo "========================================" +echo " Alias Distribution Script" +echo "========================================" +echo " SSH User: $SSH_USER" +echo " Max Parallel: $MAX_PARALLEL" +echo " Aliases: ${#ALIASES[@]}" +echo "" + +# Parse arguments +DRY_RUN=false +SHOW_ALIASES=false +while [[ $# -gt 0 ]]; do + case $1 in + -n|--dry-run) DRY_RUN=true; shift ;; + -l|--list) SHOW_ALIASES=true; shift ;; + -p|--parallel) MAX_PARALLEL="$2"; shift 2 ;; + -u|--user) SSH_USER="$2"; shift 2 ;; + -h|--help) + echo "Usage: $0 [options]" + echo " -n, --dry-run Show what would be done without deploying" + echo " -l, --list List all aliases" + echo " -p, --parallel N Max parallel jobs (default: 10)" + echo " -u, --user USER SSH user (default: current user)" + echo " -h, --help Show this help" + exit 0 + ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +if $SHOW_ALIASES; then + show_aliases + exit 0 +fi + +# Get list of hosts +HOSTS=($(get_hosts)) + +if [[ ${#HOSTS[@]} -eq 0 ]]; then + echo "No hosts found in /etc/hosts" + exit 1 +fi + +echo "Found ${#HOSTS[@]} hosts in /etc/hosts:" +printf ' - %s\n' "${HOSTS[@]}" +echo "" + +if $DRY_RUN; then + echo "[DRY RUN] Would deploy to the above hosts." + show_aliases + exit 0 +fi + +read -p "Deploy ${#ALIASES[@]} aliases to ${#HOSTS[@]} hosts? (y/N): " confirm +if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 +fi + +echo "" +echo "Starting parallel deployment..." +START_TIME=$(date +%s) + +run_parallel "${HOSTS[@]}" + +END_TIME=$(date +%s) +ELAPSED=$((END_TIME - START_TIME)) + +print_results echo " Completed in ${ELAPSED}s" \ No newline at end of file diff --git a/chown_throttled/chown_throttled.sh b/chown_throttled/chown_throttled.sh old mode 100644 new mode 100755 diff --git a/cloudflare-ip-logger/.hadolint.yaml b/cloudflare-ip-logger/.hadolint.yaml new file mode 100644 index 0000000..a088d23 --- /dev/null +++ b/cloudflare-ip-logger/.hadolint.yaml @@ -0,0 +1,3 @@ +ignored: + - DL3008 +failure-threshold: warning diff --git a/cloudflare-ip-logger/Dockerfile b/cloudflare-ip-logger/Dockerfile index bb38a8a..0f0a172 100644 --- a/cloudflare-ip-logger/Dockerfile +++ b/cloudflare-ip-logger/Dockerfile @@ -4,15 +4,16 @@ FROM golang:1.21-bookworm AS builder WORKDIR /build # Install SQLite dependencies for CGO -RUN apt-get update && apt-get install -y gcc libsqlite3-dev +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libsqlite3-dev \ + && rm -rf /var/lib/apt/lists/* -COPY go.mod go.sum* ./ +COPY go.mod go.sum ./ RUN go mod download COPY *.go ./ -RUN go mod tidy - # Build main app RUN CGO_ENABLED=1 go build -ldflags="-s -w" -o cf-ip-logger . diff --git a/cloudflare-ip-logger/README.md b/cloudflare-ip-logger/README.md index dd6dab7..5eb620a 100644 --- a/cloudflare-ip-logger/README.md +++ b/cloudflare-ip-logger/README.md @@ -147,6 +147,26 @@ Data is stored in `/data`: - `connections.log` - Plain text log file - `proxy-config.json` - Backend routing config +## Companion Tool: cf-log-parser + +A separate binary in `cmd/logparser/` that ingests `cloudflared`'s own JSON logs into the same SQLite database used by the proxy. Useful when you want to capture connection metadata that cloudflared sees but never reaches the proxy (denied by Access, served from Cloudflare's cache, etc.). + +Build and use: + +```bash +go build -o cf-log-parser ./cmd/logparser + +# Pipe cloudflared logs into the parser: +./run-with-logging.sh + +# Or as a systemd unit alongside an existing cloudflared service: +sudo cp cf-log-parser /usr/local/bin/ +sudo cp cf-log-parser.service /etc/systemd/system/ +sudo systemctl daemon-reload && sudo systemctl enable --now cf-log-parser +``` + +See `cf-log-parser.service` for the unit file and `run-with-logging.sh` for the standalone wrapper. + ## Querying SQLite Directly ```bash diff --git a/cloudflare-ip-logger/cmd/logparser/main.go b/cloudflare-ip-logger/cmd/logparser/main.go index d73a94b..ea60789 100644 --- a/cloudflare-ip-logger/cmd/logparser/main.go +++ b/cloudflare-ip-logger/cmd/logparser/main.go @@ -16,27 +16,27 @@ import ( // CloudflaredLogEntry represents a JSON log line from cloudflared type CloudflaredLogEntry struct { - Time string `json:"time"` - Level string `json:"level"` - Message string `json:"message"` - Msg string `json:"msg"` - Origin string `json:"originURL"` - ClientIP string `json:"clientIP"` - CFRay string `json:"cfRay"` - IP string `json:"ip"` - Location string `json:"location"` - FlowID string `json:"flowId"` - Dest string `json:"dest"` - Rule int `json:"ingressRule"` - Hostname string `json:"hostname"` - Error string `json:"error"` - ConnIndex int `json:"connIndex"` - TraceID string `json:"traceId"` - Status int `json:"status"` - Duration int64 `json:"duration"` - Method string `json:"method"` - Path string `json:"path"` - RuleName string `json:"ruleName"` + Time string `json:"time"` + Level string `json:"level"` + Message string `json:"message"` + Msg string `json:"msg"` + Origin string `json:"originURL"` + ClientIP string `json:"clientIP"` + CFRay string `json:"cfRay"` + IP string `json:"ip"` + Location string `json:"location"` + FlowID string `json:"flowId"` + Dest string `json:"dest"` + Rule int `json:"ingressRule"` + Hostname string `json:"hostname"` + Error string `json:"error"` + ConnIndex int `json:"connIndex"` + TraceID string `json:"traceId"` + Status int `json:"status"` + Duration int64 `json:"duration"` + Method string `json:"method"` + Path string `json:"path"` + RuleName string `json:"ruleName"` } // Regex patterns for non-JSON logs @@ -250,7 +250,7 @@ func (p *LogParser) insertConnection(timestamp, clientIP, country, method, path, INSERT INTO connections (timestamp, client_ip, country, method, path, host, user_agent, referer) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, timestamp, clientIP, country, method, path, host, userAgent, referer) - + if err != nil { log.Printf("Failed to insert: %v", err) return diff --git a/cloudflare-ip-logger/go.sum b/cloudflare-ip-logger/go.sum new file mode 100644 index 0000000..e8d092a --- /dev/null +++ b/cloudflare-ip-logger/go.sum @@ -0,0 +1,2 @@ +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= diff --git a/cloudflare-ip-logger/main.go b/cloudflare-ip-logger/main.go index 0166b91..8a0fb9d 100644 --- a/cloudflare-ip-logger/main.go +++ b/cloudflare-ip-logger/main.go @@ -34,11 +34,11 @@ type ConnectionLog struct { } type IPStats struct { - ClientIP string `json:"client_ip"` - Country string `json:"country"` - HitCount int `json:"hit_count"` - FirstSeen string `json:"first_seen"` - LastSeen string `json:"last_seen"` + ClientIP string `json:"client_ip"` + Country string `json:"country"` + HitCount int `json:"hit_count"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` } type ProxyConfig struct { diff --git a/cloudflare-ip-logger/run-with-logging.sh b/cloudflare-ip-logger/run-with-logging.sh old mode 100644 new mode 100755 index 65a1fc1..367c1a2 --- a/cloudflare-ip-logger/run-with-logging.sh +++ b/cloudflare-ip-logger/run-with-logging.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # run-with-logging.sh # Runs cloudflared and pipes logs to the parser # diff --git a/cluster-ssh-key-setup/cluster-sshKey-setup.sh b/cluster-ssh-key-setup/cluster-sshKey-setup.sh old mode 100644 new mode 100755 index cd9dd24..7092542 --- a/cluster-ssh-key-setup/cluster-sshKey-setup.sh +++ b/cluster-ssh-key-setup/cluster-sshKey-setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Kubernetes Cluster SSH Mesh Setup # Automates SSH key distribution and /etc/hosts synchronization diff --git a/cluster-system-update/update-sys.sh b/cluster-system-update/update-sys.sh old mode 100644 new mode 100755 index 626f781..6a53095 --- a/cluster-system-update/update-sys.sh +++ b/cluster-system-update/update-sys.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # NODES=( diff --git a/dirsync/dirsync.py b/dirsync/dirsync.py old mode 100644 new mode 100755 index 1a8faf5..48a4f1b --- a/dirsync/dirsync.py +++ b/dirsync/dirsync.py @@ -171,4 +171,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/docker-container-update/docker-container-update.sh b/docker-container-update/docker-container-update.sh old mode 100644 new mode 100755 index 7cb14bc..6b3ae59 --- a/docker-container-update/docker-container-update.sh +++ b/docker-container-update/docker-container-update.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Usage: ./docker-update.sh diff --git a/dotfiles/.bash_aliases b/dotfiles/.bash_aliases index 4078527..76eee35 100644 --- a/dotfiles/.bash_aliases +++ b/dotfiles/.bash_aliases @@ -1,6 +1,7 @@ #=============================================================================== # .bash_aliases - Alias Definitions #=============================================================================== +# shellcheck shell=bash #------------------------------------------------------------------------------- # Navigation @@ -193,6 +194,7 @@ alias pipinstall='pip install -r requirements.txt' alias ping='ping -c 5' alias fastping='ping -c 100 -s.2' alias myip='curl -s ifconfig.me && echo' +# shellcheck disable=SC2142 # $1 is awk's, not a shell positional alias localip="hostname -I | awk '{print \$1}'" alias ips="ip -c a" alias listening='netstat -tlnp' diff --git a/dotfiles/.bashrc b/dotfiles/.bashrc index e7f3914..8481859 100644 --- a/dotfiles/.bashrc +++ b/dotfiles/.bashrc @@ -1,6 +1,7 @@ #=============================================================================== # .bashrc - Bash Configuration #=============================================================================== +# shellcheck shell=bash # If not running interactively, don't do anything case $- in diff --git a/dotfiles/bootstrap.sh b/dotfiles/bootstrap.sh old mode 100644 new mode 100755 index b0bd1f7..04662b4 --- a/dotfiles/bootstrap.sh +++ b/dotfiles/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash #=============================================================================== # Bootstrap Script for Linux Environment Setup # Author: Brian Fritzinger diff --git a/git-update/git-update.sh b/git-update/git-update.sh old mode 100644 new mode 100755 index 872b553..e1cbe01 --- a/git-update/git-update.sh +++ b/git-update/git-update.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # ============================================================================= # Git Repository Manager diff --git a/github-star-repos/github-stars.py b/github-star-repos/github-stars.py old mode 100644 new mode 100755 index 9dd31d6..64d18dc --- a/github-star-repos/github-stars.py +++ b/github-star-repos/github-stars.py @@ -5,7 +5,6 @@ import json from collections import Counter from datetime import datetime -from pathlib import Path import unicodedata # Configure these @@ -22,25 +21,25 @@ def get_starred_repos(username, token=None): repos = [] page = 1 headers = {"Accept": "application/vnd.github.v3.star+json"} # Includes starred_at timestamp - + if token: headers["Authorization"] = f"token {token}" - + while True: url = f"https://api.github.com/users/{username}/starred" params = {"page": page, "per_page": 100} - + response = requests.get(url, headers=headers, params=params) response.raise_for_status() - + data = response.json() if not data: break - + repos.extend(data) page += 1 print(f"Fetched page {page - 1} ({len(repos)} repos so far)") - + return repos @@ -52,7 +51,7 @@ def export_to_json(repos, filename): "total_count": len(repos), "repositories": [] } - + for item in repos: # Handle both formats (with and without starred_at) if "repo" in item: @@ -61,7 +60,7 @@ def export_to_json(repos, filename): else: r = item starred_at = None - + repo_data = { "name": r["full_name"], "description": r.get("description"), @@ -81,10 +80,10 @@ def export_to_json(repos, filename): "homepage": r.get("homepage"), } export_data["repositories"].append(repo_data) - + with open(filename, "w", encoding="utf-8") as f: json.dump(export_data, f, indent=2, ensure_ascii=False) - + print(f"\n💾 Exported to {filename}") return export_data @@ -118,7 +117,7 @@ def pad_line(content, total_width): def print_recap(export_data): """Print a nicely formatted recap to screen.""" repos = export_data["repositories"] - + # Header print("\n") print("+" + "=" * 70 + "+") @@ -127,7 +126,7 @@ def print_recap(export_data): print("|" + pad_line(f" Total: {len(repos)} repositories", 70) + "|") print("|" + pad_line(f" Exported: {export_data['exported_at'][:19]}", 70) + "|") print("+" + "=" * 70 + "+") - + # Languages breakdown languages = Counter(r["language"] for r in repos if r["language"]) print("\n+" + "-" * 50 + "+") @@ -138,7 +137,7 @@ def print_recap(export_data): line = f" {lang}: {count} {bar}" print("|" + pad_line(line, 50) + "|") print("+" + "-" * 50 + "+") - + # Most popular repos by_stars = sorted(repos, key=lambda r: r["stars"], reverse=True)[:10] print("\n+" + "-" * 70 + "+") @@ -150,7 +149,7 @@ def print_recap(export_data): line = f" {name:<45} ⭐ {stars:>10}" print("|" + pad_line(line, 70) + "|") print("+" + "-" * 70 + "+") - + # Recently updated by_updated = sorted(repos, key=lambda r: r["updated_at"] or "", reverse=True)[:10] print("\n+" + "-" * 70 + "+") @@ -162,7 +161,7 @@ def print_recap(export_data): line = f" {name:<55} {updated}" print("|" + pad_line(line, 70) + "|") print("+" + "-" * 70 + "+") - + # Topics all_topics = [] for r in repos: @@ -176,12 +175,12 @@ def print_recap(export_data): line = f" {topic}: {count}" print("|" + pad_line(line, 50) + "|") print("+" + "-" * 50 + "+") - + # Stats summary archived = sum(1 for r in repos if r["archived"]) orgs = sum(1 for r in repos if r["owner_type"] == "Organization") users = len(repos) - orgs - + print("\n+" + "-" * 40 + "+") print("|" + pad_line(" 📈 QUICK STATS", 40) + "|") print("+" + "-" * 40 + "+") @@ -191,13 +190,13 @@ def print_recap(export_data): print("|" + pad_line(f" 💻 Languages: {len(languages)}", 40) + "|") print("|" + pad_line(f" 🔖 Topics: {len(topics)}", 40) + "|") print("+" + "-" * 40 + "+") - + # Full list print("\n") print("+" + "=" * 78 + "+") print("|" + pad_line(" 📋 ALL STARRED REPOS", 78) + "|") print("+" + "=" * 78 + "+") - + for r in sorted(repos, key=lambda r: r["name"].lower()): print() print(f" 📁 {r['name']}") @@ -210,7 +209,7 @@ def print_recap(export_data): stats += " 📦 ARCHIVED" print(stats) print(f" 🔗 {r['url']}") - + print("\n" + "-" * 78) print(f"⭐ Total: {len(repos)} starred repositories") print("-" * 78) @@ -219,13 +218,13 @@ def print_recap(export_data): def main(): print(f"Fetching starred repos for {GITHUB_USERNAME}...") repos = get_starred_repos(GITHUB_USERNAME, GITHUB_TOKEN) - + # Export to JSON export_data = export_to_json(repos, OUTPUT_JSON) - + # Print formatted recap print_recap(export_data) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/ollama-updater/ollama-updater.py b/ollama-updater/ollama-updater.py old mode 100644 new mode 100755 index 35e6916..22d1ba8 --- a/ollama-updater/ollama-updater.py +++ b/ollama-updater/ollama-updater.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import subprocess from datetime import datetime @@ -27,4 +28,4 @@ def check_updates(): else: print("All models are up-to-date.") -check_updates() \ No newline at end of file +check_updates() diff --git a/pwr-temp-monitor/deploy_to_nodes.sh b/pwr-temp-monitor/deploy_to_nodes.sh old mode 100644 new mode 100755 index d6974b5..3f2422a --- a/pwr-temp-monitor/deploy_to_nodes.sh +++ b/pwr-temp-monitor/deploy_to_nodes.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Deploy Pi Monitoring to All Nodes # Run this from your management machine (where you have SSH access to all Pis) diff --git a/pwr-temp-monitor/jetson_metrics.sh b/pwr-temp-monitor/jetson_metrics.sh old mode 100644 new mode 100755 index 2f3689a..a75da3d --- a/pwr-temp-monitor/jetson_metrics.sh +++ b/pwr-temp-monitor/jetson_metrics.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Nvidia Jetson Metrics Collector for Prometheus Node Exporter Textfile Collector OUTPUT_DIR="/var/lib/node_exporter/textfile_collector" diff --git a/pwr-temp-monitor/pi_metrics.sh b/pwr-temp-monitor/pi_metrics.sh old mode 100644 new mode 100755 index 568f38f..21f4a2a --- a/pwr-temp-monitor/pi_metrics.sh +++ b/pwr-temp-monitor/pi_metrics.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Pi Metrics Collector for Prometheus Node Exporter Textfile Collector # Place output in /var/lib/node_exporter/textfile_collector/ diff --git a/pwr-temp-monitor/setup.sh b/pwr-temp-monitor/setup.sh old mode 100644 new mode 100755 index 7697256..821cecb --- a/pwr-temp-monitor/setup.sh +++ b/pwr-temp-monitor/setup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Universal Metrics Setup Script # Detects system type and installs appropriate metrics collector diff --git a/pwr-temp-monitor/x86_metrics.sh b/pwr-temp-monitor/x86_metrics.sh old mode 100644 new mode 100755 index e8d0c9c..7e4e1d4 --- a/pwr-temp-monitor/x86_metrics.sh +++ b/pwr-temp-monitor/x86_metrics.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # x86 Metrics Collector for Prometheus Node Exporter Textfile Collector OUTPUT_DIR="/var/lib/node_exporter/textfile_collector" diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..91ee383 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,15 @@ +target-version = "py310" +line-length = 120 + +[lint] +select = ["E", "F", "W", "B"] +ignore = [ + "E501", + "E402", + "E722", + "B007", + "B008", +] + +[lint.per-file-ignores] +"**/__init__.py" = ["F401"]