diff --git a/.github/workflows/optimize_publish.yml b/.github/workflows/optimize_publish.yml new file mode 100644 index 000000000000..da7787138ba3 --- /dev/null +++ b/.github/workflows/optimize_publish.yml @@ -0,0 +1,22 @@ +name: Optimize Publish Settings +on: + workflow_dispatch: + +jobs: + optimize: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: pip install requests + + - name: Run optimization script + run: python3 Tools/_Forge/Publish/calculate_publish_settings.py --files-dir release --server-url "https://cdn.corvaxforge.ru/" diff --git a/.github/workflows/publish-testing.yml b/.github/workflows/publish-testing.yml index f56f9e753969..fafce40488f1 100644 --- a/.github/workflows/publish-testing.yml +++ b/.github/workflows/publish-testing.yml @@ -17,6 +17,12 @@ jobs: - uses: actions/checkout@v3.6.0 with: submodules: 'recursive' + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install discord-webhook requests + - name: Setup .NET Core uses: actions/setup-dotnet@v3.2.0 with: @@ -40,7 +46,7 @@ jobs: run: dotnet run --project Content.Packaging client --no-wipe-release - name: Publish version - run: Tools/publish_multi_request.py --fork-id wizards-testing + run: python3 Tools/_Forge/Publish/advanced_publish.py --fork-id wizards-testing --publish-token PUBLISH_TOKEN env: PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }} GITHUB_REPOSITORY: ${{ vars.GITHUB_REPOSITORY }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b4ca1699c982..77f033f34064 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,6 +17,11 @@ jobs: - name: Install dependencies run: sudo apt-get install -y python3-paramiko python3-lxml + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install discord-webhook requests - uses: actions/checkout@v4.2.2 with: @@ -45,9 +50,10 @@ jobs: run: dotnet run --project Content.Packaging client --no-wipe-release - name: Publish version - run: Tools/publish_multi_request.py + run: python3 Tools/_Forge/Publish/advanced_publish.py --fork-id frontier --publish-token PUBLISH_TOKEN --publish-webhook PUBLISH_WEBHOOK env: PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }} + PUBLISH_WEBHOOK: ${{ secrets.PUBLISH_WEBHOOK }} GITHUB_REPOSITORY: ${{ vars.GITHUB_REPOSITORY }} # - name: Publish changelog (Discord) diff --git a/.github/workflows/publish_mapping.yml b/.github/workflows/publish_mapping.yml index e017f0af2174..96dff581397d 100644 --- a/.github/workflows/publish_mapping.yml +++ b/.github/workflows/publish_mapping.yml @@ -15,6 +15,11 @@ jobs: with: submodules: 'recursive' + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install discord-webhook requests + - name: Setup .NET Core uses: actions/setup-dotnet@v4.1.0 with: @@ -38,7 +43,8 @@ jobs: run: dotnet run --project Content.Packaging client --no-wipe-release - name: Publish version - run: Tools/publish_multi_request_mapping.py + run: python3 Tools/_Forge/Publish/advanced_publish.py --fork-id frontier_mapping --publish-token PUBLISH_MAPPING_TOKEN --publish-webhook PUBLISH_WEBHOOK env: PUBLISH_MAPPING_TOKEN: ${{ secrets.PUBLISH_MAPPING_TOKEN }} + PUBLISH_WEBHOOK: ${{ secrets.PUBLISH_WEBHOOK }} FORK_ID_MAPPING: ${{ vars.FORK_ID_MAPPING }} diff --git a/Tools/_Forge/Publish/advanced_publish.py b/Tools/_Forge/Publish/advanced_publish.py new file mode 100644 index 000000000000..b11989bcb1e5 --- /dev/null +++ b/Tools/_Forge/Publish/advanced_publish.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 + +""" +Продвинутый паблиш с параллельной загрузкой, аргументами и публикацией статуса паблиша в дискорд +Github: FireFoxPhoenix +""" + +import argparse +import requests +import os +import subprocess +import threading +import logging +import sys +from discord_webhook import DiscordWebhook, DiscordEmbed +from typing import Iterable +from concurrent.futures import ThreadPoolExecutor, as_completed +from urllib3.util.retry import Retry + +thread_session = threading.local() +logger = logging.getLogger(__name__) + +# +# CONFIGURATION PARAMETERS +# Forks should change these to publish to their own infrastructure. +# +ROBUST_CDN_URL = "https://cdn.corvaxforge.ru/" # добавить в аругмент + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--fork-id", required=True) + parser.add_argument("--publish-token", required=True) + parser.add_argument("--publish-webhook", required=False, default=None) + parser.add_argument("--max-workers", type=int, default=4) + parser.add_argument("--pool-connections", type=int, default=3) + parser.add_argument("--pool-maxsize", type=int, default=10) + parser.add_argument("--max-retries", type=int, default=3) + parser.add_argument("--release_dir", default="release") + + args = parser.parse_args() + fork_id = args.fork_id + publish_token = args.publish_token + publish_webhook = args.publish_webhook + max_workers = args.max_workers + pool_connections = args.pool_connections + pool_maxsize = args.pool_maxsize + max_retries = args.max_retries + release_dir = args.release_dir + + if publish_webhook: + if publish_webhook.startswith("https://discord.com/api/webhooks/"): + pass + else: + if publish_webhook in os.environ: + publish_webhook = os.environ[publish_webhook] + else: + publish_webhook = None + logger.warning("Publish webhook not found") + else: + publish_webhook = None + logger.warning(f"Publish webhook is empty") + + if fork_id == "" or fork_id == None: + message = "Fork id was not entered" + logger.critical(message) + send_discord_message(message, "Critical", "ffa500", fork_id, publish_webhook) + raise KeyError() + + if publish_token not in os.environ: + message = "Publish token not found" + logger.critical(message) + send_discord_message(message, "Critical", "ffa500", fork_id, publish_webhook) + sys.exit(1) + publish_token = os.environ[publish_token] + if not publish_token: + message = f"Publish token is empty" + logger.critical(message) + # send_discord_message(message, "Critical", "ffa500", fork_id, publish_webhook) + sys.exit(1) + + #if "GITHUB_SHA" not in os.environ: + # logger.critical("GITHUB_SHA environment variable not set") + # sys.exit(1) + version = os.environ["GITHUB_SHA"] + logger.info(f"Starting publish on Robust.Cdn for version {version}") + + session = create_session(publish_token, pool_connections, pool_maxsize, max_retries) + data = { + "version": version, + "engineVersion": get_engine_version(), + } + headers = { "Content-Type": "application/json" } + resp = session.post(f"{ROBUST_CDN_URL}fork/{fork_id}/publish/start", json=data, headers=headers) + resp.raise_for_status() + logger.info("Publish successfully started, adding files...") + + files = list(get_files_to_publish(release_dir)) + if not files: + message = "No files found to publish" + logger.warning(message) + send_discord_message(message, "Warning", "ffff00", fork_id, publish_webhook) + + logger.info(f"Uploading {len(files)} files using {max_workers} parallel workers...") + successful = 0 + failed = 0 + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_files = { + executor.submit(upload_file, str(file), fork_id, publish_token, pool_connections, pool_maxsize, max_retries, version): file for file in files + } + for future in as_completed(future_files): + file_path = future_files[future] + try: + result = future.result() + successful += 1 + # logger.info(f"Successfully published {os.path.basename(file_path)} ({successful}/{len(files)}") + except Exception as e: + failed += 1 + logger.warning(f"Failed to publish {os.path.basename(file_path)}: {e}") + if failed: + message = f"Upload completed with {failed} failures" + logger.warning(message) + send_discord_message(message, "Warning", "ffff00", fork_id, publish_webhook) + # sys.exit(1) + else: + message = f"All {successful} files uploaded successfully" + logger.info(message) + # send_discord_message(message, "Info", "03b2f8", fork_id, publish_webhook) + + logger.info("Finishing publish...") + data = { "version": version } + headers = { "Content-Type": "application/json" } + resp = session.post(f"{ROBUST_CDN_URL}fork/{fork_id}/publish/finish", json=data, headers=headers) + resp.raise_for_status() + message = "Publish completed" + logger.info(message) + send_discord_message(message, "Info", "03b2f8", fork_id, publish_webhook) + +def get_files_to_publish(release_dir: str) -> Iterable[str]: + try: + for root, dirs, files in os.walk(release_dir): + for file in files: + yield os.path.join(root, file) + except FileNotFoundError: + logger.error(f"Release directory '{release_dir}' not found") + return [] + except PermissionError: + logger.error(f"No permission to read directory '{release_dir}'") + return [] + +def get_engine_version() -> str: + try: + proc = subprocess.run(["git", "describe","--tags", "--abbrev=0"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd="RobustToolbox", check=True, encoding="UTF-8", timeout=20) + tag = proc.stdout.strip() + if not tag.startswith("v"): + logger.warning(f"Unexpected tag format: {tag}") + return tag + return tag[1:] + except subprocess.CalledProcessError as e: + stderr = (e.stderr or "").strip() + logger.error(f"Failed to get engine version: {stderr[:300]}") + return "unknown" + except FileNotFoundError: + logger.error("RobustToolbox directory not found") + return "unknown" + except subprocess.TimeoutExpired: + logger.error("Git command timed out") + return "unknown" + +def upload_file(file_path: str, fork_id: str, publish_token: str, pool_connections: int, pool_maxsize: int, max_retries: int, version: str): + try: + if not hasattr(thread_session, "session"): + thread_session.session = create_session(publish_token, pool_connections, pool_maxsize, max_retries) + session = thread_session.session + with open(file_path, "rb") as file: + headers = { + "Content-Type": "application/octet-stream", + "Robust-Cdn-Publish-File": os.path.basename(file_path), + "Robust-Cdn-Publish-Version": version + } + resp = session.post(f"{ROBUST_CDN_URL}fork/{fork_id}/publish/file", data=file, headers=headers, timeout=(15,30)) + resp.raise_for_status() + return file_path + except FileNotFoundError: + logger.error(f"File '{file_path}' not found") + raise + except IOError as e: + logger.error(f"IO error reading '{file_path}': {e}") + raise + except Exception as e: + logger.error(f"Unexpected error with '{file_path}': {e}") + raise + +def create_session(publish_token: str, pool_connections: int, pool_maxsize: int, max_retries: int) -> requests.Session: + session = requests.Session() + r = Retry( + total=max_retries, + backoff_factor=0.5, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["HEAD", "GET", "PUT", "POST", "DELETE", "OPTIONS", "TRACE"] + ) + adapter = requests.adapters.HTTPAdapter( + pool_connections=pool_connections, + pool_maxsize=pool_maxsize, + max_retries=r + ) + session.mount("https://", adapter) + session.mount("http://", adapter) + session.headers.update({ "Authorization": f"Bearer {publish_token}" }) + session.request = lambda method, url, **kwargs: requests.Session.request( + session, method, url, timeout=(5,30), **kwargs + ) + return session + +def send_discord_message(message: str, status: str, fork_id: str = None, color: str = "00ff00", publish_webhook: str = None): + if not publish_webhook: + return + if not fork_id: + fork_id = "unknown" + try: + webhook = DiscordWebhook( + url=publish_webhook, + username="Publish Status", + rate_limit_retry=True + ) + embed = DiscordEmbed( + title=f"Publish for {fork_id}", + color=color + ) + embed.add_embed_field(name=status, value=message) + embed.set_timestamp() + webhook.add_embed(embed) + response = webhook.execute() + if response.status_code not in [200, 204]: + logger.warning("The Discord message was not sent") + except Exception as e: + logger.error(f"The Discord message was not sent: {e}") + return + +if __name__ == '__main__': + main() diff --git a/Tools/_Forge/Publish/calculate_optimal_settings.py b/Tools/_Forge/Publish/calculate_optimal_settings.py new file mode 100644 index 000000000000..2903d781645e --- /dev/null +++ b/Tools/_Forge/Publish/calculate_optimal_settings.py @@ -0,0 +1,190 @@ +""" +Считает оптимальные настройки для максимизации скорости паблиша +Github: FireFoxPhoenix +""" + +#!/usr/bin/env python3 + +import argparse +import os +import time +import requests +import statistics +from pathlib import Path +import socket +import json + +def measure_network_speed(url: str) -> float: + try: + test_file = os.urandom(1024 * 1024) + start = time.time() + response = requests.post(f"{url}fork/test/publish/file", data=test_file, headers={"Content-Type": "application/octet-stream"}, timeout=5) + elapsed = time.time() - start + if response.status_code < 500: + speed_mbps = (1 * 8) / elapsed + return speed_mbps + except: + pass + return 100.0 + +def measure_server_latency(url: str) -> float: + try: + times = [] + for _ in range(3): + start = time.perf_counter() + requests.get(f"{url}fork/test/publish/start", timeout=3) + elapsed = (time.perf_counter() - start) * 1000 + times.append(elapsed) + return statistics.median(times) + except: + return 100.0 + +def analyze_files(files_dir: str): + total_size = 0 + file_count = 0 + sizes = [] + for root, dirs, files in os.walk(files_dir): + for file in files: + filepath = os.path.join(root, file) + try: + size = os.path.getsize(filepath) + total_size += size + sizes.append(size) + file_count += 1 + except: + continue + if file_count == 0: + return 0, 0.0, 0.0, [] + avg_size = total_size / file_count / (1024 * 1024) + median_size = statistics.median(sizes) / (1024 * 1024) if sizes else 0 + size_distribution = { + 'tiny': sum(1 for s in sizes if s < 100 * 1024), + 'small': sum(1 for s in sizes if 100 * 1024 <= s < 1 * 1024 * 1024), + 'medium': sum(1 for s in sizes if 1 * 1024 * 1024 <= s < 10 * 1024 * 1024), + 'large': sum(1 for s in sizes if 10 * 1024 * 1024 <= s < 100 * 1024 * 1024), + 'huge': sum(1 for s in sizes if s >= 100 * 1024 * 1024) + } + return file_count, avg_size, median_size, size_distribution + +def calculate_optimal_settings(file_count, avg_size_mb, network_speed_mbps, latency_ms): + base_threads = min(file_count, 16) + if latency_ms > 200: + network_factor = 0.5 + elif latency_ms > 100: + network_factor = 0.7 + elif latency_ms > 50: + network_factor = 0.9 + else: + network_factor = 1.0 + + if avg_size_mb < 0.1: + size_factor = 2.0 + optimal_threads = min(base_threads, 16) + elif avg_size_mb < 1: + size_factor = 1.5 + optimal_threads = min(base_threads, 12) + elif avg_size_mb < 10: + size_factor = 1.0 + optimal_threads = min(base_threads, 8) + elif avg_size_mb < 50: + size_factor = 0.7 + optimal_threads = min(base_threads, 4) + else: + size_factor = 0.5 + optimal_threads = min(base_threads, 2) + + bandwidth_per_thread = network_speed_mbps / optimal_threads + if bandwidth_per_thread < 1: + optimal_threads = max(1, int(network_speed_mbps)) + + adjusted_threads = int(optimal_threads * network_factor * size_factor) + adjusted_threads = max(1, min(adjusted_threads, file_count, 16)) + + pool_connections = 3 + + if adjusted_threads <= 2: + pool_maxsize = 4 + elif adjusted_threads <= 4: + pool_maxsize = 8 + elif adjusted_threads <= 8: + pool_maxsize = 12 + else: + pool_maxsize = 16 + + total_size_mb = file_count * avg_size_mb + upload_time_single = (total_size_mb * 8) / network_speed_mbps + estimated_time = upload_time_single / adjusted_threads + estimated_time += (latency_ms / 1000) * file_count / adjusted_threads + + return { + 'max_workers': adjusted_threads, + 'pool_connections': pool_connections, + 'pool_maxsize': pool_maxsize, + 'estimated_time_minutes': estimated_time / 60, + 'speedup': upload_time_single / estimated_time if estimated_time > 0 else 1, + 'bandwidth_per_thread_mbps': network_speed_mbps / adjusted_threads + } + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--files-dir", default="release") + parser.add_argument("--server-url", required=True) + parser.add_argument("--network-speed", type=float) + parser.add_argument("--skip-measure", action="store_true") + + args = parser.parse_args() + + print("Analyzing files...") + file_count, avg_size, median_size, size_dist = analyze_files(args.files_dir) + + if file_count == 0: + print("No files found.") + return + + if args.skip_measure: + latency = 100.0 + network_speed = args.network_speed or 100.0 + print("Using default measurements (skipped)") + else: + print("Measuring server latency...") + latency = measure_server_latency(args.server_url) + + if args.network_speed: + network_speed = args.network_speed + print(f"Using provided network speed: {network_speed} Mbps") + else: + print("Measuring network speed...") + network_speed = measure_network_speed(args.server_url) + + print("\n" + "="*10) + print(f"Total files: {file_count}") + print(f"Total size: {file_count * avg_size:.1f} MB") + print(f"Average size: {avg_size:.2f} MB") + print(f"Median size: {median_size:.2f} MB") + print(f"Size distribution:") + print(f" <100KB: {size_dist['tiny']} files") + print(f" 100KB-1MB: {size_dist['small']} files") + print(f" 1-10MB: {size_dist['medium']} files") + print(f" 10-100MB: {size_dist['large']} files") + print(f" >100MB: {size_dist['huge']} files") + + print("\n" + "="*10) + print(f"Network speed: {network_speed:.1f} Mbps") + print(f"Server latency: {latency:.1f} ms") + + print("\n" + "="*10) + print("OPTIMAL SETTINGS") + print("="*10) + + optimal = calculate_optimal_settings(file_count, avg_size, network_speed, latency) + + print(f"Recommended --max-workers: {optimal['max_workers']}") + print(f"Recommended --pool-connections: {optimal['pool_connections']}") + print(f"Recommended --pool-maxsize: {optimal['pool_maxsize']}") + print(f"Estimated upload time: {optimal['estimated_time_minutes']:.1f} minutes") + print(f"Speedup vs single thread: {optimal['speedup']:.1f}x") + print(f"Bandwidth per thread: {optimal['bandwidth_per_thread_mbps']:.1f} Mbps") + print(f"Network quality: {'Good' if latency < 50 else 'Average' if latency < 100 else 'Poor'}") + +if __name__ == "__main__": + main()