Fix hyphen in extra options bug, bump, add some benchmark commands (#366

) * allow use of hyphen mark in MODEL_SPLIT_EQUATION_PATTERN Signed-off-by: rafsaf <[email protected]> * bump pydantic Signed-off-by: rafsaf <[email protected]> * add some benchmark commands, bump rsa Signed-off-by: rafsaf <[email protected]> * add -vv in pytest Signed-off-by: rafsaf <[email protected]> * fix test_core Signed-off-by: rafsaf <[email protected]> --------- Signed-off-by: rafsaf <[email protected]>
rafsaf · Mar 1, 2025 · c727697 · c727697
1 parent 7bdddcc
commit c727697
Show file tree

Hide file tree

Showing 12 changed files with 565 additions and 487 deletions.
diff --git a/.env.example b/.env.example
@@ -1,5 +1,5 @@
 POSTGRESQL_DB_PG_16=host=localhost port=10016 password=password-_-12!@#%^&*()/;><.,]}{[ user=user-_-12!@#%^&*()/;><.,]}{[ db=database-_-12!@#%^&*()/;><.,]}{[ cron_rule=* * * * *
-MARIADB_DB_TEST_1011=host=localhost port=12011 password=password-_-12!@#%^&*()/;><.,]}{[ user=user-_-12!@#%^&*()/;><.,]}{[ db=database-_-12!@#%^&*()/;><.,]}{[ cron_rule=* * * * *
+MARIADB_DB_TEST_1011=host=localhost port=12011 client_skip-ssl=true password=password-_-12!@#%^&*()/;><.,]}{[ user=user-_-12!@#%^&*()/;><.,]}{[ db=database-_-12!@#%^&*()/;><.,]}{[ cron_rule=* * * * *
 #SINGLEFILE_MY_TEST_FILE=abs_path=/home/somefolder/somefile cron_rule=* * * * *
 #DIRECTORY_MY_TEST_FOLDER=abs_path=/home/somefolder/someotherfolder cron_rule=* * * * *
 LOG_LEVEL=INFO

diff --git a/.gitignore b/.gitignore
@@ -137,3 +137,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# benchmark
+*massif*
+benchmark_files
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,12 +5,12 @@ repos:
       - id: check-yaml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.9
     hooks:
       - id: ruff-format
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.9
     hooks:
       - id: ruff
         args: [--fix]

diff --git a/Makefile b/Makefile
@@ -30,4 +30,18 @@ acceptance_tests:
 
 .PHONY: update_compose_db_file
 update_compose_db_file:
-	poetry run python ogion/tools/compose_file_generator.py > docker/docker-compose.dbs.yml
+	poetry run python ogion/tools/compose_file_generator.py > docker/docker-compose.dbs.yml
+
+.PHONY: benchmark-mem-massif-main
+benchmark-mem-massif-main:
+	valgrind --massif-out-file=massif.benchmark-main.out --tool=massif python -m ogion.main -s
+	massif-visualizer massif.benchmark-main.out
+
+.PHONY: benchmark-time-encrypt-2gb
+benchmark-time-encrypt-2gb:
+	$(MAKE) benchmark_files/test_file_2gb
+	time -f "User: %U seconds, System: %S seconds, Real: %e seconds" python -c "import pathlib;import ogion.core;ogion.core.run_create_age_archive(pathlib.Path('./benchmark_files/test_file_2gb'))"
+
+benchmark_files/test_file_2gb:
+	mkdir -p benchmark_files
+	dd if=/dev/urandom of=benchmark_files/test_file_2gb bs=2G count=1 iflag=fullblock
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -56,7 +56,7 @@ COPY --from=poetry /requirements-tests.txt .
 RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements-tests.txt
 COPY pyproject.toml .
 COPY tests tests
-CMD ["pytest"]
+CMD ["pytest", "-vv"]
 
 FROM common AS build
 CMD ["python", "-m", "ogion.main"] 
diff --git a/ogion/core.py b/ogion/core.py
@@ -21,7 +21,7 @@
 
 SAFE_LETTER_PATTERN = re.compile(r"[^A-Za-z0-9_]*")
 DATETIME_BACKUP_FILE_PATTERN = re.compile(r"_[0-9]{8}_[0-9]{4}_")
-MODEL_SPLIT_EQUATION_PATTERN = re.compile(r"( \w*\=|^\w*\=)")
+MODEL_SPLIT_EQUATION_PATTERN = re.compile(r"( (\w|\-)*\=|^(\w|\-)*\=)")
 
 _BM = TypeVar("_BM", bound=BaseModel)
 
@@ -102,7 +102,7 @@ def run_create_age_archive(backup_file: Path) -> Path:
 
     recipients = config.options.age_recipients_file
 
-    shell_create_age_archive = f"age -R {recipients} " f"-o {out_file} {backup_file}"
+    shell_create_age_archive = f"age -R {recipients} -o {out_file} {backup_file}"
     run_subprocess(shell_create_age_archive)
     log.info("finished age archive creating")
 

diff --git a/ogion/tools/compose_file_generator.py b/ogion/tools/compose_file_generator.py
@@ -35,7 +35,7 @@
 
 def mariadb_db_generator(cycle: EOLApiProductCycle) -> ComposeDatabase:
     host_port = 11000 + int(cycle.cycle.replace(".", ""))
-    name = f"ogion_mariadb_{cycle.cycle.replace('.','_')}"
+    name = f"ogion_mariadb_{cycle.cycle.replace('.', '_')}"
     compose_db = ComposeDatabase(
         name=name,
         restart="no",
@@ -56,7 +56,7 @@ def mariadb_db_generator(cycle: EOLApiProductCycle) -> ComposeDatabase:
 
 def mysql_db_generator(cycle: EOLApiProductCycle) -> ComposeDatabase:
     host_port = 9000 + int(cycle.cycle.replace(".", ""))
-    name = f"ogion_mysql_{cycle.cycle.replace('.','_')}"
+    name = f"ogion_mysql_{cycle.cycle.replace('.', '_')}"
     compose_db = ComposeDatabase(
         name=name,
         restart="no",
@@ -77,7 +77,7 @@ def mysql_db_generator(cycle: EOLApiProductCycle) -> ComposeDatabase:
 
 def postgres_db_generator(cycle: EOLApiProductCycle) -> ComposeDatabase:
     host_port = 10000 + int(cycle.cycle.replace(".", ""))
-    name = f"ogion_postgres_{cycle.cycle.replace('.','_')}"
+    name = f"ogion_postgres_{cycle.cycle.replace('.', '_')}"
     compose_db = ComposeDatabase(
         name=name,
         restart="no",

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -5,49 +5,49 @@ dynamic = ["dependencies"]
 license = "GNU GPLv3"
 name = "ogion"
 readme = "README.md"
-requires-python = ">=3.13"
+requires-python = ">=3.13,<4"
 version = "7.3"
 
 [tool.poetry]
 package-mode = false
 
 [tool.poetry.dependencies]
-python = ">=3.13"
+python = ">=3.13,<4"
 
-azure-identity = "^1.17.1"
-azure-storage-blob = "^12.20.0"
-croniter = "^6.0.0"
-google-cloud-storage = "^2.18.2"
-minio = "^7.2.13"
-pydantic = "^2.10.4"
-pydantic-settings = "^2.7.1"
+azure-identity = ">=1.17.1"
+azure-storage-blob = ">=12.20.0"
+croniter = ">=6.0.0"
+google-cloud-storage = ">=2.18.2"
+minio = ">=7.2.13"
+pydantic = { version = ">=2.11.0a2", allow-prereleases = true }
+pydantic-settings = ">=2.7.1"
 
 [tool.poetry.group.dev.dependencies]
-coverage = "^7.6.10"
-markdown-include = "^0.8.1"
-mike = "^2.1.3"
-mkdocs = "^1.6.1"
-mkdocs-material = "^9.5.49"
-mkdocstrings = { extras = ["python"], version = "^0.27.0" }
-mypy = "^1.14.1"
-pre-commit = "^4.0.1"
-pymdown-extensions = "^10.13"
-pyyaml = "^6.0.2"
-requests = "^2.32.3"
-ruff = "^0.8.4"
-types-croniter = "^5.0.1.20241205"
-types-google-cloud-ndb = "^2.3.0.20240813"
-types-pyyaml = "^6.0.12.20241230"
-types-requests = "^2.32.0.20240907"
+coverage = ">=7.6.10"
+markdown-include = ">=0.8.1"
+mike = ">=2.1.3"
+mkdocs = ">=1.6.1"
+mkdocs-material = ">=9.5.49"
+mkdocstrings = { extras = ["python"], version = ">=0.27.0" }
+mypy = ">=1.14.1"
+pre-commit = ">=4.0.1"
+pymdown-extensions = ">=10.13"
+pyyaml = ">=6.0.2"
+requests = ">=2.32.3"
+ruff = ">=0.9.9"
+types-croniter = ">=5.0.1.20241205"
+types-google-cloud-ndb = ">=2.3.0.20240813"
+types-pyyaml = ">=6.0.12.20241230"
+types-requests = ">=2.32.0.20240907"
 
 [tool.poetry.group.tests.dependencies]
-freezegun = "^1.5.1"
-pytest = "^8.3.2"
-pytest-cov = "^6.0.0"
-pytest-env = "^1.1.4"
-pytest-xdist = "^3.6.1"
-python-dotenv = "^1.0.1"
-responses = "^0.25.0"
+freezegun = ">=1.5.1"
+pytest = ">=8.3.2"
+pytest-cov = ">=6.0.0"
+pytest-env = ">=1.1.4"
+pytest-xdist = ">=3.6.1"
+python-dotenv = ">=1.0.1"
+responses = ">=0.25.0"
 
 [tool.pytest.ini_options]
 addopts = "-v --cov --cov-report term-missing --cov-fail-under 100 -n auto"
@@ -57,6 +57,7 @@ env = [
   "BACKUP_PROVIDER=",
   "FAKE_GCS_PORT=4443",
   "LOG_FOLDER_PATH=/tmp/pytest_ogion_env_vars_hook_logs_folder",
+  "LOG_LEVEL=DEBUG",
   "STORAGE_EMULATOR_HOST=http://localhost:4443",
   "SUBPROCESS_TIMEOUT_SECS=5",
 ]

diff --git a/tests/test_backup_target_mariadb.py b/tests/test_backup_target_mariadb.py
@@ -85,8 +85,7 @@ def test_end_to_end_successful_restore_after_backup(
     )
 
     insert_query = shlex.quote(
-        "INSERT INTO my_table (name, age) "
-        "VALUES ('Geralt z Rivii', 60),('rafsaf', 24);"
+        "INSERT INTO my_table (name, age) VALUES ('Geralt z Rivii', 60),('rafsaf', 24);"
     )
 
     core.run_subprocess(
@@ -115,7 +114,7 @@ def test_end_to_end_successful_restore_after_backup(
         " --execute='select * from my_table order by id asc;'",
     )
 
-    assert result == ("id\tname\tage\n" "1\tGeralt z Rivii\t60\n" "2\trafsaf\t24\n")
+    assert result == ("id\tname\tage\n1\tGeralt z Rivii\t60\n2\trafsaf\t24\n")
 
     result = core.run_subprocess(
         f"mariadb --defaults-file={test_db.option_file} {test_db.db_name}"
@@ -126,7 +125,7 @@ def test_end_to_end_successful_restore_after_backup(
         " --execute='select * from my_table order by id asc;'",
     )
 
-    assert result == ("id\tname\tage\n" "1\tGeralt z Rivii\t60\n")
+    assert result == ("id\tname\tage\n1\tGeralt z Rivii\t60\n")
 
     test_db.restore(str(test_db_backup))
 
@@ -135,4 +134,4 @@ def test_end_to_end_successful_restore_after_backup(
         " --execute='select * from my_table order by id asc;'",
     )
 
-    assert result == ("id\tname\tage\n" "1\tGeralt z Rivii\t60\n" "2\trafsaf\t24\n")
+    assert result == ("id\tname\tage\n1\tGeralt z Rivii\t60\n2\trafsaf\t24\n")
diff --git a/tests/test_backup_target_postgresql.py b/tests/test_backup_target_postgresql.py
@@ -78,8 +78,7 @@ def test_end_to_end_successful_restore_after_backup(
     )
 
     insert_query = shlex.quote(
-        "INSERT INTO my_table (name, age) "
-        "VALUES ('Geralt z Rivii', 60),('rafsaf', 24);"
+        "INSERT INTO my_table (name, age) VALUES ('Geralt z Rivii', 60),('rafsaf', 24);"
     )
 
     core.run_subprocess(