diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..7ac807b1 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,8 @@ +FROM mcr.microsoft.com/devcontainers/python:3.9-bookworm + +# [Optional] Uncomment if you want to install more tools +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment if you want to install ydb cli +RUN curl -fsSL https://raw.githubusercontent.com/ydb-platform/ydb/refs/heads/main/ydb/apps/ydb/install/install.sh | bash diff --git a/.devcontainer/commands/initialize.sh b/.devcontainer/commands/initialize.sh new file mode 100755 index 00000000..926031b6 --- /dev/null +++ b/.devcontainer/commands/initialize.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +git config --local format.signoff true +git config --local user.name "$(git config user.name)" +git config --local user.email "$(git config user.email)" diff --git a/.devcontainer/commands/postCreate.sh b/.devcontainer/commands/postCreate.sh new file mode 100755 index 00000000..40078bf5 --- /dev/null +++ b/.devcontainer/commands/postCreate.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +if git config --get commit.gpgsign | grep -q true; then + git config --global gpg.format ssh + git config --global gpg.ssh.defaultKeyCommand 'ssh-add -L' + git config --global gpg.ssh.allowedSigners '~/.ssh/allowed_signers' +fi + +# Set up YDB profile if ydb cli exists +if which ydb > /dev/null 2>&1; then + ENDPOINT=$(echo ${YDB_CONNECTION_STRING_SECURE:-$YDB_CONNECTION_STRING} | awk -F/ '{print $1 "//" $3}') + DATABASE=$(echo ${YDB_CONNECTION_STRING_SECURE:-$YDB_CONNECTION_STRING} | cut -d/ -f4-) + CA_FILE_OPTION="" + + if [ -n "$YDB_SSL_ROOT_CERTIFICATES_FILE" ]; then + ENDPOINT="${ENDPOINT/grpc:/grpcs:}" + CA_FILE_OPTION="--ca-file ${YDB_SSL_ROOT_CERTIFICATES_FILE}" + fi + + ydb config profile replace local \ + --endpoint "$ENDPOINT" \ + --database "/$DATABASE" \ + $CA_FILE_OPTION + + ydb config profile activate local +fi diff --git a/.devcontainer/commands/postStart.sh b/.devcontainer/commands/postStart.sh new file mode 100755 index 00000000..bf38ac35 --- /dev/null +++ b/.devcontainer/commands/postStart.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + + +# Install dependencies +pip install -r requirements.txt +# Install ydb package +pip install -e . +# Install tox for CI +pip install tox + diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml new file mode 100644 index 00000000..54a6795e --- /dev/null +++ b/.devcontainer/compose.yml @@ -0,0 +1,55 @@ +name: ydb-python-sdk + +volumes: + ydb-data: + # driver: local + # driver_opts: + # type: tmpfs + # device: tmpfs + # o: size=80g + ydb-certs: + +services: + sdk: + build: + context: . + dockerfile: Dockerfile + hostname: sdk + + volumes: + - ydb-certs:/ydb_certs + - ../:/workspaces/ydb-python-sdk:cached + + environment: + - YDB_VERSION=25.1 + - YDB_STATIC_CREDENTIALS_USER=root + - YDB_STATIC_CREDENTIALS_PASSWORD=1234 + - YDB_STATIC_CREDENTIALS_ENDPOINT=grpc://ydb:2136 + - YDB_CONNECTION_STRING=grpc://ydb:2136/local + - YDB_CONNECTION_STRING_SECURE=grpcs://ydb:2135/local + - YDB_SSL_ROOT_CERTIFICATES_FILE=/ydb_certs/ca.pem + - TEST_COMPOSE_FILE=compose.remote.yml + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + ydb: + image: ydbplatform/local-ydb:25.1 + restart: unless-stopped + hostname: ydb + platform: linux/amd64 + + ports: + - 2135 + - 2136 + - 8765 + + volumes: + - ydb-data:/ydb_data + - ydb-certs:/ydb_certs + + environment: + - YDB_USE_IN_MEMORY_PDISKS=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..5af5f41f --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,53 @@ +{ + "name": "Python & YDB", + "service": "sdk", + "dockerComposeFile": "compose.yml", + "workspaceFolder": "/workspaces/ydb-python-sdk", + // Allows the container to use ptrace, which is useful for debugging. + "capAdd": [ + "SYS_PTRACE" + ], + // Disables seccomp, which can be necessary for some debugging tools to function correctly. + "securityOpt": [ + "seccomp=unconfined" + ], + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/git": {}, + "ghcr.io/devcontainers/features/common-utils": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + "ydb:2135", + "ydb:2136", + "ydb:8765" + ], + // Use 'initializeCommand' to run commands before the container is created. + "initializeCommand": "chmod +x .devcontainer/commands/initialize.sh && .devcontainer/commands/initialize.sh", + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "chmod +x .devcontainer/commands/postCreate.sh && .devcontainer/commands/postCreate.sh", + // Use 'postStartCommand' to run commands after the container is started. + "postStartCommand": "chmod +x .devcontainer/commands/postStart.sh && .devcontainer/commands/postStart.sh", + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "ms-python.autopep8", + "ms-python.debugpy", + "ms-python.flake8", + "ms-python.isort", + "ms-python.pylint", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.vscode-python-envs" + ] + } + }, + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + "remoteUser": "root", + "mounts": [ + "source=${localEnv:HOME}/.ssh/id_ed25519_signing,target=/root/.ssh/id_ed25519_signing,type=bind,readonly" + ] +} diff --git a/compose.remote.yml b/compose.remote.yml new file mode 100644 index 00000000..505d5706 --- /dev/null +++ b/compose.remote.yml @@ -0,0 +1,22 @@ +networks: + default: + name: ydb-python-sdk_default + external: true + +services: + py-sdk-ydb: + image: ydbplatform/local-ydb:trunk + restart: always + hostname: py-sdk-ydb + ports: + - 2135 + - 2136 + - 8765 + volumes: + - ./ydb_certs:/ydb_certs + environment: + - YDB_USE_IN_MEMORY_PDISKS=true + - YDB_ENABLE_COLUMN_TABLES=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/docker-compose-tls.yml b/compose.yml similarity index 65% rename from docker-compose-tls.yml rename to compose.yml index f0a4b328..ba1fe12e 100644 --- a/docker-compose-tls.yml +++ b/compose.yml @@ -1,14 +1,17 @@ -version: "3.9" services: - ydb: + py-sdk-ydb: image: ydbplatform/local-ydb:trunk restart: always - ports: - - 2136:2136 - - 2135:2135 hostname: localhost + ports: + - 2135 + - 2136 + - 8765 volumes: - ./ydb_certs:/ydb_certs environment: - YDB_USE_IN_MEMORY_PDISKS=true - YDB_ENABLE_COLUMN_TABLES=true + - GRPC_TLS_PORT=2135 + - GRPC_PORT=2136 + - MON_PORT=8765 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 1a466fab..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: "3.3" -services: - ydb: - image: ydbplatform/local-ydb:trunk - restart: always - ports: - - 2136:2136 - hostname: localhost - environment: - - YDB_USE_IN_MEMORY_PDISKS=true - - YDB_ENABLE_COLUMN_TABLES=true diff --git a/tests/conftest.py b/tests/conftest.py index a8177f46..f88d7149 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,11 @@ @pytest.fixture(scope="module") def docker_compose_file(pytestconfig): - return os.path.join(str(pytestconfig.rootdir), "docker-compose.yml") + f = "compose.yml" + if os.environ.get('REMOTE_CONTAINERS') is not None: + f = "compose.remote.yml" + + return os.path.join(str(pytestconfig.rootdir), f) def wait_container_ready(driver): @@ -32,9 +36,14 @@ def wait_container_ready(driver): @pytest.fixture(scope="module") def endpoint(pytestconfig, module_scoped_container_getter): - with ydb.Driver(endpoint="localhost:2136", database="/local") as driver: + e = "grpc://localhost:2136" + if os.environ.get('REMOTE_CONTAINERS') is not None: + e = "grpc://py-sdk-ydb:2136" + + with ydb.Driver(endpoint=e, database="/local") as driver: wait_container_ready(driver) - yield "localhost:2136" + + yield e @pytest.fixture(scope="session") @@ -47,13 +56,19 @@ def secure_endpoint(pytestconfig, session_scoped_container_getter): assert os.path.exists(ca_path) os.environ["YDB_SSL_ROOT_CERTIFICATES_FILE"] = ca_path + + e = "grpcs://localhost:2135" + if os.environ.get('REMOTE_CONTAINERS') is not None: + e = "grpcs://py-sdk-ydb:2135" + with ydb.Driver( - endpoint="grpcs://localhost:2135", + endpoint=e, database="/local", root_certificates=ydb.load_ydb_root_certificate(), ) as driver: wait_container_ready(driver) - yield "localhost:2135" + + yield e @pytest.fixture(scope="module") diff --git a/tests/ssl/test_ssl.py b/tests/ssl/test_ssl.py index 231f082a..204f36f8 100644 --- a/tests/ssl/test_ssl.py +++ b/tests/ssl/test_ssl.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import ydb import pytest @@ -6,8 +7,8 @@ @pytest.mark.tls def test_connect_secure(secure_endpoint, database): with ydb.Driver( - endpoint="grpcs://localhost:2135", - database="/local", + endpoint=secure_endpoint, + database=database, root_certificates=ydb.load_ydb_root_certificate(), ) as driver: driver.wait(timeout=10) diff --git a/tox.ini b/tox.ini index f91e7d8a..6fbe024f 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,12 @@ usedevelop = True install_command = pip install {opts} {packages} setenv = PYTHONPATH = {env:PYTHONPATH}{:}{toxinidir} + REMOTE_CONTAINERS = {env:REMOTE_CONTAINERS} + TEST_COMPOSE_FILE = {env:TEST_COMPOSE_FILE:compose.yml} deps = -r{toxinidir}/test-requirements.txt + [testenv:dev-proto5] commands = deps = @@ -32,14 +35,14 @@ deps = [testenv:py] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<6.0.0 [testenv:py-proto4] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 @@ -48,35 +51,35 @@ deps = commands = pytest -v -m "not tls" \ --cov-report html:cov_html --cov=ydb \ - --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 [testenv:py-proto3] commands = - pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs} + pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<4.0.0 [testenv:py-tls] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<6.0.0 [testenv:py-tls-proto4] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<5.0.0 [testenv:py-tls-proto3] commands = - pytest -v -m tls --docker-compose-remove-volumes --docker-compose=docker-compose-tls.yml {posargs} + pytest -v -m tls --docker-compose-remove-volumes --docker-compose={env:TEST_COMPOSE_FILE} {posargs} deps = -r{toxinidir}/test-requirements.txt protobuf<4.0.0