Skip to content

Commit 2cb2137

Browse files
olevskiolevski
committed
feat: add cloud storage via rclone
1 parent 0c9e693 commit 2cb2137

File tree

7 files changed

+194
-72
lines changed

7 files changed

+194
-72
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,5 +161,5 @@ install_amaltheas: ## Installs both version of amalthea in the. NOTE: It uses t
161161

162162
# TODO: Add the version variables from the top of the file here when the charts are fully published
163163
amalthea_schema: ## Updates generates pydantic classes from CRDs
164-
curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/fix-missing-status/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
164+
curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/feat-add-cloud-storage/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
165165
curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/controller/crds/jupyter_server.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_jupyter_server.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg

‎bases/renku_data_services/data_api/app.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
142142
nb_config=config.nb_config,
143143
internal_gitlab_authenticator=config.gitlab_authenticator,
144144
git_repo=config.git_repositories_repo,
145+
rp_repo=config.rp_repo,
145146
)
146147
notebooks_new = NotebooksNewBP(
147148
name="notebooks",

‎components/renku_data_services/notebooks/api/classes/cloud_storage/__init__.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,15 @@
66
class ICloudStorageRequest(Protocol):
77
"""The abstract class for cloud storage."""
88

9-
exists: bool
109
mount_folder: str
11-
source_folder: str
12-
bucket: str
10+
source_path: str
1311

1412
def get_manifest_patch(
1513
self,
1614
base_name: str,
1715
namespace: str,
18-
labels: dict[str, str] = {},
19-
annotations: dict[str, str] = {},
16+
labels: dict[str, str] | None = None,
17+
annotations: dict[str, str] | None = None,
2018
) -> list[dict[str, Any]]:
2119
"""The patches applied to a jupyter server to insert the storage in the session."""
2220
...

‎components/renku_data_services/notebooks/api/schemas/cloud_storage.py

Lines changed: 78 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,18 @@
22

33
from configparser import ConfigParser
44
from io import StringIO
5-
from pathlib import Path
6-
from typing import Any, Optional, Self
5+
from pathlib import PurePosixPath
6+
from typing import Any, Final, Optional, Self
77

8+
from kubernetes import client
89
from marshmallow import EXCLUDE, Schema, ValidationError, fields, validates_schema
910

1011
from renku_data_services.base_models import APIUser
1112
from renku_data_services.notebooks.api.classes.cloud_storage import ICloudStorageRequest
1213
from renku_data_services.notebooks.config import _NotebooksConfig
1314

15+
_sanitize_for_serialization = client.ApiClient().sanitize_for_serialization
16+
1417

1518
class RCloneStorageRequest(Schema):
1619
"""Request for RClone based storage."""
@@ -36,6 +39,8 @@ def validate_storage(self, data: dict, **kwargs: dict) -> None:
3639
class RCloneStorage(ICloudStorageRequest):
3740
"""RClone based storage."""
3841

42+
pvc_secret_annotation_name: Final[str] = "csi-rclone.dev/secretName"
43+
3944
def __init__(
4045
self,
4146
source_path: str,
@@ -60,7 +65,7 @@ async def storage_from_schema(
6065
user: APIUser,
6166
internal_gitlab_user: APIUser,
6267
project_id: int,
63-
work_dir: Path,
68+
work_dir: PurePosixPath,
6469
config: _NotebooksConfig,
6570
) -> Self:
6671
"""Create storage object from request."""
@@ -92,8 +97,73 @@ async def storage_from_schema(
9297
await config.storage_validator.validate_storage_configuration(configuration, source_path)
9398
return cls(source_path, configuration, readonly, mount_folder, name, config)
9499

100+
def pvc(
101+
self,
102+
base_name: str,
103+
namespace: str,
104+
labels: dict[str, str] | None = None,
105+
annotations: dict[str, str] | None = None,
106+
) -> client.V1PersistentVolumeClaim:
107+
"""The PVC for mounting cloud storage."""
108+
return client.V1PersistentVolumeClaim(
109+
metadata=client.V1ObjectMeta(
110+
name=base_name,
111+
namespace=namespace,
112+
annotations={self.pvc_secret_annotation_name: base_name} | (annotations or {}),
113+
labels={"name": base_name} | (labels or {}),
114+
),
115+
spec=client.V1PersistentVolumeClaimSpec(
116+
access_modes=["ReadOnlyMany" if self.readonly else "ReadWriteMany"],
117+
resources=client.V1VolumeResourceRequirements(requests={"storage": "10Gi"}),
118+
storage_class_name=self.config.cloud_storage.storage_class,
119+
),
120+
)
121+
122+
def volume_mount(self, base_name: str) -> client.V1VolumeMount:
123+
"""The volume mount for cloud storage."""
124+
return client.V1VolumeMount(
125+
mount_path=self.mount_folder,
126+
name=base_name,
127+
read_only=self.readonly,
128+
)
129+
130+
def volume(self, base_name: str) -> client.V1Volume:
131+
"""The volume entry for the statefulset specification."""
132+
return client.V1Volume(
133+
name=base_name,
134+
persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
135+
claim_name=base_name, read_only=self.readonly
136+
),
137+
)
138+
139+
def secret(
140+
self,
141+
base_name: str,
142+
namespace: str,
143+
labels: dict[str, str] | None = None,
144+
annotations: dict[str, str] | None = None,
145+
) -> client.V1Secret:
146+
"""The secret containing the configuration for the rclone csi driver."""
147+
return client.V1Secret(
148+
metadata=client.V1ObjectMeta(
149+
name=base_name,
150+
namespace=namespace,
151+
annotations=annotations,
152+
labels={"name": base_name} | (labels or {}),
153+
),
154+
string_data={
155+
"remote": self.name or base_name,
156+
"remotePath": self.source_path,
157+
"configData": self.config_string(self.name or base_name),
158+
},
159+
)
160+
95161
def get_manifest_patch(
96-
self, base_name: str, namespace: str, labels: dict = {}, annotations: dict = {}
162+
self,
163+
base_name: str,
164+
namespace: str,
165+
labels: dict[str, str] | None = None,
166+
annotations: dict[str, str] | None = None,
97167
) -> list[dict[str, Any]]:
98168
"""Get server manifest patch."""
99169
patches = []
@@ -104,57 +174,22 @@ def get_manifest_patch(
104174
{
105175
"op": "add",
106176
"path": f"/{base_name}-pv",
107-
"value": {
108-
"apiVersion": "v1",
109-
"kind": "PersistentVolumeClaim",
110-
"metadata": {
111-
"name": base_name,
112-
"labels": {"name": base_name},
113-
},
114-
"spec": {
115-
"accessModes": ["ReadOnlyMany" if self.readonly else "ReadWriteMany"],
116-
"resources": {"requests": {"storage": "10Gi"}},
117-
"storageClassName": self.config.cloud_storage.storage_class,
118-
},
119-
},
177+
"value": _sanitize_for_serialization(self.pvc(base_name, namespace, labels, annotations)),
120178
},
121179
{
122180
"op": "add",
123181
"path": f"/{base_name}-secret",
124-
"value": {
125-
"apiVersion": "v1",
126-
"kind": "Secret",
127-
"metadata": {
128-
"name": base_name,
129-
"labels": {"name": base_name},
130-
},
131-
"type": "Opaque",
132-
"stringData": {
133-
"remote": self.name or base_name,
134-
"remotePath": self.source_path,
135-
"configData": self.config_string(self.name or base_name),
136-
},
137-
},
182+
"value": _sanitize_for_serialization(self.secret(base_name, namespace, labels, annotations)),
138183
},
139184
{
140185
"op": "add",
141186
"path": "/statefulset/spec/template/spec/containers/0/volumeMounts/-",
142-
"value": {
143-
"mountPath": self.mount_folder,
144-
"name": base_name,
145-
"readOnly": self.readonly,
146-
},
187+
"value": _sanitize_for_serialization(self.volume_mount(base_name)),
147188
},
148189
{
149190
"op": "add",
150191
"path": "/statefulset/spec/template/spec/volumes/-",
151-
"value": {
152-
"name": base_name,
153-
"persistentVolumeClaim": {
154-
"claimName": base_name,
155-
"readOnly": self.readonly,
156-
},
157-
},
192+
"value": _sanitize_for_serialization(self.volume(base_name)),
158193
},
159194
],
160195
}

0 commit comments

Comments
 (0)