Skip to content

Commit 219c86f

Browse files
travagliadtravagliad
authored
PMM-7 Fix Valkey Deployment (#168)
* Update PMM admin commands to include 'valkey-server' for monitoring setup * Update PMM admin commands to include 'valkey-server' for monitoring setup * PMM-7 Change role name * PMM-7 Fix node name * Fix * fix * fix * fix * fix * add cluster * fix * fix * fix * fix * fix * fix * retry * fix * fix conf.j2 * Generate workload for latency tests * fix command
1 parent b0cd747 commit 219c86f

File tree

3 files changed

+317
-19
lines changed

3 files changed

+317
-19
lines changed

‎pmm_qa/pmm-framework.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -800,18 +800,23 @@ def setup_valkey(db_type, db_version=None, db_config=None, args=None):
800800

801801
# Gather Version details
802802
valkey_version = os.getenv('VALKEY_VERSION') or db_version or database_configs[db_type]["versions"][-1]
803+
setup_type_value = get_value('SETUP_TYPE', db_type, args, db_config).lower()
803804

804805
# Define environment variables for playbook
805806
env_vars = {
806807
'PMM_SERVER_IP': args.pmm_server_ip or container_name or '127.0.0.1',
807808
'VALKEY_VERSION': valkey_version,
808809
'CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config),
809810
'ADMIN_PASSWORD': os.getenv('ADMIN_PASSWORD') or args.pmm_server_password or 'admin',
810-
'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3'
811+
'PMM_QA_GIT_BRANCH': os.getenv('PMM_QA_GIT_BRANCH') or 'v3',
812+
'SETUP_TYPE': setup_type_value
811813
}
812814

813-
# Ansible playbook filename
814-
playbook_filename = 'valkey/valkey.yml'
815+
# Choose playbook based on SETUP_TYPE (cluster is default; sentinel only when explicitly requested)
816+
if setup_type_value in ("sentinel", "sentinels"):
817+
playbook_filename = 'valkey/valkey-sentinel.yml'
818+
else:
819+
playbook_filename = 'valkey/valkey-cluster.yml'
815820

816821
# Call the function to run the Ansible playbook
817822
run_ansible_playbook(playbook_filename, env_vars, args)

‎pmm_qa/valkey/valkey-cluster.yml‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
---
2+
- name: Deploy Valkey Native Cluster
3+
hosts: localhost
4+
gather_facts: false
5+
vars:
6+
pmm_server_ip: "{{ lookup('vars', 'extra_pmm_server_ip', default=lookup('env','PMM_SERVER_IP') | default('127.0.0.1', true) ) }}"
7+
metrics_mode: "{{ lookup('env', 'metrics_mode') }}"
8+
client_version: "{{ lookup('vars', 'extra_client_version', default=lookup('env','CLIENT_VERSION') | default('3-dev-latest', true) ) }}"
9+
admin_password: "{{ lookup('vars', 'extra_admin_password', default=lookup('env','ADMIN_PASSWORD') | default('admin', true) ) }}"
10+
valkey_version: "{{ lookup('env', 'VALKEY_VERSION') | default('7', true) }}"
11+
valkey_image: "valkey/valkey:{{ valkey_version }}"
12+
valkey_network_name: "pmm-qa"
13+
valkey_password: "VKvl41568AsE"
14+
valkey_cluster_node_count: 6 # total nodes
15+
valkey_cluster_primaries: 3 # number of primary nodes
16+
valkey_cluster_replicas: 1 # replicas per primary
17+
valkey_cluster_start_port: 6379 # Base host port to map sequentially
18+
valkey_config_dir: "{{ lookup('env', 'HOME') }}/valkey/cluster-config"
19+
valkey_primary_prefix: "valkey-primary-"
20+
valkey_replica_prefix: "valkey-replica-"
21+
pmm_server_name: "pmm-server"
22+
23+
tasks:
24+
- name: Set Random Number Fact
25+
set_fact:
26+
random_number: "{{ (10000 | random) | int }}"
27+
28+
- name: Validate cluster counts
29+
assert:
30+
that:
31+
- valkey_cluster_node_count == valkey_cluster_primaries + (valkey_cluster_primaries * valkey_cluster_replicas)
32+
fail_msg: "Mismatch: total nodes must equal primaries + primaries*replicas"
33+
34+
- name: Create Docker network
35+
community.docker.docker_network:
36+
name: "{{ valkey_network_name }}"
37+
driver: bridge
38+
state: present
39+
40+
- name: Create cluster config directory
41+
file:
42+
path: "{{ valkey_config_dir }}"
43+
state: directory
44+
mode: "0755"
45+
46+
- name: Create per-primary config directories
47+
file:
48+
path: "{{ valkey_config_dir }}/{{ valkey_primary_prefix }}{{ item }}"
49+
state: directory
50+
mode: "0755"
51+
loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}"
52+
53+
- name: Create per-replica config directories
54+
file:
55+
path: "{{ valkey_config_dir }}/{{ valkey_replica_prefix }}{{ item }}"
56+
state: directory
57+
mode: "0755"
58+
loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}"
59+
60+
- name: Generate base configuration for each primary
61+
copy:
62+
dest: "{{ valkey_config_dir }}/{{ valkey_primary_prefix }}{{ item }}/valkey.conf"
63+
mode: "0644"
64+
content: |
65+
port 6379
66+
requirepass {{ valkey_password }}
67+
masterauth {{ valkey_password }}
68+
protected-mode no
69+
appendonly yes
70+
cluster-enabled yes
71+
cluster-config-file nodes.conf
72+
cluster-node-timeout 5000
73+
# Minimal persistence & logging
74+
save 900 1
75+
save 300 10
76+
save 60 10000
77+
loglevel notice
78+
loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}"
79+
80+
- name: Generate base configuration for each replica
81+
copy:
82+
dest: "{{ valkey_config_dir }}/{{ valkey_replica_prefix }}{{ item }}/valkey.conf"
83+
mode: "0644"
84+
content: |
85+
port 6379
86+
requirepass {{ valkey_password }}
87+
masterauth {{ valkey_password }}
88+
protected-mode no
89+
appendonly yes
90+
cluster-enabled yes
91+
cluster-config-file nodes.conf
92+
cluster-node-timeout 5000
93+
# Minimal persistence & logging
94+
save 900 1
95+
save 300 10
96+
save 60 10000
97+
loglevel notice
98+
loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}"
99+
100+
- name: Create docker volumes for primary nodes
101+
community.docker.docker_volume:
102+
name: "{{ valkey_primary_prefix }}{{ item }}-data"
103+
state: present
104+
loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}"
105+
106+
- name: Create docker volumes for replica nodes
107+
community.docker.docker_volume:
108+
name: "{{ valkey_replica_prefix }}{{ item }}-data"
109+
state: present
110+
loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}"
111+
112+
- name: Start primary node containers
113+
community.docker.docker_container:
114+
name: "{{ valkey_primary_prefix }}{{ item }}"
115+
hostname: "{{ valkey_primary_prefix }}{{ item }}-node-{{ random_number }}"
116+
image: "{{ valkey_image }}"
117+
state: started
118+
restart_policy: unless-stopped
119+
networks:
120+
- name: "{{ valkey_network_name }}"
121+
ports:
122+
- "{{ valkey_cluster_start_port + item - 1 }}:6379"
123+
volumes:
124+
- "{{ valkey_primary_prefix }}{{ item }}-data:/data"
125+
- "{{ valkey_config_dir }}/{{ valkey_primary_prefix }}{{ item }}/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro"
126+
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
127+
healthcheck:
128+
test: ["CMD", "valkey-cli", "-a", "{{ valkey_password }}", "ping"]
129+
interval: 10s
130+
timeout: 5s
131+
retries: 5
132+
loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}"
133+
134+
- name: Start replica node containers
135+
community.docker.docker_container:
136+
name: "{{ valkey_replica_prefix }}{{ item }}"
137+
hostname: "{{ valkey_replica_prefix }}{{ item }}-node-{{ random_number }}"
138+
image: "{{ valkey_image }}"
139+
state: started
140+
restart_policy: unless-stopped
141+
networks:
142+
- name: "{{ valkey_network_name }}"
143+
ports:
144+
- "{{ (valkey_cluster_start_port + valkey_cluster_primaries - 1) + item }}:6379"
145+
volumes:
146+
- "{{ valkey_replica_prefix }}{{ item }}-data:/data"
147+
- "{{ valkey_config_dir }}/{{ valkey_replica_prefix }}{{ item }}/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro"
148+
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
149+
healthcheck:
150+
test: ["CMD", "valkey-cli", "-a", "{{ valkey_password }}", "ping"]
151+
interval: 10s
152+
timeout: 5s
153+
retries: 5
154+
loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}"
155+
156+
- name: Wait for primary node ports
157+
wait_for:
158+
host: localhost
159+
port: "{{ valkey_cluster_start_port + item - 1 }}"
160+
timeout: 30
161+
delay: 1
162+
loop: "{{ range(1, valkey_cluster_primaries + 1) | list }}"
163+
164+
- name: Wait for replica node ports
165+
wait_for:
166+
host: localhost
167+
port: "{{ (valkey_cluster_start_port + valkey_cluster_primaries - 1) + item }}"
168+
timeout: 30
169+
delay: 1
170+
loop: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | list }}"
171+
172+
- name: Build list of internal container addresses
173+
set_fact:
174+
primary_nodes: "{{ range(1, valkey_cluster_primaries + 1) | map('string') | map('regex_replace', '^(.*)$', valkey_primary_prefix ~ '\\1') | list }}"
175+
replica_nodes: "{{ range(valkey_cluster_primaries + 1, valkey_cluster_node_count + 1) | map('string') | map('regex_replace', '^(.*)$', valkey_replica_prefix ~ '\\1') | list }}"
176+
177+
- name: Build combined list of internal container addresses
178+
set_fact:
179+
cluster_node_addresses: "{{ (primary_nodes | map('regex_replace', '^(.*)$', '\\1:6379') | list) + (replica_nodes | map('regex_replace', '^(.*)$', '\\1:6379') | list) }}"
180+
181+
- name: Display cluster node addresses
182+
debug:
183+
var: cluster_node_addresses
184+
185+
- name: Create the cluster (run once)
186+
community.docker.docker_container_exec:
187+
container: "{{ valkey_primary_prefix }}1"
188+
command: >-
189+
bash -c "yes 'yes' | valkey-cli --cluster create {{ cluster_node_addresses | join(' ') }} --cluster-replicas {{ valkey_cluster_replicas }} -a {{ valkey_password }}"
190+
register: cluster_create_output
191+
changed_when: "'[OK]' in cluster_create_output.stdout"
192+
193+
- name: Show cluster creation output
194+
debug:
195+
msg: "{{ cluster_create_output.stdout_lines }}"
196+
197+
- name: Check cluster info on first primary
198+
community.docker.docker_container_exec:
199+
container: "{{ valkey_primary_prefix }}1"
200+
command: valkey-cli -a "{{ valkey_password }}" cluster info
201+
register: cluster_info
202+
203+
- name: Display cluster info
204+
debug:
205+
msg: "{{ cluster_info.stdout_lines }}"
206+
207+
- name: Install PMM Client in each container
208+
ansible.builtin.include_tasks: ../tasks/install_pmm_client.yml
209+
loop: "{{ primary_nodes + replica_nodes }}"
210+
loop_control:
211+
loop_var: current_container_name
212+
vars:
213+
container_name: "{{ current_container_name }}"
214+
215+
- name: Add primary nodes to monitoring
216+
community.docker.docker_container_exec:
217+
container: "{{ item }}"
218+
command: >-
219+
pmm-admin add valkey --cluster=valkey-native-cluster --environment=valkey-test --username=default
220+
--password="{{ valkey_password }}" --service-name={{ item }}-svc-{{ random_number }}
221+
--host={{ item }} --port=6379 --custom-labels='role=primary'
222+
loop: "{{ primary_nodes }}"
223+
ignore_errors: yes
224+
225+
- name: Add replica nodes to monitoring
226+
community.docker.docker_container_exec:
227+
container: "{{ item }}"
228+
command: >-
229+
pmm-admin add valkey --cluster=valkey-native-cluster --environment=valkey-test --username=default
230+
--password="{{ valkey_password }}" --service-name={{ item }}-svc-{{ random_number }}
231+
--host={{ item }} --port=6379 --custom-labels='role=replica'
232+
loop: "{{ replica_nodes }}"
233+
ignore_errors: yes
234+
235+
- name: Seed sample list data on first primary
236+
community.docker.docker_container_exec:
237+
container: "{{ valkey_primary_prefix }}1"
238+
command: valkey-cli -a "{{ valkey_password }}" RPUSH mylist "one" "two" "three" "four" "five"
239+
240+
- name: Pop one item from sample list on first primary
241+
community.docker.docker_container_exec:
242+
container: "{{ valkey_primary_prefix }}1"
243+
command: valkey-cli -a "{{ valkey_password }}" RPOP mylist
244+
245+
- name: Generate workload on primary nodes (latency metrics)
246+
community.docker.docker_container_exec:
247+
container: "{{ item }}"
248+
command: >-
249+
bash -c "for i in $(seq 1 50); do valkey-cli -a {{ valkey_password }} SET k$i v$i >/dev/null; valkey-cli -a {{ valkey_password }} GET k$i >/dev/null; valkey-cli -a {{ valkey_password }} HSET h$i f v >/dev/null; valkey-cli -a {{ valkey_password }} LPUSH l$i a b c >/dev/null; valkey-cli -a {{ valkey_password }} RPUSH l$i d e f >/dev/null; valkey-cli -a {{ valkey_password }} LRANGE l$i 0 -1 >/dev/null; valkey-cli -a {{ valkey_password }} LPOP l$i >/dev/null || true; valkey-cli -a {{ valkey_password }} RPOP l$i >/dev/null || true; done"
250+
loop: "{{ primary_nodes }}"
251+
ignore_errors: yes
252+
253+
- name: Generate read workload on replica nodes (latency metrics)
254+
community.docker.docker_container_exec:
255+
container: "{{ item }}"
256+
command: >-
257+
bash -c "for i in $(seq 1 50); do valkey-cli -a {{ valkey_password }} GET k$i >/dev/null || true; valkey-cli -a {{ valkey_password }} LRANGE l$i 0 -1 >/dev/null || true; done"
258+
loop: "{{ replica_nodes }}"
259+
ignore_errors: yes

0 commit comments

Comments
 (0)