Skip to content

Commit

Permalink
Merge pull request #19 from soketi/feature/use-annotations-instead-of…
Browse files Browse the repository at this point in the history 
…-probes-token

[2.x] Removed probes tokens
  • Loading branch information
@rennokki
rennokki authored May 23, 2021
2 parents 407aeff + 2819dd5 commit b77b751
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 91 deletions.
1 change: 0 additions & 1 deletion .env.example
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
KUBE_CONNECTION=http
PROBES_TOKEN=probes-token
3 changes: 1 addition & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ Soketi is the service name for the [soketi/echo-server](https://github.com/soket

- PHP 8.0+
- Kubernetes v1.20.2 (optional; for Kubernetes-like testing)
- [Echo Server](https://github.com/soketi/echo-server) 4.2+
- [Echo Server](https://github.com/soketi/echo-server) 5.0.1+

## 🚀 Installation

Expand All @@ -37,7 +37,6 @@ $ php application network:watch
| - | - | - | - |
| `POD_NAMESPACE` | `--pod-namespace` | `default` | The Pod namespce to watch. |
| `POD_NAME` | `--pod-name` | `some-pod` | The Pod name to watch. |
| `PROBES_TOKEN` | `--probes-token` | `probes-token` | The token used for the [probes API](https://github.com/soketi/echo-server/blob/master/docs/ENV.md#probes-api). |
| `ECHO_APP_PORT` | `--echo-app-port` | `6001` | The port number for the [Echo Server](https://github.com/soketi/echo-server) app. |
| `MEMORY_PERCENT` | `--memory-percent` | `75` | The threshold (in percent) that, once reached, the Pod will be marked as "not ready" to evict any new connections or requests. |
| `CHECKING_INTERVAL` | `--checking-interval` | `1` | The amount of seconds to wait between API checks. |
Expand Down
123 changes: 87 additions & 36 deletions app/Commands/WatchNetworkCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ class WatchNetworkCommand extends Command
protected $signature = 'network:watch
{--pod-namespace=default : The Pod namespace. Defaults to the current Pod namespace.}
{--pod-name=some-pod : The Pod name to watch. Defaults to the current Pod name.}
{--probes-token=probes-token : The Probes API token used to update the network probing status.}
{--echo-app-port=6001 : The Echo App socket port.}
{--memory-percent=75 : The threshold at which new connections close for a specific server.}
{--interval=1 : The interval in seconds between each checks.}
Expand All @@ -42,9 +41,10 @@ public function handle()
{
$this->line('Starting the watcher...');

$this->registerPodMacros();

$podNamespace = env('POD_NAMESPACE') ?: $this->option('pod-namespace');
$podName = env('POD_NAME') ?: $this->option('pod-name');
$probesToken = env('PROBES_TOKEN') ?: $this->option('probes-token');
$echoAppPort = env('ECHO_APP_PORT') ?: $this->option('echo-app-port');
$memoryThreshold = env('MEMORY_PERCENT') ?: $this->option('memory-percent');
$interval = env('CHECKING_INTERVAL') ?: $this->option('interval');
Expand All @@ -63,7 +63,7 @@ public function handle()
throw new Exception("Pod {$podNamespace}/{$podName} not found.");
}

$this->checkPod($pod, $memoryThreshold, $probesToken, $echoAppPort);
$this->checkPod($pod, $memoryThreshold, $echoAppPort);

sleep($interval);

Expand All @@ -84,68 +84,96 @@ public function schedule(Schedule $schedule): void
// $schedule->command(static::class)->everyMinute();
}

/**
* Register macros for the K8sPod instance.
*
* @return void
*/
protected function registerPodMacros(): void
{
K8sPod::macro('acceptsConnections', function () {
/** @var K8sPod $this */
return $this->getLabel('echo.soketi.app/accepts-new-connections', 'yes') === 'yes';
});

K8sPod::macro('rejectsConnections', function () {
/** @var K8sPod $this */
return $this->getLabel('echo.soketi.app/accepts-new-connections', 'yes') === 'no';
});

K8sPod::macro('acceptNewConnections', function () {
/** @var K8sPod $this */
$labels = array_merge($this->getLabels(), [
'echo.soketi.app/accepts-new-connections' => 'yes',
]);

$this->refresh()->setLabels($labels)->update();

return true;
});

K8sPod::macro('rejectNewConnections', function () {
/** @var K8sPod $this */
$labels = array_merge($this->getLabels(), [
'echo.soketi.app/accepts-new-connections' => 'no',
]);

$this->refresh()->setLabels($labels)->update();

return true;
});
}

/**
* Check the pod metrics to adjust new connection allowance.
*
* @param \RenokiCo\PhpK8s\K8sResources\K8sPod $pod
* @param int $memoryThreshold
* @param string $probesToken
* @param int $echoAppPort
* @return void
*/
protected function checkPod(K8sPod $pod, int $memoryThreshold, string $probesToken, int $echoAppPort): void
protected function checkPod(K8sPod $pod, int $memoryThreshold, int $echoAppPort): void
{
$memoryUsagePercentage = $this->getMemoryUsagePercentage($this->getPodMetrics($pod, $echoAppPort));
$rejectsNewConnections = $pod->getLabel('echo.soketi.app/rejects-new-connections', 'no');
$memoryUsagePercentage = $this->getMemoryUsagePercentage($this->getEchoServerMetrics($echoAppPort));
$dateTime = now()->toDateTimeString();

$this->line("[{$dateTime}] Current memory usage is {$memoryUsagePercentage}%. Checking...", null, 'v');

if ($memoryUsagePercentage >= $memoryThreshold) {
if ($rejectsNewConnections === 'no') {
if ($pod->acceptsConnections()) {
$this->info("[{$dateTime}] Pod now rejects connections.");
$this->info("[{$dateTime}] Echo container uses {$memoryUsagePercentage}%, threshold is {$memoryThreshold}%");

$this->rejectNewConnections($pod, $probesToken, $echoAppPort);
$pod->rejectNewConnections();
}
} else {
if ($rejectsNewConnections === 'yes') {
if ($pod->rejectsConnections()) {
$this->info("[{$dateTime}] Pod now accepts connections.");
$this->info("[{$dateTime}] Echo container uses {$memoryUsagePercentage}%, threshold is {$memoryThreshold}%");

$this->acceptNewConnections($pod, $probesToken, $echoAppPort);
$pod->acceptNewConnections();
}
}
}

protected function getPodMetrics(K8sPod $pod, int $echoAppPort): array
/**
* Get the pod metrics from Prometheus.
*
* @param int $echoAppPort
* @return array
*/
protected function getEchoServerMetrics(int $echoAppPort): array
{
return Http::get("http://localhost:{$echoAppPort}/metrics?json=1")->json()['data'] ?? [];
}

protected function rejectNewConnections(K8sPod $pod, string $probesToken, int $echoAppPort): void
{
Http::post("http://localhost:{$echoAppPort}/probes/reject-new-connections?token={$probesToken}");

$this->updatePodLabels($pod, ['echo.soketi.app/rejects-new-connections' => 'yes']);
}

protected function acceptNewConnections(K8sPod $pod, string $probesToken, int $echoAppPort): void
{
Http::post("http://localhost:{$echoAppPort}/probes/accept-new-connections?token={$probesToken}");

$this->updatePodLabels($pod, ['echo.soketi.app/rejects-new-connections' => 'no']);
}

protected function updatePodLabels(K8sPod $pod, array $newLabels = []): K8sPod
{
$labels = array_merge($pod->getLabels(), $newLabels);

$pod->setLabels($labels)->update();

return $pod;
}

/**
* Get the memory usage as percentage,
* based on the given metrics from Prometheus.
*
* @param array $metrics
* @return float
*/
protected function getMemoryUsagePercentage(array $metrics): float
{
$totalMemoryBytes = $this->getTotalMemoryBytes($metrics);
Expand All @@ -157,19 +185,42 @@ protected function getMemoryUsagePercentage(array $metrics): float
return $this->getUsedMemoryBytes($metrics) * 100 / $totalMemoryBytes;
}

/**
* Get the total amount of memory allocated to the Echo Server container,
* based on the given metrics from Prometheus.
*
* @param array $metrics
* @return int
*/
protected function getTotalMemoryBytes(array $metrics): int
{
return $this->getMetricValue($metrics, 'echo_server_process_virtual_memory_bytes');
}

/**
* Get the total amount of memory that's being used by the Echo Server container,
* based on the given metrics from Prometheus.
*
* @param array $metrics
* @return int
*/
protected function getUsedMemoryBytes(array $metrics): int
{
return $this->getMetricValue($metrics, 'echo_server_nodejs_external_memory_bytes') +
$this->getMetricValue($metrics, 'echo_server_process_resident_memory_bytes');
}

/**
* Get the Prometheus metric value from the list of metrics.
*
* @param array $metrics
* @param string $name
* @return int
*/
protected function getMetricValue(array $metrics, string $name): int
{
return collect($metrics)->where('name', $name)->first()['values'][0]['value'] ?? 0;
return collect($metrics)->first(function ($metric) use ($name) {
return $metric['name'] === $name;
})['values'][0]['value'] ?? 0;
}
}
53 changes: 16 additions & 37 deletions tests/Feature/NetworkWatchTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

namespace Tests\Feature;

use Illuminate\Http\Client\Request;
use Illuminate\Support\Facades\Http;
use RenokiCo\LaravelK8s\LaravelK8sFacade as LaravelK8s;
use Tests\TestCase;
Expand All @@ -21,36 +20,26 @@ public function test_watch_pod_rejecting_connections()

$pod = $deployment->getPods()->first();

Http::fakeSequence()
->push([
'data' => [
['name' => 'echo_server_process_virtual_memory_bytes', 'values' => [['value' => 104857600]]], // 100 MB
['name' => 'echo_server_process_resident_memory_bytes', 'values' => [['value' => 83886080]]], // 80 MB @ 80% usage
['name' => 'echo_server_nodejs_external_memory_bytes', 'values' => [['value' => 0]]],
],
])
->push(['acknowledged' => true]);
Http::fakeSequence()->push([
'data' => [
['name' => 'echo_server_process_virtual_memory_bytes', 'values' => [['value' => 104857600]]], // 100 MB
['name' => 'echo_server_process_resident_memory_bytes', 'values' => [['value' => 83886080]]], // 80 MB @ 80% usage
['name' => 'echo_server_nodejs_external_memory_bytes', 'values' => [['value' => 0]]],
],
]);

$this->artisan('network:watch', [
'--pod-namespace' => 'default',
'--pod-name' => $pod->getName(),
'--probes-token' => 'probes-token',
'--echo-app-port' => 6001,
'--memory-percent' => 80,
'--interval' => 1,
'--test' => true,
]);

Http::assertSent(function (Request $request) {
return in_array($request->url(), [
'http://localhost:6001/metrics?json=1',
'http://localhost:6001/probes/reject-new-connections?token=probes-token',
]);
});

$pod->refresh();

$this->assertEquals('yes', $pod->getLabel('echo.soketi.app/rejects-new-connections'));
$this->assertEquals('no', $pod->getLabel('echo.soketi.app/accepts-new-connections'));
}

public function test_watch_pod_accepting_connections()
Expand All @@ -65,35 +54,25 @@ public function test_watch_pod_accepting_connections()

$pod = $deployment->getPods()->first();

Http::fakeSequence()
->push([
'data' => [
['name' => 'echo_server_process_virtual_memory_bytes', 'values' => [['value' => 104857600]]], // 100 MB
['name' => 'echo_server_process_resident_memory_bytes', 'values' => [['value' => 83886080]]], // 80 MB @ 80% usage
['name' => 'echo_server_nodejs_external_memory_bytes', 'values' => [['value' => 0]]],
],
])
->push(['acknowledged' => true]);
Http::fakeSequence()->push([
'data' => [
['name' => 'echo_server_process_virtual_memory_bytes', 'values' => [['value' => 104857600]]], // 100 MB
['name' => 'echo_server_process_resident_memory_bytes', 'values' => [['value' => 83886080]]], // 80 MB @ 80% usage
['name' => 'echo_server_nodejs_external_memory_bytes', 'values' => [['value' => 0]]],
],
]);

$this->artisan('network:watch', [
'--pod-namespace' => 'default',
'--pod-name' => $pod->getName(),
'--probes-token' => 'probes-token',
'--echo-app-port' => 6001,
'--memory-percent' => 90,
'--interval' => 1,
'--test' => true,
]);

Http::assertSent(function (Request $request) {
return in_array($request->url(), [
'http://localhost:6001/metrics?json=1',
'http://localhost:6001/probes/accept-new-connections?token=probes-token',
]);
});

$pod->refresh();

$this->assertEquals('no', $pod->getLabel('echo.soketi.app/rejects-new-connections'));
$this->assertEquals('yes', $pod->getLabel('echo.soketi.app/accepts-new-connections'));
}
}
16 changes: 1 addition & 15 deletions tests/fixtures/echo.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,3 @@
apiVersion: v1
kind: Secret
metadata:
name: echo-server-test-tokens
stringData:
PROBES_TOKEN: probes-token
---
apiVersion: apps/v1
kind: Deployment
metadata:
Expand All @@ -27,21 +20,14 @@ spec:
spec:
containers:
- name: echo
image: soketi/echo-server:4.3.0-14.16-alpine
image: soketi/echo-server:5.0-14-alpine
env:
- name: PRESENCE_STORAGE_DATABASE
value: socket
- name: PROMETHEUS_ENABLED
value: "1"
- name: STATS_DRIVER
value: "prometheus"
- name: NETWORK_PROBES_API_ENABLED
value: "1"
- name: NETWORK_PROBES_API_TOKEN
valueFrom:
secretKeyRef:
name: echo-server-test-tokens
key: PROBES_TOKEN
command:
- node
- --max-old-space-size=256
Expand Down

0 comments on commit b77b751

Please sign in to comment.